terdut-server/internal/api/deadman.go

package api

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"
)

// deadmanGroupPrefix namespaces the incidents this file opens. Alertmanager
// group keys always contain braces, so this can never collide with one, and the
// partial unique index on open group_key (see 008_incidents.sql) gives one open
// incident per switch for free.
const deadmanGroupPrefix = "deadman:"

// DeadmanMatcher selects the alerts that are heartbeats rather than problems.
// Every condition has to match, and Name — the alertname label — is mandatory:
// it is what lets the sweeper find candidate rows through alerts_name_idx
// instead of JSON-extracting labels from every row in the table.
type DeadmanMatcher struct {
	Name   string
	Labels map[string]string
}

// String renders the matcher the way it was configured, which is also how it
// reads in an incident title.
func (m DeadmanMatcher) String() string {
	if len(m.Labels) == 0 {
		return m.Name
	}
	parts := make([]string, 0, len(m.Labels))
	for k, v := range m.Labels {
		parts = append(parts, k+"="+v)
	}
	sort.Strings(parts)
	return m.Name + " (" + strings.Join(parts, ", ") + ")"
}

// config renders the matcher in the form parseDeadmanMatcher reads, which is
// what a switch row stores: `alertname=Watchdog,cluster=prod`.
func (m DeadmanMatcher) config() string {
	parts := make([]string, 0, len(m.Labels))
	for k, v := range m.Labels {
		parts = append(parts, k+"="+v)
	}
	sort.Strings(parts)
	return strings.Join(append([]string{"alertname=" + m.Name}, parts...), ",")
}

// matches reports whether an alert's labels satisfy every condition.
func (m DeadmanMatcher) matches(labels map[string]string) bool {
	if labels["alertname"] != m.Name {
		return false
	}
	for k, v := range m.Labels {
		if labels[k] != v {
			return false
		}
	}
	return true
}

// DeadmanConfig is the server-wide default a team's switches are seeded from:
// the environment's matchers, timeout and severity. Switches themselves are rows
// of a team's own — see DeadmanSwitch — and this is only how a fresh install
// starts out.
type DeadmanConfig struct {
	Matchers []DeadmanMatcher

	// Timeout is how long a matched alert may go without a refreshing webhook
	// before it is declared dead. It must be shorter than Alertmanager's
	// repeat_interval for the heartbeat's route, which is what refreshes it.
	// Zero disables dead man's switch handling entirely.
	Timeout time.Duration

	// Severity is the severity every dead man's switch incident opens at. These
	// incidents have no member alerts to derive one from, and the heartbeat's
	// own severity label is meaningless — Watchdog ships as "none".
	Severity string
}

// enabled reports whether there is anything to watch.
func (c DeadmanConfig) enabled() bool { return c.Timeout > 0 && len(c.Matchers) > 0 }

// DeadmanSwitch inverts the handling of the alerts it matches: receiving one
// opens nothing, and the absence of one opens an incident.
//
// The unit of monitoring is the fingerprint, not the switch — two clusters
// sending the same heartbeat alertname are two independent heartbeats under one
// switch, so one healthy cluster cannot mask a dead one.
type DeadmanSwitch struct {
	ID      int64
	Name    string
	Matcher DeadmanMatcher

	// Timeout is how long a heartbeat may go unheard before it is declared dead.
	Timeout time.Duration

	// Severity is what the incident opens at.
	Severity string
}

// deadmanSet is one team's switches.
type deadmanSet []DeadmanSwitch

// match returns the first switch an alert satisfies.
func (d deadmanSet) match(labels map[string]string) (DeadmanSwitch, bool) {
	for _, sw := range d {
		if sw.Matcher.matches(labels) {
			return sw, true
		}
	}
	return DeadmanSwitch{}, false
}

// isDeadman is match without the switch, for the ingest path.
func (d deadmanSet) isDeadman(labels map[string]string) bool {
	_, ok := d.match(labels)
	return ok
}

// names lists the distinct alertnames worth loading from the database.
func (d deadmanSet) names() []string {
	seen := map[string]bool{}
	out := make([]string, 0, len(d))
	for _, sw := range d {
		if !seen[sw.Matcher.Name] {
			seen[sw.Matcher.Name] = true
			out = append(out, sw.Matcher.Name)
		}
	}
	return out
}

// parseDeadmanMatcher reads one matcher from its configured form: "," separates
// the conditions and "=" is exact label equality — `alertname=Watchdog,cluster=prod`.
// The error says what is wrong with it, in words a form can show.
func parseDeadmanMatcher(entry string) (DeadmanMatcher, error) {
	m := DeadmanMatcher{Labels: map[string]string{}}
	for _, cond := range strings.Split(strings.TrimSpace(entry), ",") {
		k, v, ok := strings.Cut(cond, "=")
		k, v = strings.TrimSpace(k), strings.TrimSpace(v)
		if !ok || k == "" || v == "" {
			return DeadmanMatcher{}, fmt.Errorf("%q is not label=value", strings.TrimSpace(cond))
		}
		if k == "alertname" {
			m.Name = v
			continue
		}
		m.Labels[k] = v
	}
	if m.Name == "" {
		return DeadmanMatcher{}, errors.New("no alertname condition")
	}
	return m, nil
}

// ParseDeadmanConfig reads the matcher list from its configured form:
// ";" separates matchers, and each is parsed as parseDeadmanMatcher does.
//
// A malformed or alertname-less entry is dropped rather than fatal, following
// config.duration's rule that one bad tuning knob should not take the server
// down. Silence would be worse here than elsewhere, though — a typo that
// disarms the switch is exactly the failure this feature exists to catch — so
// the matchers that survived are logged.
func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string) DeadmanConfig {
	cfg := DeadmanConfig{Timeout: timeout, Severity: severity}

	for _, entry := range strings.Split(matchers, ";") {
		entry = strings.TrimSpace(entry)
		if entry == "" {
			continue
		}
		m, err := parseDeadmanMatcher(entry)
		if err != nil {
			log.Printf("deadman: ignoring matcher %q: %v", entry, err)
			continue
		}
		cfg.Matchers = append(cfg.Matchers, m)
	}

	switch {
	case timeout <= 0:
		log.Print("deadman: disabled (timeout is zero)")
	case len(cfg.Matchers) == 0:
		log.Print("deadman: disabled (no usable matchers)")
	default:
		rendered := make([]string, 0, len(cfg.Matchers))
		for _, m := range cfg.Matchers {
			rendered = append(rendered, m.String())
		}
		log.Printf("deadman: default for new teams: %s, timeout %s, severity %s",
			strings.Join(rendered, "; "), timeout, severity)
	}
	return cfg
}

// deadmanAlert is one heartbeat: the alert row carrying its last sighting, and
// the switch that claimed it.
type deadmanAlert struct {
	id          int64
	teamID      int64
	fingerprint string
	labels      map[string]string
	sw          DeadmanSwitch
	resolved    bool
	receivedAt  int64
}

// dead is the one rule for a silent heartbeat, shared by the sweeper that pages
// on it and the status the Switches page shows, so the page cannot disagree
// with the pager.
//
// An explicit resolved from Alertmanager is a stronger death signal than mere
// absence: the sender is telling us the heartbeat stopped, so there is nothing
// left to wait out.
func (a deadmanAlert) dead(now time.Time) bool {
	return a.resolved || a.receivedAt < now.Add(-a.sw.Timeout).Unix()
}

// groupKey is the switch's identity as an incident. Per fingerprint, so each
// source is tracked on its own.
func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerprint }

// sweepDeadman is the whole point of the feature: it opens an incident for every
// switch that has stopped chirping, and closes one whose switch came back.
//
// It returns the ids of the alerts it owns, because the generic staleness
// expiry must leave them alone — staleAfter and ends_at would otherwise resolve
// a heartbeat long before its own, much tighter, timeout ever fired.
// Each team is swept against its own switches, each with its own matcher,
// timeout and severity. A team watching nothing is skipped entirely, which is
// most of them.
func sweepDeadman(ctx context.Context, db *sql.DB, notify NotifyConfig) map[int64]bool {
	owned := map[int64]bool{}

	configs, err := deadmanSets(ctx, db)
	if err != nil {
		log.Printf("deadman: load configs: %v", err)
		return owned
	}

	now := time.Now()
	for teamID, cfg := range configs {
		heartbeats, err := deadmanAlerts(ctx, db, teamID, cfg)
		if err != nil {
			log.Printf("deadman: load heartbeats for team %d: %v", teamID, err)
			continue
		}

		for _, hb := range heartbeats {
			owned[hb.id] = true

			if hb.dead(now) {
				if err := deadmanDied(ctx, db, notify, hb, now); err != nil {
					log.Printf("deadman: open incident for %s: %v", hb.sw.Matcher.Name, err)
				}
				continue
			}
			if err := deadmanRecovered(ctx, db, hb); err != nil {
				log.Printf("deadman: resolve incident for %s: %v", hb.sw.Matcher.Name, err)
			}
		}
	}
	return owned
}

// deadmanAlerts loads every alert row that one of a team's switches claims. The candidate query
// is narrowed by alertname so it rides alerts_name_idx; the rest of the matching
// happens in Go, which keeps one implementation of the rules. The rows are read
// in full before the caller writes, so the writes do not run against an open
// cursor over the same table.
func deadmanAlerts(ctx context.Context, db *sql.DB, teamID int64, cfg deadmanSet) ([]deadmanAlert, error) {
	names := cfg.names()
	args := &sqlArgs{}
	nameList := make([]any, len(names))
	for i, n := range names {
		nameList[i] = n
	}

	rows, err := db.QueryContext(ctx, `
		SELECT id, team_id, fingerprint, labels, status, received_at
		FROM   alerts
		WHERE  team_id = `+args.add(teamID)+`
		  AND  name IN (`+args.addList(nameList)+`)
		  AND  archived_at IS NULL`, args.all()...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var out []deadmanAlert
	for rows.Next() {
		var a deadmanAlert
		var labelsJSON, status string
		if err := rows.Scan(&a.id, &a.teamID, &a.fingerprint, &labelsJSON, &status, &a.receivedAt); err != nil {
			return nil, err
		}
		json.Unmarshal([]byte(labelsJSON), &a.labels) //nolint:errcheck

		sw, ok := cfg.match(a.labels)
		if !ok {
			continue
		}
		a.sw = sw
		a.resolved = status == "resolved"
		out = append(out, a)
	}
	return out, rows.Err()
}

// deadmanDied raises the incident for a switch that has gone quiet.
//
// Two conditions gate it, and both matter. There must be no open incident for
// the switch already — the partial unique index enforces that anyway, but a
// second one would be a wasted page. And the heartbeat must have been seen since
// the last incident was raised, which is the re-arm rule: resolving a dead man's
// switch incident sticks, exactly as resolving an alert-backed one does (see
// incidentForGroup), and a source that is gone for good is a one-time page
// rather than a nag. Only a heartbeat that comes back and dies again earns a new
// incident.
func deadmanDied(ctx context.Context, db *sql.DB, notify NotifyConfig, hb deadmanAlert, now time.Time) error {
	var lastTriggered, open int64
	if err := db.QueryRowContext(ctx, `
		SELECT COALESCE(MAX(triggered_at), 0),
		       COUNT(*) FILTER (WHERE resolved_at IS NULL)
		FROM   incidents WHERE team_id = $1 AND group_key = $2`,
		hb.teamID, hb.groupKey()).Scan(&lastTriggered, &open); err != nil {
		return err
	}
	if open > 0 || hb.receivedAt <= lastTriggered {
		return nil
	}

	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback() //nolint:errcheck

	// A heartbeat nobody has heard from is not firing, and saying otherwise in
	// the alert list would be a lie. An Alertmanager-sourced resolution keeps its
	// own source: it told us the truth first.
	if !hb.resolved {
		if _, err := tx.ExecContext(ctx, `
			UPDATE alerts
			   SET status            = 'resolved',
			       resolution_source = $1,
			       ends_at           = COALESCE(ends_at, `+nowEpoch+`)
			 WHERE id = $2 AND status = 'firing'`, resolutionDeadman, hb.id); err != nil {
			return err
		}
	}

	severity := hb.sw.Severity
	var sev *string
	if severity != "" {
		sev = &severity
	}

	// The incident opens in the team whose integration received the heartbeat:
	// the switch belongs to whoever is watching that source, not to the install.
	incidentID, err := openIncident(ctx, tx, notify, hb.teamID, hb.groupKey(),
		"No heartbeat from "+hb.sw.Matcher.String(), hb.labels, sev)
	if err != nil {
		return err
	}

	alertID := hb.id
	detail := "last heartbeat " + humanDuration(now.Sub(time.Unix(hb.receivedAt, 0))) + " ago"
	if err := logEvent(ctx, tx, incidentID, evDeadmanSilent, nil, &alertID, &detail); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}
	log.Printf("deadman: %s went silent, opened incident %d", hb.sw.Matcher.String(), incidentID)
	return nil
}

// deadmanRecovered closes the incident for a switch that started chirping again.
//
// It cannot go through resolveIfSettled: a dead man's switch incident has no
// member alerts (linking the heartbeat would have the settled-incident cascade
// close it on the very same sweep that opened it), so the alert-driven cascade
// ignores it entirely and recovery is the only automatic way out.
func deadmanRecovered(ctx context.Context, db *sql.DB, hb deadmanAlert) error {
	var incidentID int64
	switch err := db.QueryRowContext(ctx, `
		SELECT id FROM incidents
		 WHERE team_id = $1 AND group_key = $2 AND resolved_at IS NULL`,
		hb.teamID, hb.groupKey()).Scan(&incidentID); {
	case err == sql.ErrNoRows:
		return nil
	case err != nil:
		return err
	}

	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback() //nolint:errcheck

	if _, err := tx.ExecContext(ctx, `
		UPDATE incidents
		   SET status = 'resolved', resolved_at = $1, resolution_source = $2
		 WHERE id = $3 AND resolved_at IS NULL`,
		time.Now().Unix(), incidentResolutionRecovered, incidentID); err != nil {
		return err
	}
	if err := logEvent(ctx, tx, incidentID, evResolved, nil, nil, nil); err != nil {
		return err
	}
	// The all-clear goes to whoever was paged, which enqueueResolved works out
	// from the incident's own notification history.
	if err := enqueueResolved(ctx, tx, incidentID); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}
	log.Printf("deadman: %s is back, resolved incident %d", hb.sw.Matcher.String(), incidentID)
	return nil
}

// ---------------------------------------------------------------------------
// A team's switches
// ---------------------------------------------------------------------------

const deadmanSwitchColumns = "id, team_id, name, matcher, timeout_seconds, severity"

// scanDeadmanSwitches reads switch rows into per-team sets. A row whose matcher
// no longer parses is skipped rather than fatal: the API refuses to store one,
// so it can only mean a hand edit, and one bad row must not stop the others
// from being watched.
func scanDeadmanSwitches(rows *sql.Rows) (map[int64]deadmanSet, error) {
	defer rows.Close()
	out := map[int64]deadmanSet{}
	for rows.Next() {
		var sw DeadmanSwitch
		var teamID, timeout int64
		var matcher string
		if err := rows.Scan(&sw.ID, &teamID, &sw.Name, &matcher, &timeout, &sw.Severity); err != nil {
			return nil, err
		}
		m, err := parseDeadmanMatcher(matcher)
		if err != nil {
			log.Printf("deadman: switch %d has an unusable matcher %q: %v", sw.ID, matcher, err)
			continue
		}
		sw.Matcher = m
		sw.Timeout = time.Duration(timeout) * time.Second
		out[teamID] = append(out[teamID], sw)
	}
	return out, rows.Err()
}

// deadmanSetForTeam reads one team's switches. A team with none gets an empty
// set — which is the right answer rather than an error: most teams watch no
// heartbeat at all.
func deadmanSetForTeam(ctx context.Context, q querier, teamID int64) (deadmanSet, error) {
	rows, err := q.QueryContext(ctx,
		"SELECT "+deadmanSwitchColumns+" FROM deadman_switches WHERE team_id = $1 ORDER BY id", teamID)
	if err != nil {
		return nil, err
	}
	sets, err := scanDeadmanSwitches(rows)
	return sets[teamID], err
}

// deadmanSets reads every team's switches in one query, for the sweeper.
func deadmanSets(ctx context.Context, db *sql.DB) (map[int64]deadmanSet, error) {
	rows, err := db.QueryContext(ctx,
		"SELECT "+deadmanSwitchColumns+" FROM deadman_switches ORDER BY id")
	if err != nil {
		return nil, err
	}
	return scanDeadmanSwitches(rows)
}

// deadmanSeededKey is the settings row that records the environment defaults
// were handed out. Without it, a team that deleted its last switch would get
// the default back on the next restart.
const deadmanSeededKey = "deadman_seeded"

// SeedDeadmanConfigs gives every team the server's environment defaults as
// switches, exactly once per install, so a fresh install watches Watchdog
// without anybody setting it up.
//
// Once seeded it never runs again: a team's switches are its own, and a redeploy
// must not quietly put the environment's value back over an owner's edit or
// deletion. Installs that upgraded from per-team configuration were already
// seeded, which migration 009 records.
//
// A team created after that gets none and watches nothing until its owner says
// otherwise. That is deliberate: inheriting an install-wide heartbeat would page
// a new team about a source it has never heard of, and a switch nobody chose is
// the kind that gets muted rather than fixed.
func SeedDeadmanConfigs(ctx context.Context, db *sql.DB, cfg DeadmanConfig) error {
	if !cfg.enabled() {
		return nil
	}

	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback() //nolint:errcheck

	res, err := tx.ExecContext(ctx,
		"INSERT INTO settings (key, value) VALUES ($1, '1') ON CONFLICT (key) DO NOTHING",
		deadmanSeededKey)
	if err != nil {
		return err
	}
	if n, _ := res.RowsAffected(); n == 0 {
		return nil
	}

	for _, m := range cfg.Matchers {
		if _, err := tx.ExecContext(ctx, `
			INSERT INTO deadman_switches (team_id, name, matcher, timeout_seconds, severity)
			SELECT id, $1, $1, $2, $3 FROM teams`,
			m.config(), int64(cfg.Timeout.Seconds()), cfg.Severity); err != nil {
			return err
		}
	}
	return tx.Commit()
}

// ---------------------------------------------------------------------------
// Status
// ---------------------------------------------------------------------------

const (
	switchHealthy = "healthy"
	switchDead    = "dead"
	switchDormant = "dormant"
)

// deadmanSource is one heartbeat under a switch: a fingerprint that matched.
type deadmanSource struct {
	Fingerprint     string            `json:"fingerprint"`
	Labels          map[string]string `json:"labels"`
	Status          string            `json:"status"`
	LastHeartbeatAt time.Time         `json:"last_heartbeat_at"`
	LastTriggeredAt *time.Time        `json:"last_triggered_at"`
	IncidentID      *int64            `json:"incident_id"`
}

// deadmanSwitchStatus is a switch as the Switches page shows it.
type deadmanSwitchStatus struct {
	ID             int64  `json:"id"`
	Name           string `json:"name"`
	Matcher        string `json:"matcher"`
	TimeoutSeconds int64  `json:"timeout_seconds"`
	Severity       string `json:"severity"`

	// Status is dead when any source is, dormant when none has ever been heard
	// from, healthy otherwise — a live cluster must not hide a dead one.
	Status          string          `json:"status"`
	LastHeartbeatAt *time.Time      `json:"last_heartbeat_at"`
	LastTriggeredAt *time.Time      `json:"last_triggered_at"`
	OpenIncidentID  *int64          `json:"open_incident_id"`
	Sources         []deadmanSource `json:"sources"`
}

// deadmanStatuses reports every switch of a team with what its heartbeats are
// doing. The liveness verdict is deadmanAlert.dead, the sweeper's own.
func deadmanStatuses(ctx context.Context, db *sql.DB, teamID int64, set deadmanSet, now time.Time) ([]deadmanSwitchStatus, error) {
	out := make([]deadmanSwitchStatus, 0, len(set))
	if len(set) == 0 {
		return out, nil
	}

	heartbeats, err := deadmanAlerts(ctx, db, teamID, set)
	if err != nil {
		return nil, err
	}

	// One query for every switch's incident history, keyed the way the sweeper
	// keys it.
	type history struct {
		triggeredAt int64
		openID      int64
	}
	incidents := map[string]history{}
	rows, err := db.QueryContext(ctx, `
		SELECT group_key, MAX(triggered_at), COALESCE(MAX(id) FILTER (WHERE resolved_at IS NULL), 0)
		FROM   incidents
		WHERE  team_id = $1 AND group_key LIKE $2
		GROUP  BY group_key`, teamID, deadmanGroupPrefix+"%")
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		var key string
		var h history
		if err := rows.Scan(&key, &h.triggeredAt, &h.openID); err != nil {
			return nil, err
		}
		incidents[key] = h
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	bySwitch := map[int64][]deadmanAlert{}
	for _, hb := range heartbeats {
		bySwitch[hb.sw.ID] = append(bySwitch[hb.sw.ID], hb)
	}

	later := func(cur *time.Time, unix int64) *time.Time {
		t := time.Unix(unix, 0).UTC()
		if cur == nil || t.After(*cur) {
			return &t
		}
		return cur
	}

	for _, sw := range set {
		st := deadmanSwitchStatus{
			ID: sw.ID, Name: sw.Name, Matcher: sw.Matcher.config(),
			TimeoutSeconds: int64(sw.Timeout.Seconds()), Severity: sw.Severity,
			Status: switchDormant, Sources: []deadmanSource{},
		}

		for _, hb := range bySwitch[sw.ID] {
			src := deadmanSource{
				Fingerprint:     hb.fingerprint,
				Labels:          hb.labels,
				Status:          switchHealthy,
				LastHeartbeatAt: time.Unix(hb.receivedAt, 0).UTC(),
			}
			if hb.dead(now) {
				src.Status = switchDead
			}
			if h, ok := incidents[hb.groupKey()]; ok {
				t := time.Unix(h.triggeredAt, 0).UTC()
				src.LastTriggeredAt = &t
				st.LastTriggeredAt = later(st.LastTriggeredAt, h.triggeredAt)
				if h.openID != 0 {
					id := h.openID
					src.IncidentID = &id
					if st.OpenIncidentID == nil || id > *st.OpenIncidentID {
						st.OpenIncidentID = &id
					}
				}
			}
			st.LastHeartbeatAt = later(st.LastHeartbeatAt, hb.receivedAt)
			st.Sources = append(st.Sources, src)

			switch {
			case src.Status == switchDead:
				st.Status = switchDead
			case st.Status == switchDormant:
				st.Status = switchHealthy
			}
		}

		// Dead ones first, then by fingerprint: what needs attention leads, and
		// the order does not shuffle between refreshes.
		sort.Slice(st.Sources, func(i, j int) bool {
			a, b := st.Sources[i], st.Sources[j]
			if (a.Status == switchDead) != (b.Status == switchDead) {
				return a.Status == switchDead
			}
			return a.Fingerprint < b.Fingerprint
		})
		out = append(out, st)
	}
	return out, nil
}