terdut-server/internal/api/deadman.go

package api

import (
	"context"
	"database/sql"
	"encoding/json"
	"log"
	"sort"
	"strings"
	"time"
)

// deadmanGroupPrefix namespaces the incidents this file opens. Alertmanager
// group keys always contain braces, so this can never collide with one, and the
// partial unique index on open group_key (see 008_incidents.sql) gives one open
// incident per switch for free.
const deadmanGroupPrefix = "deadman:"

// DeadmanMatcher selects the alerts that are heartbeats rather than problems.
// Every condition has to match, and Name — the alertname label — is mandatory:
// it is what lets the sweeper find candidate rows through alerts_name_idx
// instead of JSON-extracting labels from every row in the table.
type DeadmanMatcher struct {
	Name   string
	Labels map[string]string
}

// String renders the matcher the way it was configured, which is also how it
// reads in an incident title.
func (m DeadmanMatcher) String() string {
	if len(m.Labels) == 0 {
		return m.Name
	}
	parts := make([]string, 0, len(m.Labels))
	for k, v := range m.Labels {
		parts = append(parts, k+"="+v)
	}
	sort.Strings(parts)
	return m.Name + " (" + strings.Join(parts, ", ") + ")"
}

// matches reports whether an alert's labels satisfy every condition.
func (m DeadmanMatcher) matches(labels map[string]string) bool {
	if labels["alertname"] != m.Name {
		return false
	}
	for k, v := range m.Labels {
		if labels[k] != v {
			return false
		}
	}
	return true
}

// DeadmanConfig inverts the handling of the alerts it matches: receiving one
// opens nothing, and the absence of one opens an incident.
//
// The unit of monitoring is the fingerprint, not the matcher — two clusters
// sending the same heartbeat alertname are two independent switches, so one
// healthy cluster cannot mask a dead one.
type DeadmanConfig struct {
	Matchers []DeadmanMatcher

	// Timeout is how long a matched alert may go without a refreshing webhook
	// before it is declared dead. It must be shorter than Alertmanager's
	// repeat_interval for the heartbeat's route, which is what refreshes it.
	// Zero disables dead man's switch handling entirely.
	Timeout time.Duration

	// Severity is the severity every dead man's switch incident opens at. These
	// incidents have no member alerts to derive one from, and the heartbeat's
	// own severity label is meaningless — Watchdog ships as "none".
	Severity string
}

// enabled reports whether there is anything to watch.
func (c DeadmanConfig) enabled() bool { return c.Timeout > 0 && len(c.Matchers) > 0 }

// match returns the first matcher an alert satisfies.
func (c DeadmanConfig) match(labels map[string]string) (DeadmanMatcher, bool) {
	if !c.enabled() {
		return DeadmanMatcher{}, false
	}
	for _, m := range c.Matchers {
		if m.matches(labels) {
			return m, true
		}
	}
	return DeadmanMatcher{}, false
}

// isDeadman is match without the matcher, for the ingest path.
func (c DeadmanConfig) isDeadman(labels map[string]string) bool {
	_, ok := c.match(labels)
	return ok
}

// names lists the distinct alertnames worth loading from the database.
func (c DeadmanConfig) names() []string {
	seen := map[string]bool{}
	out := make([]string, 0, len(c.Matchers))
	for _, m := range c.Matchers {
		if !seen[m.Name] {
			seen[m.Name] = true
			out = append(out, m.Name)
		}
	}
	return out
}

// ParseDeadmanConfig reads the matcher list from its configured form:
// ";" separates matchers, "," separates the conditions within one, and "=" is
// exact label equality — `alertname=Watchdog,cluster=prod; alertname=Heartbeat`.
//
// A malformed or alertname-less entry is dropped rather than fatal, following
// config.duration's rule that one bad tuning knob should not take the server
// down. Silence would be worse here than elsewhere, though — a typo that
// disarms the switch is exactly the failure this feature exists to catch — so
// the matchers that survived are logged.
func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string) DeadmanConfig {
	cfg := DeadmanConfig{Timeout: timeout, Severity: severity}

	for _, entry := range strings.Split(matchers, ";") {
		entry = strings.TrimSpace(entry)
		if entry == "" {
			continue
		}

		m := DeadmanMatcher{Labels: map[string]string{}}
		malformed := false
		for _, cond := range strings.Split(entry, ",") {
			k, v, ok := strings.Cut(cond, "=")
			k, v = strings.TrimSpace(k), strings.TrimSpace(v)
			if !ok || k == "" || v == "" {
				log.Printf("deadman: ignoring matcher %q: %q is not label=value", entry, strings.TrimSpace(cond))
				malformed = true
				break
			}
			if k == "alertname" {
				m.Name = v
				continue
			}
			m.Labels[k] = v
		}
		if malformed {
			continue
		}
		if m.Name == "" {
			log.Printf("deadman: ignoring matcher %q: no alertname condition", entry)
			continue
		}
		cfg.Matchers = append(cfg.Matchers, m)
	}

	switch {
	case timeout <= 0:
		log.Print("deadman: disabled (timeout is zero)")
	case len(cfg.Matchers) == 0:
		log.Print("deadman: disabled (no usable matchers)")
	default:
		rendered := make([]string, 0, len(cfg.Matchers))
		for _, m := range cfg.Matchers {
			rendered = append(rendered, m.String())
		}
		log.Printf("deadman: watching %s, timeout %s, severity %s",
			strings.Join(rendered, "; "), timeout, severity)
	}
	return cfg
}

// deadmanAlert is one switch: the alert row carrying its last heartbeat.
type deadmanAlert struct {
	id          int64
	teamID      int64
	fingerprint string
	labels      map[string]string
	matcher     DeadmanMatcher
	resolved    bool
	receivedAt  int64
}

// groupKey is the switch's identity as an incident. Per fingerprint, so each
// source is tracked on its own.
func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerprint }

// sweepDeadman is the whole point of the feature: it opens an incident for every
// switch that has stopped chirping, and closes one whose switch came back.
//
// It returns the ids of the alerts it owns, because the generic staleness
// expiry must leave them alone — staleAfter and ends_at would otherwise resolve
// a heartbeat long before its own, much tighter, timeout ever fired.
// Each team is swept against its own configuration: its own matchers, its own
// timeout, its own severity. A team watching nothing is skipped entirely, which
// is most of them.
func sweepDeadman(ctx context.Context, db *sql.DB, notify NotifyConfig) map[int64]bool {
	owned := map[int64]bool{}

	configs, err := deadmanConfigs(ctx, db)
	if err != nil {
		log.Printf("deadman: load configs: %v", err)
		return owned
	}

	now := time.Now()
	for teamID, cfg := range configs {
		switches, err := deadmanAlerts(ctx, db, teamID, cfg)
		if err != nil {
			log.Printf("deadman: load switches for team %d: %v", teamID, err)
			continue
		}
		cutoff := now.Add(-cfg.Timeout).Unix()

		for _, sw := range switches {
			owned[sw.id] = true

			// An explicit resolved from Alertmanager is a stronger death signal
			// than mere absence: the sender is telling us the heartbeat
			// stopped, so there is nothing left to wait out.
			if sw.resolved || sw.receivedAt < cutoff {
				if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil {
					log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err)
				}
				continue
			}
			if err := deadmanRecovered(ctx, db, sw); err != nil {
				log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err)
			}
		}
	}
	return owned
}

// deadmanAlerts loads every alert row that a matcher claims. The candidate query
// is narrowed by alertname so it rides alerts_name_idx; the rest of the matching
// happens in Go, which keeps one implementation of the rules. The rows are read
// in full before the caller writes, so the writes do not run against an open
// cursor over the same table.
func deadmanAlerts(ctx context.Context, db *sql.DB, teamID int64, cfg DeadmanConfig) ([]deadmanAlert, error) {
	names := cfg.names()
	args := &sqlArgs{}
	nameList := make([]any, len(names))
	for i, n := range names {
		nameList[i] = n
	}

	rows, err := db.QueryContext(ctx, `
		SELECT id, team_id, fingerprint, labels, status, received_at
		FROM   alerts
		WHERE  team_id = `+args.add(teamID)+`
		  AND  name IN (`+args.addList(nameList)+`)
		  AND  archived_at IS NULL`, args.all()...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var out []deadmanAlert
	for rows.Next() {
		var a deadmanAlert
		var labelsJSON, status string
		if err := rows.Scan(&a.id, &a.teamID, &a.fingerprint, &labelsJSON, &status, &a.receivedAt); err != nil {
			return nil, err
		}
		json.Unmarshal([]byte(labelsJSON), &a.labels) //nolint:errcheck

		m, ok := cfg.match(a.labels)
		if !ok {
			continue
		}
		a.matcher = m
		a.resolved = status == "resolved"
		out = append(out, a)
	}
	return out, rows.Err()
}

// deadmanDied raises the incident for a switch that has gone quiet.
//
// Two conditions gate it, and both matter. There must be no open incident for
// the switch already — the partial unique index enforces that anyway, but a
// second one would be a wasted page. And the heartbeat must have been seen since
// the last incident was raised, which is the re-arm rule: resolving a dead man's
// switch incident sticks, exactly as resolving an alert-backed one does (see
// incidentForGroup), and a source that is gone for good is a one-time page
// rather than a nag. Only a heartbeat that comes back and dies again earns a new
// incident.
func deadmanDied(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig, sw deadmanAlert, now time.Time) error {
	var lastTriggered, open int64
	if err := db.QueryRowContext(ctx, `
		SELECT COALESCE(MAX(triggered_at), 0),
		       COUNT(*) FILTER (WHERE resolved_at IS NULL)
		FROM   incidents WHERE team_id = $1 AND group_key = $2`,
		sw.teamID, sw.groupKey()).Scan(&lastTriggered, &open); err != nil {
		return err
	}
	if open > 0 || sw.receivedAt <= lastTriggered {
		return nil
	}

	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback() //nolint:errcheck

	// A heartbeat nobody has heard from is not firing, and saying otherwise in
	// the alert list would be a lie. An Alertmanager-sourced resolution keeps its
	// own source: it told us the truth first.
	if !sw.resolved {
		if _, err := tx.ExecContext(ctx, `
			UPDATE alerts
			   SET status            = 'resolved',
			       resolution_source = $1,
			       ends_at           = COALESCE(ends_at, `+nowEpoch+`)
			 WHERE id = $2 AND status = 'firing'`, resolutionDeadman, sw.id); err != nil {
			return err
		}
	}

	severity := cfg.Severity
	var sev *string
	if severity != "" {
		sev = &severity
	}

	// The incident opens in the team whose integration received the heartbeat:
	// the switch belongs to whoever is watching that source, not to the install.
	incidentID, err := openIncident(ctx, tx, notify, sw.teamID, sw.groupKey(),
		"No heartbeat from "+sw.matcher.String(), sw.labels, sev)
	if err != nil {
		return err
	}

	alertID := sw.id
	detail := "last heartbeat " + humanDuration(now.Sub(time.Unix(sw.receivedAt, 0))) + " ago"
	if err := logEvent(ctx, tx, incidentID, evDeadmanSilent, nil, &alertID, &detail); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}
	log.Printf("deadman: %s went silent, opened incident %d", sw.matcher.String(), incidentID)
	return nil
}

// deadmanRecovered closes the incident for a switch that started chirping again.
//
// It cannot go through resolveIfSettled: a dead man's switch incident has no
// member alerts (linking the heartbeat would have the settled-incident cascade
// close it on the very same sweep that opened it), so the alert-driven cascade
// ignores it entirely and recovery is the only automatic way out.
func deadmanRecovered(ctx context.Context, db *sql.DB, sw deadmanAlert) error {
	var incidentID int64
	switch err := db.QueryRowContext(ctx, `
		SELECT id FROM incidents
		 WHERE team_id = $1 AND group_key = $2 AND resolved_at IS NULL`,
		sw.teamID, sw.groupKey()).Scan(&incidentID); {
	case err == sql.ErrNoRows:
		return nil
	case err != nil:
		return err
	}

	tx, err := db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback() //nolint:errcheck

	if _, err := tx.ExecContext(ctx, `
		UPDATE incidents
		   SET status = 'resolved', resolved_at = $1, resolution_source = $2
		 WHERE id = $3 AND resolved_at IS NULL`,
		time.Now().Unix(), incidentResolutionRecovered, incidentID); err != nil {
		return err
	}
	if err := logEvent(ctx, tx, incidentID, evResolved, nil, nil, nil); err != nil {
		return err
	}
	// The all-clear goes to whoever was paged, which enqueueResolved works out
	// from the incident's own notification history.
	if err := enqueueResolved(ctx, tx, incidentID); err != nil {
		return err
	}

	if err := tx.Commit(); err != nil {
		return err
	}
	log.Printf("deadman: %s is back, resolved incident %d", sw.matcher.String(), incidentID)
	return nil
}

// ---------------------------------------------------------------------------
// Per-team configuration
// ---------------------------------------------------------------------------

// deadmanConfigForTeam reads one team's switches. A team with no row, or with
// nothing configured, gets a disabled config — which is the right answer rather
// than an error: most teams watch no heartbeat at all.
func deadmanConfigForTeam(ctx context.Context, q querier, teamID int64) (DeadmanConfig, error) {
	var matchers, severity string
	var timeout int64
	err := q.QueryRowContext(ctx,
		"SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1",
		teamID).Scan(&matchers, &timeout, &severity)
	if err == sql.ErrNoRows {
		return DeadmanConfig{}, nil
	}
	if err != nil {
		return DeadmanConfig{}, err
	}
	return parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity), nil
}

// deadmanConfigs reads every team's switches in one query, for the sweeper.
func deadmanConfigs(ctx context.Context, db *sql.DB) (map[int64]DeadmanConfig, error) {
	rows, err := db.QueryContext(ctx,
		"SELECT team_id, matchers, timeout_seconds, severity FROM deadman_configs")
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	out := map[int64]DeadmanConfig{}
	for rows.Next() {
		var teamID, timeout int64
		var matchers, severity string
		if err := rows.Scan(&teamID, &matchers, &timeout, &severity); err != nil {
			return nil, err
		}
		cfg := parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity)
		if cfg.enabled() {
			out[teamID] = cfg
		}
	}
	return out, rows.Err()
}

// SeedDeadmanConfigs gives every team without a row the server's environment
// configuration, so the install that upgrades into per-team switches keeps
// watching exactly what it was watching before.
//
// Idempotent, and never overwrites: once a team has a row it owns its own
// configuration, and a redeploy must not quietly put the environment's value
// back over an owner's edit.
//
// A team created after startup gets no row and therefore watches nothing until
// its owner says otherwise. That is deliberate: inheriting an install-wide
// heartbeat would page a new team about a source it has never heard of, and a
// switch nobody chose is the kind that gets muted rather than fixed.
func SeedDeadmanConfigs(ctx context.Context, db *sql.DB, cfg DeadmanConfig) error {
	matchers := make([]string, 0, len(cfg.Matchers))
	for _, m := range cfg.Matchers {
		parts := []string{"alertname=" + m.Name}
		for k, v := range m.Labels {
			parts = append(parts, k+"="+v)
		}
		sort.Strings(parts[1:])
		matchers = append(matchers, strings.Join(parts, ","))
	}

	_, err := db.ExecContext(ctx, `
		INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity)
		SELECT id, $1, $2, $3 FROM teams
		ON CONFLICT (team_id) DO NOTHING`,
		strings.Join(matchers, "; "), int64(cfg.Timeout.Seconds()), cfg.Severity)
	return err
}

// parseDeadmanQuietly is ParseDeadmanConfig without the startup logging: a
// team's configuration is read on every sweep and every webhook, and logging it
// each time would bury everything else.
func parseDeadmanQuietly(matchers string, timeout time.Duration, severity string) DeadmanConfig {
	cfg := DeadmanConfig{Timeout: timeout, Severity: severity}
	for _, entry := range strings.Split(matchers, ";") {
		entry = strings.TrimSpace(entry)
		if entry == "" {
			continue
		}
		m := DeadmanMatcher{Labels: map[string]string{}}
		malformed := false
		for _, cond := range strings.Split(entry, ",") {
			k, v, ok := strings.Cut(cond, "=")
			k, v = strings.TrimSpace(k), strings.TrimSpace(v)
			if !ok || k == "" || v == "" {
				malformed = true
				break
			}
			if k == "alertname" {
				m.Name = v
				continue
			}
			m.Labels[k] = v
		}
		if malformed || m.Name == "" {
			continue
		}
		cfg.Matchers = append(cfg.Matchers, m)
	}
	return cfg
}