package api import ( "context" "database/sql" "encoding/json" "errors" "fmt" "log" "sort" "strings" "time" ) // deadmanGroupPrefix namespaces the incidents this file opens. Alertmanager // group keys always contain braces, so this can never collide with one, and the // partial unique index on open group_key (see 008_incidents.sql) gives one open // incident per switch for free. const deadmanGroupPrefix = "deadman:" // DeadmanMatcher selects the alerts that are heartbeats rather than problems. // Every condition has to match, and Name — the alertname label — is mandatory: // it is what lets the sweeper find candidate rows through alerts_name_idx // instead of JSON-extracting labels from every row in the table. type DeadmanMatcher struct { Name string Labels map[string]string } // String renders the matcher the way it was configured, which is also how it // reads in an incident title. func (m DeadmanMatcher) String() string { if len(m.Labels) == 0 { return m.Name } parts := make([]string, 0, len(m.Labels)) for k, v := range m.Labels { parts = append(parts, k+"="+v) } sort.Strings(parts) return m.Name + " (" + strings.Join(parts, ", ") + ")" } // config renders the matcher in the form parseDeadmanMatcher reads, which is // what a switch row stores: `alertname=Watchdog,cluster=prod`. func (m DeadmanMatcher) config() string { parts := make([]string, 0, len(m.Labels)) for k, v := range m.Labels { parts = append(parts, k+"="+v) } sort.Strings(parts) return strings.Join(append([]string{"alertname=" + m.Name}, parts...), ",") } // matches reports whether an alert's labels satisfy every condition. func (m DeadmanMatcher) matches(labels map[string]string) bool { if labels["alertname"] != m.Name { return false } for k, v := range m.Labels { if labels[k] != v { return false } } return true } // DeadmanConfig is the server-wide default a team's switches are seeded from: // the environment's matchers, timeout and severity. Switches themselves are rows // of a team's own — see DeadmanSwitch — and this is only how a fresh install // starts out. type DeadmanConfig struct { Matchers []DeadmanMatcher // Timeout is how long a matched alert may go without a refreshing webhook // before it is declared dead. It must be shorter than Alertmanager's // repeat_interval for the heartbeat's route, which is what refreshes it. // Zero disables dead man's switch handling entirely. Timeout time.Duration // Severity is the severity every dead man's switch incident opens at. These // incidents have no member alerts to derive one from, and the heartbeat's // own severity label is meaningless — Watchdog ships as "none". Severity string } // enabled reports whether there is anything to watch. func (c DeadmanConfig) enabled() bool { return c.Timeout > 0 && len(c.Matchers) > 0 } // DeadmanSwitch inverts the handling of the alerts it matches: receiving one // opens nothing, and the absence of one opens an incident. // // The unit of monitoring is the fingerprint, not the switch — two clusters // sending the same heartbeat alertname are two independent heartbeats under one // switch, so one healthy cluster cannot mask a dead one. type DeadmanSwitch struct { ID int64 Name string Matcher DeadmanMatcher // Timeout is how long a heartbeat may go unheard before it is declared dead. Timeout time.Duration // Severity is what the incident opens at. Severity string } // deadmanSet is one team's switches. type deadmanSet []DeadmanSwitch // match returns the first switch an alert satisfies. func (d deadmanSet) match(labels map[string]string) (DeadmanSwitch, bool) { for _, sw := range d { if sw.Matcher.matches(labels) { return sw, true } } return DeadmanSwitch{}, false } // isDeadman is match without the switch, for the ingest path. func (d deadmanSet) isDeadman(labels map[string]string) bool { _, ok := d.match(labels) return ok } // names lists the distinct alertnames worth loading from the database. func (d deadmanSet) names() []string { seen := map[string]bool{} out := make([]string, 0, len(d)) for _, sw := range d { if !seen[sw.Matcher.Name] { seen[sw.Matcher.Name] = true out = append(out, sw.Matcher.Name) } } return out } // parseDeadmanMatcher reads one matcher from its configured form: "," separates // the conditions and "=" is exact label equality — `alertname=Watchdog,cluster=prod`. // The error says what is wrong with it, in words a form can show. func parseDeadmanMatcher(entry string) (DeadmanMatcher, error) { m := DeadmanMatcher{Labels: map[string]string{}} for _, cond := range strings.Split(strings.TrimSpace(entry), ",") { k, v, ok := strings.Cut(cond, "=") k, v = strings.TrimSpace(k), strings.TrimSpace(v) if !ok || k == "" || v == "" { return DeadmanMatcher{}, fmt.Errorf("%q is not label=value", strings.TrimSpace(cond)) } if k == "alertname" { m.Name = v continue } m.Labels[k] = v } if m.Name == "" { return DeadmanMatcher{}, errors.New("no alertname condition") } return m, nil } // ParseDeadmanConfig reads the matcher list from its configured form: // ";" separates matchers, and each is parsed as parseDeadmanMatcher does. // // A malformed or alertname-less entry is dropped rather than fatal, following // config.duration's rule that one bad tuning knob should not take the server // down. Silence would be worse here than elsewhere, though — a typo that // disarms the switch is exactly the failure this feature exists to catch — so // the matchers that survived are logged. func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string) DeadmanConfig { cfg := DeadmanConfig{Timeout: timeout, Severity: severity} for _, entry := range strings.Split(matchers, ";") { entry = strings.TrimSpace(entry) if entry == "" { continue } m, err := parseDeadmanMatcher(entry) if err != nil { log.Printf("deadman: ignoring matcher %q: %v", entry, err) continue } cfg.Matchers = append(cfg.Matchers, m) } switch { case timeout <= 0: log.Print("deadman: disabled (timeout is zero)") case len(cfg.Matchers) == 0: log.Print("deadman: disabled (no usable matchers)") default: rendered := make([]string, 0, len(cfg.Matchers)) for _, m := range cfg.Matchers { rendered = append(rendered, m.String()) } log.Printf("deadman: default for new teams: %s, timeout %s, severity %s", strings.Join(rendered, "; "), timeout, severity) } return cfg } // deadmanAlert is one heartbeat: the alert row carrying its last sighting, and // the switch that claimed it. type deadmanAlert struct { id int64 teamID int64 fingerprint string labels map[string]string sw DeadmanSwitch resolved bool receivedAt int64 } // dead is the one rule for a silent heartbeat, shared by the sweeper that pages // on it and the status the Switches page shows, so the page cannot disagree // with the pager. // // An explicit resolved from Alertmanager is a stronger death signal than mere // absence: the sender is telling us the heartbeat stopped, so there is nothing // left to wait out. func (a deadmanAlert) dead(now time.Time) bool { return a.resolved || a.receivedAt < now.Add(-a.sw.Timeout).Unix() } // groupKey is the switch's identity as an incident. Per fingerprint, so each // source is tracked on its own. func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerprint } // sweepDeadman is the whole point of the feature: it opens an incident for every // switch that has stopped chirping, and closes one whose switch came back. // // It returns the ids of the alerts it owns, because the generic staleness // expiry must leave them alone — staleAfter and ends_at would otherwise resolve // a heartbeat long before its own, much tighter, timeout ever fired. // Each team is swept against its own switches, each with its own matcher, // timeout and severity. A team watching nothing is skipped entirely, which is // most of them. func sweepDeadman(ctx context.Context, db *sql.DB, notify NotifyConfig) map[int64]bool { owned := map[int64]bool{} configs, err := deadmanSets(ctx, db) if err != nil { log.Printf("deadman: load configs: %v", err) return owned } now := time.Now() for teamID, cfg := range configs { heartbeats, err := deadmanAlerts(ctx, db, teamID, cfg) if err != nil { log.Printf("deadman: load heartbeats for team %d: %v", teamID, err) continue } for _, hb := range heartbeats { owned[hb.id] = true if hb.dead(now) { if err := deadmanDied(ctx, db, notify, hb, now); err != nil { log.Printf("deadman: open incident for %s: %v", hb.sw.Matcher.Name, err) } continue } if err := deadmanRecovered(ctx, db, hb); err != nil { log.Printf("deadman: resolve incident for %s: %v", hb.sw.Matcher.Name, err) } } } return owned } // deadmanAlerts loads every alert row that one of a team's switches claims. The candidate query // is narrowed by alertname so it rides alerts_name_idx; the rest of the matching // happens in Go, which keeps one implementation of the rules. The rows are read // in full before the caller writes, so the writes do not run against an open // cursor over the same table. func deadmanAlerts(ctx context.Context, db *sql.DB, teamID int64, cfg deadmanSet) ([]deadmanAlert, error) { names := cfg.names() args := &sqlArgs{} nameList := make([]any, len(names)) for i, n := range names { nameList[i] = n } rows, err := db.QueryContext(ctx, ` SELECT id, team_id, fingerprint, labels, status, received_at FROM alerts WHERE team_id = `+args.add(teamID)+` AND name IN (`+args.addList(nameList)+`) AND archived_at IS NULL`, args.all()...) if err != nil { return nil, err } defer rows.Close() var out []deadmanAlert for rows.Next() { var a deadmanAlert var labelsJSON, status string if err := rows.Scan(&a.id, &a.teamID, &a.fingerprint, &labelsJSON, &status, &a.receivedAt); err != nil { return nil, err } json.Unmarshal([]byte(labelsJSON), &a.labels) //nolint:errcheck sw, ok := cfg.match(a.labels) if !ok { continue } a.sw = sw a.resolved = status == "resolved" out = append(out, a) } return out, rows.Err() } // deadmanDied raises the incident for a switch that has gone quiet. // // Two conditions gate it, and both matter. There must be no open incident for // the switch already — the partial unique index enforces that anyway, but a // second one would be a wasted page. And the heartbeat must have been seen since // the last incident was raised, which is the re-arm rule: resolving a dead man's // switch incident sticks, exactly as resolving an alert-backed one does (see // incidentForGroup), and a source that is gone for good is a one-time page // rather than a nag. Only a heartbeat that comes back and dies again earns a new // incident. func deadmanDied(ctx context.Context, db *sql.DB, notify NotifyConfig, hb deadmanAlert, now time.Time) error { var lastTriggered, open int64 if err := db.QueryRowContext(ctx, ` SELECT COALESCE(MAX(triggered_at), 0), COUNT(*) FILTER (WHERE resolved_at IS NULL) FROM incidents WHERE team_id = $1 AND group_key = $2`, hb.teamID, hb.groupKey()).Scan(&lastTriggered, &open); err != nil { return err } if open > 0 || hb.receivedAt <= lastTriggered { return nil } tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck // A heartbeat nobody has heard from is not firing, and saying otherwise in // the alert list would be a lie. An Alertmanager-sourced resolution keeps its // own source: it told us the truth first. if !hb.resolved { if _, err := tx.ExecContext(ctx, ` UPDATE alerts SET status = 'resolved', resolution_source = $1, ends_at = COALESCE(ends_at, `+nowEpoch+`) WHERE id = $2 AND status = 'firing'`, resolutionDeadman, hb.id); err != nil { return err } } severity := hb.sw.Severity var sev *string if severity != "" { sev = &severity } // The incident opens in the team whose integration received the heartbeat: // the switch belongs to whoever is watching that source, not to the install. incidentID, err := openIncident(ctx, tx, notify, hb.teamID, hb.groupKey(), "No heartbeat from "+hb.sw.Matcher.String(), hb.labels, sev) if err != nil { return err } alertID := hb.id detail := "last heartbeat " + humanDuration(now.Sub(time.Unix(hb.receivedAt, 0))) + " ago" if err := logEvent(ctx, tx, incidentID, evDeadmanSilent, nil, &alertID, &detail); err != nil { return err } if err := tx.Commit(); err != nil { return err } log.Printf("deadman: %s went silent, opened incident %d", hb.sw.Matcher.String(), incidentID) return nil } // deadmanRecovered closes the incident for a switch that started chirping again. // // It cannot go through resolveIfSettled: a dead man's switch incident has no // member alerts (linking the heartbeat would have the settled-incident cascade // close it on the very same sweep that opened it), so the alert-driven cascade // ignores it entirely and recovery is the only automatic way out. func deadmanRecovered(ctx context.Context, db *sql.DB, hb deadmanAlert) error { var incidentID int64 switch err := db.QueryRowContext(ctx, ` SELECT id FROM incidents WHERE team_id = $1 AND group_key = $2 AND resolved_at IS NULL`, hb.teamID, hb.groupKey()).Scan(&incidentID); { case err == sql.ErrNoRows: return nil case err != nil: return err } tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck if _, err := tx.ExecContext(ctx, ` UPDATE incidents SET status = 'resolved', resolved_at = $1, resolution_source = $2 WHERE id = $3 AND resolved_at IS NULL`, time.Now().Unix(), incidentResolutionRecovered, incidentID); err != nil { return err } if err := logEvent(ctx, tx, incidentID, evResolved, nil, nil, nil); err != nil { return err } // The all-clear goes to whoever was paged, which enqueueResolved works out // from the incident's own notification history. if err := enqueueResolved(ctx, tx, incidentID); err != nil { return err } if err := tx.Commit(); err != nil { return err } log.Printf("deadman: %s is back, resolved incident %d", hb.sw.Matcher.String(), incidentID) return nil } // --------------------------------------------------------------------------- // A team's switches // --------------------------------------------------------------------------- const deadmanSwitchColumns = "id, team_id, name, matcher, timeout_seconds, severity" // scanDeadmanSwitches reads switch rows into per-team sets. A row whose matcher // no longer parses is skipped rather than fatal: the API refuses to store one, // so it can only mean a hand edit, and one bad row must not stop the others // from being watched. func scanDeadmanSwitches(rows *sql.Rows) (map[int64]deadmanSet, error) { defer rows.Close() out := map[int64]deadmanSet{} for rows.Next() { var sw DeadmanSwitch var teamID, timeout int64 var matcher string if err := rows.Scan(&sw.ID, &teamID, &sw.Name, &matcher, &timeout, &sw.Severity); err != nil { return nil, err } m, err := parseDeadmanMatcher(matcher) if err != nil { log.Printf("deadman: switch %d has an unusable matcher %q: %v", sw.ID, matcher, err) continue } sw.Matcher = m sw.Timeout = time.Duration(timeout) * time.Second out[teamID] = append(out[teamID], sw) } return out, rows.Err() } // deadmanSetForTeam reads one team's switches. A team with none gets an empty // set — which is the right answer rather than an error: most teams watch no // heartbeat at all. func deadmanSetForTeam(ctx context.Context, q querier, teamID int64) (deadmanSet, error) { rows, err := q.QueryContext(ctx, "SELECT "+deadmanSwitchColumns+" FROM deadman_switches WHERE team_id = $1 ORDER BY id", teamID) if err != nil { return nil, err } sets, err := scanDeadmanSwitches(rows) return sets[teamID], err } // deadmanSets reads every team's switches in one query, for the sweeper. func deadmanSets(ctx context.Context, db *sql.DB) (map[int64]deadmanSet, error) { rows, err := db.QueryContext(ctx, "SELECT "+deadmanSwitchColumns+" FROM deadman_switches ORDER BY id") if err != nil { return nil, err } return scanDeadmanSwitches(rows) } // deadmanSeededKey is the settings row that records the environment defaults // were handed out. Without it, a team that deleted its last switch would get // the default back on the next restart. const deadmanSeededKey = "deadman_seeded" // SeedDeadmanConfigs gives every team the server's environment defaults as // switches, exactly once per install, so a fresh install watches Watchdog // without anybody setting it up. // // Once seeded it never runs again: a team's switches are its own, and a redeploy // must not quietly put the environment's value back over an owner's edit or // deletion. Installs that upgraded from per-team configuration were already // seeded, which migration 009 records. // // A team created after that gets none and watches nothing until its owner says // otherwise. That is deliberate: inheriting an install-wide heartbeat would page // a new team about a source it has never heard of, and a switch nobody chose is // the kind that gets muted rather than fixed. func SeedDeadmanConfigs(ctx context.Context, db *sql.DB, cfg DeadmanConfig) error { if !cfg.enabled() { return nil } tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck res, err := tx.ExecContext(ctx, "INSERT INTO settings (key, value) VALUES ($1, '1') ON CONFLICT (key) DO NOTHING", deadmanSeededKey) if err != nil { return err } if n, _ := res.RowsAffected(); n == 0 { return nil } for _, m := range cfg.Matchers { if _, err := tx.ExecContext(ctx, ` INSERT INTO deadman_switches (team_id, name, matcher, timeout_seconds, severity) SELECT id, $1, $1, $2, $3 FROM teams`, m.config(), int64(cfg.Timeout.Seconds()), cfg.Severity); err != nil { return err } } return tx.Commit() } // --------------------------------------------------------------------------- // Status // --------------------------------------------------------------------------- const ( switchHealthy = "healthy" switchDead = "dead" switchDormant = "dormant" ) // deadmanSource is one heartbeat under a switch: a fingerprint that matched. type deadmanSource struct { Fingerprint string `json:"fingerprint"` Labels map[string]string `json:"labels"` Status string `json:"status"` LastHeartbeatAt time.Time `json:"last_heartbeat_at"` LastTriggeredAt *time.Time `json:"last_triggered_at"` IncidentID *int64 `json:"incident_id"` } // deadmanSwitchStatus is a switch as the Switches page shows it. type deadmanSwitchStatus struct { ID int64 `json:"id"` Name string `json:"name"` Matcher string `json:"matcher"` TimeoutSeconds int64 `json:"timeout_seconds"` Severity string `json:"severity"` // Status is dead when any source is, dormant when none has ever been heard // from, healthy otherwise — a live cluster must not hide a dead one. Status string `json:"status"` LastHeartbeatAt *time.Time `json:"last_heartbeat_at"` LastTriggeredAt *time.Time `json:"last_triggered_at"` OpenIncidentID *int64 `json:"open_incident_id"` Sources []deadmanSource `json:"sources"` } // deadmanStatuses reports every switch of a team with what its heartbeats are // doing. The liveness verdict is deadmanAlert.dead, the sweeper's own. func deadmanStatuses(ctx context.Context, db *sql.DB, teamID int64, set deadmanSet, now time.Time) ([]deadmanSwitchStatus, error) { out := make([]deadmanSwitchStatus, 0, len(set)) if len(set) == 0 { return out, nil } heartbeats, err := deadmanAlerts(ctx, db, teamID, set) if err != nil { return nil, err } // One query for every switch's incident history, keyed the way the sweeper // keys it. type history struct { triggeredAt int64 openID int64 } incidents := map[string]history{} rows, err := db.QueryContext(ctx, ` SELECT group_key, MAX(triggered_at), COALESCE(MAX(id) FILTER (WHERE resolved_at IS NULL), 0) FROM incidents WHERE team_id = $1 AND group_key LIKE $2 GROUP BY group_key`, teamID, deadmanGroupPrefix+"%") if err != nil { return nil, err } defer rows.Close() for rows.Next() { var key string var h history if err := rows.Scan(&key, &h.triggeredAt, &h.openID); err != nil { return nil, err } incidents[key] = h } if err := rows.Err(); err != nil { return nil, err } bySwitch := map[int64][]deadmanAlert{} for _, hb := range heartbeats { bySwitch[hb.sw.ID] = append(bySwitch[hb.sw.ID], hb) } later := func(cur *time.Time, unix int64) *time.Time { t := time.Unix(unix, 0).UTC() if cur == nil || t.After(*cur) { return &t } return cur } for _, sw := range set { st := deadmanSwitchStatus{ ID: sw.ID, Name: sw.Name, Matcher: sw.Matcher.config(), TimeoutSeconds: int64(sw.Timeout.Seconds()), Severity: sw.Severity, Status: switchDormant, Sources: []deadmanSource{}, } for _, hb := range bySwitch[sw.ID] { src := deadmanSource{ Fingerprint: hb.fingerprint, Labels: hb.labels, Status: switchHealthy, LastHeartbeatAt: time.Unix(hb.receivedAt, 0).UTC(), } if hb.dead(now) { src.Status = switchDead } if h, ok := incidents[hb.groupKey()]; ok { t := time.Unix(h.triggeredAt, 0).UTC() src.LastTriggeredAt = &t st.LastTriggeredAt = later(st.LastTriggeredAt, h.triggeredAt) if h.openID != 0 { id := h.openID src.IncidentID = &id if st.OpenIncidentID == nil || id > *st.OpenIncidentID { st.OpenIncidentID = &id } } } st.LastHeartbeatAt = later(st.LastHeartbeatAt, hb.receivedAt) st.Sources = append(st.Sources, src) switch { case src.Status == switchDead: st.Status = switchDead case st.Status == switchDormant: st.Status = switchHealthy } } // Dead ones first, then by fingerprint: what needs attention leads, and // the order does not shuffle between refreshes. sort.Slice(st.Sources, func(i, j int) bool { a, b := st.Sources[i], st.Sources[j] if (a.Status == switchDead) != (b.Status == switchDead) { return a.Status == switchDead } return a.Fingerprint < b.Fingerprint }) out = append(out, st) } return out, nil }