package api import ( "context" "database/sql" "encoding/json" "log" "sort" "strings" "time" ) // deadmanGroupPrefix namespaces the incidents this file opens. Alertmanager // group keys always contain braces, so this can never collide with one, and the // partial unique index on open group_key (see 008_incidents.sql) gives one open // incident per switch for free. const deadmanGroupPrefix = "deadman:" // DeadmanMatcher selects the alerts that are heartbeats rather than problems. // Every condition has to match, and Name — the alertname label — is mandatory: // it is what lets the sweeper find candidate rows through alerts_name_idx // instead of JSON-extracting labels from every row in the table. type DeadmanMatcher struct { Name string Labels map[string]string } // String renders the matcher the way it was configured, which is also how it // reads in an incident title. func (m DeadmanMatcher) String() string { if len(m.Labels) == 0 { return m.Name } parts := make([]string, 0, len(m.Labels)) for k, v := range m.Labels { parts = append(parts, k+"="+v) } sort.Strings(parts) return m.Name + " (" + strings.Join(parts, ", ") + ")" } // matches reports whether an alert's labels satisfy every condition. func (m DeadmanMatcher) matches(labels map[string]string) bool { if labels["alertname"] != m.Name { return false } for k, v := range m.Labels { if labels[k] != v { return false } } return true } // DeadmanConfig inverts the handling of the alerts it matches: receiving one // opens nothing, and the absence of one opens an incident. // // The unit of monitoring is the fingerprint, not the matcher — two clusters // sending the same heartbeat alertname are two independent switches, so one // healthy cluster cannot mask a dead one. type DeadmanConfig struct { Matchers []DeadmanMatcher // Timeout is how long a matched alert may go without a refreshing webhook // before it is declared dead. It must be shorter than Alertmanager's // repeat_interval for the heartbeat's route, which is what refreshes it. // Zero disables dead man's switch handling entirely. Timeout time.Duration // Severity is the severity every dead man's switch incident opens at. These // incidents have no member alerts to derive one from, and the heartbeat's // own severity label is meaningless — Watchdog ships as "none". Severity string } // enabled reports whether there is anything to watch. func (c DeadmanConfig) enabled() bool { return c.Timeout > 0 && len(c.Matchers) > 0 } // match returns the first matcher an alert satisfies. func (c DeadmanConfig) match(labels map[string]string) (DeadmanMatcher, bool) { if !c.enabled() { return DeadmanMatcher{}, false } for _, m := range c.Matchers { if m.matches(labels) { return m, true } } return DeadmanMatcher{}, false } // isDeadman is match without the matcher, for the ingest path. func (c DeadmanConfig) isDeadman(labels map[string]string) bool { _, ok := c.match(labels) return ok } // names lists the distinct alertnames worth loading from the database. func (c DeadmanConfig) names() []string { seen := map[string]bool{} out := make([]string, 0, len(c.Matchers)) for _, m := range c.Matchers { if !seen[m.Name] { seen[m.Name] = true out = append(out, m.Name) } } return out } // ParseDeadmanConfig reads the matcher list from its configured form: // ";" separates matchers, "," separates the conditions within one, and "=" is // exact label equality — `alertname=Watchdog,cluster=prod; alertname=Heartbeat`. // // A malformed or alertname-less entry is dropped rather than fatal, following // config.duration's rule that one bad tuning knob should not take the server // down. Silence would be worse here than elsewhere, though — a typo that // disarms the switch is exactly the failure this feature exists to catch — so // the matchers that survived are logged. func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string) DeadmanConfig { cfg := DeadmanConfig{Timeout: timeout, Severity: severity} for _, entry := range strings.Split(matchers, ";") { entry = strings.TrimSpace(entry) if entry == "" { continue } m := DeadmanMatcher{Labels: map[string]string{}} malformed := false for _, cond := range strings.Split(entry, ",") { k, v, ok := strings.Cut(cond, "=") k, v = strings.TrimSpace(k), strings.TrimSpace(v) if !ok || k == "" || v == "" { log.Printf("deadman: ignoring matcher %q: %q is not label=value", entry, strings.TrimSpace(cond)) malformed = true break } if k == "alertname" { m.Name = v continue } m.Labels[k] = v } if malformed { continue } if m.Name == "" { log.Printf("deadman: ignoring matcher %q: no alertname condition", entry) continue } cfg.Matchers = append(cfg.Matchers, m) } switch { case timeout <= 0: log.Print("deadman: disabled (timeout is zero)") case len(cfg.Matchers) == 0: log.Print("deadman: disabled (no usable matchers)") default: rendered := make([]string, 0, len(cfg.Matchers)) for _, m := range cfg.Matchers { rendered = append(rendered, m.String()) } log.Printf("deadman: watching %s, timeout %s, severity %s", strings.Join(rendered, "; "), timeout, severity) } return cfg } // deadmanAlert is one switch: the alert row carrying its last heartbeat. type deadmanAlert struct { id int64 fingerprint string labels map[string]string matcher DeadmanMatcher resolved bool receivedAt int64 } // groupKey is the switch's identity as an incident. Per fingerprint, so each // source is tracked on its own. func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerprint } // sweepDeadman is the whole point of the feature: it opens an incident for every // switch that has stopped chirping, and closes one whose switch came back. // // It returns the ids of the alerts it owns, because the generic staleness // expiry must leave them alone — staleAfter and ends_at would otherwise resolve // a heartbeat long before its own, much tighter, timeout ever fired. func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig) map[int64]bool { owned := map[int64]bool{} if !cfg.enabled() { return owned } switches, err := deadmanAlerts(ctx, db, cfg) if err != nil { log.Printf("deadman: load switches: %v", err) return owned } now := time.Now() cutoff := now.Add(-cfg.Timeout).Unix() for _, sw := range switches { owned[sw.id] = true // An explicit resolved from Alertmanager is a stronger death signal than // mere absence: the sender is telling us the heartbeat stopped, so there // is nothing left to wait out. if sw.resolved || sw.receivedAt < cutoff { if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil { log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err) } continue } if err := deadmanRecovered(ctx, db, sw); err != nil { log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err) } } return owned } // deadmanAlerts loads every alert row that a matcher claims. The candidate query // is narrowed by alertname so it rides alerts_name_idx; the rest of the matching // happens in Go, which keeps one implementation of the rules. The rows are read // in full before the caller writes, because the pool holds a single connection. func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadmanAlert, error) { names := cfg.names() args := make([]any, 0, len(names)) for _, n := range names { args = append(args, n) } rows, err := db.QueryContext(ctx, ` SELECT id, fingerprint, labels, status, received_at FROM alerts WHERE name IN (`+placeholders(len(names))+`) AND archived_at IS NULL`, args...) if err != nil { return nil, err } defer rows.Close() var out []deadmanAlert for rows.Next() { var a deadmanAlert var labelsJSON, status string if err := rows.Scan(&a.id, &a.fingerprint, &labelsJSON, &status, &a.receivedAt); err != nil { return nil, err } json.Unmarshal([]byte(labelsJSON), &a.labels) //nolint:errcheck m, ok := cfg.match(a.labels) if !ok { continue } a.matcher = m a.resolved = status == "resolved" out = append(out, a) } return out, rows.Err() } // deadmanDied raises the incident for a switch that has gone quiet. // // Two conditions gate it, and both matter. There must be no open incident for // the switch already — the partial unique index enforces that anyway, but a // second one would be a wasted page. And the heartbeat must have been seen since // the last incident was raised, which is the re-arm rule: resolving a dead man's // switch incident sticks, exactly as resolving an alert-backed one does (see // incidentForGroup), and a source that is gone for good is a one-time page // rather than a nag. Only a heartbeat that comes back and dies again earns a new // incident. func deadmanDied(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig, sw deadmanAlert, now time.Time) error { var lastTriggered, open int64 if err := db.QueryRowContext(ctx, ` SELECT COALESCE(MAX(triggered_at), 0), COALESCE(SUM(resolved_at IS NULL), 0) FROM incidents WHERE group_key = ?`, sw.groupKey()).Scan(&lastTriggered, &open); err != nil { return err } if open > 0 || sw.receivedAt <= lastTriggered { return nil } tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck // A heartbeat nobody has heard from is not firing, and saying otherwise in // the alert list would be a lie. An Alertmanager-sourced resolution keeps its // own source: it told us the truth first. if !sw.resolved { if _, err := tx.ExecContext(ctx, ` UPDATE alerts SET status = 'resolved', resolution_source = ?, ends_at = COALESCE(ends_at, unixepoch()) WHERE id = ? AND status = 'firing'`, resolutionDeadman, sw.id); err != nil { return err } } severity := cfg.Severity var sev *string if severity != "" { sev = &severity } incidentID, err := openIncident(ctx, tx, notify, sw.groupKey(), "No heartbeat from "+sw.matcher.String(), sw.labels, sev) if err != nil { return err } alertID := sw.id detail := "last heartbeat " + humanDuration(now.Sub(time.Unix(sw.receivedAt, 0))) + " ago" if err := logEvent(ctx, tx, incidentID, evDeadmanSilent, nil, &alertID, &detail); err != nil { return err } if err := tx.Commit(); err != nil { return err } log.Printf("deadman: %s went silent, opened incident %d", sw.matcher.String(), incidentID) return nil } // deadmanRecovered closes the incident for a switch that started chirping again. // // It cannot go through resolveIfSettled: a dead man's switch incident has no // member alerts (linking the heartbeat would have the settled-incident cascade // close it on the very same sweep that opened it), so the alert-driven cascade // ignores it entirely and recovery is the only automatic way out. func deadmanRecovered(ctx context.Context, db *sql.DB, sw deadmanAlert) error { var incidentID int64 switch err := db.QueryRowContext(ctx, ` SELECT id FROM incidents WHERE group_key = ? AND resolved_at IS NULL`, sw.groupKey()).Scan(&incidentID); { case err == sql.ErrNoRows: return nil case err != nil: return err } tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck if _, err := tx.ExecContext(ctx, ` UPDATE incidents SET status = 'resolved', resolved_at = ?, resolution_source = ? WHERE id = ? AND resolved_at IS NULL`, time.Now().Unix(), incidentResolutionRecovered, incidentID); err != nil { return err } if err := logEvent(ctx, tx, incidentID, evResolved, nil, nil, nil); err != nil { return err } // The all-clear goes to whoever was paged, which enqueueResolved works out // from the incident's own notification history. if err := enqueueResolved(ctx, tx, incidentID); err != nil { return err } if err := tx.Commit(); err != nil { return err } log.Printf("deadman: %s is back, resolved incident %d", sw.matcher.String(), incidentID) return nil }