package api_test import ( "net/http" "strings" "testing" "time" "git.ryuvia.com/niklas/terdut-server/internal/api" ) // --------------------------------------------------------------------------- // Harness // --------------------------------------------------------------------------- // watchdogGroupKey is what Alertmanager sends for a Watchdog grouped by // alertname, which is how the deployed route is configured. const watchdogGroupKey = `{}:{alertname="Watchdog"}` // deadmanCfg watches Watchdog with a timeout short enough to reason about and // long enough that a fresh heartbeat is never accidentally stale. func deadmanCfg() api.DeadmanConfig { return api.ParseDeadmanConfig("alertname=Watchdog", time.Hour, "critical") } // deadmanTS is notifyTS with dead man's switch handling on: notifications // enabled against a fake ntfy, the admin on call today with a topic. func deadmanTS(t *testing.T, cfg api.DeadmanConfig) (*ts, *fakeNtfy) { t.Helper() f := newFakeNtfy(t) s := newDeadmanTS(t, cfg, api.NotifyConfig{ BaseURL: f.URL, PublicURL: "https://terdut.example.com", }) putOnCall(t, s, 1) setTopic(t, s, 1, "terdut-admin") return s, f } // heartbeat posts one Watchdog webhook. Its startsAt never changes: a dead man's // switch alert fires once and is re-sent unchanged forever, which is precisely // what makes its absence meaningful. func heartbeat(t *testing.T, s *ts, fingerprint string, labels map[string]string) { t.Helper() postWebhook(t, s, []map[string]any{ amAlert(fingerprint, "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, labels), }, watchdogGroupKey) } // silence back-dates a heartbeat's received_at, which is the only clock the // sweeper reads. There is no fake clock in this package. func silence(t *testing.T, s *ts, fingerprint string, ago time.Duration) { t.Helper() s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2", time.Now().Add(-ago).Unix(), fingerprint) } // ageIncidents back-dates every incident. The re-arm rule compares a heartbeat // against the last incident raised for its switch, so a test that wants a second // episode has to put the first one in the past — there is no fake clock here. func ageIncidents(t *testing.T, s *ts, ago time.Duration) { t.Helper() past := time.Now().Add(-ago).Unix() // $2 is cast explicitly: with NULL in the other branch Postgres has nothing // to infer the parameter's type from and defaults it to text, which the // bigint column then refuses. s.exec(t, `UPDATE incidents SET triggered_at = $1, resolved_at = CASE WHEN resolved_at IS NULL THEN NULL ELSE $2::bigint END`, past, past) } // incidentByGroup reads the incident for a group key, resolved ones included. func incidentByGroup(t *testing.T, s *ts, groupKey string) (id int64, status, severity string, source *string) { t.Helper() err := s.db.QueryRow(` SELECT id, status, COALESCE(severity, ''), resolution_source FROM incidents WHERE group_key = $1 ORDER BY id DESC LIMIT 1`, groupKey).Scan(&id, &status, &severity, &source) if err != nil { t.Fatalf("read incident for group %s: %v", groupKey, err) } return id, status, severity, source } // --------------------------------------------------------------------------- // Receiving a heartbeat // --------------------------------------------------------------------------- // The whole inversion: arrival of a dead man's switch alert is good news, and // good news is not an incident. func TestDeadman_HeartbeatOpensNoIncident(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) if got := s.countIncidents(t); got != 0 { t.Fatalf("expected a heartbeat to open no incident, got %d", got) } if got := s.countNotifications(t, ""); got != 0 { t.Errorf("expected no notification for a heartbeat, got %d", got) } if status, _, _ := s.alertRow(t, "fp-watchdog"); status != "firing" { t.Errorf("expected the heartbeat to be stored firing, got %q", status) } } // A heartbeat routed into a group alongside real alerts must not join their // incident: it is not a symptom of anything. func TestDeadman_MixedGroupExcludesHeartbeat(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) postWebhook(t, s, []map[string]any{ amAlert("fp-mixed-wd", "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), amAlert("fp-mixed-disk", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"}), }, `{}:{namespace="prod"}`) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected 1 incident for the real alert, got %d", got) } var alerts []map[string]any decode(t, s.req(t, http.MethodGet, "/api/incidents/1/alerts", nil), &alerts) if len(alerts) != 1 { t.Fatalf("expected 1 member alert, got %d", len(alerts)) } if name := alerts[0]["name"]; name != "DiskFull" { t.Errorf("expected only the real alert linked, got %v", name) } } // A matcher scoped by label only claims the alerts it names, so a heartbeat from // somewhere else stays an ordinary alert. func TestDeadman_LabelScopedMatcherIgnoresOthers(t *testing.T) { s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=Watchdog,cluster=prod", time.Hour, "critical")) heartbeat(t, s, "fp-dev", map[string]string{"cluster": "dev"}) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected an unmatched Watchdog to behave like any other alert, got %d incidents", got) } } // --------------------------------------------------------------------------- // Silence // --------------------------------------------------------------------------- func TestDeadman_SilenceOpensIncident(t *testing.T) { s, f := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 2*time.Hour) sweep(t, s, noArchive) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected silence to open 1 incident, got %d", got) } id, status, severity, _ := incidentByGroup(t, s, "deadman:fp-watchdog") if status != "triggered" { t.Errorf("expected a triggered incident, got %q", status) } if severity != "critical" { t.Errorf("expected the configured severity, got %q", severity) } // The alert list must not keep claiming a dead heartbeat is firing. alertStatus, source, _ := s.alertRow(t, "fp-watchdog") if alertStatus != "resolved" || source == nil || *source != "deadman" { t.Errorf("expected the heartbeat resolved as deadman, got %q / %v", alertStatus, source) } // Nobody was told anything by an alert here, so the page has to come from // the switch itself. s.sweepNotify(t) msgs := f.messages() if len(msgs) != 1 { t.Fatalf("expected 1 page, got %d", len(msgs)) } if msgs[0].Topic != "terdut-admin" { t.Errorf("expected the on-call topic, got %q", msgs[0].Topic) } if msgs[0].Priority != 5 { t.Errorf("expected a critical page to override quiet hours (priority 5), got %d", msgs[0].Priority) } // The timeline says why, with the age of the last heartbeat. types := eventTypes(timeline(t, s, int(id))) found := false for _, ty := range types { if ty == "deadman_silent" { found = true } } if !found { t.Errorf("expected a deadman_silent event, got %v", types) } } // The generic staleness sweep must keep its hands off heartbeats: they answer to // their own, much tighter, timeout, and an 'expiry' resolution here would be // both wrong and unrecoverable. func TestDeadman_GenericExpiryLeavesHeartbeatAlone(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 5*time.Minute) // staleAfter far tighter than the dead man's switch timeout. sweep(t, s, time.Minute) status, source, _ := s.alertRow(t, "fp-watchdog") if status != "firing" || source != nil { t.Errorf("expected a live heartbeat left alone, got %q / %v", status, source) } if got := s.countIncidents(t); got != 0 { t.Errorf("expected no incident for a heartbeat that is still fresh, got %d", got) } } // An explicit resolved from Alertmanager is the sender telling us the heartbeat // stopped. There is nothing left to wait out. func TestDeadman_AlertmanagerResolvedIsImmediateDeath(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) postWebhook(t, s, []map[string]any{ amAlert("fp-watchdog", "Watchdog", "resolved", "2026-05-20T10:00:00Z", zeroTime, nil), }, watchdogGroupKey) // No ageing: received_at is seconds old, well inside the timeout. sweep(t, s, noArchive) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected a resolved heartbeat to open an incident at once, got %d", got) } // Alertmanager told the truth first, so its resolution source stands. if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "alertmanager" { t.Errorf("expected the Alertmanager resolution source kept, got %v", source) } } // Each label set is its own switch, so one healthy source cannot mask a dead one. func TestDeadman_TracksEachFingerprintSeparately(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-a", map[string]string{"cluster": "a"}) heartbeat(t, s, "fp-b", map[string]string{"cluster": "b"}) silence(t, s, "fp-b", 2*time.Hour) sweep(t, s, noArchive) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected only the silent switch to page, got %d incidents", got) } if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-b"); status != "triggered" { t.Errorf("expected the incident to belong to the silent switch, got %q", status) } if status, _, _ := s.alertRow(t, "fp-a"); status != "firing" { t.Errorf("expected the live switch untouched, got %q", status) } } // A switch nothing has ever been heard from is dormant. A fresh deploy, a // restored database or a typo'd alertname must not page. func TestDeadman_UnheardOfSwitchIsDormant(t *testing.T) { s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=NeverSent", time.Hour, "critical")) sweep(t, s, noArchive) if got := s.countIncidents(t); got != 0 { t.Fatalf("expected a switch that never chirped to be dormant, got %d incidents", got) } } // --------------------------------------------------------------------------- // Recovery and re-arming // --------------------------------------------------------------------------- // The returning heartbeat carries the unchanged startsAt of an alert that never // stopped firing, so this also covers the ingest guard exemption: without it the // upsert would discard the payload and the switch could die exactly once. func TestDeadman_RecoveryResolvesIncident(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 2*time.Hour) sweep(t, s, noArchive) heartbeat(t, s, "fp-watchdog", nil) if status, source, _ := s.alertRow(t, "fp-watchdog"); status != "firing" || source != nil { t.Fatalf("expected the returning heartbeat to be accepted, got %q / %v", status, source) } sweep(t, s, noArchive) _, status, _, source := incidentByGroup(t, s, "deadman:fp-watchdog") if status != "resolved" { t.Errorf("expected recovery to close the incident, got %q", status) } if source == nil || *source != "recovered" { t.Errorf("expected resolution_source recovered, got %v", source) } if got := s.countNotifications(t, "resolved"); got != 1 { t.Errorf("expected 1 all-clear, got %d", got) } } // Resolving a dead man's switch incident sticks, exactly as it does for an // alert-backed one. A source that is gone for good is a one-time page. func TestDeadman_ManualResolveSticksWhileSilent(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 2*time.Hour) sweep(t, s, noArchive) s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close() // Still silent, several sweeps later. sweep(t, s, noArchive) sweep(t, s, noArchive) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected a manually resolved incident to stay closed, got %d", got) } } // ...but the switch re-arms, so a heartbeat that comes back and dies again is a // new incident rather than silence forever. func TestDeadman_ReArmsAfterHeartbeatReturns(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 2*time.Hour) sweep(t, s, noArchive) s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close() // That episode is yesterday's news; the heartbeat now returns after it. ageIncidents(t, s, 10*time.Hour) heartbeat(t, s, "fp-watchdog", nil) sweep(t, s, noArchive) if got := s.countIncidents(t); got != 1 { t.Fatalf("expected the live switch to open nothing, got %d incidents", got) } silence(t, s, "fp-watchdog", 2*time.Hour) sweep(t, s, noArchive) if got := s.countIncidents(t); got != 2 { t.Fatalf("expected a second death to open a second incident, got %d", got) } } // A dead man's switch incident has no member alerts — linking the heartbeat // would have the settled-incident cascade close it on the very sweep that opened // it — so the cascade has to leave it alone. func TestDeadman_SettledCascadeLeavesIncidentOpen(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 2*time.Hour) sweep(t, s, noArchive) sweep(t, s, noArchive) if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-watchdog"); status != "triggered" { t.Fatalf("expected the incident to stay open until the switch recovers, got %q", status) } } // --------------------------------------------------------------------------- // Configuration // --------------------------------------------------------------------------- func TestParseDeadmanConfig(t *testing.T) { tests := []struct { name string matchers string timeout time.Duration want []api.DeadmanMatcher enabled bool }{ { name: "single alertname", matchers: "alertname=Watchdog", timeout: time.Hour, want: []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}}, enabled: true, }, { name: "several matchers with extra labels and whitespace", matchers: " alertname=Watchdog, cluster=prod ; alertname=EdgeHeartbeat ", timeout: time.Hour, want: []api.DeadmanMatcher{ {Name: "Watchdog", Labels: map[string]string{"cluster": "prod"}}, {Name: "EdgeHeartbeat", Labels: map[string]string{}}, }, enabled: true, }, { // Mandatory: it is what keeps the sweeper's candidate query on an index. name: "matcher without alertname is dropped", matchers: "cluster=prod; alertname=Watchdog", timeout: time.Hour, want: []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}}, enabled: true, }, { name: "malformed condition drops only its matcher", matchers: "alertname=Watchdog,garbage; alertname=Other", timeout: time.Hour, want: []api.DeadmanMatcher{{Name: "Other", Labels: map[string]string{}}}, enabled: true, }, { name: "zero timeout disables", matchers: "alertname=Watchdog", timeout: 0, want: []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}}, enabled: false, }, { name: "no usable matchers disables", matchers: "", timeout: time.Hour, want: nil, enabled: false, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { got := api.ParseDeadmanConfig(tc.matchers, tc.timeout, "critical") if len(got.Matchers) != len(tc.want) { t.Fatalf("got %d matchers %v, want %d", len(got.Matchers), got.Matchers, len(tc.want)) } for i, w := range tc.want { if got.Matchers[i].Name != w.Name { t.Errorf("matcher %d: name %q, want %q", i, got.Matchers[i].Name, w.Name) } if len(got.Matchers[i].Labels) != len(w.Labels) { t.Errorf("matcher %d: labels %v, want %v", i, got.Matchers[i].Labels, w.Labels) continue } for k, v := range w.Labels { if got.Matchers[i].Labels[k] != v { t.Errorf("matcher %d: label %s=%q, want %q", i, k, got.Matchers[i].Labels[k], v) } } } }) } } // A zero config is off, which is what keeps the feature opt-in for anything // building a router without one. func TestDeadman_DisabledConfigIsInert(t *testing.T) { s, _ := deadmanTS(t, api.DeadmanConfig{}) heartbeat(t, s, "fp-watchdog", nil) silence(t, s, "fp-watchdog", 48*time.Hour) sweep(t, s, time.Hour) // Ordinary alert handling: an incident from the arrival, not the absence. if got := s.countIncidents(t); got != 1 { t.Fatalf("expected plain alert handling with deadman off, got %d incidents", got) } if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "expiry" { t.Errorf("expected the generic sweeper to own the alert, got %v", source) } } // --------------------------------------------------------------------------- // Per-team configuration // --------------------------------------------------------------------------- // Each team decides for itself what a heartbeat is. The same alert is a // heartbeat in one team and an ordinary problem in another. func TestDeadman_ConfigurationIsPerTeam(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) watched := newTeam(t, s, "watched") unwatched := newTeam(t, s, "unwatched") // Only the first team calls Watchdog a heartbeat. resp := s.req(t, http.MethodPut, "/api/teams/"+id64(watched.id)+"/deadman", map[string]any{ "matchers": "alertname=Watchdog", "timeout_seconds": 3600, "severity": "critical", }) resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("configure the watched team: %d", resp.StatusCode) } postToIntegration(t, s, watched.key, "fp-watched", "Watchdog") postToIntegration(t, s, unwatched.key, "fp-unwatched", "Watchdog") // A heartbeat opens nothing where it is one; an ordinary alert opens an // incident where it is not. if got := len(list(t, watched.call(http.MethodGet, "/api/incidents", nil))); got != 0 { t.Errorf("the watched team's heartbeat opened %d incident(s), want 0", got) } if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 { t.Errorf("the unwatched team's Watchdog opened %d incident(s), want 1", got) } // Silence pages only the team that is watching. s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2", time.Now().Add(-2*time.Hour).Unix(), "fp-watched") s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2", time.Now().Add(-2*time.Hour).Unix(), "fp-unwatched") sweep(t, s, noArchive) watchedIncidents := list(t, watched.call(http.MethodGet, "/api/incidents", nil)) if len(watchedIncidents) != 1 { t.Fatalf("silence opened %d incident(s) for the watching team, want 1", len(watchedIncidents)) } if title := watchedIncidents[0]["title"].(string); title != "No heartbeat from Watchdog" { t.Errorf("unexpected incident title %q", title) } if teamID := int64(watchedIncidents[0]["team_id"].(float64)); teamID != watched.id { t.Errorf("the incident opened in team %d, want %d", teamID, watched.id) } // The unwatched team's alert went stale the ordinary way, so it has the one // incident it always had — not a second, dead man's switch one. if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 { t.Errorf("the unwatched team ended with %d incident(s), want 1", got) } } // Configuration is an owner's to change and a member's to read, like the rest of // a team's settings. func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) team := newTeam(t, s, "red") // A plain member of that team. var user struct { ID int64 `json:"id"` } decode(t, s.req(t, http.MethodPost, "/api/users", map[string]string{"username": "plain", "email": "plain@test.com"}), &user) s.req(t, http.MethodPost, "/api/teams/"+id64(team.id)+"/members", map[string]any{"user_id": user.ID, "role": "member"}).Body.Close() var key struct { Key string `json:"key"` } decode(t, s.req(t, http.MethodPost, "/api/users/"+id64(user.ID)+"/api-keys", map[string]string{"name": "test"}), &key) req, _ := http.NewRequest(http.MethodPut, s.URL+"/api/teams/"+id64(team.id)+"/deadman", strings.NewReader(`{"matchers":"alertname=Watchdog","timeout_seconds":60}`)) req.Header.Set("Authorization", "Bearer "+key.Key) req.Header.Set("Content-Type", "application/json") resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("put: %v", err) } resp.Body.Close() if resp.StatusCode != http.StatusForbidden { t.Errorf("a member editing the switches: expected 403, got %d", resp.StatusCode) } read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman", nil) read.Header.Set("Authorization", "Bearer "+key.Key) got, err := http.DefaultClient.Do(read) if err != nil { t.Fatalf("get: %v", err) } got.Body.Close() if got.StatusCode != http.StatusOK { t.Errorf("a member reading the switches: expected 200, got %d", got.StatusCode) } } // A matcher with no alertname watches nothing, silently, which is the failure // this feature exists to prevent — so it is refused at the door. func TestDeadman_UnusableMatchersAreRejected(t *testing.T) { s, _ := deadmanTS(t, deadmanCfg()) resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{ "matchers": "cluster=prod", "timeout_seconds": 900, }) resp.Body.Close() if resp.StatusCode != http.StatusBadRequest { t.Errorf("expected 400 for a matcher with no alertname, got %d", resp.StatusCode) } }