terdut-server/internal/api/deadman_test.go

package api_test

import (
	"net/http"
	"strings"
	"testing"
	"time"

	"git.ryuvia.com/niklas/terdut-server/internal/api"
)

// ---------------------------------------------------------------------------
// Harness
// ---------------------------------------------------------------------------

// watchdogGroupKey is what Alertmanager sends for a Watchdog grouped by
// alertname, which is how the deployed route is configured.
const watchdogGroupKey = `{}:{alertname="Watchdog"}`

// deadmanCfg watches Watchdog with a timeout short enough to reason about and
// long enough that a fresh heartbeat is never accidentally stale.
func deadmanCfg() api.DeadmanConfig {
	return api.ParseDeadmanConfig("alertname=Watchdog", time.Hour, "critical")
}

// deadmanTS is notifyTS with dead man's switch handling on: notifications
// enabled against a fake ntfy, the admin on call today with a topic.
func deadmanTS(t *testing.T, cfg api.DeadmanConfig) (*ts, *fakeNtfy) {
	t.Helper()
	f := newFakeNtfy(t)
	s := newDeadmanTS(t, cfg, api.NotifyConfig{
		BaseURL:   f.URL,
		PublicURL: "https://terdut.example.com",
	})

	putOnCall(t, s, 1)
	setTopic(t, s, 1, "terdut-admin")
	return s, f
}

// heartbeat posts one Watchdog webhook. Its startsAt never changes: a dead man's
// switch alert fires once and is re-sent unchanged forever, which is precisely
// what makes its absence meaningful.
func heartbeat(t *testing.T, s *ts, fingerprint string, labels map[string]string) {
	t.Helper()
	postWebhook(t, s, []map[string]any{
		amAlert(fingerprint, "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, labels),
	}, watchdogGroupKey)
}

// silence back-dates a heartbeat's received_at, which is the only clock the
// sweeper reads. There is no fake clock in this package.
func silence(t *testing.T, s *ts, fingerprint string, ago time.Duration) {
	t.Helper()
	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
		time.Now().Add(-ago).Unix(), fingerprint)
}

// ageIncidents back-dates every incident. The re-arm rule compares a heartbeat
// against the last incident raised for its switch, so a test that wants a second
// episode has to put the first one in the past — there is no fake clock here.
func ageIncidents(t *testing.T, s *ts, ago time.Duration) {
	t.Helper()
	past := time.Now().Add(-ago).Unix()
	// $2 is cast explicitly: with NULL in the other branch Postgres has nothing
	// to infer the parameter's type from and defaults it to text, which the
	// bigint column then refuses.
	s.exec(t, `UPDATE incidents
	              SET triggered_at = $1,
	                  resolved_at  = CASE WHEN resolved_at IS NULL THEN NULL ELSE $2::bigint END`,
		past, past)
}

// incidentByGroup reads the incident for a group key, resolved ones included.
func incidentByGroup(t *testing.T, s *ts, groupKey string) (id int64, status, severity string, source *string) {
	t.Helper()
	err := s.db.QueryRow(`
		SELECT id, status, COALESCE(severity, ''), resolution_source
		FROM   incidents WHERE group_key = $1 ORDER BY id DESC LIMIT 1`,
		groupKey).Scan(&id, &status, &severity, &source)
	if err != nil {
		t.Fatalf("read incident for group %s: %v", groupKey, err)
	}
	return id, status, severity, source
}

// ---------------------------------------------------------------------------
// Receiving a heartbeat
// ---------------------------------------------------------------------------

// The whole inversion: arrival of a dead man's switch alert is good news, and
// good news is not an incident.
func TestDeadman_HeartbeatOpensNoIncident(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)

	if got := s.countIncidents(t); got != 0 {
		t.Fatalf("expected a heartbeat to open no incident, got %d", got)
	}
	if got := s.countNotifications(t, ""); got != 0 {
		t.Errorf("expected no notification for a heartbeat, got %d", got)
	}
	if status, _, _ := s.alertRow(t, "fp-watchdog"); status != "firing" {
		t.Errorf("expected the heartbeat to be stored firing, got %q", status)
	}
}

// A heartbeat routed into a group alongside real alerts must not join their
// incident: it is not a symptom of anything.
func TestDeadman_MixedGroupExcludesHeartbeat(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	postWebhook(t, s, []map[string]any{
		amAlert("fp-mixed-wd", "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
		amAlert("fp-mixed-disk", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime,
			map[string]string{"severity": "critical"}),
	}, `{}:{namespace="prod"}`)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected 1 incident for the real alert, got %d", got)
	}

	var alerts []map[string]any
	decode(t, s.req(t, http.MethodGet, "/api/incidents/1/alerts", nil), &alerts)
	if len(alerts) != 1 {
		t.Fatalf("expected 1 member alert, got %d", len(alerts))
	}
	if name := alerts[0]["name"]; name != "DiskFull" {
		t.Errorf("expected only the real alert linked, got %v", name)
	}
}

// A matcher scoped by label only claims the alerts it names, so a heartbeat from
// somewhere else stays an ordinary alert.
func TestDeadman_LabelScopedMatcherIgnoresOthers(t *testing.T) {
	s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=Watchdog,cluster=prod", time.Hour, "critical"))

	heartbeat(t, s, "fp-dev", map[string]string{"cluster": "dev"})

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected an unmatched Watchdog to behave like any other alert, got %d incidents", got)
	}
}

// ---------------------------------------------------------------------------
// Silence
// ---------------------------------------------------------------------------

func TestDeadman_SilenceOpensIncident(t *testing.T) {
	s, f := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected silence to open 1 incident, got %d", got)
	}
	id, status, severity, _ := incidentByGroup(t, s, "deadman:fp-watchdog")
	if status != "triggered" {
		t.Errorf("expected a triggered incident, got %q", status)
	}
	if severity != "critical" {
		t.Errorf("expected the configured severity, got %q", severity)
	}

	// The alert list must not keep claiming a dead heartbeat is firing.
	alertStatus, source, _ := s.alertRow(t, "fp-watchdog")
	if alertStatus != "resolved" || source == nil || *source != "deadman" {
		t.Errorf("expected the heartbeat resolved as deadman, got %q / %v", alertStatus, source)
	}

	// Nobody was told anything by an alert here, so the page has to come from
	// the switch itself.
	s.sweepNotify(t)
	msgs := f.messages()
	if len(msgs) != 1 {
		t.Fatalf("expected 1 page, got %d", len(msgs))
	}
	if msgs[0].Topic != "terdut-admin" {
		t.Errorf("expected the on-call topic, got %q", msgs[0].Topic)
	}
	if msgs[0].Priority != 5 {
		t.Errorf("expected a critical page to override quiet hours (priority 5), got %d", msgs[0].Priority)
	}

	// The timeline says why, with the age of the last heartbeat.
	types := eventTypes(timeline(t, s, int(id)))
	found := false
	for _, ty := range types {
		if ty == "deadman_silent" {
			found = true
		}
	}
	if !found {
		t.Errorf("expected a deadman_silent event, got %v", types)
	}
}

// The generic staleness sweep must keep its hands off heartbeats: they answer to
// their own, much tighter, timeout, and an 'expiry' resolution here would be
// both wrong and unrecoverable.
func TestDeadman_GenericExpiryLeavesHeartbeatAlone(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 5*time.Minute)

	// staleAfter far tighter than the dead man's switch timeout.
	sweep(t, s, time.Minute)

	status, source, _ := s.alertRow(t, "fp-watchdog")
	if status != "firing" || source != nil {
		t.Errorf("expected a live heartbeat left alone, got %q / %v", status, source)
	}
	if got := s.countIncidents(t); got != 0 {
		t.Errorf("expected no incident for a heartbeat that is still fresh, got %d", got)
	}
}

// An explicit resolved from Alertmanager is the sender telling us the heartbeat
// stopped. There is nothing left to wait out.
func TestDeadman_AlertmanagerResolvedIsImmediateDeath(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	postWebhook(t, s, []map[string]any{
		amAlert("fp-watchdog", "Watchdog", "resolved", "2026-05-20T10:00:00Z", zeroTime, nil),
	}, watchdogGroupKey)

	// No ageing: received_at is seconds old, well inside the timeout.
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected a resolved heartbeat to open an incident at once, got %d", got)
	}
	// Alertmanager told the truth first, so its resolution source stands.
	if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "alertmanager" {
		t.Errorf("expected the Alertmanager resolution source kept, got %v", source)
	}
}

// Each label set is its own switch, so one healthy source cannot mask a dead one.
func TestDeadman_TracksEachFingerprintSeparately(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-a", map[string]string{"cluster": "a"})
	heartbeat(t, s, "fp-b", map[string]string{"cluster": "b"})
	silence(t, s, "fp-b", 2*time.Hour)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected only the silent switch to page, got %d incidents", got)
	}
	if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-b"); status != "triggered" {
		t.Errorf("expected the incident to belong to the silent switch, got %q", status)
	}
	if status, _, _ := s.alertRow(t, "fp-a"); status != "firing" {
		t.Errorf("expected the live switch untouched, got %q", status)
	}
}

// A switch nothing has ever been heard from is dormant. A fresh deploy, a
// restored database or a typo'd alertname must not page.
func TestDeadman_UnheardOfSwitchIsDormant(t *testing.T) {
	s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=NeverSent", time.Hour, "critical"))

	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 0 {
		t.Fatalf("expected a switch that never chirped to be dormant, got %d incidents", got)
	}
}

// ---------------------------------------------------------------------------
// Recovery and re-arming
// ---------------------------------------------------------------------------

// The returning heartbeat carries the unchanged startsAt of an alert that never
// stopped firing, so this also covers the ingest guard exemption: without it the
// upsert would discard the payload and the switch could die exactly once.
func TestDeadman_RecoveryResolvesIncident(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	heartbeat(t, s, "fp-watchdog", nil)
	if status, source, _ := s.alertRow(t, "fp-watchdog"); status != "firing" || source != nil {
		t.Fatalf("expected the returning heartbeat to be accepted, got %q / %v", status, source)
	}

	sweep(t, s, noArchive)

	_, status, _, source := incidentByGroup(t, s, "deadman:fp-watchdog")
	if status != "resolved" {
		t.Errorf("expected recovery to close the incident, got %q", status)
	}
	if source == nil || *source != "recovered" {
		t.Errorf("expected resolution_source recovered, got %v", source)
	}
	if got := s.countNotifications(t, "resolved"); got != 1 {
		t.Errorf("expected 1 all-clear, got %d", got)
	}
}

// Resolving a dead man's switch incident sticks, exactly as it does for an
// alert-backed one. A source that is gone for good is a one-time page.
func TestDeadman_ManualResolveSticksWhileSilent(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()

	// Still silent, several sweeps later.
	sweep(t, s, noArchive)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected a manually resolved incident to stay closed, got %d", got)
	}
}

// ...but the switch re-arms, so a heartbeat that comes back and dies again is a
// new incident rather than silence forever.
func TestDeadman_ReArmsAfterHeartbeatReturns(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)
	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()

	// That episode is yesterday's news; the heartbeat now returns after it.
	ageIncidents(t, s, 10*time.Hour)

	heartbeat(t, s, "fp-watchdog", nil)
	sweep(t, s, noArchive)
	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected the live switch to open nothing, got %d incidents", got)
	}

	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 2 {
		t.Fatalf("expected a second death to open a second incident, got %d", got)
	}
}

// A dead man's switch incident has no member alerts — linking the heartbeat
// would have the settled-incident cascade close it on the very sweep that opened
// it — so the cascade has to leave it alone.
func TestDeadman_SettledCascadeLeavesIncidentOpen(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)
	sweep(t, s, noArchive)

	if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-watchdog"); status != "triggered" {
		t.Fatalf("expected the incident to stay open until the switch recovers, got %q", status)
	}
}

// ---------------------------------------------------------------------------
// Configuration
// ---------------------------------------------------------------------------

func TestParseDeadmanConfig(t *testing.T) {
	tests := []struct {
		name     string
		matchers string
		timeout  time.Duration
		want     []api.DeadmanMatcher
		enabled  bool
	}{
		{
			name:     "single alertname",
			matchers: "alertname=Watchdog",
			timeout:  time.Hour,
			want:     []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
			enabled:  true,
		},
		{
			name:     "several matchers with extra labels and whitespace",
			matchers: " alertname=Watchdog, cluster=prod ; alertname=EdgeHeartbeat ",
			timeout:  time.Hour,
			want: []api.DeadmanMatcher{
				{Name: "Watchdog", Labels: map[string]string{"cluster": "prod"}},
				{Name: "EdgeHeartbeat", Labels: map[string]string{}},
			},
			enabled: true,
		},
		{
			// Mandatory: it is what keeps the sweeper's candidate query on an index.
			name:     "matcher without alertname is dropped",
			matchers: "cluster=prod; alertname=Watchdog",
			timeout:  time.Hour,
			want:     []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
			enabled:  true,
		},
		{
			name:     "malformed condition drops only its matcher",
			matchers: "alertname=Watchdog,garbage; alertname=Other",
			timeout:  time.Hour,
			want:     []api.DeadmanMatcher{{Name: "Other", Labels: map[string]string{}}},
			enabled:  true,
		},
		{
			name:     "zero timeout disables",
			matchers: "alertname=Watchdog",
			timeout:  0,
			want:     []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
			enabled:  false,
		},
		{
			name:     "no usable matchers disables",
			matchers: "",
			timeout:  time.Hour,
			want:     nil,
			enabled:  false,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			got := api.ParseDeadmanConfig(tc.matchers, tc.timeout, "critical")
			if len(got.Matchers) != len(tc.want) {
				t.Fatalf("got %d matchers %v, want %d", len(got.Matchers), got.Matchers, len(tc.want))
			}
			for i, w := range tc.want {
				if got.Matchers[i].Name != w.Name {
					t.Errorf("matcher %d: name %q, want %q", i, got.Matchers[i].Name, w.Name)
				}
				if len(got.Matchers[i].Labels) != len(w.Labels) {
					t.Errorf("matcher %d: labels %v, want %v", i, got.Matchers[i].Labels, w.Labels)
					continue
				}
				for k, v := range w.Labels {
					if got.Matchers[i].Labels[k] != v {
						t.Errorf("matcher %d: label %s=%q, want %q", i, k, got.Matchers[i].Labels[k], v)
					}
				}
			}
		})
	}
}

// A zero config is off, which is what keeps the feature opt-in for anything
// building a router without one.
func TestDeadman_DisabledConfigIsInert(t *testing.T) {
	s, _ := deadmanTS(t, api.DeadmanConfig{})

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 48*time.Hour)
	sweep(t, s, time.Hour)

	// Ordinary alert handling: an incident from the arrival, not the absence.
	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected plain alert handling with deadman off, got %d incidents", got)
	}
	if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "expiry" {
		t.Errorf("expected the generic sweeper to own the alert, got %v", source)
	}
}

// ---------------------------------------------------------------------------
// Per-team configuration
// ---------------------------------------------------------------------------

// Each team decides for itself what a heartbeat is. The same alert is a
// heartbeat in one team and an ordinary problem in another.
func TestDeadman_ConfigurationIsPerTeam(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())
	watched := newTeam(t, s, "watched")
	unwatched := newTeam(t, s, "unwatched")

	// Only the first team calls Watchdog a heartbeat.
	resp := s.req(t, http.MethodPut, "/api/teams/"+id64(watched.id)+"/deadman", map[string]any{
		"matchers":        "alertname=Watchdog",
		"timeout_seconds": 3600,
		"severity":        "critical",
	})
	resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("configure the watched team: %d", resp.StatusCode)
	}

	postToIntegration(t, s, watched.key, "fp-watched", "Watchdog")
	postToIntegration(t, s, unwatched.key, "fp-unwatched", "Watchdog")

	// A heartbeat opens nothing where it is one; an ordinary alert opens an
	// incident where it is not.
	if got := len(list(t, watched.call(http.MethodGet, "/api/incidents", nil))); got != 0 {
		t.Errorf("the watched team's heartbeat opened %d incident(s), want 0", got)
	}
	if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 {
		t.Errorf("the unwatched team's Watchdog opened %d incident(s), want 1", got)
	}

	// Silence pages only the team that is watching.
	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
		time.Now().Add(-2*time.Hour).Unix(), "fp-watched")
	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
		time.Now().Add(-2*time.Hour).Unix(), "fp-unwatched")
	sweep(t, s, noArchive)

	watchedIncidents := list(t, watched.call(http.MethodGet, "/api/incidents", nil))
	if len(watchedIncidents) != 1 {
		t.Fatalf("silence opened %d incident(s) for the watching team, want 1", len(watchedIncidents))
	}
	if title := watchedIncidents[0]["title"].(string); title != "No heartbeat from Watchdog" {
		t.Errorf("unexpected incident title %q", title)
	}
	if teamID := int64(watchedIncidents[0]["team_id"].(float64)); teamID != watched.id {
		t.Errorf("the incident opened in team %d, want %d", teamID, watched.id)
	}

	// The unwatched team's alert went stale the ordinary way, so it has the one
	// incident it always had — not a second, dead man's switch one.
	if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 {
		t.Errorf("the unwatched team ended with %d incident(s), want 1", got)
	}
}

// Configuration is an owner's to change and a member's to read, like the rest of
// a team's settings.
func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())
	team := newTeam(t, s, "red")

	// A plain member of that team.
	var user struct {
		ID int64 `json:"id"`
	}
	decode(t, s.req(t, http.MethodPost, "/api/users",
		map[string]string{"username": "plain", "email": "plain@test.com"}), &user)
	s.req(t, http.MethodPost, "/api/teams/"+id64(team.id)+"/members",
		map[string]any{"user_id": user.ID, "role": "member"}).Body.Close()
	var key struct {
		Key string `json:"key"`
	}
	decode(t, s.req(t, http.MethodPost, "/api/users/"+id64(user.ID)+"/api-keys",
		map[string]string{"name": "test"}), &key)

	req, _ := http.NewRequest(http.MethodPut,
		s.URL+"/api/teams/"+id64(team.id)+"/deadman",
		strings.NewReader(`{"matchers":"alertname=Watchdog","timeout_seconds":60}`))
	req.Header.Set("Authorization", "Bearer "+key.Key)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatalf("put: %v", err)
	}
	resp.Body.Close()
	if resp.StatusCode != http.StatusForbidden {
		t.Errorf("a member editing the switches: expected 403, got %d", resp.StatusCode)
	}

	read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman", nil)
	read.Header.Set("Authorization", "Bearer "+key.Key)
	got, err := http.DefaultClient.Do(read)
	if err != nil {
		t.Fatalf("get: %v", err)
	}
	got.Body.Close()
	if got.StatusCode != http.StatusOK {
		t.Errorf("a member reading the switches: expected 200, got %d", got.StatusCode)
	}
}

// A matcher with no alertname watches nothing, silently, which is the failure
// this feature exists to prevent — so it is refused at the door.
func TestDeadman_UnusableMatchersAreRejected(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{
		"matchers":        "cluster=prod",
		"timeout_seconds": 900,
	})
	resp.Body.Close()
	if resp.StatusCode != http.StatusBadRequest {
		t.Errorf("expected 400 for a matcher with no alertname, got %d", resp.StatusCode)
	}
}