terdut-server/internal/api/deadman_test.go

package api_test

import (
	"net/http"
	"strings"
	"testing"
	"time"

	"git.ryuvia.com/niklas/terdut-server/internal/api"
)

// ---------------------------------------------------------------------------
// Harness
// ---------------------------------------------------------------------------

// watchdogGroupKey is what Alertmanager sends for a Watchdog grouped by
// alertname, which is how the deployed route is configured.
const watchdogGroupKey = `{}:{alertname="Watchdog"}`

// deadmanCfg watches Watchdog with a timeout short enough to reason about and
// long enough that a fresh heartbeat is never accidentally stale.
func deadmanCfg() api.DeadmanConfig {
	return api.ParseDeadmanConfig("alertname=Watchdog", time.Hour, "critical")
}

// deadmanTS is notifyTS with dead man's switch handling on: notifications
// enabled against a fake ntfy, the admin on call today with a topic.
func deadmanTS(t *testing.T, cfg api.DeadmanConfig) (*ts, *fakeNtfy) {
	t.Helper()
	f := newFakeNtfy(t)
	s := newDeadmanTS(t, cfg, api.NotifyConfig{
		BaseURL:   f.URL,
		PublicURL: "https://terdut.example.com",
	})

	putOnCall(t, s, 1)
	setTopic(t, s, 1, "terdut-admin")
	return s, f
}

// heartbeat posts one Watchdog webhook. Its startsAt never changes: a dead man's
// switch alert fires once and is re-sent unchanged forever, which is precisely
// what makes its absence meaningful.
func heartbeat(t *testing.T, s *ts, fingerprint string, labels map[string]string) {
	t.Helper()
	postWebhook(t, s, []map[string]any{
		amAlert(fingerprint, "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, labels),
	}, watchdogGroupKey)
}

// silence back-dates a heartbeat's received_at, which is the only clock the
// sweeper reads. There is no fake clock in this package.
func silence(t *testing.T, s *ts, fingerprint string, ago time.Duration) {
	t.Helper()
	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
		time.Now().Add(-ago).Unix(), fingerprint)
}

// ageIncidents back-dates every incident. The re-arm rule compares a heartbeat
// against the last incident raised for its switch, so a test that wants a second
// episode has to put the first one in the past — there is no fake clock here.
func ageIncidents(t *testing.T, s *ts, ago time.Duration) {
	t.Helper()
	past := time.Now().Add(-ago).Unix()
	// $2 is cast explicitly: with NULL in the other branch Postgres has nothing
	// to infer the parameter's type from and defaults it to text, which the
	// bigint column then refuses.
	s.exec(t, `UPDATE incidents
	              SET triggered_at = $1,
	                  resolved_at  = CASE WHEN resolved_at IS NULL THEN NULL ELSE $2::bigint END`,
		past, past)
}

// incidentByGroup reads the incident for a group key, resolved ones included.
func incidentByGroup(t *testing.T, s *ts, groupKey string) (id int64, status, severity string, source *string) {
	t.Helper()
	err := s.db.QueryRow(`
		SELECT id, status, COALESCE(severity, ''), resolution_source
		FROM   incidents WHERE group_key = $1 ORDER BY id DESC LIMIT 1`,
		groupKey).Scan(&id, &status, &severity, &source)
	if err != nil {
		t.Fatalf("read incident for group %s: %v", groupKey, err)
	}
	return id, status, severity, source
}

// ---------------------------------------------------------------------------
// Receiving a heartbeat
// ---------------------------------------------------------------------------

// The whole inversion: arrival of a dead man's switch alert is good news, and
// good news is not an incident.
func TestDeadman_HeartbeatOpensNoIncident(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)

	if got := s.countIncidents(t); got != 0 {
		t.Fatalf("expected a heartbeat to open no incident, got %d", got)
	}
	if got := s.countNotifications(t, ""); got != 0 {
		t.Errorf("expected no notification for a heartbeat, got %d", got)
	}
	if status, _, _ := s.alertRow(t, "fp-watchdog"); status != "firing" {
		t.Errorf("expected the heartbeat to be stored firing, got %q", status)
	}
}

// A heartbeat routed into a group alongside real alerts must not join their
// incident: it is not a symptom of anything.
func TestDeadman_MixedGroupExcludesHeartbeat(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	postWebhook(t, s, []map[string]any{
		amAlert("fp-mixed-wd", "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
		amAlert("fp-mixed-disk", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime,
			map[string]string{"severity": "critical"}),
	}, `{}:{namespace="prod"}`)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected 1 incident for the real alert, got %d", got)
	}

	var incident struct {
		Alerts []map[string]any `json:"alerts"`
	}
	decode(t, s.req(t, http.MethodGet, "/api/incidents/1", nil), &incident)
	alerts := incident.Alerts
	if len(alerts) != 1 {
		t.Fatalf("expected 1 member alert, got %d", len(alerts))
	}
	if name := alerts[0]["name"]; name != "DiskFull" {
		t.Errorf("expected only the real alert linked, got %v", name)
	}
}

// A matcher scoped by label only claims the alerts it names, so a heartbeat from
// somewhere else stays an ordinary alert.
func TestDeadman_LabelScopedMatcherIgnoresOthers(t *testing.T) {
	s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=Watchdog,cluster=prod", time.Hour, "critical"))

	heartbeat(t, s, "fp-dev", map[string]string{"cluster": "dev"})

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected an unmatched Watchdog to behave like any other alert, got %d incidents", got)
	}
}

// ---------------------------------------------------------------------------
// Silence
// ---------------------------------------------------------------------------

func TestDeadman_SilenceOpensIncident(t *testing.T) {
	s, f := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected silence to open 1 incident, got %d", got)
	}
	id, status, severity, _ := incidentByGroup(t, s, "deadman:fp-watchdog")
	if status != "triggered" {
		t.Errorf("expected a triggered incident, got %q", status)
	}
	if severity != "critical" {
		t.Errorf("expected the configured severity, got %q", severity)
	}

	// The alert list must not keep claiming a dead heartbeat is firing.
	alertStatus, source, _ := s.alertRow(t, "fp-watchdog")
	if alertStatus != "resolved" || source == nil || *source != "deadman" {
		t.Errorf("expected the heartbeat resolved as deadman, got %q / %v", alertStatus, source)
	}

	// Nobody was told anything by an alert here, so the page has to come from
	// the switch itself.
	s.sweepNotify(t)
	msgs := f.messages()
	if len(msgs) != 1 {
		t.Fatalf("expected 1 page, got %d", len(msgs))
	}
	if msgs[0].Topic != "terdut-admin" {
		t.Errorf("expected the on-call topic, got %q", msgs[0].Topic)
	}
	if msgs[0].Priority != 5 {
		t.Errorf("expected a critical page to override quiet hours (priority 5), got %d", msgs[0].Priority)
	}

	// The timeline says why, with the age of the last heartbeat.
	types := eventTypes(timeline(t, s, int(id)))
	found := false
	for _, ty := range types {
		if ty == "deadman_silent" {
			found = true
		}
	}
	if !found {
		t.Errorf("expected a deadman_silent event, got %v", types)
	}
}

// The generic staleness sweep must keep its hands off heartbeats: they answer to
// their own, much tighter, timeout, and an 'expiry' resolution here would be
// both wrong and unrecoverable.
func TestDeadman_GenericExpiryLeavesHeartbeatAlone(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 5*time.Minute)

	// staleAfter far tighter than the dead man's switch timeout.
	sweep(t, s, time.Minute)

	status, source, _ := s.alertRow(t, "fp-watchdog")
	if status != "firing" || source != nil {
		t.Errorf("expected a live heartbeat left alone, got %q / %v", status, source)
	}
	if got := s.countIncidents(t); got != 0 {
		t.Errorf("expected no incident for a heartbeat that is still fresh, got %d", got)
	}
}

// An explicit resolved from Alertmanager is the sender telling us the heartbeat
// stopped. There is nothing left to wait out.
func TestDeadman_AlertmanagerResolvedIsImmediateDeath(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	postWebhook(t, s, []map[string]any{
		amAlert("fp-watchdog", "Watchdog", "resolved", "2026-05-20T10:00:00Z", zeroTime, nil),
	}, watchdogGroupKey)

	// No ageing: received_at is seconds old, well inside the timeout.
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected a resolved heartbeat to open an incident at once, got %d", got)
	}
	// Alertmanager told the truth first, so its resolution source stands.
	if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "alertmanager" {
		t.Errorf("expected the Alertmanager resolution source kept, got %v", source)
	}
}

// Each label set is its own switch, so one healthy source cannot mask a dead one.
func TestDeadman_TracksEachFingerprintSeparately(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-a", map[string]string{"cluster": "a"})
	heartbeat(t, s, "fp-b", map[string]string{"cluster": "b"})
	silence(t, s, "fp-b", 2*time.Hour)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected only the silent switch to page, got %d incidents", got)
	}
	if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-b"); status != "triggered" {
		t.Errorf("expected the incident to belong to the silent switch, got %q", status)
	}
	if status, _, _ := s.alertRow(t, "fp-a"); status != "firing" {
		t.Errorf("expected the live switch untouched, got %q", status)
	}
}

// A switch nothing has ever been heard from is dormant. A fresh deploy, a
// restored database or a typo'd alertname must not page.
func TestDeadman_UnheardOfSwitchIsDormant(t *testing.T) {
	s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=NeverSent", time.Hour, "critical"))

	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 0 {
		t.Fatalf("expected a switch that never chirped to be dormant, got %d incidents", got)
	}
}

// ---------------------------------------------------------------------------
// Recovery and re-arming
// ---------------------------------------------------------------------------

// The returning heartbeat carries the unchanged startsAt of an alert that never
// stopped firing, so this also covers the ingest guard exemption: without it the
// upsert would discard the payload and the switch could die exactly once.
func TestDeadman_RecoveryResolvesIncident(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	heartbeat(t, s, "fp-watchdog", nil)
	if status, source, _ := s.alertRow(t, "fp-watchdog"); status != "firing" || source != nil {
		t.Fatalf("expected the returning heartbeat to be accepted, got %q / %v", status, source)
	}

	sweep(t, s, noArchive)

	_, status, _, source := incidentByGroup(t, s, "deadman:fp-watchdog")
	if status != "resolved" {
		t.Errorf("expected recovery to close the incident, got %q", status)
	}
	if source == nil || *source != "recovered" {
		t.Errorf("expected resolution_source recovered, got %v", source)
	}
	if got := s.countNotifications(t, "resolved"); got != 1 {
		t.Errorf("expected 1 all-clear, got %d", got)
	}
}

// Resolving a dead man's switch incident sticks, exactly as it does for an
// alert-backed one. A source that is gone for good is a one-time page.
func TestDeadman_ManualResolveSticksWhileSilent(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()

	// Still silent, several sweeps later.
	sweep(t, s, noArchive)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected a manually resolved incident to stay closed, got %d", got)
	}
}

// ...but the switch re-arms, so a heartbeat that comes back and dies again is a
// new incident rather than silence forever.
func TestDeadman_ReArmsAfterHeartbeatReturns(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)
	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()

	// That episode is yesterday's news; the heartbeat now returns after it.
	ageIncidents(t, s, 10*time.Hour)

	heartbeat(t, s, "fp-watchdog", nil)
	sweep(t, s, noArchive)
	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected the live switch to open nothing, got %d incidents", got)
	}

	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)

	if got := s.countIncidents(t); got != 2 {
		t.Fatalf("expected a second death to open a second incident, got %d", got)
	}
}

// A dead man's switch incident has no member alerts — linking the heartbeat
// would have the settled-incident cascade close it on the very sweep that opened
// it — so the cascade has to leave it alone.
func TestDeadman_SettledCascadeLeavesIncidentOpen(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)
	sweep(t, s, noArchive)

	if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-watchdog"); status != "triggered" {
		t.Fatalf("expected the incident to stay open until the switch recovers, got %q", status)
	}
}

// ---------------------------------------------------------------------------
// Configuration
// ---------------------------------------------------------------------------

func TestParseDeadmanConfig(t *testing.T) {
	tests := []struct {
		name     string
		matchers string
		timeout  time.Duration
		want     []api.DeadmanMatcher
		enabled  bool
	}{
		{
			name:     "single alertname",
			matchers: "alertname=Watchdog",
			timeout:  time.Hour,
			want:     []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
			enabled:  true,
		},
		{
			name:     "several matchers with extra labels and whitespace",
			matchers: " alertname=Watchdog, cluster=prod ; alertname=EdgeHeartbeat ",
			timeout:  time.Hour,
			want: []api.DeadmanMatcher{
				{Name: "Watchdog", Labels: map[string]string{"cluster": "prod"}},
				{Name: "EdgeHeartbeat", Labels: map[string]string{}},
			},
			enabled: true,
		},
		{
			// Mandatory: it is what keeps the sweeper's candidate query on an index.
			name:     "matcher without alertname is dropped",
			matchers: "cluster=prod; alertname=Watchdog",
			timeout:  time.Hour,
			want:     []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
			enabled:  true,
		},
		{
			name:     "malformed condition drops only its matcher",
			matchers: "alertname=Watchdog,garbage; alertname=Other",
			timeout:  time.Hour,
			want:     []api.DeadmanMatcher{{Name: "Other", Labels: map[string]string{}}},
			enabled:  true,
		},
		{
			name:     "zero timeout disables",
			matchers: "alertname=Watchdog",
			timeout:  0,
			want:     []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
			enabled:  false,
		},
		{
			name:     "no usable matchers disables",
			matchers: "",
			timeout:  time.Hour,
			want:     nil,
			enabled:  false,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			got := api.ParseDeadmanConfig(tc.matchers, tc.timeout, "critical")
			if len(got.Matchers) != len(tc.want) {
				t.Fatalf("got %d matchers %v, want %d", len(got.Matchers), got.Matchers, len(tc.want))
			}
			for i, w := range tc.want {
				if got.Matchers[i].Name != w.Name {
					t.Errorf("matcher %d: name %q, want %q", i, got.Matchers[i].Name, w.Name)
				}
				if len(got.Matchers[i].Labels) != len(w.Labels) {
					t.Errorf("matcher %d: labels %v, want %v", i, got.Matchers[i].Labels, w.Labels)
					continue
				}
				for k, v := range w.Labels {
					if got.Matchers[i].Labels[k] != v {
						t.Errorf("matcher %d: label %s=%q, want %q", i, k, got.Matchers[i].Labels[k], v)
					}
				}
			}
		})
	}
}

// A zero config is off, which is what keeps the feature opt-in for anything
// building a router without one.
func TestDeadman_DisabledConfigIsInert(t *testing.T) {
	s, _ := deadmanTS(t, api.DeadmanConfig{})

	heartbeat(t, s, "fp-watchdog", nil)
	silence(t, s, "fp-watchdog", 48*time.Hour)
	sweep(t, s, time.Hour)

	// Ordinary alert handling: an incident from the arrival, not the absence.
	if got := s.countIncidents(t); got != 1 {
		t.Fatalf("expected plain alert handling with deadman off, got %d incidents", got)
	}
	if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "expiry" {
		t.Errorf("expected the generic sweeper to own the alert, got %v", source)
	}
}

// ---------------------------------------------------------------------------
// Per-team configuration
// ---------------------------------------------------------------------------

// Each team decides for itself what a heartbeat is. The same alert is a
// heartbeat in one team and an ordinary problem in another.
func TestDeadman_ConfigurationIsPerTeam(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())
	watched := newTeam(t, s, "watched")
	unwatched := newTeam(t, s, "unwatched")

	// Only the first team calls Watchdog a heartbeat.
	resp := s.req(t, http.MethodPost, "/api/teams/"+id64(watched.id)+"/deadman/switches", map[string]any{
		"matcher":         "alertname=Watchdog",
		"timeout_seconds": 3600,
		"severity":        "critical",
	})
	resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		t.Fatalf("configure the watched team: %d", resp.StatusCode)
	}

	postToIntegration(t, s, watched.key, "fp-watched", "Watchdog")
	postToIntegration(t, s, unwatched.key, "fp-unwatched", "Watchdog")

	// A heartbeat opens nothing where it is one; an ordinary alert opens an
	// incident where it is not.
	if got := len(list(t, watched.call(http.MethodGet, "/api/incidents", nil))); got != 0 {
		t.Errorf("the watched team's heartbeat opened %d incident(s), want 0", got)
	}
	if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 {
		t.Errorf("the unwatched team's Watchdog opened %d incident(s), want 1", got)
	}

	// Silence pages only the team that is watching.
	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
		time.Now().Add(-2*time.Hour).Unix(), "fp-watched")
	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
		time.Now().Add(-2*time.Hour).Unix(), "fp-unwatched")
	sweep(t, s, noArchive)

	watchedIncidents := list(t, watched.call(http.MethodGet, "/api/incidents", nil))
	if len(watchedIncidents) != 1 {
		t.Fatalf("silence opened %d incident(s) for the watching team, want 1", len(watchedIncidents))
	}
	if title := watchedIncidents[0]["title"].(string); title != "No heartbeat from Watchdog" {
		t.Errorf("unexpected incident title %q", title)
	}
	if teamID := int64(watchedIncidents[0]["team_id"].(float64)); teamID != watched.id {
		t.Errorf("the incident opened in team %d, want %d", teamID, watched.id)
	}

	// The unwatched team's alert went stale the ordinary way, so it has the one
	// incident it always had — not a second, dead man's switch one.
	if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 {
		t.Errorf("the unwatched team ended with %d incident(s), want 1", got)
	}
}

// Configuration is an owner's to change and a member's to read, like the rest of
// a team's settings.
func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())
	team := newTeam(t, s, "red")

	// A plain member of that team.
	var user struct {
		ID int64 `json:"id"`
	}
	decode(t, s.req(t, http.MethodPost, "/api/users",
		map[string]string{"username": "plain", "email": "plain@test.com"}), &user)
	s.req(t, http.MethodPost, "/api/teams/"+id64(team.id)+"/members",
		map[string]any{"user_id": user.ID, "role": "member"}).Body.Close()
	var key struct {
		Key string `json:"key"`
	}
	decode(t, s.req(t, http.MethodPost, "/api/users/"+id64(user.ID)+"/api-keys",
		map[string]string{"name": "test"}), &key)

	req, _ := http.NewRequest(http.MethodPost,
		s.URL+"/api/teams/"+id64(team.id)+"/deadman/switches",
		strings.NewReader(`{"matcher":"alertname=Watchdog","timeout_seconds":60}`))
	req.Header.Set("Authorization", "Bearer "+key.Key)
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatalf("put: %v", err)
	}
	resp.Body.Close()
	if resp.StatusCode != http.StatusForbidden {
		t.Errorf("a member editing the switches: expected 403, got %d", resp.StatusCode)
	}

	read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman/switches", nil)
	read.Header.Set("Authorization", "Bearer "+key.Key)
	got, err := http.DefaultClient.Do(read)
	if err != nil {
		t.Fatalf("get: %v", err)
	}
	got.Body.Close()
	if got.StatusCode != http.StatusOK {
		t.Errorf("a member reading the switches: expected 200, got %d", got.StatusCode)
	}
}

// A matcher with no alertname watches nothing, silently, which is the failure
// this feature exists to prevent — so it is refused at the door, along with the
// other things that would make a switch unable to fire.
func TestDeadman_UnusableSwitchesAreRejected(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	for name, body := range map[string]map[string]any{
		"no alertname":  {"matcher": "cluster=prod", "timeout_seconds": 900},
		"malformed":     {"matcher": "alertname=Watchdog,garbage", "timeout_seconds": 900},
		"several":       {"matcher": "alertname=A; alertname=B", "timeout_seconds": 900},
		"zero timeout":  {"matcher": "alertname=Watchdog", "timeout_seconds": 0},
		"bad severity":  {"matcher": "alertname=Watchdog", "timeout_seconds": 900, "severity": "loud"},
		"empty matcher": {"matcher": "", "timeout_seconds": 900},
	} {
		resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/deadman/switches", body)
		resp.Body.Close()
		if resp.StatusCode != http.StatusBadRequest {
			t.Errorf("%s: expected 400, got %d", name, resp.StatusCode)
		}
	}
}

// ---------------------------------------------------------------------------
// The switch list
// ---------------------------------------------------------------------------

// listSwitches reads the default team's switches as the Switches page does.
func listSwitches(t *testing.T, s *ts) []map[string]any {
	t.Helper()
	return list(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/deadman/switches", nil))
}

// A switch is healthy while its heartbeat is fresh, dead once it is silent, and
// dormant until the first one arrives.
func TestDeadman_ListReportsStatus(t *testing.T) {
	s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=Watchdog; alertname=NeverSent", time.Hour, "critical"))

	got := listSwitches(t, s)
	if len(got) != 2 {
		t.Fatalf("expected 2 switches, got %d", len(got))
	}
	for _, sw := range got {
		if sw["status"] != "dormant" || sw["last_heartbeat_at"] != nil || sw["last_triggered_at"] != nil {
			t.Errorf("a switch nobody has heard from should be dormant and blank, got %v", sw)
		}
	}

	heartbeat(t, s, "fp-watchdog", nil)
	got = listSwitches(t, s)
	if got[0]["status"] != "healthy" || got[0]["last_heartbeat_at"] == nil {
		t.Errorf("a fresh heartbeat should be healthy with a timestamp, got %v", got[0])
	}
	if got[1]["status"] != "dormant" {
		t.Errorf("the other switch is still dormant, got %v", got[1]["status"])
	}

	silence(t, s, "fp-watchdog", 2*time.Hour)
	sweep(t, s, noArchive)
	got = listSwitches(t, s)
	if got[0]["status"] != "dead" {
		t.Fatalf("a silent heartbeat should be dead, got %v", got[0]["status"])
	}
	if got[0]["last_triggered_at"] == nil || got[0]["open_incident_id"] == nil {
		t.Errorf("a dead switch should show when it triggered and its open incident, got %v", got[0])
	}
}

// One matcher, several clusters: the switch is as bad as its worst heartbeat and
// each heartbeat is listed on its own.
func TestDeadman_ListBreaksDownByFingerprint(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())

	heartbeat(t, s, "fp-a", map[string]string{"cluster": "a"})
	heartbeat(t, s, "fp-b", map[string]string{"cluster": "b"})
	silence(t, s, "fp-b", 2*time.Hour)

	sw := listSwitches(t, s)[0]
	if sw["status"] != "dead" {
		t.Errorf("one dead cluster makes the switch dead, got %v", sw["status"])
	}
	sources := sw["sources"].([]any)
	if len(sources) != 2 {
		t.Fatalf("expected 2 sources, got %d", len(sources))
	}
	first, second := sources[0].(map[string]any), sources[1].(map[string]any)
	if first["fingerprint"] != "fp-b" || first["status"] != "dead" || second["status"] != "healthy" {
		t.Errorf("the dead source should lead, got %v then %v", first, second)
	}
}

// Every switch keeps its own deadline.
func TestDeadman_TimeoutsArePerSwitch(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())
	resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/deadman/switches", map[string]any{
		"matcher": "alertname=Edge", "timeout_seconds": 300,
	})
	resp.Body.Close()

	heartbeat(t, s, "fp-watchdog", nil)
	postWebhook(t, s, []map[string]any{
		amAlert("fp-edge", "Edge", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	}, `{}:{alertname="Edge"}`)

	// Ten minutes of silence: past the Edge switch's five, inside Watchdog's hour.
	silence(t, s, "fp-watchdog", 10*time.Minute)
	silence(t, s, "fp-edge", 10*time.Minute)

	got := listSwitches(t, s)
	if got[0]["status"] != "healthy" || got[1]["status"] != "dead" {
		t.Errorf("want Watchdog healthy and Edge dead, got %v and %v", got[0]["status"], got[1]["status"])
	}
}

// Deleting is an owner's, is scoped to the team, and leaves what the switch
// already opened alone.
func TestDeadman_DeleteIsScopedToTheTeam(t *testing.T) {
	s, _ := deadmanTS(t, deadmanCfg())
	other := newTeam(t, s, "other")

	id := int64(listSwitches(t, s)[0]["id"].(float64))

	// Another team's owner cannot reach it.
	resp := other.call(http.MethodDelete, "/api/teams/"+id64(other.id)+"/deadman/switches/"+id64(id), nil)
	resp.Body.Close()
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("deleting another team's switch: expected 404, got %d", resp.StatusCode)
	}
	if got := len(listSwitches(t, s)); got != 1 {
		t.Fatalf("the switch should have survived, %d left", got)
	}

	resp = s.req(t, http.MethodDelete, "/api/teams/"+defaultTeam+"/deadman/switches/"+id64(id), nil)
	resp.Body.Close()
	if resp.StatusCode != http.StatusNoContent {
		t.Fatalf("deleting: expected 204, got %d", resp.StatusCode)
	}
	if got := len(listSwitches(t, s)); got != 0 {
		t.Errorf("expected no switches, got %d", got)
	}
}