List dead man's switches with their status on Team -> Switches

The page was a bare form: it did not say which switches existed or whether they were alive. It now lists them, each with a Healthy, Dead or Dormant badge, when its heartbeat was last heard and when it last opened an incident (linked while that incident is open). A matcher that several clusters satisfy is broken down per cluster, since a live cluster must not hide a dead one. The form moved into a "New switch" sheet, and each row has a Remove with a confirm. That needed a switch to be a thing, so switches are rows now (migration 009) with their own name, matcher, timeout and severity, instead of one string with one team-wide timeout in deadman_configs. Existing configuration is split into one row per matcher; a team whose timeout was zero simply has none. The sweeper and the status endpoint share one death rule (deadmanAlert.dead), so the page cannot disagree with the pager. Incident group keys are unchanged, so incidents that are open across the upgrade keep working. The environment defaults (TERDUT_DEADMAN_*) are seeded into teams once per install, recorded in settings, so a team that deletes its last switch does not get it back on the next restart. Installs that already had per-team rows are marked as seeded by the migration. Removing a switch stops the watching but leaves an incident it already opened open until someone resolves it. API: GET/PUT /api/teams/{id}/deadman are replaced by GET/POST /deadman/switches and DELETE /deadman/switches/{switchID}. terdut-tui does not call them, so nothing to mirror there.
2026-09-25 23:40:55 +02:00
parent 3ee8583f6f
commit f3918b863c
11 changed files with 868 additions and 313 deletions
@@ -1,6 +1,7 @@
 package api_test

 import (
+	"context"
 	"net/http"
 	"strings"
 	"testing"
@@ -483,13 +484,13 @@ func TestDeadman_ConfigurationIsPerTeam(t *testing.T) {
 	unwatched := newTeam(t, s, "unwatched")

 	// Only the first team calls Watchdog a heartbeat.
-	resp := s.req(t, http.MethodPut, "/api/teams/"+id64(watched.id)+"/deadman", map[string]any{
-		"matchers":        "alertname=Watchdog",
+	resp := s.req(t, http.MethodPost, "/api/teams/"+id64(watched.id)+"/deadman/switches", map[string]any{
+		"matcher":         "alertname=Watchdog",
 		"timeout_seconds": 3600,
 		"severity":        "critical",
 	})
 	resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode != http.StatusCreated {
 		t.Fatalf("configure the watched team: %d", resp.StatusCode)
 	}

@@ -550,9 +551,9 @@ func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) {
 	decode(t, s.req(t, http.MethodPost, "/api/users/"+id64(user.ID)+"/api-keys",
 		map[string]string{"name": "test"}), &key)

-	req, _ := http.NewRequest(http.MethodPut,
-		s.URL+"/api/teams/"+id64(team.id)+"/deadman",
-		strings.NewReader(`{"matchers":"alertname=Watchdog","timeout_seconds":60}`))
+	req, _ := http.NewRequest(http.MethodPost,
+		s.URL+"/api/teams/"+id64(team.id)+"/deadman/switches",
+		strings.NewReader(`{"matcher":"alertname=Watchdog","timeout_seconds":60}`))
 	req.Header.Set("Authorization", "Bearer "+key.Key)
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := http.DefaultClient.Do(req)
@@ -564,7 +565,7 @@ func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) {
 		t.Errorf("a member editing the switches: expected 403, got %d", resp.StatusCode)
 	}

-	read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman", nil)
+	read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman/switches", nil)
 	read.Header.Set("Authorization", "Bearer "+key.Key)
 	got, err := http.DefaultClient.Do(read)
 	if err != nil {
@@ -577,16 +578,165 @@ func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) {
 }

 // A matcher with no alertname watches nothing, silently, which is the failure
-// this feature exists to prevent — so it is refused at the door.
-func TestDeadman_UnusableMatchersAreRejected(t *testing.T) {
+// this feature exists to prevent — so it is refused at the door, along with the
+// other things that would make a switch unable to fire.
+func TestDeadman_UnusableSwitchesAreRejected(t *testing.T) {
 	s, _ := deadmanTS(t, deadmanCfg())

-	resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{
-		"matchers":        "cluster=prod",
-		"timeout_seconds": 900,
-	})
-	resp.Body.Close()
-	if resp.StatusCode != http.StatusBadRequest {
-		t.Errorf("expected 400 for a matcher with no alertname, got %d", resp.StatusCode)
+	for name, body := range map[string]map[string]any{
+		"no alertname":  {"matcher": "cluster=prod", "timeout_seconds": 900},
+		"malformed":     {"matcher": "alertname=Watchdog,garbage", "timeout_seconds": 900},
+		"several":       {"matcher": "alertname=A; alertname=B", "timeout_seconds": 900},
+		"zero timeout":  {"matcher": "alertname=Watchdog", "timeout_seconds": 0},
+		"bad severity":  {"matcher": "alertname=Watchdog", "timeout_seconds": 900, "severity": "loud"},
+		"empty matcher": {"matcher": "", "timeout_seconds": 900},
+	} {
+		resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/deadman/switches", body)
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusBadRequest {
+			t.Errorf("%s: expected 400, got %d", name, resp.StatusCode)
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// The switch list
+// ---------------------------------------------------------------------------
+
+// listSwitches reads the default team's switches as the Switches page does.
+func listSwitches(t *testing.T, s *ts) []map[string]any {
+	t.Helper()
+	return list(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/deadman/switches", nil))
+}
+
+// A switch is healthy while its heartbeat is fresh, dead once it is silent, and
+// dormant until the first one arrives.
+func TestDeadman_ListReportsStatus(t *testing.T) {
+	s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=Watchdog; alertname=NeverSent", time.Hour, "critical"))
+
+	got := listSwitches(t, s)
+	if len(got) != 2 {
+		t.Fatalf("expected 2 switches, got %d", len(got))
+	}
+	for _, sw := range got {
+		if sw["status"] != "dormant" || sw["last_heartbeat_at"] != nil || sw["last_triggered_at"] != nil {
+			t.Errorf("a switch nobody has heard from should be dormant and blank, got %v", sw)
+		}
+	}
+
+	heartbeat(t, s, "fp-watchdog", nil)
+	got = listSwitches(t, s)
+	if got[0]["status"] != "healthy" || got[0]["last_heartbeat_at"] == nil {
+		t.Errorf("a fresh heartbeat should be healthy with a timestamp, got %v", got[0])
+	}
+	if got[1]["status"] != "dormant" {
+		t.Errorf("the other switch is still dormant, got %v", got[1]["status"])
+	}
+
+	silence(t, s, "fp-watchdog", 2*time.Hour)
+	sweep(t, s, noArchive)
+	got = listSwitches(t, s)
+	if got[0]["status"] != "dead" {
+		t.Fatalf("a silent heartbeat should be dead, got %v", got[0]["status"])
+	}
+	if got[0]["last_triggered_at"] == nil || got[0]["open_incident_id"] == nil {
+		t.Errorf("a dead switch should show when it triggered and its open incident, got %v", got[0])
+	}
+}
+
+// One matcher, several clusters: the switch is as bad as its worst heartbeat and
+// each heartbeat is listed on its own.
+func TestDeadman_ListBreaksDownByFingerprint(t *testing.T) {
+	s, _ := deadmanTS(t, deadmanCfg())
+
+	heartbeat(t, s, "fp-a", map[string]string{"cluster": "a"})
+	heartbeat(t, s, "fp-b", map[string]string{"cluster": "b"})
+	silence(t, s, "fp-b", 2*time.Hour)
+
+	sw := listSwitches(t, s)[0]
+	if sw["status"] != "dead" {
+		t.Errorf("one dead cluster makes the switch dead, got %v", sw["status"])
+	}
+	sources := sw["sources"].([]any)
+	if len(sources) != 2 {
+		t.Fatalf("expected 2 sources, got %d", len(sources))
+	}
+	first, second := sources[0].(map[string]any), sources[1].(map[string]any)
+	if first["fingerprint"] != "fp-b" || first["status"] != "dead" || second["status"] != "healthy" {
+		t.Errorf("the dead source should lead, got %v then %v", first, second)
+	}
+}
+
+// Every switch keeps its own deadline.
+func TestDeadman_TimeoutsArePerSwitch(t *testing.T) {
+	s, _ := deadmanTS(t, deadmanCfg())
+	resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/deadman/switches", map[string]any{
+		"matcher": "alertname=Edge", "timeout_seconds": 300,
+	})
+	resp.Body.Close()
+
+	heartbeat(t, s, "fp-watchdog", nil)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-edge", "Edge", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	}, `{}:{alertname="Edge"}`)
+
+	// Ten minutes of silence: past the Edge switch's five, inside Watchdog's hour.
+	silence(t, s, "fp-watchdog", 10*time.Minute)
+	silence(t, s, "fp-edge", 10*time.Minute)
+
+	got := listSwitches(t, s)
+	if got[0]["status"] != "healthy" || got[1]["status"] != "dead" {
+		t.Errorf("want Watchdog healthy and Edge dead, got %v and %v", got[0]["status"], got[1]["status"])
+	}
+}
+
+// Deleting is an owner's, is scoped to the team, and leaves what the switch
+// already opened alone.
+func TestDeadman_DeleteIsScopedToTheTeam(t *testing.T) {
+	s, _ := deadmanTS(t, deadmanCfg())
+	other := newTeam(t, s, "other")
+
+	id := int64(listSwitches(t, s)[0]["id"].(float64))
+
+	// Another team's owner cannot reach it.
+	resp := other.call(http.MethodDelete, "/api/teams/"+id64(other.id)+"/deadman/switches/"+id64(id), nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("deleting another team's switch: expected 404, got %d", resp.StatusCode)
+	}
+	if got := len(listSwitches(t, s)); got != 1 {
+		t.Fatalf("the switch should have survived, %d left", got)
+	}
+
+	resp = s.req(t, http.MethodDelete, "/api/teams/"+defaultTeam+"/deadman/switches/"+id64(id), nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Fatalf("deleting: expected 204, got %d", resp.StatusCode)
+	}
+	if got := len(listSwitches(t, s)); got != 0 {
+		t.Errorf("expected no switches, got %d", got)
+	}
+}
+
+// The environment's defaults are handed out once and then belong to the teams.
+func TestDeadman_SeedRunsOnce(t *testing.T) {
+	s := newTS(t)
+	cfg := api.ParseDeadmanConfig("alertname=Watchdog", time.Hour, "critical")
+
+	if err := api.SeedDeadmanConfigs(context.Background(), s.db, cfg); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+	if got := len(listSwitches(t, s)); got != 1 {
+		t.Fatalf("the first seed should add the default, got %d switches", got)
+	}
+
+	// The owner deletes it; a restart must not put it back.
+	id := int64(listSwitches(t, s)[0]["id"].(float64))
+	s.req(t, http.MethodDelete, "/api/teams/"+defaultTeam+"/deadman/switches/"+id64(id), nil).Body.Close()
+	if err := api.SeedDeadmanConfigs(context.Background(), s.db, cfg); err != nil {
+		t.Fatalf("seed again: %v", err)
+	}
+	if got := len(listSwitches(t, s)); got != 0 {
+		t.Errorf("a second seed resurrected %d switch(es)", got)
 	}
 }