package api_test import ( "net/http" "strings" "testing" "time" "git.ryuvia.com/niklas/terdut-server/internal/api" ) // teamUser creates a user in the default team with an ntfy topic, so they can // actually be paged. func teamUser(t *testing.T, s *ts, username, topic string) int64 { t.Helper() var user struct { ID int64 `json:"id"` } decode(t, s.req(t, http.MethodPost, "/api/users", map[string]string{"username": username, "email": username + "@test.com"}), &user) resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members", map[string]any{"user_id": user.ID, "role": "member"}) resp.Body.Close() setTopic(t, s, int(user.ID), topic) return user.ID } // Escalation is all timeouts, and there is no fake clock in this package. The // tests back-date escalation_level_at instead, which is the same trick the dead // man's switch tests use on received_at: the sweeper reads a stored timestamp, // so moving the timestamp is moving the clock. // ladder configures the default team with two levels: the rota first, then a // named person, then the fallback topic. func ladder(t *testing.T, s *ts, secondUserID int64, repeat int64, fallback string) { t.Helper() resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", map[string]any{ "repeat_count": repeat, "fallback_topic": fallback, "levels": []map[string]any{ {"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}}, {"timeout_seconds": 300, "targets": []map[string]any{{"kind": "user", "user_id": secondUserID}}}, }, }) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("configure the ladder: %d", resp.StatusCode) } } // overdue back-dates an incident's current level so its timeout has passed. func overdue(t *testing.T, s *ts, incidentID int64) { t.Helper() s.exec(t, "UPDATE incidents SET escalation_level_at = $1 WHERE id = $2", time.Now().Add(-time.Hour).Unix(), incidentID) } func escalationLevel(t *testing.T, s *ts, incidentID int64) (level, round int64) { t.Helper() if err := s.db.QueryRow( "SELECT escalation_level, escalation_round FROM incidents WHERE id = $1", incidentID).Scan(&level, &round); err != nil { t.Fatalf("read escalation state: %v", err) } return level, round } // The whole point: nobody answers, so somebody else is woken. func TestEscalation_PagesTheNextLevel(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-esc", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) // Level 1 is the rota, so the first page went to the admin. if level, _ := escalationLevel(t, s, 1); level != 1 { t.Fatalf("a new incident should start at level 1, got %d", level) } if got := f.topicsSince(t); len(got) == 0 || got[0] != "terdut-admin" { t.Fatalf("the first page should go to the on-call user, went to %v", got) } // Time passes with no acknowledgement. f.forget() overdue(t, s, 1) s.sweepNotify(t) if level, _ := escalationLevel(t, s, 1); level != 2 { t.Errorf("expected level 2, got %d", level) } if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-second" { t.Errorf("level 2 should page the named user, paged %v", got) } // And the timeline says so, which is what somebody reads afterwards to // understand why their phone rang at 04:00. timeline := list(t, s.req(t, http.MethodGet, "/api/incidents/1/timeline", nil)) found := "" for _, e := range timeline { if e["type"] == "escalated" { found, _ = e["detail"].(string) } } if found == "" { t.Error("the timeline should record the escalation") } else if !strings.HasPrefix(found, "level 2") || !strings.Contains(found, "second") { t.Errorf("the escalation entry should say which level and who: %q", found) } } // Acknowledging is somebody saying "I have this". Nobody else should be woken. func TestEscalation_AcknowledgementStopsIt(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-ack", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close() if level, _ := escalationLevel(t, s, 1); level != 0 { t.Errorf("acknowledging should take the incident off the ladder, level is %d", level) } f.forget() overdue(t, s, 1) // no-op: level is 0, so there is nothing due s.sweepNotify(t) if got := f.topicsSince(t); len(got) != 0 { t.Errorf("an acknowledged incident should page nobody, paged %v", got) } } // Resolving stops it too, and by the same mechanism. func TestEscalation_ResolutionStopsIt(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-res", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close() f.forget() overdue(t, s, 1) s.sweepNotify(t) if level, _ := escalationLevel(t, s, 1); level != 0 { t.Errorf("a resolved incident should be off the ladder, level is %d", level) } } // Snoozing is a deliberate "not now", so the ladder waits rather than carrying // on without the person who asked for quiet. func TestEscalation_SnoozePausesIt(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-snooze", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]any{"duration": "1h"}) resp.Body.Close() f.forget() overdue(t, s, 1) s.sweepNotify(t) if level, _ := escalationLevel(t, s, 1); level != 1 { t.Errorf("a snoozed incident should stay where it is, level is %d", level) } if got := f.topicsSince(t); len(got) != 0 { t.Errorf("a snoozed incident should page nobody, paged %v", got) } // When the snooze ends, the ladder picks up where it left off. s.exec(t, "UPDATE incidents SET snoozed_until = $1 WHERE id = 1", time.Now().Add(-time.Minute).Unix()) s.sweepNotify(t) if level, _ := escalationLevel(t, s, 1); level != 2 { t.Errorf("after the snooze the ladder should resume, level is %d", level) } } // Running out of ladder pages the team's fallback topic once, and says so. func TestEscalation_ExhaustionPagesTheFallback(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-end", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) overdue(t, s, 1) s.sweepNotify(t) // level 2 f.forget() overdue(t, s, 1) s.sweepNotify(t) // off the end if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-fallback" { t.Errorf("exhaustion should page the fallback topic once, paged %v", got) } level, _ := escalationLevel(t, s, 1) if level != 0 { t.Errorf("an exhausted ladder should stop asking, level is %d", level) } // The incident is still open: running out of people is not an answer. var status string if err := s.db.QueryRow("SELECT status FROM incidents WHERE id = 1").Scan(&status); err != nil { t.Fatal(err) } if status != "triggered" { t.Errorf("exhaustion must not resolve the incident, status is %q", status) } } // repeat_count walks the whole ladder again before giving up. func TestEscalation_RepeatsTheChain(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 1, "terdut-fallback") // one extra round postWebhook(t, s, []map[string]any{ amAlert("fp-repeat", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) overdue(t, s, 1) s.sweepNotify(t) // level 2 f.forget() overdue(t, s, 1) s.sweepNotify(t) // back to level 1, round 2 level, round := escalationLevel(t, s, 1) if level != 1 || round != 1 { t.Errorf("expected level 1 round 1, got level %d round %d", level, round) } if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-admin" { t.Errorf("the second round should start at the top again, paged %v", got) } } // A team without a ladder keeps exactly the behaviour it had, and never gets // both a reminder and an escalation for the same silence. func TestEscalation_WithoutAPolicyRemindersStillRun(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) postWebhook(t, s, []map[string]any{ amAlert("fp-noesc", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) // Age the first notification past the repeat interval. f.forget() s.exec(t, "UPDATE notifications SET created_at = $1, sent_at = $1", time.Now().Add(-time.Hour).Unix()) s.sweepNotify(t) if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-admin" { t.Errorf("without a ladder the reminder should still fire, paged %v", got) } if level, _ := escalationLevel(t, s, 1); level != 0 { t.Errorf("an incident in a team with no ladder should not be on one, level is %d", level) } } // With a ladder, reminders stop: two pages for one silence is how people learn // to mute the tool. func TestEscalation_WithAPolicyRemindersDoNotAlsoFire(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-both", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) f.forget() // Old enough for a reminder, but not yet due for escalation. s.exec(t, "UPDATE notifications SET created_at = $1, sent_at = $1", time.Now().Add(-time.Hour).Unix()) s.sweepNotify(t) if got := f.topicsSince(t); len(got) != 0 { t.Errorf("a team with a ladder should not also get reminders, paged %v", got) } } // The API refuses a ladder that cannot page anybody. func TestEscalation_RejectsAnUnusablePolicy(t *testing.T) { s := newTS(t) for _, c := range []struct { name string body map[string]any }{ {"a level with no targets", map[string]any{ "levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{}}}, }}, {"a level with no timeout", map[string]any{ "levels": []map[string]any{{"timeout_seconds": 0, "targets": []map[string]any{{"kind": "oncall"}}}}, }}, {"a user target with no user", map[string]any{ "levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "user"}}}}, }}, {"an unknown target kind", map[string]any{ "levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "everybody"}}}}, }}, {"an absurd repeat count", map[string]any{ "repeat_count": 99, "levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}}}, }}, } { resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", c.body) resp.Body.Close() if resp.StatusCode != http.StatusBadRequest { t.Errorf("%s: expected 400, got %d", c.name, resp.StatusCode) } } } // Editing the ladder is an owner's job; reading it is any member's. func TestEscalation_OwnerOnlyToEdit(t *testing.T) { s := newTS(t) _, call := member(t, s, "plain") resp := call(http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", map[string]any{ "levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}}}, }) resp.Body.Close() if resp.StatusCode != http.StatusForbidden { t.Errorf("a member editing the ladder: expected 403, got %d", resp.StatusCode) } resp = call(http.MethodGet, "/api/teams/"+defaultTeam+"/escalation", nil) resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Errorf("a member reading the ladder: expected 200, got %d", resp.StatusCode) } } // A target who cannot be woken is not a reason to stop: the next level is the // answer to an unreachable one. func TestEscalation_SkipsUnreachableTargets(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) // Second user has no ntfy topic at all. var user struct { ID int64 `json:"id"` } decode(t, s.req(t, http.MethodPost, "/api/users", map[string]string{"username": "silent", "email": "silent@test.com"}), &user) s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members", map[string]any{"user_id": user.ID, "role": "member"}).Body.Close() ladder(t, s, user.ID, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-silent", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) f.forget() overdue(t, s, 1) s.sweepNotify(t) // Level 2 was entered even though it woke nobody, so the ladder keeps // moving toward the fallback rather than stalling on a silent rung. if level, _ := escalationLevel(t, s, 1); level != 2 { t.Errorf("expected the ladder to advance past an unreachable target, level is %d", level) } if got := f.topicsSince(t); len(got) != 0 { t.Errorf("a target with no topic should page nothing, paged %v", got) } } // --------------------------------------------------------------------------- // The ladder as the Escalation page reads it // --------------------------------------------------------------------------- type ladderLevel struct { Status string `json:"status"` Waiting []int64 `json:"waiting"` Targets []struct { Kind string `json:"kind"` Username string `json:"username"` Reachable bool `json:"reachable"` Problem string `json:"problem"` } `json:"targets"` } type ladderView struct { Levels []ladderLevel `json:"levels"` LastEscalatedAt *string `json:"last_escalated_at"` LastEscalatedIncidentID *int64 `json:"last_escalated_incident_id"` } func readLadder(t *testing.T, s *ts) ladderView { t.Helper() var v ladderView decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/escalation", nil), &v) return v } // Targets say who they mean today, so "whoever is on call" is a name and not a // promise. func TestEscalation_StatusResolvesTargets(t *testing.T) { s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") v := readLadder(t, s) if len(v.Levels) != 2 { t.Fatalf("expected 2 levels, got %d", len(v.Levels)) } if got := v.Levels[0].Targets[0]; got.Kind != "oncall" || got.Username != "admin" || !got.Reachable { t.Errorf("the rota target should resolve to the person on call, got %+v", got) } if got := v.Levels[1].Targets[0]; got.Username != "second" || !got.Reachable { t.Errorf("the named target should be reachable, got %+v", got) } if v.Levels[0].Status != "ready" || v.Levels[1].Status != "ready" || v.LastEscalatedAt != nil { t.Errorf("an idle, healthy ladder is ready and has never escalated, got %+v", v) } } // A rung that would page nobody is called out before an incident finds it. func TestEscalation_StatusFlagsUnreachableLevels(t *testing.T) { s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) silent := teamUser(t, s, "silent", "terdut-silent") ladder(t, s, silent, 0, "terdut-fallback") // Nobody on call today, and the named person loses their topic. s.exec(t, "DELETE FROM schedule_entries") s.exec(t, "UPDATE users SET ntfy_topic = NULL WHERE id = $1", silent) v := readLadder(t, s) if v.Levels[0].Status != "unreachable" || v.Levels[0].Targets[0].Problem != "nobody is on call today" { t.Errorf("an empty rota should make level 1 unreachable, got %+v", v.Levels[0]) } if v.Levels[1].Status != "unreachable" || v.Levels[1].Targets[0].Problem != "has no ntfy topic" { t.Errorf("a person with no topic should make level 2 unreachable, got %+v", v.Levels[1]) } s.exec(t, "UPDATE users SET disabled_at = 1 WHERE id = $1", silent) if p := readLadder(t, s).Levels[1].Targets[0].Problem; p != "account is disabled" { t.Errorf("a disabled account should say so, got %q", p) } } // Where unanswered incidents are right now, and when the ladder last did its // job. func TestEscalation_StatusShowsWhoIsWaitingAndLastEscalation(t *testing.T) { s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) second := teamUser(t, s, "second", "terdut-second") ladder(t, s, second, 0, "terdut-fallback") postWebhook(t, s, []map[string]any{ amAlert("fp-wait", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil), }) s.sweepNotify(t) // On level 1 it is waiting, which is normal and not yet an escalation. v := readLadder(t, s) if len(v.Levels[0].Waiting) != 1 || v.Levels[0].Status != "ready" || v.LastEscalatedAt != nil { t.Fatalf("a fresh incident waits on level 1 quietly, got %+v", v) } overdue(t, s, 1) s.sweepNotify(t) v = readLadder(t, s) if v.Levels[1].Status != "escalating" || len(v.Levels[1].Waiting) != 1 || v.Levels[1].Waiting[0] != 1 { t.Errorf("level 2 should be escalating with the incident on it, got %+v", v.Levels[1]) } if v.LastEscalatedAt == nil || v.LastEscalatedIncidentID == nil || *v.LastEscalatedIncidentID != 1 { t.Errorf("the escalation should be recorded, got %+v", v) } // Somebody answers: nothing is waiting, but the history stays. s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close() v = readLadder(t, s) if v.Levels[1].Status != "ready" || len(v.Levels[1].Waiting) != 0 || v.LastEscalatedAt == nil { t.Errorf("an acknowledged incident stops waiting but stays in the history, got %+v", v) } } // No ladder is a real answer, not an error. func TestEscalation_StatusWithoutALadder(t *testing.T) { s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute}) v := readLadder(t, s) if len(v.Levels) != 0 || v.LastEscalatedAt != nil { t.Errorf("a team with no ladder should read as empty, got %+v", v) } }