terdut-server/internal/api/escalation_test.go

package api_test

import (
	"net/http"
	"strings"
	"testing"
	"time"

	"git.ryuvia.com/niklas/terdut-server/internal/api"
)

// teamUser creates a user in the default team with an ntfy topic, so they can
// actually be paged.
func teamUser(t *testing.T, s *ts, username, topic string) int64 {
	t.Helper()
	var user struct {
		ID int64 `json:"id"`
	}
	decode(t, s.req(t, http.MethodPost, "/api/users",
		map[string]string{"username": username, "email": username + "@test.com"}), &user)
	resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members",
		map[string]any{"user_id": user.ID, "role": "member"})
	resp.Body.Close()
	setTopic(t, s, int(user.ID), topic)
	return user.ID
}

// Escalation is all timeouts, and there is no fake clock in this package. The
// tests back-date escalation_level_at instead, which is the same trick the dead
// man's switch tests use on received_at: the sweeper reads a stored timestamp,
// so moving the timestamp is moving the clock.

// ladder configures the default team with two levels: the rota first, then a
// named person, then the fallback topic.
func ladder(t *testing.T, s *ts, secondUserID int64, repeat int64, fallback string) {
	t.Helper()
	resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", map[string]any{
		"repeat_count":   repeat,
		"fallback_topic": fallback,
		"levels": []map[string]any{
			{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}},
			{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "user", "user_id": secondUserID}}},
		},
	})
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("configure the ladder: %d", resp.StatusCode)
	}
}

// overdue back-dates an incident's current level so its timeout has passed.
func overdue(t *testing.T, s *ts, incidentID int64) {
	t.Helper()
	s.exec(t, "UPDATE incidents SET escalation_level_at = $1 WHERE id = $2",
		time.Now().Add(-time.Hour).Unix(), incidentID)
}

func escalationLevel(t *testing.T, s *ts, incidentID int64) (level, round int64) {
	t.Helper()
	if err := s.db.QueryRow(
		"SELECT escalation_level, escalation_round FROM incidents WHERE id = $1",
		incidentID).Scan(&level, &round); err != nil {
		t.Fatalf("read escalation state: %v", err)
	}
	return level, round
}

// The whole point: nobody answers, so somebody else is woken.
func TestEscalation_PagesTheNextLevel(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-esc", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	// Level 1 is the rota, so the first page went to the admin.
	if level, _ := escalationLevel(t, s, 1); level != 1 {
		t.Fatalf("a new incident should start at level 1, got %d", level)
	}
	if got := f.topicsSince(t); len(got) == 0 || got[0] != "terdut-admin" {
		t.Fatalf("the first page should go to the on-call user, went to %v", got)
	}

	// Time passes with no acknowledgement.
	f.forget()
	overdue(t, s, 1)
	s.sweepNotify(t)

	if level, _ := escalationLevel(t, s, 1); level != 2 {
		t.Errorf("expected level 2, got %d", level)
	}
	if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-second" {
		t.Errorf("level 2 should page the named user, paged %v", got)
	}

	// And the timeline says so, which is what somebody reads afterwards to
	// understand why their phone rang at 04:00.
	timeline := list(t, s.req(t, http.MethodGet, "/api/incidents/1/timeline", nil))
	found := ""
	for _, e := range timeline {
		if e["type"] == "escalated" {
			found, _ = e["detail"].(string)
		}
	}
	if found == "" {
		t.Error("the timeline should record the escalation")
	} else if !strings.HasPrefix(found, "level 2") || !strings.Contains(found, "second") {
		t.Errorf("the escalation entry should say which level and who: %q", found)
	}
}

// Acknowledging is somebody saying "I have this". Nobody else should be woken.
func TestEscalation_AcknowledgementStopsIt(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-ack", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
	if level, _ := escalationLevel(t, s, 1); level != 0 {
		t.Errorf("acknowledging should take the incident off the ladder, level is %d", level)
	}

	f.forget()
	overdue(t, s, 1) // no-op: level is 0, so there is nothing due
	s.sweepNotify(t)
	if got := f.topicsSince(t); len(got) != 0 {
		t.Errorf("an acknowledged incident should page nobody, paged %v", got)
	}
}

// Resolving stops it too, and by the same mechanism.
func TestEscalation_ResolutionStopsIt(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-res", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)
	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()

	f.forget()
	overdue(t, s, 1)
	s.sweepNotify(t)
	if level, _ := escalationLevel(t, s, 1); level != 0 {
		t.Errorf("a resolved incident should be off the ladder, level is %d", level)
	}
}

// Snoozing is a deliberate "not now", so the ladder waits rather than carrying
// on without the person who asked for quiet.
func TestEscalation_SnoozePausesIt(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-snooze", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]any{"duration": "1h"})
	resp.Body.Close()

	f.forget()
	overdue(t, s, 1)
	s.sweepNotify(t)

	if level, _ := escalationLevel(t, s, 1); level != 1 {
		t.Errorf("a snoozed incident should stay where it is, level is %d", level)
	}
	if got := f.topicsSince(t); len(got) != 0 {
		t.Errorf("a snoozed incident should page nobody, paged %v", got)
	}

	// When the snooze ends, the ladder picks up where it left off.
	s.exec(t, "UPDATE incidents SET snoozed_until = $1 WHERE id = 1", time.Now().Add(-time.Minute).Unix())
	s.sweepNotify(t)
	if level, _ := escalationLevel(t, s, 1); level != 2 {
		t.Errorf("after the snooze the ladder should resume, level is %d", level)
	}
}

// Running out of ladder pages the team's fallback topic once, and says so.
func TestEscalation_ExhaustionPagesTheFallback(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-end", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	overdue(t, s, 1)
	s.sweepNotify(t) // level 2
	f.forget()
	overdue(t, s, 1)
	s.sweepNotify(t) // off the end

	if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-fallback" {
		t.Errorf("exhaustion should page the fallback topic once, paged %v", got)
	}
	level, _ := escalationLevel(t, s, 1)
	if level != 0 {
		t.Errorf("an exhausted ladder should stop asking, level is %d", level)
	}

	// The incident is still open: running out of people is not an answer.
	var status string
	if err := s.db.QueryRow("SELECT status FROM incidents WHERE id = 1").Scan(&status); err != nil {
		t.Fatal(err)
	}
	if status != "triggered" {
		t.Errorf("exhaustion must not resolve the incident, status is %q", status)
	}
}

// repeat_count walks the whole ladder again before giving up.
func TestEscalation_RepeatsTheChain(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 1, "terdut-fallback") // one extra round

	postWebhook(t, s, []map[string]any{
		amAlert("fp-repeat", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	overdue(t, s, 1)
	s.sweepNotify(t) // level 2
	f.forget()
	overdue(t, s, 1)
	s.sweepNotify(t) // back to level 1, round 2

	level, round := escalationLevel(t, s, 1)
	if level != 1 || round != 1 {
		t.Errorf("expected level 1 round 1, got level %d round %d", level, round)
	}
	if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-admin" {
		t.Errorf("the second round should start at the top again, paged %v", got)
	}
}

// A team without a ladder keeps exactly the behaviour it had, and never gets
// both a reminder and an escalation for the same silence.
func TestEscalation_WithoutAPolicyRemindersStillRun(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	postWebhook(t, s, []map[string]any{
		amAlert("fp-noesc", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	// Age the first notification past the repeat interval.
	f.forget()
	s.exec(t, "UPDATE notifications SET created_at = $1, sent_at = $1",
		time.Now().Add(-time.Hour).Unix())
	s.sweepNotify(t)

	if got := f.topicsSince(t); len(got) != 1 || got[0] != "terdut-admin" {
		t.Errorf("without a ladder the reminder should still fire, paged %v", got)
	}
	if level, _ := escalationLevel(t, s, 1); level != 0 {
		t.Errorf("an incident in a team with no ladder should not be on one, level is %d", level)
	}
}

// With a ladder, reminders stop: two pages for one silence is how people learn
// to mute the tool.
func TestEscalation_WithAPolicyRemindersDoNotAlsoFire(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-both", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	f.forget()
	// Old enough for a reminder, but not yet due for escalation.
	s.exec(t, "UPDATE notifications SET created_at = $1, sent_at = $1",
		time.Now().Add(-time.Hour).Unix())
	s.sweepNotify(t)

	if got := f.topicsSince(t); len(got) != 0 {
		t.Errorf("a team with a ladder should not also get reminders, paged %v", got)
	}
}

// The API refuses a ladder that cannot page anybody.
func TestEscalation_RejectsAnUnusablePolicy(t *testing.T) {
	s := newTS(t)

	for _, c := range []struct {
		name string
		body map[string]any
	}{
		{"a level with no targets", map[string]any{
			"levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{}}},
		}},
		{"a level with no timeout", map[string]any{
			"levels": []map[string]any{{"timeout_seconds": 0, "targets": []map[string]any{{"kind": "oncall"}}}},
		}},
		{"a user target with no user", map[string]any{
			"levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "user"}}}},
		}},
		{"an unknown target kind", map[string]any{
			"levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "everybody"}}}},
		}},
		{"an absurd repeat count", map[string]any{
			"repeat_count": 99,
			"levels":       []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}}},
		}},
	} {
		resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", c.body)
		resp.Body.Close()
		if resp.StatusCode != http.StatusBadRequest {
			t.Errorf("%s: expected 400, got %d", c.name, resp.StatusCode)
		}
	}
}

// Editing the ladder is an owner's job; reading it is any member's.
func TestEscalation_OwnerOnlyToEdit(t *testing.T) {
	s := newTS(t)
	_, call := member(t, s, "plain")

	resp := call(http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", map[string]any{
		"levels": []map[string]any{{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}}},
	})
	resp.Body.Close()
	if resp.StatusCode != http.StatusForbidden {
		t.Errorf("a member editing the ladder: expected 403, got %d", resp.StatusCode)
	}

	resp = call(http.MethodGet, "/api/teams/"+defaultTeam+"/escalation", nil)
	resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Errorf("a member reading the ladder: expected 200, got %d", resp.StatusCode)
	}
}

// A target who cannot be woken is not a reason to stop: the next level is the
// answer to an unreachable one.
func TestEscalation_SkipsUnreachableTargets(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	// Second user has no ntfy topic at all.
	var user struct {
		ID int64 `json:"id"`
	}
	decode(t, s.req(t, http.MethodPost, "/api/users",
		map[string]string{"username": "silent", "email": "silent@test.com"}), &user)
	s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members",
		map[string]any{"user_id": user.ID, "role": "member"}).Body.Close()

	ladder(t, s, user.ID, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-silent", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	f.forget()
	overdue(t, s, 1)
	s.sweepNotify(t)

	// Level 2 was entered even though it woke nobody, so the ladder keeps
	// moving toward the fallback rather than stalling on a silent rung.
	if level, _ := escalationLevel(t, s, 1); level != 2 {
		t.Errorf("expected the ladder to advance past an unreachable target, level is %d", level)
	}
	if got := f.topicsSince(t); len(got) != 0 {
		t.Errorf("a target with no topic should page nothing, paged %v", got)
	}
}

// ---------------------------------------------------------------------------
// The ladder as the Escalation page reads it
// ---------------------------------------------------------------------------

type ladderLevel struct {
	Status  string  `json:"status"`
	Waiting []int64 `json:"waiting"`
	Targets []struct {
		Kind      string `json:"kind"`
		Username  string `json:"username"`
		Reachable bool   `json:"reachable"`
		Problem   string `json:"problem"`
	} `json:"targets"`
}

type ladderView struct {
	Levels                  []ladderLevel `json:"levels"`
	LastEscalatedAt         *string       `json:"last_escalated_at"`
	LastEscalatedIncidentID *int64        `json:"last_escalated_incident_id"`
}

func readLadder(t *testing.T, s *ts) ladderView {
	t.Helper()
	var v ladderView
	decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/escalation", nil), &v)
	return v
}

// Targets say who they mean today, so "whoever is on call" is a name and not a
// promise.
func TestEscalation_StatusResolvesTargets(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	v := readLadder(t, s)
	if len(v.Levels) != 2 {
		t.Fatalf("expected 2 levels, got %d", len(v.Levels))
	}
	if got := v.Levels[0].Targets[0]; got.Kind != "oncall" || got.Username != "admin" || !got.Reachable {
		t.Errorf("the rota target should resolve to the person on call, got %+v", got)
	}
	if got := v.Levels[1].Targets[0]; got.Username != "second" || !got.Reachable {
		t.Errorf("the named target should be reachable, got %+v", got)
	}
	if v.Levels[0].Status != "ready" || v.Levels[1].Status != "ready" || v.LastEscalatedAt != nil {
		t.Errorf("an idle, healthy ladder is ready and has never escalated, got %+v", v)
	}
}

// A rung that would page nobody is called out before an incident finds it.
func TestEscalation_StatusFlagsUnreachableLevels(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	silent := teamUser(t, s, "silent", "terdut-silent")
	ladder(t, s, silent, 0, "terdut-fallback")

	// Nobody on call today, and the named person loses their topic.
	s.exec(t, "DELETE FROM schedule_entries")
	s.exec(t, "UPDATE users SET ntfy_topic = NULL WHERE id = $1", silent)

	v := readLadder(t, s)
	if v.Levels[0].Status != "unreachable" || v.Levels[0].Targets[0].Problem != "nobody is on call today" {
		t.Errorf("an empty rota should make level 1 unreachable, got %+v", v.Levels[0])
	}
	if v.Levels[1].Status != "unreachable" || v.Levels[1].Targets[0].Problem != "has no ntfy topic" {
		t.Errorf("a person with no topic should make level 2 unreachable, got %+v", v.Levels[1])
	}

	s.exec(t, "UPDATE users SET disabled_at = 1 WHERE id = $1", silent)
	if p := readLadder(t, s).Levels[1].Targets[0].Problem; p != "account is disabled" {
		t.Errorf("a disabled account should say so, got %q", p)
	}
}

// Where unanswered incidents are right now, and when the ladder last did its
// job.
func TestEscalation_StatusShowsWhoIsWaitingAndLastEscalation(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	second := teamUser(t, s, "second", "terdut-second")
	ladder(t, s, second, 0, "terdut-fallback")

	postWebhook(t, s, []map[string]any{
		amAlert("fp-wait", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
	})
	s.sweepNotify(t)

	// On level 1 it is waiting, which is normal and not yet an escalation.
	v := readLadder(t, s)
	if len(v.Levels[0].Waiting) != 1 || v.Levels[0].Status != "ready" || v.LastEscalatedAt != nil {
		t.Fatalf("a fresh incident waits on level 1 quietly, got %+v", v)
	}

	overdue(t, s, 1)
	s.sweepNotify(t)
	v = readLadder(t, s)
	if v.Levels[1].Status != "escalating" || len(v.Levels[1].Waiting) != 1 || v.Levels[1].Waiting[0] != 1 {
		t.Errorf("level 2 should be escalating with the incident on it, got %+v", v.Levels[1])
	}
	if v.LastEscalatedAt == nil || v.LastEscalatedIncidentID == nil || *v.LastEscalatedIncidentID != 1 {
		t.Errorf("the escalation should be recorded, got %+v", v)
	}

	// Somebody answers: nothing is waiting, but the history stays.
	s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
	v = readLadder(t, s)
	if v.Levels[1].Status != "ready" || len(v.Levels[1].Waiting) != 0 || v.LastEscalatedAt == nil {
		t.Errorf("an acknowledged incident stops waiting but stays in the history, got %+v", v)
	}
}

// No ladder is a real answer, not an error.
func TestEscalation_StatusWithoutALadder(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com", RepeatEvery: 15 * time.Minute})
	v := readLadder(t, s)
	if len(v.Levels) != 0 || v.LastEscalatedAt != nil {
		t.Errorf("a team with no ladder should read as empty, got %+v", v)
	}
}