Files
terdut-server/internal/api/incidents_test.go
T
Niklas Ye a4fbd60441
CI / chart (pull_request) Successful in 1s
CI / security (pull_request) Successful in 13s
CI / test (pull_request) Successful in 1m49s
Scope everything to a team, and route alerts by integration key
The core of #4, and what #1 is for: terdut stops being one shared space.
A team owns its incidents, alerts, schedule and integrations; a user sees
exactly the teams they are in. Everything that existed moves into one
Default team and every existing user becomes an owner of it, so the
upgrade is a no-op for the people using it.

Ingestion is the load-bearing half. An alert arrives on a team's
integration key, and the key is both the credential and the routing: it
says that the sender may post, and which team the alerts belong to. That
also closes the unauthenticated webhook -- the old path stays for one
release, deprecated and routed to the oldest team, so an upgrade does not
stop delivering while somebody edits the Alertmanager config.

Scoping is enforced in as few places as possible, because the failure
mode is silent. serveAs loads the caller's memberships once; list queries
carry `team_id = ANY(...)`; and every incident route goes through
incidentIDParam, which now parses the id AND checks the team in the same
call, so a new handler cannot remember the first half and forget the
second. Anything in another team is 404, never 403: whether an incident
exists is that team's business.

Two bugs this found, both of which would have been silent:

  * upsertAlerts decided "is this a new occurrence" by looking up the
    fingerprint alone. Across teams that made team B's first alert look
    like a re-send of team A's, so it opened no incident at all. The
    lookups are keyed on (team_id, fingerprint) now, as the index is.

  * Every uniqueness rule was written for one tenant. Two teams watching
    two clusters legitimately see the same fingerprint, the same
    groupKey, and want somebody on call on the same day; all three
    constraints move to include team_id.

Roles inside a team are separate from the system administrator flag: an
owner configures the team, a member works its incidents, and an admin is
NOT implicitly in every team -- administration is about accounts, not
about reading other people's incidents. An admin can still repair a team
whose owner has left, which is why requireTeamOwner lets them through.

A shift can only be given to somebody in the team. Paging a person who
cannot open the incident is worse than paging nobody.

The UI is updated only as far as keeping it working: it loads the
viewer's teams with the session and uses the first one, since nobody has
a second yet. "On call now" shows every team the viewer is in, named only
when there is more than one, so the common case reads exactly as before.
The team switcher, badges and per-team settings pages are the next step.

Breaking for API clients: the schedule endpoints moved under the team,
and /api/schedule/current returns an array rather than an object or a
404. terdut-tui will need a version for that.

Per-team dead-man configuration is deliberately not here. A heartbeat's
incident already opens in the team whose key received it, which is the
part that matters for isolation; moving the matchers out of env into
per-team rows is a change to how deadman.go is configured rather than to
who sees what.

Claude-Session: https://claude.ai/code/session_01RHPj4ggeFdEjKKfm4SHbD7
2026-09-20 13:36:24 +02:00

730 lines
24 KiB
Go

package api_test
import (
"context"
"fmt"
"net/http"
"testing"
"time"
"git.ryuvia.com/niklas/terdut-server/internal/api"
)
// amAlert builds one alert of a webhook payload.
func amAlert(fingerprint, name, status, startsAt, endsAt string, labels map[string]string) map[string]any {
l := map[string]string{"alertname": name}
for k, v := range labels {
l[k] = v
}
return map[string]any{
"status": status,
"labels": l,
"annotations": map[string]string{},
"startsAt": startsAt,
"endsAt": endsAt,
"generatorURL": "",
"fingerprint": fingerprint,
}
}
func listIncidents(t *testing.T, s *ts, query string) []map[string]any {
t.Helper()
var out []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/incidents"+query, nil), &out)
return out
}
func getIncident(t *testing.T, s *ts, id int) map[string]any {
t.Helper()
var out map[string]any
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d", id), nil), &out)
return out
}
func timeline(t *testing.T, s *ts, id int) []map[string]any {
t.Helper()
var out []map[string]any
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id), nil), &out)
return out
}
// eventTypes flattens a timeline to its event types, which is what the ordering
// assertions actually care about.
func eventTypes(events []map[string]any) []string {
types := make([]string, len(events))
for i, e := range events {
types[i] = e["type"].(string)
}
return types
}
// countIncidents counts rows directly, including resolved and archived ones that
// no list view returns.
func (s *ts) countIncidents(t *testing.T) int {
t.Helper()
var n int
if err := s.db.QueryRow("SELECT COUNT(*) FROM incidents").Scan(&n); err != nil {
t.Fatalf("count incidents: %v", err)
}
return n
}
// ---------------------------------------------------------------------------
// Ingest: alerts becoming incidents
// ---------------------------------------------------------------------------
func TestWebhook_FiringOpensIncident(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-1", "HighCPU", "firing", "2026-05-20T10:00:00Z", zeroTime,
map[string]string{"severity": "critical"}),
}, "{}:{alertname=\"HighCPU\"}")
incidents := listIncidents(t, s, "")
if len(incidents) != 1 {
t.Fatalf("expected 1 incident, got %d", len(incidents))
}
inc := incidents[0]
if inc["status"] != "triggered" {
t.Errorf("expected status triggered, got %v", inc["status"])
}
if inc["severity"] != "critical" {
t.Errorf("expected severity critical, got %v", inc["severity"])
}
if inc["title"] != "HighCPU" {
t.Errorf("expected title from groupLabels, got %v", inc["title"])
}
// The alert points back at the incident it opened.
var alerts []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 || alerts[0]["incident_id"] == nil {
t.Fatalf("expected the alert to carry an incident_id, got %v", alerts)
}
}
// Alertmanager already grouped these; we adopt its answer rather than
// correlating again.
func TestWebhook_SameGroupKeyJoinsOneIncident(t *testing.T) {
s := newTS(t)
const groupKey = "{}:{alertname=\"DiskFull\"}"
postWebhook(t, s, []map[string]any{
amAlert("fp-a", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
amAlert("fp-b", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
}, groupKey)
incidents := listIncidents(t, s, "")
if len(incidents) != 1 {
t.Fatalf("expected 1 incident for one groupKey, got %d", len(incidents))
}
id := int(incidents[0]["id"].(float64))
inc := getIncident(t, s, id)
members, _ := inc["alerts"].([]any)
if len(members) != 2 {
t.Fatalf("expected 2 alerts under the incident, got %d", len(members))
}
added := 0
for _, ty := range eventTypes(timeline(t, s, id)) {
if ty == "alert_added" {
added++
}
}
if added != 2 {
t.Errorf("expected 2 alert_added events, got %d", added)
}
}
// The load-bearing rule. Alertmanager re-sends firing notifications every
// repeat_interval; if those re-sends reopened incidents, resolving one by hand
// would mean nothing.
func TestWebhook_HeartbeatDoesNotReopenResolvedIncident(t *testing.T) {
s := newTS(t)
const groupKey = "{}:{alertname=\"Flapper\"}"
alert := amAlert("fp-hb", "Flapper", "firing", "2026-05-20T10:00:00Z", zeroTime, nil)
postWebhook(t, s, []map[string]any{alert}, groupKey)
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("resolve returned %d", resp.StatusCode)
}
resp.Body.Close()
// Same startsAt, same fingerprint: a re-send, not a new occurrence.
postWebhook(t, s, []map[string]any{alert}, groupKey)
if n := s.countIncidents(t); n != 1 {
t.Fatalf("expected the heartbeat to open no incident, got %d total", n)
}
if inc := getIncident(t, s, 1); inc["resolved_at"] == nil {
t.Error("expected incident 1 to stay resolved")
}
// The alert itself is still firing and still being tracked — only the work
// item is closed.
status, _, _ := s.alertRow(t, "fp-hb")
if status != "firing" {
t.Errorf("expected the alert to still be firing, got %q", status)
}
}
func TestWebhook_NewOccurrenceOpensNewIncident(t *testing.T) {
s := newTS(t)
const groupKey = "{}:{alertname=\"Recurring\"}"
postWebhook(t, s, []map[string]any{
amAlert("fp-new", "Recurring", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
}, groupKey)
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
// A newer startsAt is a genuinely new occurrence, not a re-send.
postWebhook(t, s, []map[string]any{
amAlert("fp-new", "Recurring", "firing", "2026-05-21T09:00:00Z", zeroTime, nil),
}, groupKey)
if n := s.countIncidents(t); n != 2 {
t.Fatalf("expected a second incident for the new occurrence, got %d total", n)
}
open := listIncidents(t, s, "")
if len(open) != 1 || int(open[0]["id"].(float64)) != 2 {
t.Fatalf("expected incident 2 to be the open one, got %v", open)
}
// The new incident starts unacknowledged: that is the point of the split.
if open[0]["acknowledged_by"] != nil {
t.Error("expected a fresh occurrence to start unacknowledged")
}
}
func TestWebhook_ResolvedOnlyPayloadOpensNothing(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-res", "AlreadyOver", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
}, "{}:{alertname=\"AlreadyOver\"}")
if n := s.countIncidents(t); n != 0 {
t.Errorf("expected no incident from a resolved-only payload, got %d", n)
}
if status, _, _ := s.alertRow(t, "fp-res"); status != "resolved" {
t.Errorf("expected the alert itself to be stored, got %q", status)
}
}
// An incident that hit critical was a critical incident, even once the critical
// alert clears and only a warning is left.
func TestIncident_SeverityIsHighWaterMark(t *testing.T) {
s := newTS(t)
const groupKey = "{}:{alertname=\"Mixed\"}"
postWebhook(t, s, []map[string]any{
amAlert("fp-warn", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
map[string]string{"severity": "warning"}),
amAlert("fp-crit", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
map[string]string{"severity": "critical"}),
}, groupKey)
if inc := getIncident(t, s, 1); inc["severity"] != "critical" {
t.Fatalf("expected severity critical, got %v", inc["severity"])
}
// The critical alert clears; the warning keeps the incident open.
postWebhook(t, s, []map[string]any{
amAlert("fp-crit", "Mixed", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z",
map[string]string{"severity": "critical"}),
}, groupKey)
inc := getIncident(t, s, 1)
if inc["resolved_at"] != nil {
t.Fatal("expected the incident to stay open")
}
if inc["severity"] != "critical" {
t.Errorf("expected severity to stay critical, got %v", inc["severity"])
}
}
// ---------------------------------------------------------------------------
// Resolution cascade
// ---------------------------------------------------------------------------
func TestIncident_AllAlertsResolvedAutoResolves(t *testing.T) {
s := newTS(t)
const groupKey = "{}:{alertname=\"Pair\"}"
postWebhook(t, s, []map[string]any{
amAlert("fp-p1", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
amAlert("fp-p2", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
}, groupKey)
// One down, one still firing: the work is not done.
postWebhook(t, s, []map[string]any{
amAlert("fp-p1", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
}, groupKey)
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
t.Fatal("expected the incident to stay open while an alert is firing")
}
postWebhook(t, s, []map[string]any{
amAlert("fp-p2", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:30:00Z", nil),
}, groupKey)
inc := getIncident(t, s, 1)
if inc["status"] != "resolved" {
t.Errorf("expected status resolved, got %v", inc["status"])
}
if inc["resolution_source"] != "alerts" {
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
}
}
// Expiry is inference, not observation, but it still has to close the work item
// — otherwise a lost resolved notification leaves an incident open forever.
func TestExpiry_CascadesToIncidentResolution(t *testing.T) {
s := newTS(t)
postAlert(t, s, "fp-exp", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'fp-exp'",
time.Now().Add(-10*time.Hour).Unix())
sweep(t, s, 6*time.Hour)
inc := getIncident(t, s, 1)
if inc["status"] != "resolved" {
t.Errorf("expected the incident to resolve after expiry, got %v", inc["status"])
}
if inc["resolution_source"] != "alerts" {
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
}
// The expiry is recorded against the alert, not the incident.
if _, source, _ := s.alertRow(t, "fp-exp"); source == nil || *source != "expiry" {
t.Errorf("expected the alert's resolution_source to stay expiry, got %v", source)
}
if types := eventTypes(timeline(t, s, 1)); !contains(types, "alert_resolved") {
t.Errorf("expected an alert_resolved event on the timeline, got %v", types)
}
}
// ---------------------------------------------------------------------------
// Workflow actions
// ---------------------------------------------------------------------------
func TestIncident_Acknowledge(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-ack", "X", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("acknowledge returned %d", resp.StatusCode)
}
var inc map[string]any
decode(t, resp, &inc)
if inc["acknowledged_by"] == nil {
t.Error("expected acknowledged_by to be set")
}
if inc["status"] != "acknowledged" {
t.Errorf("expected status acknowledged, got %v", inc["status"])
}
resp = s.req(t, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unacknowledge returned %d", resp.StatusCode)
}
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
t.Errorf("expected status back to triggered, got %v", inc["status"])
}
}
func TestIncident_ManualResolveIsTerminal(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-term", "Terminal", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("first resolve returned %d", resp.StatusCode)
}
var inc map[string]any
decode(t, resp, &inc)
if inc["resolution_source"] != "manual" {
t.Errorf("expected resolution_source manual, got %v", inc["resolution_source"])
}
resp = s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
resp.Body.Close()
if resp.StatusCode != http.StatusConflict {
t.Errorf("expected 409 on re-resolve, got %d", resp.StatusCode)
}
// Acknowledging a closed incident is equally meaningless.
resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
resp.Body.Close()
if resp.StatusCode != http.StatusConflict {
t.Errorf("expected 409 acknowledging a resolved incident, got %d", resp.StatusCode)
}
}
func TestIncident_SnoozeHiddenFromDefaultList(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-snz", "Noisy", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]string{"duration": "2h"})
if resp.StatusCode != http.StatusOK {
t.Fatalf("snooze returned %d", resp.StatusCode)
}
var inc map[string]any
decode(t, resp, &inc)
if inc["snoozed_until"] == nil {
t.Error("expected snoozed_until to be set")
}
if got := listIncidents(t, s, ""); len(got) != 0 {
t.Errorf("expected the snoozed incident to be hidden, got %d", len(got))
}
if got := listIncidents(t, s, "?snoozed=true"); len(got) != 1 {
t.Errorf("expected snoozed=true to show it, got %d", len(got))
}
// A snooze is not a resolution: the incident is still open work.
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
t.Error("expected a snoozed incident to stay open")
}
resp = s.req(t, http.MethodDelete, "/api/incidents/1/snooze", nil)
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unsnooze returned %d", resp.StatusCode)
}
if got := listIncidents(t, s, ""); len(got) != 1 {
t.Errorf("expected the incident back in the default list, got %d", len(got))
}
}
func TestIncident_SnoozeRejectsPastDeadline(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-past", "Past", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze",
map[string]string{"until": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)})
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("expected 400 for a snooze in the past, got %d", resp.StatusCode)
}
}
// The schedule stops being decorative here: it is read at trigger time.
func TestIncident_AutoAssignedToCurrentOnCall(t *testing.T) {
s := newTS(t)
today := time.Now().UTC().Format("2006-01-02")
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{today}})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("schedule assignment returned %d", resp.StatusCode)
}
resp.Body.Close()
postWebhook(t, s, []map[string]any{
amAlert("fp-oncall", "PageMe", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
inc := getIncident(t, s, 1)
if inc["assigned_to"] != "admin" {
t.Errorf("expected the incident assigned to today's on-call, got %v", inc["assigned_to"])
}
if types := eventTypes(timeline(t, s, 1)); !contains(types, "assigned") {
t.Errorf("expected an assigned event, got %v", types)
}
}
func TestIncident_AssignToUser(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-asg", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
s.req(t, http.MethodPost, "/api/users",
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
resp := s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2})
if resp.StatusCode != http.StatusOK {
t.Fatalf("assign returned %d", resp.StatusCode)
}
var inc map[string]any
decode(t, resp, &inc)
if inc["assigned_to"] != "alice" {
t.Errorf("expected assigned_to alice, got %v", inc["assigned_to"])
}
resp = s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 99})
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("expected 404 assigning an unknown user, got %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Timeline and notes
// ---------------------------------------------------------------------------
func TestIncident_TimelineOrdering(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-tl", "Storyline", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
s.req(t, http.MethodPost, "/api/incidents/1/notes",
map[string]string{"content": "looking into it"}).Body.Close()
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
events := timeline(t, s, 1)
want := []string{"triggered", "alert_added", "acknowledged", "note", "resolved"}
got := eventTypes(events)
if len(got) != len(want) {
t.Fatalf("expected timeline %v, got %v", want, got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("expected timeline %v, got %v", want, got)
}
}
// The note carries its author; system events do not.
for _, e := range events {
if e["type"] == "note" {
if e["username"] != "admin" || e["detail"] != "looking into it" {
t.Errorf("unexpected note event: %v", e)
}
}
if e["type"] == "triggered" && e["username"] != nil {
t.Errorf("expected the triggered event to have no author, got %v", e["username"])
}
}
}
func TestIncident_NoteDeleteOwnOnly(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-note", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
s.req(t, http.MethodPost, "/api/users",
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
var keyData map[string]any
decode(t, s.req(t, http.MethodPost, "/api/users/2/api-keys",
map[string]string{"name": "alice-key"}), &keyData)
aliceKey := keyData["key"].(string)
var note map[string]any
decode(t, s.req(t, http.MethodPost, "/api/incidents/1/notes",
map[string]string{"content": "admin note"}), &note)
noteID := int(note["id"].(float64))
// Alice cannot delete admin's note.
req, _ := http.NewRequest(http.MethodDelete,
fmt.Sprintf("%s/api/incidents/1/notes/%d", s.URL, noteID), nil)
req.Header.Set("Authorization", "Bearer "+aliceKey)
resp, _ := http.DefaultClient.Do(req)
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("expected 404 deleting another user's note, got %d", resp.StatusCode)
}
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Errorf("expected 204 deleting own note, got %d", resp.StatusCode)
}
}
// Only notes are deletable — the rest of the timeline is what happened.
func TestIncident_CannotDeleteSystemEvent(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-sys", "System", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
events := timeline(t, s, 1)
id := int(events[0]["id"].(float64))
resp := s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", id), nil)
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("expected 404 deleting a system event, got %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Archive
// ---------------------------------------------------------------------------
func TestIncident_ArchiveRoundTrip(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-arc", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
resp := s.req(t, http.MethodPost, "/api/incidents/1/archive", nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("archive returned %d", resp.StatusCode)
}
var inc map[string]any
decode(t, resp, &inc)
if inc["archived_at"] == nil {
t.Error("expected archived_at to be set")
}
if got := listIncidents(t, s, "?status=resolved"); len(got) != 0 {
t.Errorf("expected the archived incident to be hidden, got %d", len(got))
}
if got := listIncidents(t, s, "?status=resolved&archived=true"); len(got) != 1 {
t.Errorf("expected archived=true to show it, got %d", len(got))
}
resp = s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil)
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unarchive returned %d", resp.StatusCode)
}
if got := listIncidents(t, s, "?status=resolved"); len(got) != 1 {
t.Errorf("expected the incident back, got %d", len(got))
}
}
func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-swp", "Old", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
s.exec(t, "UPDATE incidents SET resolved_at = $1 WHERE id = 1",
time.Now().Add(-30*24*time.Hour).Unix())
api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.deadman, s.notify)
if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
t.Error("expected the sweeper to archive a long-resolved incident")
}
}
// ---------------------------------------------------------------------------
// Stats
// ---------------------------------------------------------------------------
func TestStats_Incidents(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-s1", "One", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
}, "g1")
postWebhook(t, s, []map[string]any{
amAlert("fp-s2", "Two", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
}, "g2")
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
var stats map[string]any
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
if stats["total"].(float64) != 2 {
t.Errorf("expected total 2, got %v", stats["total"])
}
if stats["resolved"].(float64) != 1 {
t.Errorf("expected resolved 1, got %v", stats["resolved"])
}
if stats["triggered"].(float64) != 1 {
t.Errorf("expected triggered 1, got %v", stats["triggered"])
}
// One incident has been acknowledged and resolved, so both averages exist.
if stats["mtta_seconds"] == nil || stats["mttr_seconds"] == nil {
t.Errorf("expected mtta and mttr to be computable, got %v", stats)
}
}
// An empty window is a report of zero, not a failure. SUM over no rows is NULL
// in Postgres as it was in SQLite, and that used to come back as a 500 the
// moment every incident was archived — the state a quiet installation settles
// into.
func TestStats_IncidentsEmptyWindowIsZeroNotAnError(t *testing.T) {
s := newTS(t)
// No incidents at all.
resp := s.req(t, http.MethodGet, "/api/stats/incidents", nil)
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
}
var stats map[string]any
decode(t, resp, &stats)
for _, k := range []string{"total", "triggered", "acknowledged", "resolved"} {
if stats[k].(float64) != 0 {
t.Errorf("expected %s 0, got %v", k, stats[k])
}
}
// And with every incident archived out of the window.
postWebhook(t, s, []map[string]any{
amAlert("fp-s4", "Gone", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
s.req(t, http.MethodPost, "/api/incidents/1/archive", nil).Body.Close()
resp = s.req(t, http.MethodGet, "/api/stats/incidents", nil)
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
t.Fatalf("expected 200 when every incident is archived, got %d", resp.StatusCode)
}
stats = nil
decode(t, resp, &stats)
if stats["total"].(float64) != 0 {
t.Errorf("expected total 0, got %v", stats["total"])
}
}
// The alert stats share the same aggregate, and the same empty-window trap.
func TestStats_AlertsEmptyWindowIsZeroNotAnError(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
}
var stats map[string]any
decode(t, resp, &stats)
for _, k := range []string{"total", "firing", "resolved"} {
if stats[k].(float64) != 0 {
t.Errorf("expected %s 0, got %v", k, stats[k])
}
}
}
// Nothing acknowledged yet means "no data", which is not the same claim as zero.
func TestStats_IncidentsNullMTTAWhenNothingAcknowledged(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{
amAlert("fp-s3", "Untouched", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
})
var stats map[string]any
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
if stats["mtta_seconds"] != nil {
t.Errorf("expected mtta_seconds null, got %v", stats["mtta_seconds"])
}
}
func contains(haystack []string, needle string) bool {
for _, s := range haystack {
if s == needle {
return true
}
}
return false
}