94dec19976
SUM över noll rader är NULL i SQLite, inte 0. handleStatsIncidents läste de tre statusräknarna rakt in i int64, så i samma stund som filtret inte matchade någon rad föll skanningen på "converting NULL to int64 is unsupported" och hela /api/stats/incidents svarade 500. COUNT(*) ger däremot 0 utan knot, vilket är precis varför felet inte syns förrän tabellen töms — det är det enda uttrycket i satsen som klarar noll rader. Filtret är alltid på: statsFilter lägger på archived_at IS NULL (923fc8b, flyttat hit i279ef6c). En installation som varit tyst ett tag arkiverar därmed sig själv in i felet. Det är sluttillståndet för en lugn vecka, inte ett kantfall, och klustret står i det nu. Symptomet pekade åt fel håll. terdut-tui hämtar listan och statistiken i samma uppdatering, så incidentvyn såg trasig ut medan /api/incidents svarade 200 med []. Loggen i klustret visar de två anropen bredvid varandra, det ena grönt och det andra rött. Ingen ändring i terdut-tui behövs: dess ListIncidents är oförändrad sedan 0.7.2 och skickar samma parametrar som förut. handleStatsAlerts bar samma fel och rättas likadant, innan någon hittar det på samma sätt. COALESCE i SQL i stället för sql.NullInt64 i Go, eftersom jämförelserna redan bor i satserna här (severityRankSQL,279ef6c). mtta_seconds och mttr_seconds lämnas medvetet utan COALESCE. null betyder "inget att mäta ännu" och 0 skulle läsas som "omedelbart" — två olika påståenden, och testet från279ef6clåser fast skillnaden. Claude-Session: https://claude.ai/code/session_01S7R4gWTz5wh5xCY4nCSJjN
828 lines
27 KiB
Go
828 lines
27 KiB
Go
package api_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
|
"git.ryuvia.com/niklas/terdut-server/internal/db"
|
|
)
|
|
|
|
// amAlert builds one alert of a webhook payload.
|
|
func amAlert(fingerprint, name, status, startsAt, endsAt string, labels map[string]string) map[string]any {
|
|
l := map[string]string{"alertname": name}
|
|
for k, v := range labels {
|
|
l[k] = v
|
|
}
|
|
return map[string]any{
|
|
"status": status,
|
|
"labels": l,
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}
|
|
}
|
|
|
|
func listIncidents(t *testing.T, s *ts, query string) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/incidents"+query, nil), &out)
|
|
return out
|
|
}
|
|
|
|
func getIncident(t *testing.T, s *ts, id int) map[string]any {
|
|
t.Helper()
|
|
var out map[string]any
|
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d", id), nil), &out)
|
|
return out
|
|
}
|
|
|
|
func timeline(t *testing.T, s *ts, id int) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id), nil), &out)
|
|
return out
|
|
}
|
|
|
|
// eventTypes flattens a timeline to its event types, which is what the ordering
|
|
// assertions actually care about.
|
|
func eventTypes(events []map[string]any) []string {
|
|
types := make([]string, len(events))
|
|
for i, e := range events {
|
|
types[i] = e["type"].(string)
|
|
}
|
|
return types
|
|
}
|
|
|
|
// countIncidents counts rows directly, including resolved and archived ones that
|
|
// no list view returns.
|
|
func (s *ts) countIncidents(t *testing.T) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM incidents").Scan(&n); err != nil {
|
|
t.Fatalf("count incidents: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Ingest: alerts becoming incidents
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_FiringOpensIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-1", "HighCPU", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "critical"}),
|
|
}, "{}:{alertname=\"HighCPU\"}")
|
|
|
|
incidents := listIncidents(t, s, "")
|
|
if len(incidents) != 1 {
|
|
t.Fatalf("expected 1 incident, got %d", len(incidents))
|
|
}
|
|
inc := incidents[0]
|
|
if inc["status"] != "triggered" {
|
|
t.Errorf("expected status triggered, got %v", inc["status"])
|
|
}
|
|
if inc["severity"] != "critical" {
|
|
t.Errorf("expected severity critical, got %v", inc["severity"])
|
|
}
|
|
if inc["title"] != "HighCPU" {
|
|
t.Errorf("expected title from groupLabels, got %v", inc["title"])
|
|
}
|
|
|
|
// The alert points back at the incident it opened.
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 || alerts[0]["incident_id"] == nil {
|
|
t.Fatalf("expected the alert to carry an incident_id, got %v", alerts)
|
|
}
|
|
}
|
|
|
|
// Alertmanager already grouped these; we adopt its answer rather than
|
|
// correlating again.
|
|
func TestWebhook_SameGroupKeyJoinsOneIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"DiskFull\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-a", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
amAlert("fp-b", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
incidents := listIncidents(t, s, "")
|
|
if len(incidents) != 1 {
|
|
t.Fatalf("expected 1 incident for one groupKey, got %d", len(incidents))
|
|
}
|
|
id := int(incidents[0]["id"].(float64))
|
|
|
|
inc := getIncident(t, s, id)
|
|
members, _ := inc["alerts"].([]any)
|
|
if len(members) != 2 {
|
|
t.Fatalf("expected 2 alerts under the incident, got %d", len(members))
|
|
}
|
|
|
|
added := 0
|
|
for _, ty := range eventTypes(timeline(t, s, id)) {
|
|
if ty == "alert_added" {
|
|
added++
|
|
}
|
|
}
|
|
if added != 2 {
|
|
t.Errorf("expected 2 alert_added events, got %d", added)
|
|
}
|
|
}
|
|
|
|
// The load-bearing rule. Alertmanager re-sends firing notifications every
|
|
// repeat_interval; if those re-sends reopened incidents, resolving one by hand
|
|
// would mean nothing.
|
|
func TestWebhook_HeartbeatDoesNotReopenResolvedIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Flapper\"}"
|
|
alert := amAlert("fp-hb", "Flapper", "firing", "2026-05-20T10:00:00Z", zeroTime, nil)
|
|
|
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("resolve returned %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// Same startsAt, same fingerprint: a re-send, not a new occurrence.
|
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
|
|
|
if n := s.countIncidents(t); n != 1 {
|
|
t.Fatalf("expected the heartbeat to open no incident, got %d total", n)
|
|
}
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] == nil {
|
|
t.Error("expected incident 1 to stay resolved")
|
|
}
|
|
|
|
// The alert itself is still firing and still being tracked — only the work
|
|
// item is closed.
|
|
status, _, _ := s.alertRow(t, "fp-hb")
|
|
if status != "firing" {
|
|
t.Errorf("expected the alert to still be firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
func TestWebhook_NewOccurrenceOpensNewIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Recurring\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-new", "Recurring", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
// A newer startsAt is a genuinely new occurrence, not a re-send.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-new", "Recurring", "firing", "2026-05-21T09:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
if n := s.countIncidents(t); n != 2 {
|
|
t.Fatalf("expected a second incident for the new occurrence, got %d total", n)
|
|
}
|
|
open := listIncidents(t, s, "")
|
|
if len(open) != 1 || int(open[0]["id"].(float64)) != 2 {
|
|
t.Fatalf("expected incident 2 to be the open one, got %v", open)
|
|
}
|
|
|
|
// The new incident starts unacknowledged: that is the point of the split.
|
|
if open[0]["acknowledged_by"] != nil {
|
|
t.Error("expected a fresh occurrence to start unacknowledged")
|
|
}
|
|
}
|
|
|
|
func TestWebhook_ResolvedOnlyPayloadOpensNothing(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-res", "AlreadyOver", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
|
}, "{}:{alertname=\"AlreadyOver\"}")
|
|
|
|
if n := s.countIncidents(t); n != 0 {
|
|
t.Errorf("expected no incident from a resolved-only payload, got %d", n)
|
|
}
|
|
if status, _, _ := s.alertRow(t, "fp-res"); status != "resolved" {
|
|
t.Errorf("expected the alert itself to be stored, got %q", status)
|
|
}
|
|
}
|
|
|
|
// An incident that hit critical was a critical incident, even once the critical
|
|
// alert clears and only a warning is left.
|
|
func TestIncident_SeverityIsHighWaterMark(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Mixed\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-warn", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "warning"}),
|
|
amAlert("fp-crit", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "critical"}),
|
|
}, groupKey)
|
|
|
|
if inc := getIncident(t, s, 1); inc["severity"] != "critical" {
|
|
t.Fatalf("expected severity critical, got %v", inc["severity"])
|
|
}
|
|
|
|
// The critical alert clears; the warning keeps the incident open.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-crit", "Mixed", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z",
|
|
map[string]string{"severity": "critical"}),
|
|
}, groupKey)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["resolved_at"] != nil {
|
|
t.Fatal("expected the incident to stay open")
|
|
}
|
|
if inc["severity"] != "critical" {
|
|
t.Errorf("expected severity to stay critical, got %v", inc["severity"])
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Resolution cascade
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_AllAlertsResolvedAutoResolves(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Pair\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p1", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
amAlert("fp-p2", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
// One down, one still firing: the work is not done.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p1", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
|
}, groupKey)
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
|
t.Fatal("expected the incident to stay open while an alert is firing")
|
|
}
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p2", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:30:00Z", nil),
|
|
}, groupKey)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %v", inc["status"])
|
|
}
|
|
if inc["resolution_source"] != "alerts" {
|
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
|
}
|
|
}
|
|
|
|
// Expiry is inference, not observation, but it still has to close the work item
|
|
// — otherwise a lost resolved notification leaves an incident open forever.
|
|
func TestExpiry_CascadesToIncidentResolution(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fp-exp", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'fp-exp'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["status"] != "resolved" {
|
|
t.Errorf("expected the incident to resolve after expiry, got %v", inc["status"])
|
|
}
|
|
if inc["resolution_source"] != "alerts" {
|
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
|
}
|
|
|
|
// The expiry is recorded against the alert, not the incident.
|
|
if _, source, _ := s.alertRow(t, "fp-exp"); source == nil || *source != "expiry" {
|
|
t.Errorf("expected the alert's resolution_source to stay expiry, got %v", source)
|
|
}
|
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "alert_resolved") {
|
|
t.Errorf("expected an alert_resolved event on the timeline, got %v", types)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workflow actions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_Acknowledge(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-ack", "X", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["acknowledged_by"] == nil {
|
|
t.Error("expected acknowledged_by to be set")
|
|
}
|
|
if inc["status"] != "acknowledged" {
|
|
t.Errorf("expected status acknowledged, got %v", inc["status"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unacknowledge returned %d", resp.StatusCode)
|
|
}
|
|
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
|
|
t.Errorf("expected status back to triggered, got %v", inc["status"])
|
|
}
|
|
}
|
|
|
|
func TestIncident_ManualResolveIsTerminal(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-term", "Terminal", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("first resolve returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["resolution_source"] != "manual" {
|
|
t.Errorf("expected resolution_source manual, got %v", inc["resolution_source"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on re-resolve, got %d", resp.StatusCode)
|
|
}
|
|
|
|
// Acknowledging a closed incident is equally meaningless.
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 acknowledging a resolved incident, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestIncident_SnoozeHiddenFromDefaultList(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-snz", "Noisy", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]string{"duration": "2h"})
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("snooze returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["snoozed_until"] == nil {
|
|
t.Error("expected snoozed_until to be set")
|
|
}
|
|
|
|
if got := listIncidents(t, s, ""); len(got) != 0 {
|
|
t.Errorf("expected the snoozed incident to be hidden, got %d", len(got))
|
|
}
|
|
if got := listIncidents(t, s, "?snoozed=true"); len(got) != 1 {
|
|
t.Errorf("expected snoozed=true to show it, got %d", len(got))
|
|
}
|
|
|
|
// A snooze is not a resolution: the incident is still open work.
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
|
t.Error("expected a snoozed incident to stay open")
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/snooze", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unsnooze returned %d", resp.StatusCode)
|
|
}
|
|
if got := listIncidents(t, s, ""); len(got) != 1 {
|
|
t.Errorf("expected the incident back in the default list, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestIncident_SnoozeRejectsPastDeadline(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-past", "Past", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze",
|
|
map[string]string{"until": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusBadRequest {
|
|
t.Errorf("expected 400 for a snooze in the past, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// The schedule stops being decorative here: it is read at trigger time.
|
|
func TestIncident_AutoAssignedToCurrentOnCall(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
today := time.Now().UTC().Format("2006-01-02")
|
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{today}})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("schedule assignment returned %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-oncall", "PageMe", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["assigned_to"] != "admin" {
|
|
t.Errorf("expected the incident assigned to today's on-call, got %v", inc["assigned_to"])
|
|
}
|
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "assigned") {
|
|
t.Errorf("expected an assigned event, got %v", types)
|
|
}
|
|
}
|
|
|
|
func TestIncident_AssignToUser(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-asg", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2})
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("assign returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["assigned_to"] != "alice" {
|
|
t.Errorf("expected assigned_to alice, got %v", inc["assigned_to"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 99})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 assigning an unknown user, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Timeline and notes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_TimelineOrdering(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-tl", "Storyline", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "looking into it"}).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
events := timeline(t, s, 1)
|
|
want := []string{"triggered", "alert_added", "acknowledged", "note", "resolved"}
|
|
got := eventTypes(events)
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
|
}
|
|
}
|
|
|
|
// The note carries its author; system events do not.
|
|
for _, e := range events {
|
|
if e["type"] == "note" {
|
|
if e["username"] != "admin" || e["detail"] != "looking into it" {
|
|
t.Errorf("unexpected note event: %v", e)
|
|
}
|
|
}
|
|
if e["type"] == "triggered" && e["username"] != nil {
|
|
t.Errorf("expected the triggered event to have no author, got %v", e["username"])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIncident_NoteDeleteOwnOnly(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-note", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
var keyData map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
|
map[string]string{"name": "alice-key"}), &keyData)
|
|
aliceKey := keyData["key"].(string)
|
|
|
|
var note map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "admin note"}), ¬e)
|
|
noteID := int(note["id"].(float64))
|
|
|
|
// Alice cannot delete admin's note.
|
|
req, _ := http.NewRequest(http.MethodDelete,
|
|
fmt.Sprintf("%s/api/incidents/1/notes/%d", s.URL, noteID), nil)
|
|
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 deleting another user's note, got %d", resp.StatusCode)
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("expected 204 deleting own note, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// Only notes are deletable — the rest of the timeline is what happened.
|
|
func TestIncident_CannotDeleteSystemEvent(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-sys", "System", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
events := timeline(t, s, 1)
|
|
id := int(events[0]["id"].(float64))
|
|
resp := s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", id), nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 deleting a system event, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_ArchiveRoundTrip(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-arc", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/archive", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("archive returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["archived_at"] == nil {
|
|
t.Error("expected archived_at to be set")
|
|
}
|
|
|
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 0 {
|
|
t.Errorf("expected the archived incident to be hidden, got %d", len(got))
|
|
}
|
|
if got := listIncidents(t, s, "?status=resolved&archived=true"); len(got) != 1 {
|
|
t.Errorf("expected archived=true to show it, got %d", len(got))
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unarchive returned %d", resp.StatusCode)
|
|
}
|
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 1 {
|
|
t.Errorf("expected the incident back, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-swp", "Old", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
s.exec(t, "UPDATE incidents SET resolved_at = ? WHERE id = 1",
|
|
time.Now().Add(-30*24*time.Hour).Unix())
|
|
api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.deadman, s.notify)
|
|
|
|
if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
|
|
t.Error("expected the sweeper to archive a long-resolved incident")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Incidents(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s1", "One", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, "g1")
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s2", "Two", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, "g2")
|
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
var stats map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
|
|
|
if stats["total"].(float64) != 2 {
|
|
t.Errorf("expected total 2, got %v", stats["total"])
|
|
}
|
|
if stats["resolved"].(float64) != 1 {
|
|
t.Errorf("expected resolved 1, got %v", stats["resolved"])
|
|
}
|
|
if stats["triggered"].(float64) != 1 {
|
|
t.Errorf("expected triggered 1, got %v", stats["triggered"])
|
|
}
|
|
// One incident has been acknowledged and resolved, so both averages exist.
|
|
if stats["mtta_seconds"] == nil || stats["mttr_seconds"] == nil {
|
|
t.Errorf("expected mtta and mttr to be computable, got %v", stats)
|
|
}
|
|
}
|
|
|
|
// An empty window is a report of zero, not a failure. SUM over no rows is NULL
|
|
// in SQLite, which used to come back as a 500 the moment every incident was
|
|
// archived — the state a quiet installation settles into.
|
|
func TestStats_IncidentsEmptyWindowIsZeroNotAnError(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// No incidents at all.
|
|
resp := s.req(t, http.MethodGet, "/api/stats/incidents", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
|
|
}
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
for _, k := range []string{"total", "triggered", "acknowledged", "resolved"} {
|
|
if stats[k].(float64) != 0 {
|
|
t.Errorf("expected %s 0, got %v", k, stats[k])
|
|
}
|
|
}
|
|
|
|
// And with every incident archived out of the window.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s4", "Gone", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/archive", nil).Body.Close()
|
|
|
|
resp = s.req(t, http.MethodGet, "/api/stats/incidents", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
t.Fatalf("expected 200 when every incident is archived, got %d", resp.StatusCode)
|
|
}
|
|
stats = nil
|
|
decode(t, resp, &stats)
|
|
if stats["total"].(float64) != 0 {
|
|
t.Errorf("expected total 0, got %v", stats["total"])
|
|
}
|
|
}
|
|
|
|
// The alert stats share the same aggregate, and the same empty-window trap.
|
|
func TestStats_AlertsEmptyWindowIsZeroNotAnError(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
|
|
}
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
for _, k := range []string{"total", "firing", "resolved"} {
|
|
if stats[k].(float64) != 0 {
|
|
t.Errorf("expected %s 0, got %v", k, stats[k])
|
|
}
|
|
}
|
|
}
|
|
|
|
// Nothing acknowledged yet means "no data", which is not the same claim as zero.
|
|
func TestStats_IncidentsNullMTTAWhenNothingAcknowledged(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s3", "Untouched", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
var stats map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
|
if stats["mtta_seconds"] != nil {
|
|
t.Errorf("expected mtta_seconds null, got %v", stats["mtta_seconds"])
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Migration backfill
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// An upgrade must not drop the acknowledgements and comments people already
|
|
// have, so 008 is replayed here over a database left at 007.
|
|
func TestMigration_BackfillCarriesAckAndComments(t *testing.T) {
|
|
database, err := db.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
t.Cleanup(func() { database.Close() })
|
|
|
|
files, err := filepath.Glob("../db/migrations/*.sql")
|
|
if err != nil || len(files) == 0 {
|
|
t.Fatalf("find migrations: %v", err)
|
|
}
|
|
sort.Strings(files)
|
|
|
|
var incidentsMigration string
|
|
for _, f := range files {
|
|
if filepath.Base(f) >= "008" {
|
|
incidentsMigration = f
|
|
break
|
|
}
|
|
data, err := os.ReadFile(f)
|
|
if err != nil {
|
|
t.Fatalf("read %s: %v", f, err)
|
|
}
|
|
if _, err := database.Exec(string(data)); err != nil {
|
|
t.Fatalf("apply %s: %v", f, err)
|
|
}
|
|
}
|
|
if incidentsMigration == "" {
|
|
t.Fatal("008 migration not found")
|
|
}
|
|
|
|
// A database as it would look on the old schema: an acknowledged firing
|
|
// alert with a comment on it.
|
|
if _, err := database.Exec(`
|
|
INSERT INTO users (id, username, email) VALUES (1, 'admin', 'admin@test.com');
|
|
INSERT INTO alerts (id, fingerprint, name, status, labels, annotations,
|
|
starts_at, received_at, acknowledged_by, acknowledged_at)
|
|
VALUES (1, 'legacy-fp', 'LegacyAlert', 'firing',
|
|
'{"severity":"warning"}', '{}', 1000, 1000, 1, 1500);
|
|
INSERT INTO alert_comments (alert_id, user_id, content, created_at)
|
|
VALUES (1, 1, 'legacy comment', 1600);`); err != nil {
|
|
t.Fatalf("seed pre-008 data: %v", err)
|
|
}
|
|
|
|
data, err := os.ReadFile(incidentsMigration)
|
|
if err != nil {
|
|
t.Fatalf("read 008: %v", err)
|
|
}
|
|
if _, err := database.Exec(string(data)); err != nil {
|
|
t.Fatalf("apply 008: %v", err)
|
|
}
|
|
|
|
var status, groupKey string
|
|
var ackBy int64
|
|
var severity string
|
|
if err := database.QueryRow(
|
|
"SELECT status, group_key, acknowledged_by, severity FROM incidents WHERE id = 1",
|
|
).Scan(&status, &groupKey, &ackBy, &severity); err != nil {
|
|
t.Fatalf("read backfilled incident: %v", err)
|
|
}
|
|
if status != "acknowledged" {
|
|
t.Errorf("expected the ack to carry over as status, got %q", status)
|
|
}
|
|
if groupKey != "backfill:legacy-fp" {
|
|
t.Errorf("unexpected group_key %q", groupKey)
|
|
}
|
|
if ackBy != 1 {
|
|
t.Errorf("expected acknowledged_by 1, got %d", ackBy)
|
|
}
|
|
if severity != "warning" {
|
|
t.Errorf("expected severity carried from labels, got %q", severity)
|
|
}
|
|
|
|
var notes int
|
|
if err := database.QueryRow(
|
|
"SELECT COUNT(*) FROM incident_events WHERE type = 'note' AND detail = 'legacy comment'",
|
|
).Scan(¬es); err != nil {
|
|
t.Fatalf("count notes: %v", err)
|
|
}
|
|
if notes != 1 {
|
|
t.Errorf("expected the comment to become a note, got %d", notes)
|
|
}
|
|
|
|
// And the columns that caused the ack-survives-a-re-fire bug are gone.
|
|
if _, err := database.Exec("SELECT acknowledged_by FROM alerts"); err == nil {
|
|
t.Error("expected alerts.acknowledged_by to be dropped")
|
|
}
|
|
}
|
|
|
|
func contains(haystack []string, needle string) bool {
|
|
for _, s := range haystack {
|
|
if s == needle {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|