Niklas Ye
0aaea8efb5
Assign logged only the assignee (user_id), archive/unarchive logged nothing. Migration 018 adds actor_user_id/actor_service_account_id to incident_events for 'assigned'; archive/unarchive now log archived/unarchived events with the caller via callerActorIDs. Timeline JSON gains actor_* fields; web timeline renders them. Service accounts are still not assignable.
975 lines
34 KiB
Go
975 lines
34 KiB
Go
package api_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
|
)
|
|
|
|
// amAlert builds one alert of a webhook payload.
|
|
func amAlert(fingerprint, name, status, startsAt, endsAt string, labels map[string]string) map[string]any {
|
|
l := map[string]string{"alertname": name}
|
|
for k, v := range labels {
|
|
l[k] = v
|
|
}
|
|
return map[string]any{
|
|
"status": status,
|
|
"labels": l,
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}
|
|
}
|
|
|
|
func listIncidents(t *testing.T, s *ts, query string) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/incidents"+query, nil), &out)
|
|
return out
|
|
}
|
|
|
|
func getIncident(t *testing.T, s *ts, id int) map[string]any {
|
|
t.Helper()
|
|
var out map[string]any
|
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d", id), nil), &out)
|
|
return out
|
|
}
|
|
|
|
func timeline(t *testing.T, s *ts, id int) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id), nil), &out)
|
|
return out
|
|
}
|
|
|
|
// eventTypes flattens a timeline to its event types, which is what the ordering
|
|
// assertions actually care about.
|
|
func eventTypes(events []map[string]any) []string {
|
|
types := make([]string, len(events))
|
|
for i, e := range events {
|
|
types[i] = e["type"].(string)
|
|
}
|
|
return types
|
|
}
|
|
|
|
// countIncidents counts rows directly, including resolved and archived ones that
|
|
// no list view returns.
|
|
func (s *ts) countIncidents(t *testing.T) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM incidents").Scan(&n); err != nil {
|
|
t.Fatalf("count incidents: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Ingest: alerts becoming incidents
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_FiringOpensIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-1", "HighCPU", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "critical"}),
|
|
}, "{}:{alertname=\"HighCPU\"}")
|
|
|
|
incidents := listIncidents(t, s, "")
|
|
if len(incidents) != 1 {
|
|
t.Fatalf("expected 1 incident, got %d", len(incidents))
|
|
}
|
|
inc := incidents[0]
|
|
if inc["status"] != "triggered" {
|
|
t.Errorf("expected status triggered, got %v", inc["status"])
|
|
}
|
|
if inc["severity"] != "critical" {
|
|
t.Errorf("expected severity critical, got %v", inc["severity"])
|
|
}
|
|
if inc["title"] != "HighCPU" {
|
|
t.Errorf("expected title from groupLabels, got %v", inc["title"])
|
|
}
|
|
|
|
// The alert points back at the incident it opened.
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 || alerts[0]["incident_id"] == nil {
|
|
t.Fatalf("expected the alert to carry an incident_id, got %v", alerts)
|
|
}
|
|
}
|
|
|
|
// Alertmanager already grouped these; we adopt its answer rather than
|
|
// correlating again.
|
|
func TestWebhook_SameGroupKeyJoinsOneIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"DiskFull\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-a", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
amAlert("fp-b", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
incidents := listIncidents(t, s, "")
|
|
if len(incidents) != 1 {
|
|
t.Fatalf("expected 1 incident for one groupKey, got %d", len(incidents))
|
|
}
|
|
id := int(incidents[0]["id"].(float64))
|
|
|
|
inc := getIncident(t, s, id)
|
|
members, _ := inc["alerts"].([]any)
|
|
if len(members) != 2 {
|
|
t.Fatalf("expected 2 alerts under the incident, got %d", len(members))
|
|
}
|
|
|
|
added := 0
|
|
for _, ty := range eventTypes(timeline(t, s, id)) {
|
|
if ty == "alert_added" {
|
|
added++
|
|
}
|
|
}
|
|
if added != 2 {
|
|
t.Errorf("expected 2 alert_added events, got %d", added)
|
|
}
|
|
}
|
|
|
|
// The load-bearing rule. Alertmanager re-sends firing notifications every
|
|
// repeat_interval; if those re-sends reopened incidents, resolving one by hand
|
|
// would mean nothing.
|
|
func TestWebhook_HeartbeatDoesNotReopenResolvedIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Flapper\"}"
|
|
alert := amAlert("fp-hb", "Flapper", "firing", "2026-05-20T10:00:00Z", zeroTime, nil)
|
|
|
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("resolve returned %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// Same startsAt, same fingerprint: a re-send, not a new occurrence.
|
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
|
|
|
if n := s.countIncidents(t); n != 1 {
|
|
t.Fatalf("expected the heartbeat to open no incident, got %d total", n)
|
|
}
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] == nil {
|
|
t.Error("expected incident 1 to stay resolved")
|
|
}
|
|
|
|
// The alert itself is still firing and still being tracked — only the work
|
|
// item is closed.
|
|
status, _, _ := s.alertRow(t, "fp-hb")
|
|
if status != "firing" {
|
|
t.Errorf("expected the alert to still be firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
func TestWebhook_NewOccurrenceOpensNewIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Recurring\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-new", "Recurring", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
// A newer startsAt is a genuinely new occurrence, not a re-send.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-new", "Recurring", "firing", "2026-05-21T09:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
if n := s.countIncidents(t); n != 2 {
|
|
t.Fatalf("expected a second incident for the new occurrence, got %d total", n)
|
|
}
|
|
open := listIncidents(t, s, "")
|
|
if len(open) != 1 || int(open[0]["id"].(float64)) != 2 {
|
|
t.Fatalf("expected incident 2 to be the open one, got %v", open)
|
|
}
|
|
|
|
// The new incident starts unacknowledged: that is the point of the split.
|
|
if open[0]["acknowledged_by"] != nil {
|
|
t.Error("expected a fresh occurrence to start unacknowledged")
|
|
}
|
|
}
|
|
|
|
func TestWebhook_ResolvedOnlyPayloadOpensNothing(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-res", "AlreadyOver", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
|
}, "{}:{alertname=\"AlreadyOver\"}")
|
|
|
|
if n := s.countIncidents(t); n != 0 {
|
|
t.Errorf("expected no incident from a resolved-only payload, got %d", n)
|
|
}
|
|
if status, _, _ := s.alertRow(t, "fp-res"); status != "resolved" {
|
|
t.Errorf("expected the alert itself to be stored, got %q", status)
|
|
}
|
|
}
|
|
|
|
// An incident that hit critical was a critical incident, even once the critical
|
|
// alert clears and only a warning is left.
|
|
func TestIncident_SeverityIsHighWaterMark(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Mixed\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-warn", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "warning"}),
|
|
amAlert("fp-crit", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "critical"}),
|
|
}, groupKey)
|
|
|
|
if inc := getIncident(t, s, 1); inc["severity"] != "critical" {
|
|
t.Fatalf("expected severity critical, got %v", inc["severity"])
|
|
}
|
|
|
|
// The critical alert clears; the warning keeps the incident open.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-crit", "Mixed", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z",
|
|
map[string]string{"severity": "critical"}),
|
|
}, groupKey)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["resolved_at"] != nil {
|
|
t.Fatal("expected the incident to stay open")
|
|
}
|
|
if inc["severity"] != "critical" {
|
|
t.Errorf("expected severity to stay critical, got %v", inc["severity"])
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Resolution cascade
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_AllAlertsResolvedAutoResolves(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Pair\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p1", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
amAlert("fp-p2", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
// One down, one still firing: the work is not done.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p1", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
|
}, groupKey)
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
|
t.Fatal("expected the incident to stay open while an alert is firing")
|
|
}
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p2", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:30:00Z", nil),
|
|
}, groupKey)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %v", inc["status"])
|
|
}
|
|
if inc["resolution_source"] != "alerts" {
|
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
|
}
|
|
}
|
|
|
|
// Expiry is inference, not observation, but it still has to close the work item
|
|
// — otherwise a lost resolved notification leaves an incident open forever.
|
|
func TestExpiry_CascadesToIncidentResolution(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fp-exp", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'fp-exp'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["status"] != "resolved" {
|
|
t.Errorf("expected the incident to resolve after expiry, got %v", inc["status"])
|
|
}
|
|
if inc["resolution_source"] != "alerts" {
|
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
|
}
|
|
|
|
// The expiry is recorded against the alert, not the incident.
|
|
if _, source, _ := s.alertRow(t, "fp-exp"); source == nil || *source != "expiry" {
|
|
t.Errorf("expected the alert's resolution_source to stay expiry, got %v", source)
|
|
}
|
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "alert_resolved") {
|
|
t.Errorf("expected an alert_resolved event on the timeline, got %v", types)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workflow actions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_Acknowledge(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-ack", "X", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["acknowledged_by"] == nil {
|
|
t.Error("expected acknowledged_by to be set")
|
|
}
|
|
if inc["status"] != "acknowledged" {
|
|
t.Errorf("expected status acknowledged, got %v", inc["status"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unacknowledge returned %d", resp.StatusCode)
|
|
}
|
|
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
|
|
t.Errorf("expected status back to triggered, got %v", inc["status"])
|
|
}
|
|
}
|
|
|
|
// A second acknowledge — a retried request, or a stale push notification
|
|
// tapped after the web UI already acked it — must be a no-op: same state,
|
|
// no second "acknowledged" timeline entry. Regression test for the bug
|
|
// described in issue #26 ("two acknowledged entries look like a bug").
|
|
func TestIncident_AcknowledgeTwiceIsIdempotent(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-ack2", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("first acknowledge returned %d", resp.StatusCode)
|
|
}
|
|
var first map[string]any
|
|
decode(t, resp, &first)
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("second acknowledge returned %d, want 200 (idempotent)", resp.StatusCode)
|
|
}
|
|
var second map[string]any
|
|
decode(t, resp, &second)
|
|
if second["status"] != "acknowledged" {
|
|
t.Errorf("expected status still acknowledged, got %v", second["status"])
|
|
}
|
|
if second["acknowledged_by"] != first["acknowledged_by"] {
|
|
t.Errorf("expected the same acknowledged_by, got %v then %v", first["acknowledged_by"], second["acknowledged_by"])
|
|
}
|
|
|
|
types := eventTypes(timeline(t, s, 1))
|
|
n := 0
|
|
for _, ty := range types {
|
|
if ty == "acknowledged" {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("expected exactly one acknowledged event, got %d in %v", n, types)
|
|
}
|
|
}
|
|
|
|
func TestIncident_ManualResolveIsTerminal(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-term", "Terminal", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("first resolve returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["resolution_source"] != "manual" {
|
|
t.Errorf("expected resolution_source manual, got %v", inc["resolution_source"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on re-resolve, got %d", resp.StatusCode)
|
|
}
|
|
|
|
// Acknowledging a closed incident is equally meaningless.
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 acknowledging a resolved incident, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestIncident_SnoozeHiddenFromDefaultList(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-snz", "Noisy", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]string{"duration": "2h"})
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("snooze returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["snoozed_until"] == nil {
|
|
t.Error("expected snoozed_until to be set")
|
|
}
|
|
|
|
if got := listIncidents(t, s, ""); len(got) != 0 {
|
|
t.Errorf("expected the snoozed incident to be hidden, got %d", len(got))
|
|
}
|
|
if got := listIncidents(t, s, "?snoozed=true"); len(got) != 1 {
|
|
t.Errorf("expected snoozed=true to show it, got %d", len(got))
|
|
}
|
|
|
|
// A snooze is not a resolution: the incident is still open work.
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
|
t.Error("expected a snoozed incident to stay open")
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/snooze", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unsnooze returned %d", resp.StatusCode)
|
|
}
|
|
if got := listIncidents(t, s, ""); len(got) != 1 {
|
|
t.Errorf("expected the incident back in the default list, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestIncident_SnoozeRejectsPastDeadline(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-past", "Past", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze",
|
|
map[string]string{"until": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusBadRequest {
|
|
t.Errorf("expected 400 for a snooze in the past, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// The schedule stops being decorative here: it is read at trigger time.
|
|
func TestIncident_AutoAssignedToCurrentOnCall(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
today := time.Now().UTC().Format("2006-01-02")
|
|
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{today}})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("schedule assignment returned %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-oncall", "PageMe", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["assigned_to"] != "admin" {
|
|
t.Errorf("expected the incident assigned to today's on-call, got %v", inc["assigned_to"])
|
|
}
|
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "assigned") {
|
|
t.Errorf("expected an assigned event, got %v", types)
|
|
}
|
|
}
|
|
|
|
func TestIncident_AssignToUser(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-asg", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2})
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("assign returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["assigned_to"] != "alice" {
|
|
t.Errorf("expected assigned_to alice, got %v", inc["assigned_to"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 99})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 assigning an unknown user, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Timeline and notes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_TimelineOrdering(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-tl", "Storyline", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "looking into it"}).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
events := timeline(t, s, 1)
|
|
want := []string{"triggered", "alert_added", "acknowledged", "note", "resolved"}
|
|
got := eventTypes(events)
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
|
}
|
|
}
|
|
|
|
// The note carries its author; system events do not.
|
|
for _, e := range events {
|
|
if e["type"] == "note" {
|
|
if e["username"] != "admin" || e["detail"] != "looking into it" {
|
|
t.Errorf("unexpected note event: %v", e)
|
|
}
|
|
}
|
|
if e["type"] == "triggered" && e["username"] != nil {
|
|
t.Errorf("expected the triggered event to have no author, got %v", e["username"])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIncident_NoteDeleteOwnOnly(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-note", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
var keyData map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
|
map[string]string{"name": "alice-key"}), &keyData)
|
|
aliceKey := keyData["key"].(string)
|
|
|
|
var note map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "admin note"}), ¬e)
|
|
noteID := int(note["id"].(float64))
|
|
|
|
// Alice cannot delete admin's note.
|
|
req, _ := http.NewRequest(http.MethodDelete,
|
|
fmt.Sprintf("%s/api/incidents/1/notes/%d", s.URL, noteID), nil)
|
|
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 deleting another user's note, got %d", resp.StatusCode)
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("expected 204 deleting own note, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// Only notes are deletable — the rest of the timeline is what happened.
|
|
func TestIncident_CannotDeleteSystemEvent(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-sys", "System", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
events := timeline(t, s, 1)
|
|
id := int(events[0]["id"].(float64))
|
|
resp := s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", id), nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 deleting a system event, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_ArchiveRoundTrip(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-arc", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/archive", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("archive returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["archived_at"] == nil {
|
|
t.Error("expected archived_at to be set")
|
|
}
|
|
|
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 0 {
|
|
t.Errorf("expected the archived incident to be hidden, got %d", len(got))
|
|
}
|
|
if got := listIncidents(t, s, "?status=resolved&archived=true"); len(got) != 1 {
|
|
t.Errorf("expected archived=true to show it, got %d", len(got))
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unarchive returned %d", resp.StatusCode)
|
|
}
|
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 1 {
|
|
t.Errorf("expected the incident back, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Service accounts (terdut-server#25)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// TestServiceAccount_CanActOnItsTeamsIncidents is #25's regression test.
|
|
// Before the fix: acknowledge/resolve/snooze/create-note each 500'd (writing
|
|
// acknowledged_by/user_id = 0, violating the users(id) FK for a service
|
|
// account), and delete-note silently matched zero rows (WHERE user_id = 0)
|
|
// instead of deleting.
|
|
func TestServiceAccount_CanActOnItsTeamsIncidents(t *testing.T) {
|
|
s := newTS(t)
|
|
instanceKey := createServiceAccount(t, s, s.key, "operator", models.ServiceAccountScopeInstance, 0)
|
|
teamA := createTeamAs(t, s, instanceKey, "team-a")
|
|
keyA := createServiceAccount(t, s, instanceKey, "team-a-sa", models.ServiceAccountScopeTeam, teamA)
|
|
|
|
var integration struct {
|
|
Key string `json:"key"`
|
|
}
|
|
decode(t, s.reqAs(t, keyA, http.MethodPost, "/api/teams/"+id64(teamA)+"/integrations",
|
|
map[string]string{"name": "test"}), &integration)
|
|
postToIntegration(t, s, integration.Key, "fp-sa", "SAIncident") // incident 1
|
|
|
|
// Acknowledge.
|
|
resp := s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("service account acknowledge: %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["acknowledged_by_service_account_id"] == nil {
|
|
t.Error("expected acknowledged_by_service_account_id to be set")
|
|
}
|
|
if inc["acknowledged_by_id"] != nil {
|
|
t.Errorf("expected acknowledged_by_id to stay nil for a service-account actor, got %v", inc["acknowledged_by_id"])
|
|
}
|
|
|
|
// Unacknowledge.
|
|
resp = s.reqAs(t, keyA, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("service account unacknowledge: %d", resp.StatusCode)
|
|
}
|
|
|
|
// Snooze, then unsnooze.
|
|
resp = s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/snooze",
|
|
map[string]string{"duration": "1h"})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Errorf("service account snooze: %d", resp.StatusCode)
|
|
}
|
|
resp = s.reqAs(t, keyA, http.MethodDelete, "/api/incidents/1/snooze", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("service account unsnooze: %d", resp.StatusCode)
|
|
}
|
|
|
|
// Create, then delete, a note.
|
|
var note map[string]any
|
|
decode(t, s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "looking into it"}), ¬e)
|
|
if note["service_account_id"] == nil {
|
|
t.Error("expected service_account_id on the note event")
|
|
}
|
|
if note["user_id"] != nil {
|
|
t.Errorf("expected no user_id on a service-account note, got %v", note["user_id"])
|
|
}
|
|
noteID := int(note["id"].(float64))
|
|
delResp := s.reqAs(t, keyA, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
|
|
delResp.Body.Close()
|
|
if delResp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("service account deleting its own note: %d", delResp.StatusCode)
|
|
}
|
|
|
|
// Resolve.
|
|
resp = s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Errorf("service account resolve: %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// Regression guard: a human actor must still write only the human columns,
|
|
// unaffected by the service-account branch added above.
|
|
func TestIncident_AcknowledgeStillWritesOnlyHumanColumn(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-human-ack", "Z", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
var inc map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil), &inc)
|
|
if inc["acknowledged_by_id"] == nil {
|
|
t.Error("expected acknowledged_by_id to be set for a human actor")
|
|
}
|
|
if inc["acknowledged_by_service_account_id"] != nil {
|
|
t.Errorf("expected acknowledged_by_service_account_id to stay nil for a human actor, got %v",
|
|
inc["acknowledged_by_service_account_id"])
|
|
}
|
|
}
|
|
|
|
func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-swp", "Old", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
s.exec(t, "UPDATE incidents SET resolved_at = $1 WHERE id = 1",
|
|
time.Now().Add(-30*24*time.Hour).Unix())
|
|
api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.notify)
|
|
|
|
if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
|
|
t.Error("expected the sweeper to archive a long-resolved incident")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Incidents(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s1", "One", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, "g1")
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s2", "Two", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, "g2")
|
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
var stats map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
|
|
|
if stats["total"].(float64) != 2 {
|
|
t.Errorf("expected total 2, got %v", stats["total"])
|
|
}
|
|
if stats["resolved"].(float64) != 1 {
|
|
t.Errorf("expected resolved 1, got %v", stats["resolved"])
|
|
}
|
|
if stats["triggered"].(float64) != 1 {
|
|
t.Errorf("expected triggered 1, got %v", stats["triggered"])
|
|
}
|
|
// One incident has been acknowledged and resolved, so both averages exist.
|
|
if stats["mtta_seconds"] == nil || stats["mttr_seconds"] == nil {
|
|
t.Errorf("expected mtta and mttr to be computable, got %v", stats)
|
|
}
|
|
}
|
|
|
|
// An empty window is a report of zero, not a failure. SUM over no rows is NULL
|
|
// in Postgres as it was in SQLite, and that used to come back as a 500 the
|
|
// moment every incident was archived — the state a quiet installation settles
|
|
// into.
|
|
func TestStats_IncidentsEmptyWindowIsZeroNotAnError(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// No incidents at all.
|
|
resp := s.req(t, http.MethodGet, "/api/stats/incidents", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
|
|
}
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
for _, k := range []string{"total", "triggered", "acknowledged", "resolved"} {
|
|
if stats[k].(float64) != 0 {
|
|
t.Errorf("expected %s 0, got %v", k, stats[k])
|
|
}
|
|
}
|
|
|
|
// And with every incident archived out of the window.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s4", "Gone", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/archive", nil).Body.Close()
|
|
|
|
resp = s.req(t, http.MethodGet, "/api/stats/incidents", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
t.Fatalf("expected 200 when every incident is archived, got %d", resp.StatusCode)
|
|
}
|
|
stats = nil
|
|
decode(t, resp, &stats)
|
|
if stats["total"].(float64) != 0 {
|
|
t.Errorf("expected total 0, got %v", stats["total"])
|
|
}
|
|
}
|
|
|
|
// The alert stats share the same aggregate, and the same empty-window trap.
|
|
func TestStats_AlertsEmptyWindowIsZeroNotAnError(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
|
|
}
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
for _, k := range []string{"total", "firing", "resolved"} {
|
|
if stats[k].(float64) != 0 {
|
|
t.Errorf("expected %s 0, got %v", k, stats[k])
|
|
}
|
|
}
|
|
}
|
|
|
|
// Nothing acknowledged yet means "no data", which is not the same claim as zero.
|
|
func TestStats_IncidentsNullMTTAWhenNothingAcknowledged(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s3", "Untouched", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
var stats map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
|
if stats["mtta_seconds"] != nil {
|
|
t.Errorf("expected mtta_seconds null, got %v", stats["mtta_seconds"])
|
|
}
|
|
}
|
|
|
|
func contains(haystack []string, needle string) bool {
|
|
for _, s := range haystack {
|
|
if s == needle {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Actor on assign / archive / unarchive (terdut-server#35)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// lastEvent returns the newest timeline event of the given type.
|
|
func lastEvent(t *testing.T, events []map[string]any, typ string) map[string]any {
|
|
t.Helper()
|
|
for i := len(events) - 1; i >= 0; i-- {
|
|
if events[i]["type"] == typ {
|
|
return events[i]
|
|
}
|
|
}
|
|
t.Fatalf("no %q event in %v", typ, eventTypes(events))
|
|
return nil
|
|
}
|
|
|
|
func TestIncident_AssignRecordsHumanActor(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-asg-actor", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2}).Body.Close()
|
|
|
|
ev := lastEvent(t, timeline(t, s, 1), "assigned")
|
|
if ev["username"] != "alice" {
|
|
t.Errorf("expected the assignee alice in username, got %v", ev["username"])
|
|
}
|
|
if ev["actor_user_id"] == nil || ev["actor_username"] == nil {
|
|
t.Errorf("expected the assigning human in actor_*, got %v", ev)
|
|
}
|
|
if ev["actor_service_account_id"] != nil {
|
|
t.Errorf("expected no service-account actor, got %v", ev["actor_service_account_id"])
|
|
}
|
|
}
|
|
|
|
func TestIncident_ArchiveUnarchiveRecordHumanActor(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-arc-actor", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/archive", nil).Body.Close()
|
|
s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil).Body.Close()
|
|
|
|
events := timeline(t, s, 1)
|
|
for _, typ := range []string{"archived", "unarchived"} {
|
|
ev := lastEvent(t, events, typ)
|
|
if ev["user_id"] == nil || ev["service_account_id"] != nil {
|
|
t.Errorf("%s: expected only the human actor, got %v", typ, ev)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestServiceAccount_AssignArchiveUnarchiveRecordActor(t *testing.T) {
|
|
s := newTS(t)
|
|
instanceKey := createServiceAccount(t, s, s.key, "operator", models.ServiceAccountScopeInstance, 0)
|
|
teamA := createTeamAs(t, s, instanceKey, "team-a")
|
|
keyA := createServiceAccount(t, s, instanceKey, "team-a-sa", models.ServiceAccountScopeTeam, teamA)
|
|
var integration struct {
|
|
Key string `json:"key"`
|
|
}
|
|
decode(t, s.reqAs(t, keyA, http.MethodPost, "/api/teams/"+id64(teamA)+"/integrations",
|
|
map[string]string{"name": "test"}), &integration)
|
|
postToIntegration(t, s, integration.Key, "fp-sa-35", "SA35") // incident 1
|
|
|
|
resp := s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 1})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("service account assign: %d", resp.StatusCode)
|
|
}
|
|
resp = s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
resp.Body.Close()
|
|
resp = s.reqAs(t, keyA, http.MethodPost, "/api/incidents/1/archive", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("service account archive: %d", resp.StatusCode)
|
|
}
|
|
resp = s.reqAs(t, keyA, http.MethodDelete, "/api/incidents/1/archive", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("service account unarchive: %d", resp.StatusCode)
|
|
}
|
|
|
|
var events []map[string]any
|
|
decode(t, s.reqAs(t, keyA, http.MethodGet, "/api/incidents/1/timeline", nil), &events)
|
|
asg := lastEvent(t, events, "assigned")
|
|
if asg["actor_service_account_id"] == nil || asg["actor_user_id"] != nil {
|
|
t.Errorf("assigned: expected only the service-account actor, got %v", asg)
|
|
}
|
|
if asg["user_id"] == nil {
|
|
t.Errorf("assigned: user_id must stay the assignee, got %v", asg)
|
|
}
|
|
for _, typ := range []string{"archived", "unarchived"} {
|
|
ev := lastEvent(t, events, typ)
|
|
if ev["service_account_id"] == nil || ev["user_id"] != nil {
|
|
t.Errorf("%s: expected only the service-account actor, got %v", typ, ev)
|
|
}
|
|
}
|
|
}
|