14c24f8fda
Release / release (push) Has been skipped
Release / build (amd64, linux) (push) Has been skipped
Release / build (arm64, darwin) (push) Has been skipped
Release / test (push) Failing after 5s
Release / build (arm64, linux) (push) Has been skipped
Release / docker (push) Has been skipped
Release / chart (push) Has been skipped
Release / build (amd64, darwin) (push) Has been skipped
Everything this server does assumes alerts arrive. If Prometheus stops evaluating, or Alertmanager cannot reach us, nothing arrives — and silence is indistinguishable from everything being fine. The cluster has shipped the alert for exactly this case all along: Watchdog is expr: vector(1), so it fires permanently and is re-sent forever, and it is worth nothing unless something downstream notices it stop. Nothing did. It arrived, opened no incident because a repeat_interval re-send is not a new occurrence, and when the monitoring stack died the sweeper quietly expired it and paged nobody. So the handling is inverted for a configurable set of alerts: receiving one opens no incident, and the absence of one does. TERDUT_DEADMAN_MATCHERS selects them as label matchers, defaulting to alertname=Watchdog. The unit of monitoring is the fingerprint rather than the alert name. Two clusters sending the same Watchdog are two independent switches, so a healthy one can never mask a dead one. Every matcher must name an alertname, which keeps the sweeper's candidate query on alerts_name_idx instead of JSON-extracting labels from every row, and leaves matching with a single implementation. A switch is dormant until its first heartbeat: a matcher nothing has ever sent opens nothing, so a fresh deploy or a restored database does not page. Resolving the incident by hand sticks, exactly as it does for an alert-backed one, so a decommissioned source is a one-time page rather than a nag; the switch re-arms only when the heartbeat comes back, and dying again is a new incident. The incident has no member alerts on purpose. Linking the heartbeat would have the settled-incident cascade close it on the very sweep that opened it, and there is no alert describing the problem anyway — the problem is that no alert arrived. What happened is on the timeline instead, and recovery is the only automatic way out. One narrow exemption in the ingest guard makes recovery possible at all. A heartbeat we declared dead is marked resolved, and the one that proves us wrong carries the unchanged startsAt of an alert that never stopped firing — so "resolution is terminal within an instance" would discard it forever and a switch could die exactly once. The exemption is scoped to resolution_source = 'deadman', which is the only resolution this server infers from silence on a timeout of its own, so nothing another writer set can be undone by a stale retry. Matched alerts are also held back from the generic staleness expiry, which would otherwise resolve a heartbeat as 'expiry' long before its own tighter deadline. The timeout points the opposite way to TERDUT_STALE_AFTER: staleness is a generous grace period around a repeat_interval you do not control, while this is a deadline you set deliberately and configure the heartbeat's route to beat. Inheriting a 4h or 12h repeat_interval gives a dead man's switch with a twelve hour fuse, so the README spells out the route the heartbeat needs.
773 lines
26 KiB
Go
773 lines
26 KiB
Go
package api_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/yeniklas/terdut-server/internal/api"
|
|
"github.com/yeniklas/terdut-server/internal/db"
|
|
)
|
|
|
|
// amAlert builds one alert of a webhook payload.
|
|
func amAlert(fingerprint, name, status, startsAt, endsAt string, labels map[string]string) map[string]any {
|
|
l := map[string]string{"alertname": name}
|
|
for k, v := range labels {
|
|
l[k] = v
|
|
}
|
|
return map[string]any{
|
|
"status": status,
|
|
"labels": l,
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}
|
|
}
|
|
|
|
func listIncidents(t *testing.T, s *ts, query string) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/incidents"+query, nil), &out)
|
|
return out
|
|
}
|
|
|
|
func getIncident(t *testing.T, s *ts, id int) map[string]any {
|
|
t.Helper()
|
|
var out map[string]any
|
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d", id), nil), &out)
|
|
return out
|
|
}
|
|
|
|
func timeline(t *testing.T, s *ts, id int) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id), nil), &out)
|
|
return out
|
|
}
|
|
|
|
// eventTypes flattens a timeline to its event types, which is what the ordering
|
|
// assertions actually care about.
|
|
func eventTypes(events []map[string]any) []string {
|
|
types := make([]string, len(events))
|
|
for i, e := range events {
|
|
types[i] = e["type"].(string)
|
|
}
|
|
return types
|
|
}
|
|
|
|
// countIncidents counts rows directly, including resolved and archived ones that
|
|
// no list view returns.
|
|
func (s *ts) countIncidents(t *testing.T) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM incidents").Scan(&n); err != nil {
|
|
t.Fatalf("count incidents: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Ingest: alerts becoming incidents
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_FiringOpensIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-1", "HighCPU", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "critical"}),
|
|
}, "{}:{alertname=\"HighCPU\"}")
|
|
|
|
incidents := listIncidents(t, s, "")
|
|
if len(incidents) != 1 {
|
|
t.Fatalf("expected 1 incident, got %d", len(incidents))
|
|
}
|
|
inc := incidents[0]
|
|
if inc["status"] != "triggered" {
|
|
t.Errorf("expected status triggered, got %v", inc["status"])
|
|
}
|
|
if inc["severity"] != "critical" {
|
|
t.Errorf("expected severity critical, got %v", inc["severity"])
|
|
}
|
|
if inc["title"] != "HighCPU" {
|
|
t.Errorf("expected title from groupLabels, got %v", inc["title"])
|
|
}
|
|
|
|
// The alert points back at the incident it opened.
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 || alerts[0]["incident_id"] == nil {
|
|
t.Fatalf("expected the alert to carry an incident_id, got %v", alerts)
|
|
}
|
|
}
|
|
|
|
// Alertmanager already grouped these; we adopt its answer rather than
|
|
// correlating again.
|
|
func TestWebhook_SameGroupKeyJoinsOneIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"DiskFull\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-a", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
amAlert("fp-b", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
incidents := listIncidents(t, s, "")
|
|
if len(incidents) != 1 {
|
|
t.Fatalf("expected 1 incident for one groupKey, got %d", len(incidents))
|
|
}
|
|
id := int(incidents[0]["id"].(float64))
|
|
|
|
inc := getIncident(t, s, id)
|
|
members, _ := inc["alerts"].([]any)
|
|
if len(members) != 2 {
|
|
t.Fatalf("expected 2 alerts under the incident, got %d", len(members))
|
|
}
|
|
|
|
added := 0
|
|
for _, ty := range eventTypes(timeline(t, s, id)) {
|
|
if ty == "alert_added" {
|
|
added++
|
|
}
|
|
}
|
|
if added != 2 {
|
|
t.Errorf("expected 2 alert_added events, got %d", added)
|
|
}
|
|
}
|
|
|
|
// The load-bearing rule. Alertmanager re-sends firing notifications every
|
|
// repeat_interval; if those re-sends reopened incidents, resolving one by hand
|
|
// would mean nothing.
|
|
func TestWebhook_HeartbeatDoesNotReopenResolvedIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Flapper\"}"
|
|
alert := amAlert("fp-hb", "Flapper", "firing", "2026-05-20T10:00:00Z", zeroTime, nil)
|
|
|
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("resolve returned %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// Same startsAt, same fingerprint: a re-send, not a new occurrence.
|
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
|
|
|
if n := s.countIncidents(t); n != 1 {
|
|
t.Fatalf("expected the heartbeat to open no incident, got %d total", n)
|
|
}
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] == nil {
|
|
t.Error("expected incident 1 to stay resolved")
|
|
}
|
|
|
|
// The alert itself is still firing and still being tracked — only the work
|
|
// item is closed.
|
|
status, _, _ := s.alertRow(t, "fp-hb")
|
|
if status != "firing" {
|
|
t.Errorf("expected the alert to still be firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
func TestWebhook_NewOccurrenceOpensNewIncident(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Recurring\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-new", "Recurring", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
// A newer startsAt is a genuinely new occurrence, not a re-send.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-new", "Recurring", "firing", "2026-05-21T09:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
if n := s.countIncidents(t); n != 2 {
|
|
t.Fatalf("expected a second incident for the new occurrence, got %d total", n)
|
|
}
|
|
open := listIncidents(t, s, "")
|
|
if len(open) != 1 || int(open[0]["id"].(float64)) != 2 {
|
|
t.Fatalf("expected incident 2 to be the open one, got %v", open)
|
|
}
|
|
|
|
// The new incident starts unacknowledged: that is the point of the split.
|
|
if open[0]["acknowledged_by"] != nil {
|
|
t.Error("expected a fresh occurrence to start unacknowledged")
|
|
}
|
|
}
|
|
|
|
func TestWebhook_ResolvedOnlyPayloadOpensNothing(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-res", "AlreadyOver", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
|
}, "{}:{alertname=\"AlreadyOver\"}")
|
|
|
|
if n := s.countIncidents(t); n != 0 {
|
|
t.Errorf("expected no incident from a resolved-only payload, got %d", n)
|
|
}
|
|
if status, _, _ := s.alertRow(t, "fp-res"); status != "resolved" {
|
|
t.Errorf("expected the alert itself to be stored, got %q", status)
|
|
}
|
|
}
|
|
|
|
// An incident that hit critical was a critical incident, even once the critical
|
|
// alert clears and only a warning is left.
|
|
func TestIncident_SeverityIsHighWaterMark(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Mixed\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-warn", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "warning"}),
|
|
amAlert("fp-crit", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
|
map[string]string{"severity": "critical"}),
|
|
}, groupKey)
|
|
|
|
if inc := getIncident(t, s, 1); inc["severity"] != "critical" {
|
|
t.Fatalf("expected severity critical, got %v", inc["severity"])
|
|
}
|
|
|
|
// The critical alert clears; the warning keeps the incident open.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-crit", "Mixed", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z",
|
|
map[string]string{"severity": "critical"}),
|
|
}, groupKey)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["resolved_at"] != nil {
|
|
t.Fatal("expected the incident to stay open")
|
|
}
|
|
if inc["severity"] != "critical" {
|
|
t.Errorf("expected severity to stay critical, got %v", inc["severity"])
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Resolution cascade
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_AllAlertsResolvedAutoResolves(t *testing.T) {
|
|
s := newTS(t)
|
|
const groupKey = "{}:{alertname=\"Pair\"}"
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p1", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
amAlert("fp-p2", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, groupKey)
|
|
|
|
// One down, one still firing: the work is not done.
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p1", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
|
}, groupKey)
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
|
t.Fatal("expected the incident to stay open while an alert is firing")
|
|
}
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-p2", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:30:00Z", nil),
|
|
}, groupKey)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %v", inc["status"])
|
|
}
|
|
if inc["resolution_source"] != "alerts" {
|
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
|
}
|
|
}
|
|
|
|
// Expiry is inference, not observation, but it still has to close the work item
|
|
// — otherwise a lost resolved notification leaves an incident open forever.
|
|
func TestExpiry_CascadesToIncidentResolution(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fp-exp", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'fp-exp'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["status"] != "resolved" {
|
|
t.Errorf("expected the incident to resolve after expiry, got %v", inc["status"])
|
|
}
|
|
if inc["resolution_source"] != "alerts" {
|
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
|
}
|
|
|
|
// The expiry is recorded against the alert, not the incident.
|
|
if _, source, _ := s.alertRow(t, "fp-exp"); source == nil || *source != "expiry" {
|
|
t.Errorf("expected the alert's resolution_source to stay expiry, got %v", source)
|
|
}
|
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "alert_resolved") {
|
|
t.Errorf("expected an alert_resolved event on the timeline, got %v", types)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Workflow actions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_Acknowledge(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-ack", "X", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["acknowledged_by"] == nil {
|
|
t.Error("expected acknowledged_by to be set")
|
|
}
|
|
if inc["status"] != "acknowledged" {
|
|
t.Errorf("expected status acknowledged, got %v", inc["status"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unacknowledge returned %d", resp.StatusCode)
|
|
}
|
|
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
|
|
t.Errorf("expected status back to triggered, got %v", inc["status"])
|
|
}
|
|
}
|
|
|
|
func TestIncident_ManualResolveIsTerminal(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-term", "Terminal", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("first resolve returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["resolution_source"] != "manual" {
|
|
t.Errorf("expected resolution_source manual, got %v", inc["resolution_source"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on re-resolve, got %d", resp.StatusCode)
|
|
}
|
|
|
|
// Acknowledging a closed incident is equally meaningless.
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 acknowledging a resolved incident, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestIncident_SnoozeHiddenFromDefaultList(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-snz", "Noisy", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]string{"duration": "2h"})
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("snooze returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["snoozed_until"] == nil {
|
|
t.Error("expected snoozed_until to be set")
|
|
}
|
|
|
|
if got := listIncidents(t, s, ""); len(got) != 0 {
|
|
t.Errorf("expected the snoozed incident to be hidden, got %d", len(got))
|
|
}
|
|
if got := listIncidents(t, s, "?snoozed=true"); len(got) != 1 {
|
|
t.Errorf("expected snoozed=true to show it, got %d", len(got))
|
|
}
|
|
|
|
// A snooze is not a resolution: the incident is still open work.
|
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
|
t.Error("expected a snoozed incident to stay open")
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/snooze", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unsnooze returned %d", resp.StatusCode)
|
|
}
|
|
if got := listIncidents(t, s, ""); len(got) != 1 {
|
|
t.Errorf("expected the incident back in the default list, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestIncident_SnoozeRejectsPastDeadline(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-past", "Past", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze",
|
|
map[string]string{"until": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusBadRequest {
|
|
t.Errorf("expected 400 for a snooze in the past, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// The schedule stops being decorative here: it is read at trigger time.
|
|
func TestIncident_AutoAssignedToCurrentOnCall(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
today := time.Now().UTC().Format("2006-01-02")
|
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{today}})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("schedule assignment returned %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-oncall", "PageMe", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
inc := getIncident(t, s, 1)
|
|
if inc["assigned_to"] != "admin" {
|
|
t.Errorf("expected the incident assigned to today's on-call, got %v", inc["assigned_to"])
|
|
}
|
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "assigned") {
|
|
t.Errorf("expected an assigned event, got %v", types)
|
|
}
|
|
}
|
|
|
|
func TestIncident_AssignToUser(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-asg", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2})
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("assign returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["assigned_to"] != "alice" {
|
|
t.Errorf("expected assigned_to alice, got %v", inc["assigned_to"])
|
|
}
|
|
|
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 99})
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 assigning an unknown user, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Timeline and notes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_TimelineOrdering(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-tl", "Storyline", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "looking into it"}).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
events := timeline(t, s, 1)
|
|
want := []string{"triggered", "alert_added", "acknowledged", "note", "resolved"}
|
|
got := eventTypes(events)
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
|
}
|
|
}
|
|
|
|
// The note carries its author; system events do not.
|
|
for _, e := range events {
|
|
if e["type"] == "note" {
|
|
if e["username"] != "admin" || e["detail"] != "looking into it" {
|
|
t.Errorf("unexpected note event: %v", e)
|
|
}
|
|
}
|
|
if e["type"] == "triggered" && e["username"] != nil {
|
|
t.Errorf("expected the triggered event to have no author, got %v", e["username"])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIncident_NoteDeleteOwnOnly(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-note", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
|
var keyData map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
|
map[string]string{"name": "alice-key"}), &keyData)
|
|
aliceKey := keyData["key"].(string)
|
|
|
|
var note map[string]any
|
|
decode(t, s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]string{"content": "admin note"}), ¬e)
|
|
noteID := int(note["id"].(float64))
|
|
|
|
// Alice cannot delete admin's note.
|
|
req, _ := http.NewRequest(http.MethodDelete,
|
|
fmt.Sprintf("%s/api/incidents/1/notes/%d", s.URL, noteID), nil)
|
|
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 deleting another user's note, got %d", resp.StatusCode)
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("expected 204 deleting own note, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// Only notes are deletable — the rest of the timeline is what happened.
|
|
func TestIncident_CannotDeleteSystemEvent(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-sys", "System", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
events := timeline(t, s, 1)
|
|
id := int(events[0]["id"].(float64))
|
|
resp := s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", id), nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 deleting a system event, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestIncident_ArchiveRoundTrip(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-arc", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/archive", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("archive returned %d", resp.StatusCode)
|
|
}
|
|
var inc map[string]any
|
|
decode(t, resp, &inc)
|
|
if inc["archived_at"] == nil {
|
|
t.Error("expected archived_at to be set")
|
|
}
|
|
|
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 0 {
|
|
t.Errorf("expected the archived incident to be hidden, got %d", len(got))
|
|
}
|
|
if got := listIncidents(t, s, "?status=resolved&archived=true"); len(got) != 1 {
|
|
t.Errorf("expected archived=true to show it, got %d", len(got))
|
|
}
|
|
|
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unarchive returned %d", resp.StatusCode)
|
|
}
|
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 1 {
|
|
t.Errorf("expected the incident back, got %d", len(got))
|
|
}
|
|
}
|
|
|
|
func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-swp", "Old", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
s.exec(t, "UPDATE incidents SET resolved_at = ? WHERE id = 1",
|
|
time.Now().Add(-30*24*time.Hour).Unix())
|
|
api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.deadman, s.notify)
|
|
|
|
if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
|
|
t.Error("expected the sweeper to archive a long-resolved incident")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Incidents(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s1", "One", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, "g1")
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s2", "Two", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
}, "g2")
|
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
var stats map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
|
|
|
if stats["total"].(float64) != 2 {
|
|
t.Errorf("expected total 2, got %v", stats["total"])
|
|
}
|
|
if stats["resolved"].(float64) != 1 {
|
|
t.Errorf("expected resolved 1, got %v", stats["resolved"])
|
|
}
|
|
if stats["triggered"].(float64) != 1 {
|
|
t.Errorf("expected triggered 1, got %v", stats["triggered"])
|
|
}
|
|
// One incident has been acknowledged and resolved, so both averages exist.
|
|
if stats["mtta_seconds"] == nil || stats["mttr_seconds"] == nil {
|
|
t.Errorf("expected mtta and mttr to be computable, got %v", stats)
|
|
}
|
|
}
|
|
|
|
// Nothing acknowledged yet means "no data", which is not the same claim as zero.
|
|
func TestStats_IncidentsNullMTTAWhenNothingAcknowledged(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{
|
|
amAlert("fp-s3", "Untouched", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
|
})
|
|
|
|
var stats map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
|
if stats["mtta_seconds"] != nil {
|
|
t.Errorf("expected mtta_seconds null, got %v", stats["mtta_seconds"])
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Migration backfill
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// An upgrade must not drop the acknowledgements and comments people already
|
|
// have, so 008 is replayed here over a database left at 007.
|
|
func TestMigration_BackfillCarriesAckAndComments(t *testing.T) {
|
|
database, err := db.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
t.Cleanup(func() { database.Close() })
|
|
|
|
files, err := filepath.Glob("../db/migrations/*.sql")
|
|
if err != nil || len(files) == 0 {
|
|
t.Fatalf("find migrations: %v", err)
|
|
}
|
|
sort.Strings(files)
|
|
|
|
var incidentsMigration string
|
|
for _, f := range files {
|
|
if filepath.Base(f) >= "008" {
|
|
incidentsMigration = f
|
|
break
|
|
}
|
|
data, err := os.ReadFile(f)
|
|
if err != nil {
|
|
t.Fatalf("read %s: %v", f, err)
|
|
}
|
|
if _, err := database.Exec(string(data)); err != nil {
|
|
t.Fatalf("apply %s: %v", f, err)
|
|
}
|
|
}
|
|
if incidentsMigration == "" {
|
|
t.Fatal("008 migration not found")
|
|
}
|
|
|
|
// A database as it would look on the old schema: an acknowledged firing
|
|
// alert with a comment on it.
|
|
if _, err := database.Exec(`
|
|
INSERT INTO users (id, username, email) VALUES (1, 'admin', 'admin@test.com');
|
|
INSERT INTO alerts (id, fingerprint, name, status, labels, annotations,
|
|
starts_at, received_at, acknowledged_by, acknowledged_at)
|
|
VALUES (1, 'legacy-fp', 'LegacyAlert', 'firing',
|
|
'{"severity":"warning"}', '{}', 1000, 1000, 1, 1500);
|
|
INSERT INTO alert_comments (alert_id, user_id, content, created_at)
|
|
VALUES (1, 1, 'legacy comment', 1600);`); err != nil {
|
|
t.Fatalf("seed pre-008 data: %v", err)
|
|
}
|
|
|
|
data, err := os.ReadFile(incidentsMigration)
|
|
if err != nil {
|
|
t.Fatalf("read 008: %v", err)
|
|
}
|
|
if _, err := database.Exec(string(data)); err != nil {
|
|
t.Fatalf("apply 008: %v", err)
|
|
}
|
|
|
|
var status, groupKey string
|
|
var ackBy int64
|
|
var severity string
|
|
if err := database.QueryRow(
|
|
"SELECT status, group_key, acknowledged_by, severity FROM incidents WHERE id = 1",
|
|
).Scan(&status, &groupKey, &ackBy, &severity); err != nil {
|
|
t.Fatalf("read backfilled incident: %v", err)
|
|
}
|
|
if status != "acknowledged" {
|
|
t.Errorf("expected the ack to carry over as status, got %q", status)
|
|
}
|
|
if groupKey != "backfill:legacy-fp" {
|
|
t.Errorf("unexpected group_key %q", groupKey)
|
|
}
|
|
if ackBy != 1 {
|
|
t.Errorf("expected acknowledged_by 1, got %d", ackBy)
|
|
}
|
|
if severity != "warning" {
|
|
t.Errorf("expected severity carried from labels, got %q", severity)
|
|
}
|
|
|
|
var notes int
|
|
if err := database.QueryRow(
|
|
"SELECT COUNT(*) FROM incident_events WHERE type = 'note' AND detail = 'legacy comment'",
|
|
).Scan(¬es); err != nil {
|
|
t.Fatalf("count notes: %v", err)
|
|
}
|
|
if notes != 1 {
|
|
t.Errorf("expected the comment to become a note, got %d", notes)
|
|
}
|
|
|
|
// And the columns that caused the ack-survives-a-re-fire bug are gone.
|
|
if _, err := database.Exec("SELECT acknowledged_by FROM alerts"); err == nil {
|
|
t.Error("expected alerts.acknowledged_by to be dropped")
|
|
}
|
|
}
|
|
|
|
func contains(haystack []string, needle string) bool {
|
|
for _, s := range haystack {
|
|
if s == needle {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|