package api_test import ( "bytes" "context" "database/sql" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "sort" "strings" "testing" "time" "git.ryuvia.com/niklas/terdut-server/internal/api" "git.ryuvia.com/niklas/terdut-server/internal/config" ) // ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so // tests can age rows directly — the sweeper's inputs are wall-clock timestamps. type ts struct { *httptest.Server key string // ingestKey is an integration key for the default team: the only way in // since the unauthenticated webhook was removed, so the tests exercise the // same path production does. ingestKey string db *sql.DB notify api.NotifyConfig deadman api.DeadmanConfig } // newTS builds a server over a fresh database. Notifications are off // unless a NotifyConfig is passed, so tests that predate them are unaffected. // Dead man's switches are off too — see newDeadmanTS. func newTS(t *testing.T, notify ...api.NotifyConfig) *ts { t.Helper() var cfg api.NotifyConfig if len(notify) > 0 { cfg = notify[0] } return newDeadmanTS(t, api.DeadmanConfig{}, cfg) } // newDeadmanTS is newTS with the default team's dead man's switches configured. func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts { t.Helper() var cfg api.NotifyConfig if len(notify) > 0 { cfg = notify[0] } return newTSWith(t, deadman, cfg, testConfig()) } // newTSWith is newDeadmanTS with the server's own configuration supplied, for // tests of behaviour that config switches on, such as single sign-on. func newTSWith(t *testing.T, deadman api.DeadmanConfig, cfg api.NotifyConfig, conf config.Config) *ts { t.Helper() database := newTestDB(t) srv := httptest.NewServer(api.NewRouter(database, cfg, conf)) t.Cleanup(srv.Close) body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"}) resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body)) if err != nil { t.Fatalf("bootstrap: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { t.Fatalf("bootstrap returned %d", resp.StatusCode) } var result map[string]any json.NewDecoder(resp.Body).Decode(&result) key := result["api_key"].(map[string]any)["key"].(string) s := &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman} var integration struct { Key string `json:"key"` } decode(t, s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/integrations", map[string]string{"name": "test"}), &integration) if integration.Key == "" { t.Fatal("no integration key was returned") } s.ingestKey = integration.Key // Dead man's switches belong to a team now, so a test that wants them // configures the default team the way an owner would. if deadman.Timeout > 0 { setTeamDeadman(t, s, deadman) } return s } // setTeamDeadman gives the default team one switch per configured matcher, over // the API, the way an owner would add them. func setTeamDeadman(t *testing.T, s *ts, cfg api.DeadmanConfig) { t.Helper() for _, m := range cfg.Matchers { parts := []string{"alertname=" + m.Name} for k, v := range m.Labels { parts = append(parts, k+"="+v) } sort.Strings(parts[1:]) resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/deadman/switches", map[string]any{ "matcher": strings.Join(parts, ","), "timeout_seconds": int64(cfg.Timeout.Seconds()), "severity": cfg.Severity, }) resp.Body.Close() if resp.StatusCode != http.StatusCreated { t.Fatalf("add a dead man's switch: %d", resp.StatusCode) } } } // exec runs a statement against the test database. func (s *ts) exec(t *testing.T, query string, args ...any) { t.Helper() if _, err := s.db.Exec(query, args...); err != nil { t.Fatalf("exec %q: %v", query, err) } } // alertRow reads the sweeper-relevant columns of one alert straight from the DB. func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) { t.Helper() err := s.db.QueryRow( "SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = $1", fingerprint).Scan(&status, &source, &archivedAt) if err != nil { t.Fatalf("read alert %s: %v", fingerprint, err) } return status, source, archivedAt } // alertTimes reads the timestamp columns that make up the received_at contract. func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) { t.Helper() err := s.db.QueryRow( "SELECT starts_at, received_at FROM alerts WHERE fingerprint = $1", fingerprint).Scan(&startsAt, &receivedAt) if err != nil { t.Fatalf("read alert times %s: %v", fingerprint, err) } return startsAt, receivedAt } // alertEndsAt reads the nullable ends_at column of one alert. func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 { t.Helper() var endsAt *int64 if err := s.db.QueryRow( "SELECT ends_at FROM alerts WHERE fingerprint = $1", fingerprint).Scan(&endsAt); err != nil { t.Fatalf("read ends_at %s: %v", fingerprint, err) } return endsAt } // req sends an authenticated request, optionally with a JSON body. func (s *ts) req(t *testing.T, method, path string, body any) *http.Response { t.Helper() var r io.Reader if body != nil { data, _ := json.Marshal(body) r = bytes.NewReader(data) } req, _ := http.NewRequest(method, s.URL+path, r) req.Header.Set("Authorization", "Bearer "+s.key) if body != nil { req.Header.Set("Content-Type", "application/json") } resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("%s %s: %v", method, path, err) } return resp } func decode(t *testing.T, resp *http.Response, v any) { t.Helper() defer resp.Body.Close() if err := json.NewDecoder(resp.Body).Decode(v); err != nil { t.Fatalf("decode response: %v", err) } } // --------------------------------------------------------------------------- // Auth middleware // --------------------------------------------------------------------------- func TestAuthMiddleware_MissingToken(t *testing.T) { s := newTS(t) req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil) resp, _ := http.DefaultClient.Do(req) if resp.StatusCode != http.StatusUnauthorized { t.Errorf("expected 401, got %d", resp.StatusCode) } } func TestAuthMiddleware_InvalidToken(t *testing.T) { s := newTS(t) req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil) req.Header.Set("Authorization", "Bearer notavalidkey") resp, _ := http.DefaultClient.Do(req) if resp.StatusCode != http.StatusUnauthorized { t.Errorf("expected 401, got %d", resp.StatusCode) } } func TestAuthMiddleware_ValidToken(t *testing.T) { s := newTS(t) resp := s.req(t, http.MethodGet, "/api/users", nil) if resp.StatusCode != http.StatusOK { t.Errorf("expected 200, got %d", resp.StatusCode) } } // --------------------------------------------------------------------------- // Bootstrap idempotency // --------------------------------------------------------------------------- func TestBootstrap_SecondCallForbidden(t *testing.T) { s := newTS(t) // already bootstrapped body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"}) resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body)) if resp.StatusCode != http.StatusForbidden { t.Errorf("expected 403, got %d", resp.StatusCode) } } // --------------------------------------------------------------------------- // Alert upsert by fingerprint // --------------------------------------------------------------------------- // postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting // it exercises the fallback for senders that do not group, which is what most of // these tests want. func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) { t.Helper() payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts} if len(groupKey) > 0 { payload["groupKey"] = groupKey[0] // Alertmanager groups by alertname by default, so the group labels echo // the first alert's name. if len(alerts) > 0 { if labels, ok := alerts[0]["labels"].(map[string]string); ok { payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]} } } } data, _ := json.Marshal(payload) resp, err := http.Post(s.URL+"/api/integrations/"+s.ingestKey+"/alertmanager", "application/json", bytes.NewReader(data)) if err != nil { t.Fatalf("post webhook: %v", err) } resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("webhook returned %d", resp.StatusCode) } } func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) { s := newTS(t) alert := map[string]any{ "status": "firing", "labels": map[string]string{"alertname": "HighCPU"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-upsert", } postWebhook(t, s, []map[string]any{alert}) // Same fingerprint, now resolved. alert["status"] = "resolved" alert["endsAt"] = "2026-05-20T11:00:00Z" postWebhook(t, s, []map[string]any{alert}) resp := s.req(t, http.MethodGet, "/api/alerts", nil) var list []map[string]any decode(t, resp, &list) if len(list) != 1 { t.Fatalf("expected 1 alert (upsert), got %d", len(list)) } if list[0]["status"] != "resolved" { t.Errorf("expected status resolved, got %s", list[0]["status"]) } if list[0]["ends_at"] == nil { t.Error("expected ends_at to be set after resolve") } } func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) { s := newTS(t) for i := range 3 { alert := map[string]any{ "status": "firing", "labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": fmt.Sprintf("fp-%d", i), } postWebhook(t, s, []map[string]any{alert}) } resp := s.req(t, http.MethodGet, "/api/alerts", nil) var list []map[string]any decode(t, resp, &list) if len(list) != 3 { t.Errorf("expected 3 alerts, got %d", len(list)) } } // --------------------------------------------------------------------------- // Schedule conflict // --------------------------------------------------------------------------- func TestSchedule_ConflictOnSameDate(t *testing.T) { s := newTS(t) first := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}) if first.StatusCode != http.StatusCreated { t.Fatalf("first assignment returned %d", first.StatusCode) } first.Body.Close() second := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}) if second.StatusCode != http.StatusConflict { t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode) } second.Body.Close() } func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) { s := newTS(t) // Claim 2026-06-10 first. s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close() // Try to assign two dates in one request where the second conflicts. resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}}) if resp.StatusCode != http.StatusConflict { t.Fatalf("expected 409, got %d", resp.StatusCode) } resp.Body.Close() // 2026-06-09 must NOT have been committed (transaction rolled back). listResp := s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-09&to=2026-06-09", nil) var entries []any decode(t, listResp, &entries) if len(entries) != 0 { t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries)) } } // --------------------------------------------------------------------------- // Schedule reassignment // --------------------------------------------------------------------------- // addUser creates a second person to hand a shift to. The bootstrap user is // admin, id 1. // addUser creates a user and puts them in the default team, because a user who // is in no team can be paged by nobody and take no shift — which is the rule // these tests exercise around, not the one they are testing. func addUser(t *testing.T, s *ts, username string) { t.Helper() resp := s.req(t, http.MethodPost, "/api/users", map[string]any{"username": username, "email": username + "@test.com"}) if resp.StatusCode != http.StatusCreated { resp.Body.Close() t.Fatalf("create user returned %d", resp.StatusCode) } var user struct { ID int64 `json:"id"` } decode(t, resp, &user) member := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members", map[string]any{"user_id": user.ID, "role": "member"}) defer member.Body.Close() if member.StatusCode != http.StatusNoContent { t.Fatalf("add %s to the team returned %d", username, member.StatusCode) } } // scheduleHolder reports who is on call for one date, or "" for nobody. func scheduleHolder(t *testing.T, s *ts, date string) string { t.Helper() var entries []map[string]any decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from="+date+"&to="+date, nil), &entries) if len(entries) == 0 { return "" } return entries[0]["username"].(string) } // Taking a day somebody else holds is possible, but only by asking for it. func TestSchedule_ReplaceTakesAnAssignedDate(t *testing.T) { s := newTS(t) addUser(t, s, "alex") s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close() resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}, "replace": true}) if resp.StatusCode != http.StatusCreated { t.Fatalf("expected replace to succeed, got %d", resp.StatusCode) } resp.Body.Close() if got := scheduleHolder(t, s, "2026-06-01"); got != "alex" { t.Errorf("expected alex to hold the day, got %q", got) } // One row, not two: two entries for a date would mean two people believing // they are on call for it. var entries []map[string]any decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-01&to=2026-06-01", nil), &entries) if len(entries) != 1 { t.Errorf("expected exactly one entry for the date, got %d", len(entries)) } } // A week where only some days are taken is the case that was impossible before: // the free days and the taken ones have to land together. func TestSchedule_ReplaceMixedWeek(t *testing.T) { s := newTS(t) addUser(t, s, "alex") s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-02", "2026-06-04"}}).Body.Close() week := []string{"2026-06-01", "2026-06-02", "2026-06-03", "2026-06-04", "2026-06-05"} resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 2, "dates": week, "replace": true}) if resp.StatusCode != http.StatusCreated { t.Fatalf("expected the mixed week to succeed, got %d", resp.StatusCode) } resp.Body.Close() for _, d := range week { if got := scheduleHolder(t, s, d); got != "alex" { t.Errorf("%s: expected alex, got %q", d, got) } } } // Without replace the guard stands: nobody loses a shift by accident. func TestSchedule_ReplaceDefaultsOff(t *testing.T) { s := newTS(t) addUser(t, s, "alex") s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close() resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}}) if resp.StatusCode != http.StatusConflict { t.Fatalf("expected 409 without replace, got %d", resp.StatusCode) } resp.Body.Close() if got := scheduleHolder(t, s, "2026-06-01"); got != "admin" { t.Errorf("expected the original holder untouched, got %q", got) } } // Replace makes a repeated date idempotent rather than a conflict: the second // pass clears what the first wrote and rewrites it. Worth pinning down, because // the same input without replace is a 409. func TestSchedule_ReplaceCollapsesRepeatedDates(t *testing.T) { s := newTS(t) resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule", map[string]any{"user_id": 1, "dates": []string{"2026-06-01", "2026-06-01"}, "replace": true}) if resp.StatusCode != http.StatusCreated { t.Fatalf("expected a repeated date to be accepted under replace, got %d", resp.StatusCode) } resp.Body.Close() var entries []map[string]any decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-01&to=2026-06-01", nil), &entries) if len(entries) != 1 { t.Errorf("expected one entry for the repeated date, got %d", len(entries)) } } // --------------------------------------------------------------------------- // Stats // --------------------------------------------------------------------------- func TestStats_Totals(t *testing.T) { s := newTS(t) alerts := []map[string]any{ {"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"}, {"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"}, } postWebhook(t, s, alerts) resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil) var stats map[string]any decode(t, resp, &stats) if int(stats["total"].(float64)) != 2 { t.Errorf("expected total=2, got %v", stats["total"]) } if int(stats["firing"].(float64)) != 1 { t.Errorf("expected firing=1, got %v", stats["firing"]) } if int(stats["resolved"].(float64)) != 1 { t.Errorf("expected resolved=1, got %v", stats["resolved"]) } } func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) { s := newTS(t) resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil) var slots []any decode(t, resp, &slots) if len(slots) != 24 { t.Errorf("expected 24 hour slots, got %d", len(slots)) } } // --------------------------------------------------------------------------- // Archive // --------------------------------------------------------------------------- // Alert archiving is sweeper-only housekeeping now — nobody archives an alert by // hand — but the list filter it drives is still part of the API. func TestArchive_AlertListFilter(t *testing.T) { s := newTS(t) postWebhook(t, s, []map[string]any{{ "status": "resolved", "fingerprint": "arch1", "labels": map[string]string{"alertname": "Archivable"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", }}) // 1. Alert appears in the default list. var alerts []map[string]any decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts) if len(alerts) != 1 { t.Fatalf("expected 1 alert in default list, got %d", len(alerts)) } // 2. Let the sweeper archive it: ends_at is already well past archiveAfter. api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.notify) // 3. Default list excludes it. decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts) if len(alerts) != 0 { t.Errorf("expected archived alert to be hidden, got %d results", len(alerts)) } // 4. archived=true shows it. decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts) if len(alerts) != 1 { t.Fatalf("expected 1 archived alert, got %d", len(alerts)) } } // --------------------------------------------------------------------------- // Stale-alert expiry // --------------------------------------------------------------------------- // noArchive is long enough that archiving never interferes with expiry tests. const noArchive = 365 * 24 * time.Hour // zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL. const zeroTime = "0001-01-01T00:00:00Z" // postAlert sends a single-alert webhook. func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) { t.Helper() postWebhook(t, s, []map[string]any{{ "status": status, "labels": map[string]string{"alertname": "Stale"}, "annotations": map[string]string{}, "startsAt": startsAt, "endsAt": endsAt, "generatorURL": "", "fingerprint": fingerprint, }}) } func sweep(t *testing.T, s *ts, staleAfter time.Duration) { t.Helper() api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.notify) } // A firing alert Alertmanager stopped refreshing is resolved via the // received_at heartbeat, even with no ends_at watermark to go on. func TestExpiry_StaleFiringAlert(t *testing.T) { s := newTS(t) postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime) // Age the last-seen timestamp past the staleness window. s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'stale1'", time.Now().Add(-10*time.Hour).Unix()) sweep(t, s, 6*time.Hour) status, source, _ := s.alertRow(t, "stale1") if status != "resolved" { t.Errorf("expected status resolved, got %q", status) } if source == nil || *source != "expiry" { t.Errorf("expected resolution_source=expiry, got %v", source) } } // A fresh webhook whose ends_at watermark has already passed is expired without // waiting out the full staleness window. func TestExpiry_PastEndsAt(t *testing.T) { s := newTS(t) postAlert(t, s, "stale2", "firing", time.Now().Add(-2*time.Hour).Format(time.RFC3339), time.Now().Add(-30*time.Minute).Format(time.RFC3339)) sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger status, source, _ := s.alertRow(t, "stale2") if status != "resolved" { t.Errorf("expected status resolved, got %q", status) } if source == nil || *source != "expiry" { t.Errorf("expected resolution_source=expiry, got %v", source) } } // The regression that matters most: a genuinely firing alert must survive a // sweep untouched. func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) { s := newTS(t) postAlert(t, s, "fresh1", "firing", time.Now().Add(-10*time.Minute).Format(time.RFC3339), time.Now().Add(1*time.Hour).Format(time.RFC3339)) sweep(t, s, 6*time.Hour) status, source, _ := s.alertRow(t, "fresh1") if status != "firing" { t.Errorf("expected fresh alert to stay firing, got %q", status) } if source != nil { t.Errorf("expected no resolution_source, got %q", *source) } } // An ends_at only just past must not trip expiry — that grace absorbs clock skew. func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) { s := newTS(t) postAlert(t, s, "grace1", "firing", time.Now().Add(-time.Hour).Format(time.RFC3339), time.Now().Add(-1*time.Minute).Format(time.RFC3339)) sweep(t, s, 6*time.Hour) if status, _, _ := s.alertRow(t, "grace1"); status != "firing" { t.Errorf("expected alert within grace period to stay firing, got %q", status) } } // --------------------------------------------------------------------------- // Webhook resolution bookkeeping // --------------------------------------------------------------------------- func TestWebhook_ResolvedSetsSource(t *testing.T) { s := newTS(t) start := time.Now().Add(-time.Hour).Format(time.RFC3339) postAlert(t, s, "src1", "firing", start, zeroTime) if _, source, _ := s.alertRow(t, "src1"); source != nil { t.Errorf("expected firing alert to have no resolution_source, got %q", *source) } postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339)) status, source, _ := s.alertRow(t, "src1") if status != "resolved" { t.Errorf("expected status resolved, got %q", status) } if source == nil || *source != "alertmanager" { t.Errorf("expected resolution_source=alertmanager, got %v", source) } } // A re-fire under the same fingerprint must leave the archive and clear the // stale expiry marker, otherwise the alert stays invisible in the default list. func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) { s := newTS(t) postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime) // Expire it, then archive it. s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'refire1'", time.Now().Add(-10*time.Hour).Unix()) sweep(t, s, 6*time.Hour) s.exec(t, "UPDATE alerts SET archived_at = FLOOR(EXTRACT(EPOCH FROM now()))::bigint WHERE fingerprint = 'refire1'") var alerts []map[string]any decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts) if len(alerts) != 0 { t.Fatalf("expected archived alert to be hidden, got %d", len(alerts)) } // Fires again: a new alert instance, so a newer startsAt. postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime) status, source, archivedAt := s.alertRow(t, "refire1") if status != "firing" { t.Errorf("expected status firing after re-fire, got %q", status) } if source != nil { t.Errorf("expected resolution_source cleared on re-fire, got %q", *source) } if archivedAt != nil { t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt) } decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts) if len(alerts) != 1 { t.Errorf("expected re-fired alert back in default list, got %d", len(alerts)) } } // Alertmanager retries failed notifications, so a firing payload for an // already-resolved instance can arrive late. It must not resurrect the alert. func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) { s := newTS(t) start := time.Now().Add(-time.Hour).Format(time.RFC3339) end := time.Now().Format(time.RFC3339) postAlert(t, s, "ooo1", "firing", start, zeroTime) postAlert(t, s, "ooo1", "resolved", start, end) postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance status, source, _ := s.alertRow(t, "ooo1") if status != "resolved" { t.Errorf("expected alert to stay resolved after stale retry, got %q", status) } if source == nil || *source != "alertmanager" { t.Errorf("expected resolution_source=alertmanager, got %v", source) } } // An expiry resolve writes ends_at as an upper bound, not an observed end: an // Alertmanager watermark already on the row is preserved, and a row that never // carried one is stamped at sweep time. Clients are told to read it that way — // see "resolution_source says how much to trust ends_at" in the README. func TestExpiry_EndsAtIsUpperBound(t *testing.T) { s := newTS(t) // No watermark: expires on the received_at heartbeat, so the sweeper has // nothing to go on but its own clock. postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime) s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'ub-none'", time.Now().Add(-10*time.Hour).Unix()) // Stale watermark: expires on the ends_at branch, and that reported time // must survive the resolve rather than be overwritten with sweep time. watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second) postAlert(t, s, "ub-mark", "firing", time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339)) sweep(t, s, 6*time.Hour) if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" { t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source) } stamped := s.alertEndsAt(t, "ub-none") if stamped == nil { t.Fatal("expected expiry to stamp ends_at when no watermark was known") } if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 { t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew) } if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" { t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source) } switch kept := s.alertEndsAt(t, "ub-mark"); { case kept == nil: t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix()) case *kept != watermark.Unix(): t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept) } } // --------------------------------------------------------------------------- // received_at heartbeat contract // // received_at is documented as a public liveness signal, so these lock the // behaviour clients are told they may rely on. See "received_at is a liveness // heartbeat" in the README and the comment on models.Alert.ReceivedAt. // --------------------------------------------------------------------------- // The heartbeat itself: an unchanged firing notification — what Alertmanager // re-sends every repeat_interval — must advance received_at, while leaving // starts_at, which identifies the alert instance, untouched. func TestWebhook_ResendBumpsReceivedAt(t *testing.T) { s := newTS(t) start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) postAlert(t, s, "beat1", "firing", start, zeroTime) startsBefore, _ := s.alertTimes(t, "beat1") // received_at has one-second granularity, so back-date it to make the bump // observable instead of sleeping out a second. aged := time.Now().Add(-2 * time.Hour).Unix() s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat1'", aged) // Identical re-send: same fingerprint, same startsAt, still firing. postAlert(t, s, "beat1", "firing", start, zeroTime) startsAfter, receivedAfter := s.alertTimes(t, "beat1") if receivedAfter <= aged { t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter) } if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 { t.Errorf("expected received_at to track the server clock, off by %ds", skew) } if startsAfter != startsBefore { t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore) } } // received_at tracks accepted payloads, not delivery attempts: a retry // describing an already-resolved instance is discarded, so it must not register // as a heartbeat and revive the alert's apparent liveness. func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) { s := newTS(t) start := time.Now().Add(-time.Hour).Format(time.RFC3339) postAlert(t, s, "beat2", "firing", start, zeroTime) postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339)) aged := time.Now().Add(-2 * time.Hour).Unix() s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat2'", aged) postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged { t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter) } } func TestStats_ByDayReturnsSevenSlots(t *testing.T) { s := newTS(t) resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil) var slots []any decode(t, resp, &slots) if len(slots) != 7 { t.Errorf("expected 7 day slots, got %d", len(slots)) } }