Files
terdut-server/internal/api/api_test.go
T
Niklas Ye a4fbd60441
CI / chart (pull_request) Successful in 1s
CI / security (pull_request) Successful in 13s
CI / test (pull_request) Successful in 1m49s
Scope everything to a team, and route alerts by integration key
The core of #4, and what #1 is for: terdut stops being one shared space.
A team owns its incidents, alerts, schedule and integrations; a user sees
exactly the teams they are in. Everything that existed moves into one
Default team and every existing user becomes an owner of it, so the
upgrade is a no-op for the people using it.

Ingestion is the load-bearing half. An alert arrives on a team's
integration key, and the key is both the credential and the routing: it
says that the sender may post, and which team the alerts belong to. That
also closes the unauthenticated webhook -- the old path stays for one
release, deprecated and routed to the oldest team, so an upgrade does not
stop delivering while somebody edits the Alertmanager config.

Scoping is enforced in as few places as possible, because the failure
mode is silent. serveAs loads the caller's memberships once; list queries
carry `team_id = ANY(...)`; and every incident route goes through
incidentIDParam, which now parses the id AND checks the team in the same
call, so a new handler cannot remember the first half and forget the
second. Anything in another team is 404, never 403: whether an incident
exists is that team's business.

Two bugs this found, both of which would have been silent:

  * upsertAlerts decided "is this a new occurrence" by looking up the
    fingerprint alone. Across teams that made team B's first alert look
    like a re-send of team A's, so it opened no incident at all. The
    lookups are keyed on (team_id, fingerprint) now, as the index is.

  * Every uniqueness rule was written for one tenant. Two teams watching
    two clusters legitimately see the same fingerprint, the same
    groupKey, and want somebody on call on the same day; all three
    constraints move to include team_id.

Roles inside a team are separate from the system administrator flag: an
owner configures the team, a member works its incidents, and an admin is
NOT implicitly in every team -- administration is about accounts, not
about reading other people's incidents. An admin can still repair a team
whose owner has left, which is why requireTeamOwner lets them through.

A shift can only be given to somebody in the team. Paging a person who
cannot open the incident is worse than paging nobody.

The UI is updated only as far as keeping it working: it loads the
viewer's teams with the session and uses the first one, since nobody has
a second yet. "On call now" shows every team the viewer is in, named only
when there is more than one, so the common case reads exactly as before.
The team switcher, badges and per-team settings pages are the next step.

Breaking for API clients: the schedule endpoints moved under the team,
and /api/schedule/current returns an array rather than an object or a
404. terdut-tui will need a version for that.

Per-team dead-man configuration is deliberately not here. A heartbeat's
incident already opens in the team whose key received it, which is the
part that matters for isolation; moving the matchers out of env into
per-team rows is a change to how deadman.go is configured rather than to
who sees what.

Claude-Session: https://claude.ai/code/session_01RHPj4ggeFdEjKKfm4SHbD7
2026-09-20 13:36:24 +02:00

824 lines
29 KiB
Go

package api_test
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"git.ryuvia.com/niklas/terdut-server/internal/api"
)
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
type ts struct {
*httptest.Server
key string
db *sql.DB
notify api.NotifyConfig
deadman api.DeadmanConfig
}
// newTS builds a server over a fresh database. Notifications are off
// unless a NotifyConfig is passed, so tests that predate them are unaffected.
// Dead man's switches are off too — see newDeadmanTS.
func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
t.Helper()
var cfg api.NotifyConfig
if len(notify) > 0 {
cfg = notify[0]
}
return newDeadmanTS(t, api.DeadmanConfig{}, cfg)
}
// newDeadmanTS is newTS with dead man's switch handling configured.
func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts {
t.Helper()
var cfg api.NotifyConfig
if len(notify) > 0 {
cfg = notify[0]
}
database := newTestDB(t)
srv := httptest.NewServer(api.NewRouter(database, cfg, deadman))
t.Cleanup(srv.Close)
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("bootstrap: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
t.Fatalf("bootstrap returned %d", resp.StatusCode)
}
var result map[string]any
json.NewDecoder(resp.Body).Decode(&result)
key := result["api_key"].(map[string]any)["key"].(string)
return &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman}
}
// exec runs a statement against the test database.
func (s *ts) exec(t *testing.T, query string, args ...any) {
t.Helper()
if _, err := s.db.Exec(query, args...); err != nil {
t.Fatalf("exec %q: %v", query, err)
}
}
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
t.Helper()
err := s.db.QueryRow(
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = $1",
fingerprint).Scan(&status, &source, &archivedAt)
if err != nil {
t.Fatalf("read alert %s: %v", fingerprint, err)
}
return status, source, archivedAt
}
// alertTimes reads the timestamp columns that make up the received_at contract.
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
t.Helper()
err := s.db.QueryRow(
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = $1",
fingerprint).Scan(&startsAt, &receivedAt)
if err != nil {
t.Fatalf("read alert times %s: %v", fingerprint, err)
}
return startsAt, receivedAt
}
// alertEndsAt reads the nullable ends_at column of one alert.
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
t.Helper()
var endsAt *int64
if err := s.db.QueryRow(
"SELECT ends_at FROM alerts WHERE fingerprint = $1", fingerprint).Scan(&endsAt); err != nil {
t.Fatalf("read ends_at %s: %v", fingerprint, err)
}
return endsAt
}
// req sends an authenticated request, optionally with a JSON body.
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
t.Helper()
var r io.Reader
if body != nil {
data, _ := json.Marshal(body)
r = bytes.NewReader(data)
}
req, _ := http.NewRequest(method, s.URL+path, r)
req.Header.Set("Authorization", "Bearer "+s.key)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s %s: %v", method, path, err)
}
return resp
}
func decode(t *testing.T, resp *http.Response, v any) {
t.Helper()
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
t.Fatalf("decode response: %v", err)
}
}
// ---------------------------------------------------------------------------
// Auth middleware
// ---------------------------------------------------------------------------
func TestAuthMiddleware_MissingToken(t *testing.T) {
s := newTS(t)
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
resp, _ := http.DefaultClient.Do(req)
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("expected 401, got %d", resp.StatusCode)
}
}
func TestAuthMiddleware_InvalidToken(t *testing.T) {
s := newTS(t)
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
req.Header.Set("Authorization", "Bearer notavalidkey")
resp, _ := http.DefaultClient.Do(req)
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("expected 401, got %d", resp.StatusCode)
}
}
func TestAuthMiddleware_ValidToken(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/users", nil)
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Bootstrap idempotency
// ---------------------------------------------------------------------------
func TestBootstrap_SecondCallForbidden(t *testing.T) {
s := newTS(t) // already bootstrapped
body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"})
resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
if resp.StatusCode != http.StatusForbidden {
t.Errorf("expected 403, got %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Alert upsert by fingerprint
// ---------------------------------------------------------------------------
// postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting
// it exercises the fallback for senders that do not group, which is what most of
// these tests want.
func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) {
t.Helper()
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
if len(groupKey) > 0 {
payload["groupKey"] = groupKey[0]
// Alertmanager groups by alertname by default, so the group labels echo
// the first alert's name.
if len(alerts) > 0 {
if labels, ok := alerts[0]["labels"].(map[string]string); ok {
payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]}
}
}
}
data, _ := json.Marshal(payload)
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
if err != nil {
t.Fatalf("post webhook: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("webhook returned %d", resp.StatusCode)
}
}
func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) {
s := newTS(t)
alert := map[string]any{
"status": "firing",
"labels": map[string]string{"alertname": "HighCPU"},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"fingerprint": "fp-upsert",
}
postWebhook(t, s, []map[string]any{alert})
// Same fingerprint, now resolved.
alert["status"] = "resolved"
alert["endsAt"] = "2026-05-20T11:00:00Z"
postWebhook(t, s, []map[string]any{alert})
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
var list []map[string]any
decode(t, resp, &list)
if len(list) != 1 {
t.Fatalf("expected 1 alert (upsert), got %d", len(list))
}
if list[0]["status"] != "resolved" {
t.Errorf("expected status resolved, got %s", list[0]["status"])
}
if list[0]["ends_at"] == nil {
t.Error("expected ends_at to be set after resolve")
}
}
func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
s := newTS(t)
for i := range 3 {
alert := map[string]any{
"status": "firing",
"labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"fingerprint": fmt.Sprintf("fp-%d", i),
}
postWebhook(t, s, []map[string]any{alert})
}
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
var list []map[string]any
decode(t, resp, &list)
if len(list) != 3 {
t.Errorf("expected 3 alerts, got %d", len(list))
}
}
// ---------------------------------------------------------------------------
// Schedule conflict
// ---------------------------------------------------------------------------
func TestSchedule_ConflictOnSameDate(t *testing.T) {
s := newTS(t)
first := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
if first.StatusCode != http.StatusCreated {
t.Fatalf("first assignment returned %d", first.StatusCode)
}
first.Body.Close()
second := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
if second.StatusCode != http.StatusConflict {
t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode)
}
second.Body.Close()
}
func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
s := newTS(t)
// Claim 2026-06-10 first.
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close()
// Try to assign two dates in one request where the second conflicts.
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}})
if resp.StatusCode != http.StatusConflict {
t.Fatalf("expected 409, got %d", resp.StatusCode)
}
resp.Body.Close()
// 2026-06-09 must NOT have been committed (transaction rolled back).
listResp := s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-09&to=2026-06-09", nil)
var entries []any
decode(t, listResp, &entries)
if len(entries) != 0 {
t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries))
}
}
// ---------------------------------------------------------------------------
// Schedule reassignment
// ---------------------------------------------------------------------------
// addUser creates a second person to hand a shift to. The bootstrap user is
// admin, id 1.
// addUser creates a user and puts them in the default team, because a user who
// is in no team can be paged by nobody and take no shift — which is the rule
// these tests exercise around, not the one they are testing.
func addUser(t *testing.T, s *ts, username string) {
t.Helper()
resp := s.req(t, http.MethodPost, "/api/users",
map[string]any{"username": username, "email": username + "@test.com"})
if resp.StatusCode != http.StatusCreated {
resp.Body.Close()
t.Fatalf("create user returned %d", resp.StatusCode)
}
var user struct {
ID int64 `json:"id"`
}
decode(t, resp, &user)
member := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members",
map[string]any{"user_id": user.ID, "role": "member"})
defer member.Body.Close()
if member.StatusCode != http.StatusNoContent {
t.Fatalf("add %s to the team returned %d", username, member.StatusCode)
}
}
// scheduleHolder reports who is on call for one date, or "" for nobody.
func scheduleHolder(t *testing.T, s *ts, date string) string {
t.Helper()
var entries []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from="+date+"&to="+date, nil), &entries)
if len(entries) == 0 {
return ""
}
return entries[0]["username"].(string)
}
// Taking a day somebody else holds is possible, but only by asking for it.
func TestSchedule_ReplaceTakesAnAssignedDate(t *testing.T) {
s := newTS(t)
addUser(t, s, "alex")
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}, "replace": true})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected replace to succeed, got %d", resp.StatusCode)
}
resp.Body.Close()
if got := scheduleHolder(t, s, "2026-06-01"); got != "alex" {
t.Errorf("expected alex to hold the day, got %q", got)
}
// One row, not two: two entries for a date would mean two people believing
// they are on call for it.
var entries []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
if len(entries) != 1 {
t.Errorf("expected exactly one entry for the date, got %d", len(entries))
}
}
// A week where only some days are taken is the case that was impossible before:
// the free days and the taken ones have to land together.
func TestSchedule_ReplaceMixedWeek(t *testing.T) {
s := newTS(t)
addUser(t, s, "alex")
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-02", "2026-06-04"}}).Body.Close()
week := []string{"2026-06-01", "2026-06-02", "2026-06-03", "2026-06-04", "2026-06-05"}
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 2, "dates": week, "replace": true})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected the mixed week to succeed, got %d", resp.StatusCode)
}
resp.Body.Close()
for _, d := range week {
if got := scheduleHolder(t, s, d); got != "alex" {
t.Errorf("%s: expected alex, got %q", d, got)
}
}
}
// Without replace the guard stands: nobody loses a shift by accident.
func TestSchedule_ReplaceDefaultsOff(t *testing.T) {
s := newTS(t)
addUser(t, s, "alex")
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}})
if resp.StatusCode != http.StatusConflict {
t.Fatalf("expected 409 without replace, got %d", resp.StatusCode)
}
resp.Body.Close()
if got := scheduleHolder(t, s, "2026-06-01"); got != "admin" {
t.Errorf("expected the original holder untouched, got %q", got)
}
}
// Replace makes a repeated date idempotent rather than a conflict: the second
// pass clears what the first wrote and rewrites it. Worth pinning down, because
// the same input without replace is a 409.
func TestSchedule_ReplaceCollapsesRepeatedDates(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01", "2026-06-01"}, "replace": true})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected a repeated date to be accepted under replace, got %d", resp.StatusCode)
}
resp.Body.Close()
var entries []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
if len(entries) != 1 {
t.Errorf("expected one entry for the repeated date, got %d", len(entries))
}
}
// ---------------------------------------------------------------------------
// Stats
// ---------------------------------------------------------------------------
func TestStats_Totals(t *testing.T) {
s := newTS(t)
alerts := []map[string]any{
{"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"},
{"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"},
}
postWebhook(t, s, alerts)
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
var stats map[string]any
decode(t, resp, &stats)
if int(stats["total"].(float64)) != 2 {
t.Errorf("expected total=2, got %v", stats["total"])
}
if int(stats["firing"].(float64)) != 1 {
t.Errorf("expected firing=1, got %v", stats["firing"])
}
if int(stats["resolved"].(float64)) != 1 {
t.Errorf("expected resolved=1, got %v", stats["resolved"])
}
}
func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil)
var slots []any
decode(t, resp, &slots)
if len(slots) != 24 {
t.Errorf("expected 24 hour slots, got %d", len(slots))
}
}
// ---------------------------------------------------------------------------
// Archive
// ---------------------------------------------------------------------------
// Alert archiving is sweeper-only housekeeping now — nobody archives an alert by
// hand — but the list filter it drives is still part of the API.
func TestArchive_AlertListFilter(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{{
"status": "resolved", "fingerprint": "arch1",
"labels": map[string]string{"alertname": "Archivable"},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z",
"endsAt": "2026-05-20T11:00:00Z",
"generatorURL": "",
}})
// 1. Alert appears in the default list.
var alerts []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 {
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
}
// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.deadman, s.notify)
// 3. Default list excludes it.
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 0 {
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
}
// 4. archived=true shows it.
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
if len(alerts) != 1 {
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
}
}
// ---------------------------------------------------------------------------
// Stale-alert expiry
// ---------------------------------------------------------------------------
// noArchive is long enough that archiving never interferes with expiry tests.
const noArchive = 365 * 24 * time.Hour
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
const zeroTime = "0001-01-01T00:00:00Z"
// postAlert sends a single-alert webhook.
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
t.Helper()
postWebhook(t, s, []map[string]any{{
"status": status,
"labels": map[string]string{"alertname": "Stale"},
"annotations": map[string]string{},
"startsAt": startsAt,
"endsAt": endsAt,
"generatorURL": "",
"fingerprint": fingerprint,
}})
}
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
t.Helper()
api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.deadman, s.notify)
}
// A firing alert Alertmanager stopped refreshing is resolved via the
// received_at heartbeat, even with no ends_at watermark to go on.
func TestExpiry_StaleFiringAlert(t *testing.T) {
s := newTS(t)
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
// Age the last-seen timestamp past the staleness window.
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'stale1'",
time.Now().Add(-10*time.Hour).Unix())
sweep(t, s, 6*time.Hour)
status, source, _ := s.alertRow(t, "stale1")
if status != "resolved" {
t.Errorf("expected status resolved, got %q", status)
}
if source == nil || *source != "expiry" {
t.Errorf("expected resolution_source=expiry, got %v", source)
}
}
// A fresh webhook whose ends_at watermark has already passed is expired without
// waiting out the full staleness window.
func TestExpiry_PastEndsAt(t *testing.T) {
s := newTS(t)
postAlert(t, s, "stale2", "firing",
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
status, source, _ := s.alertRow(t, "stale2")
if status != "resolved" {
t.Errorf("expected status resolved, got %q", status)
}
if source == nil || *source != "expiry" {
t.Errorf("expected resolution_source=expiry, got %v", source)
}
}
// The regression that matters most: a genuinely firing alert must survive a
// sweep untouched.
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
s := newTS(t)
postAlert(t, s, "fresh1", "firing",
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
time.Now().Add(1*time.Hour).Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
status, source, _ := s.alertRow(t, "fresh1")
if status != "firing" {
t.Errorf("expected fresh alert to stay firing, got %q", status)
}
if source != nil {
t.Errorf("expected no resolution_source, got %q", *source)
}
}
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
s := newTS(t)
postAlert(t, s, "grace1", "firing",
time.Now().Add(-time.Hour).Format(time.RFC3339),
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
t.Errorf("expected alert within grace period to stay firing, got %q", status)
}
}
// ---------------------------------------------------------------------------
// Webhook resolution bookkeeping
// ---------------------------------------------------------------------------
func TestWebhook_ResolvedSetsSource(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
postAlert(t, s, "src1", "firing", start, zeroTime)
if _, source, _ := s.alertRow(t, "src1"); source != nil {
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
}
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
status, source, _ := s.alertRow(t, "src1")
if status != "resolved" {
t.Errorf("expected status resolved, got %q", status)
}
if source == nil || *source != "alertmanager" {
t.Errorf("expected resolution_source=alertmanager, got %v", source)
}
}
// A re-fire under the same fingerprint must leave the archive and clear the
// stale expiry marker, otherwise the alert stays invisible in the default list.
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
s := newTS(t)
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
// Expire it, then archive it.
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'refire1'",
time.Now().Add(-10*time.Hour).Unix())
sweep(t, s, 6*time.Hour)
s.exec(t, "UPDATE alerts SET archived_at = FLOOR(EXTRACT(EPOCH FROM now()))::bigint WHERE fingerprint = 'refire1'")
var alerts []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 0 {
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
}
// Fires again: a new alert instance, so a newer startsAt.
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
status, source, archivedAt := s.alertRow(t, "refire1")
if status != "firing" {
t.Errorf("expected status firing after re-fire, got %q", status)
}
if source != nil {
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
}
if archivedAt != nil {
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
}
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 {
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
}
}
// Alertmanager retries failed notifications, so a firing payload for an
// already-resolved instance can arrive late. It must not resurrect the alert.
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
end := time.Now().Format(time.RFC3339)
postAlert(t, s, "ooo1", "firing", start, zeroTime)
postAlert(t, s, "ooo1", "resolved", start, end)
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
status, source, _ := s.alertRow(t, "ooo1")
if status != "resolved" {
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
}
if source == nil || *source != "alertmanager" {
t.Errorf("expected resolution_source=alertmanager, got %v", source)
}
}
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
// Alertmanager watermark already on the row is preserved, and a row that never
// carried one is stamped at sweep time. Clients are told to read it that way —
// see "resolution_source says how much to trust ends_at" in the README.
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
s := newTS(t)
// No watermark: expires on the received_at heartbeat, so the sweeper has
// nothing to go on but its own clock.
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'ub-none'",
time.Now().Add(-10*time.Hour).Unix())
// Stale watermark: expires on the ends_at branch, and that reported time
// must survive the resolve rather than be overwritten with sweep time.
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
postAlert(t, s, "ub-mark", "firing",
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
}
stamped := s.alertEndsAt(t, "ub-none")
if stamped == nil {
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
}
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
}
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
}
switch kept := s.alertEndsAt(t, "ub-mark"); {
case kept == nil:
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
case *kept != watermark.Unix():
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
}
}
// ---------------------------------------------------------------------------
// received_at heartbeat contract
//
// received_at is documented as a public liveness signal, so these lock the
// behaviour clients are told they may rely on. See "received_at is a liveness
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
// ---------------------------------------------------------------------------
// The heartbeat itself: an unchanged firing notification — what Alertmanager
// re-sends every repeat_interval — must advance received_at, while leaving
// starts_at, which identifies the alert instance, untouched.
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
postAlert(t, s, "beat1", "firing", start, zeroTime)
startsBefore, _ := s.alertTimes(t, "beat1")
// received_at has one-second granularity, so back-date it to make the bump
// observable instead of sleeping out a second.
aged := time.Now().Add(-2 * time.Hour).Unix()
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat1'", aged)
// Identical re-send: same fingerprint, same startsAt, still firing.
postAlert(t, s, "beat1", "firing", start, zeroTime)
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
if receivedAfter <= aged {
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
}
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
}
if startsAfter != startsBefore {
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
}
}
// received_at tracks accepted payloads, not delivery attempts: a retry
// describing an already-resolved instance is discarded, so it must not register
// as a heartbeat and revive the alert's apparent liveness.
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
postAlert(t, s, "beat2", "firing", start, zeroTime)
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
aged := time.Now().Add(-2 * time.Hour).Unix()
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat2'", aged)
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
}
}
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
var slots []any
decode(t, resp, &slots)
if len(slots) != 7 {
t.Errorf("expected 7 day slots, got %d", len(slots))
}
}