Files
terdut-server/internal/api/api_test.go
T
Niklas Ye dc39e3a5d3
CI / chart (pull_request) Successful in 1s
CI / security (pull_request) Successful in 17s
CI / test (pull_request) Successful in 2m5s
Move the database to Postgres, before teams need the schema
First step of #1, and it goes first for one reason: #4 adds a team_id to
nearly every table, and doing that twice -- once for SQLite, once for
Postgres -- is work nobody gets paid for. The teams migrations now only
have to be written against one database.

The ten SQLite migrations are replaced by a single Postgres baseline
rather than ported one by one. They were incremental in a way that has
no value on a fresh install: 004 adds columns 008 drops again, and 008's
backfill rewrites data a Postgres database never had. The history stays
in git; the schema they add up to is now 001_baseline.sql.

Timestamps stay BIGINT unix seconds and are NOT converted to timestamptz.
Everything in Go already speaks epochs, so converting would have been a
second, larger change riding along inside this one. It is worth doing on
its own. The JSON columns did move to jsonb, because #4 will want to
filter and index on labels.

Most of the port is mechanical -- 170 placeholders from ? to $1 -- but
four things needed more than a search and replace:

  * Dynamically built WHERE clauses cannot keep their numbering straight
    by hand, so they hand out placeholders through sqlArgs instead. A
    filter can now be added or reordered without renumbering anything.

  * SUM(resolved_at IS NULL) was SQLite counting a boolean as 0 or 1.
    Postgres has no sum(boolean), and this was breaking every dead man's
    switch -- silently, since the sweeper only logs. Now COUNT(*) FILTER.

  * unixepoch() became FLOOR(EXTRACT(EPOCH FROM now()))::bigint. The
    FLOOR is load-bearing: a bare cast rounds half up, so a row written
    at .6 of a second claimed a timestamp a second in the future and
    disagreed with the time.Now().Unix() the Go side stamps.

  * The unique-violation check matched SQLite's error text. It matches
    SQLSTATE 23505 now, so a renamed constraint cannot turn a 409 back
    into a 500.

Tests need a real Postgres, because there is no in-memory Postgres the
way there was an in-memory SQLite. Each test gets its own schema on a
shared server -- cheaper than a database each, and still isolated.
TERDUT_TEST_DSN says where it is; `make test-db` starts one locally and
ci.yaml runs one as a service container. An unset DSN fails the suite
rather than skipping it: a run that quietly tests nothing is worse than
one that does not run.

TestMigration_BackfillCarriesAckAndComments is deleted along with the
migrations it replayed. What it protected -- an upgrade not losing
acknowledgements and comments -- now belongs to scripts/sqlite-to-postgres.go,
which is build-tagged so the SQLite driver stays out of the server
binary. Both are meant to be deleted once this install has migrated.

The chart loses the PVC, the data volume and the python backup sidecar,
and requires database.dsnSecret.name: it provisions no database and
cannot guess where the credentials live, so a render without it is meant
to fail. Backups move to where Postgres actually runs. The other half of
that -- the postgresql CR, the k8up pg_dump annotation and the network
policy -- is a change to the wrapper chart in Ryuvia/charts and is not in
here.

Verified rather than assumed: the gate is green with -race against
Postgres 17, govulncheck and gitleaks are clean, and the migration script
was run end to end against a SQLite database built at the old schema and
seeded in every table. Ids survive, so incidents keep their numbers and
every foreign key still points where it did; the identity sequences are
moved past the copied ids, and a webhook after the migration opened
incident 12 rather than colliding at 1.
2026-09-20 10:44:12 +02:00

810 lines
28 KiB
Go

package api_test
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"git.ryuvia.com/niklas/terdut-server/internal/api"
)
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
type ts struct {
*httptest.Server
key string
db *sql.DB
notify api.NotifyConfig
deadman api.DeadmanConfig
}
// newTS builds a server over a fresh database. Notifications are off
// unless a NotifyConfig is passed, so tests that predate them are unaffected.
// Dead man's switches are off too — see newDeadmanTS.
func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
t.Helper()
var cfg api.NotifyConfig
if len(notify) > 0 {
cfg = notify[0]
}
return newDeadmanTS(t, api.DeadmanConfig{}, cfg)
}
// newDeadmanTS is newTS with dead man's switch handling configured.
func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts {
t.Helper()
var cfg api.NotifyConfig
if len(notify) > 0 {
cfg = notify[0]
}
database := newTestDB(t)
srv := httptest.NewServer(api.NewRouter(database, cfg, deadman))
t.Cleanup(srv.Close)
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("bootstrap: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
t.Fatalf("bootstrap returned %d", resp.StatusCode)
}
var result map[string]any
json.NewDecoder(resp.Body).Decode(&result)
key := result["api_key"].(map[string]any)["key"].(string)
return &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman}
}
// exec runs a statement against the test database.
func (s *ts) exec(t *testing.T, query string, args ...any) {
t.Helper()
if _, err := s.db.Exec(query, args...); err != nil {
t.Fatalf("exec %q: %v", query, err)
}
}
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
t.Helper()
err := s.db.QueryRow(
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = $1",
fingerprint).Scan(&status, &source, &archivedAt)
if err != nil {
t.Fatalf("read alert %s: %v", fingerprint, err)
}
return status, source, archivedAt
}
// alertTimes reads the timestamp columns that make up the received_at contract.
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
t.Helper()
err := s.db.QueryRow(
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = $1",
fingerprint).Scan(&startsAt, &receivedAt)
if err != nil {
t.Fatalf("read alert times %s: %v", fingerprint, err)
}
return startsAt, receivedAt
}
// alertEndsAt reads the nullable ends_at column of one alert.
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
t.Helper()
var endsAt *int64
if err := s.db.QueryRow(
"SELECT ends_at FROM alerts WHERE fingerprint = $1", fingerprint).Scan(&endsAt); err != nil {
t.Fatalf("read ends_at %s: %v", fingerprint, err)
}
return endsAt
}
// req sends an authenticated request, optionally with a JSON body.
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
t.Helper()
var r io.Reader
if body != nil {
data, _ := json.Marshal(body)
r = bytes.NewReader(data)
}
req, _ := http.NewRequest(method, s.URL+path, r)
req.Header.Set("Authorization", "Bearer "+s.key)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("%s %s: %v", method, path, err)
}
return resp
}
func decode(t *testing.T, resp *http.Response, v any) {
t.Helper()
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
t.Fatalf("decode response: %v", err)
}
}
// ---------------------------------------------------------------------------
// Auth middleware
// ---------------------------------------------------------------------------
func TestAuthMiddleware_MissingToken(t *testing.T) {
s := newTS(t)
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
resp, _ := http.DefaultClient.Do(req)
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("expected 401, got %d", resp.StatusCode)
}
}
func TestAuthMiddleware_InvalidToken(t *testing.T) {
s := newTS(t)
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
req.Header.Set("Authorization", "Bearer notavalidkey")
resp, _ := http.DefaultClient.Do(req)
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("expected 401, got %d", resp.StatusCode)
}
}
func TestAuthMiddleware_ValidToken(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/users", nil)
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Bootstrap idempotency
// ---------------------------------------------------------------------------
func TestBootstrap_SecondCallForbidden(t *testing.T) {
s := newTS(t) // already bootstrapped
body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"})
resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
if resp.StatusCode != http.StatusForbidden {
t.Errorf("expected 403, got %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Alert upsert by fingerprint
// ---------------------------------------------------------------------------
// postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting
// it exercises the fallback for senders that do not group, which is what most of
// these tests want.
func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) {
t.Helper()
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
if len(groupKey) > 0 {
payload["groupKey"] = groupKey[0]
// Alertmanager groups by alertname by default, so the group labels echo
// the first alert's name.
if len(alerts) > 0 {
if labels, ok := alerts[0]["labels"].(map[string]string); ok {
payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]}
}
}
}
data, _ := json.Marshal(payload)
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
if err != nil {
t.Fatalf("post webhook: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("webhook returned %d", resp.StatusCode)
}
}
func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) {
s := newTS(t)
alert := map[string]any{
"status": "firing",
"labels": map[string]string{"alertname": "HighCPU"},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"fingerprint": "fp-upsert",
}
postWebhook(t, s, []map[string]any{alert})
// Same fingerprint, now resolved.
alert["status"] = "resolved"
alert["endsAt"] = "2026-05-20T11:00:00Z"
postWebhook(t, s, []map[string]any{alert})
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
var list []map[string]any
decode(t, resp, &list)
if len(list) != 1 {
t.Fatalf("expected 1 alert (upsert), got %d", len(list))
}
if list[0]["status"] != "resolved" {
t.Errorf("expected status resolved, got %s", list[0]["status"])
}
if list[0]["ends_at"] == nil {
t.Error("expected ends_at to be set after resolve")
}
}
func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
s := newTS(t)
for i := range 3 {
alert := map[string]any{
"status": "firing",
"labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"fingerprint": fmt.Sprintf("fp-%d", i),
}
postWebhook(t, s, []map[string]any{alert})
}
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
var list []map[string]any
decode(t, resp, &list)
if len(list) != 3 {
t.Errorf("expected 3 alerts, got %d", len(list))
}
}
// ---------------------------------------------------------------------------
// Schedule conflict
// ---------------------------------------------------------------------------
func TestSchedule_ConflictOnSameDate(t *testing.T) {
s := newTS(t)
first := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
if first.StatusCode != http.StatusCreated {
t.Fatalf("first assignment returned %d", first.StatusCode)
}
first.Body.Close()
second := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
if second.StatusCode != http.StatusConflict {
t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode)
}
second.Body.Close()
}
func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
s := newTS(t)
// Claim 2026-06-10 first.
s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close()
// Try to assign two dates in one request where the second conflicts.
resp := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}})
if resp.StatusCode != http.StatusConflict {
t.Fatalf("expected 409, got %d", resp.StatusCode)
}
resp.Body.Close()
// 2026-06-09 must NOT have been committed (transaction rolled back).
listResp := s.req(t, http.MethodGet, "/api/schedule?from=2026-06-09&to=2026-06-09", nil)
var entries []any
decode(t, listResp, &entries)
if len(entries) != 0 {
t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries))
}
}
// ---------------------------------------------------------------------------
// Schedule reassignment
// ---------------------------------------------------------------------------
// addUser creates a second person to hand a shift to. The bootstrap user is
// admin, id 1.
func addUser(t *testing.T, s *ts, username string) {
t.Helper()
resp := s.req(t, http.MethodPost, "/api/users",
map[string]any{"username": username, "email": username + "@test.com"})
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
t.Fatalf("create user returned %d", resp.StatusCode)
}
}
// scheduleHolder reports who is on call for one date, or "" for nobody.
func scheduleHolder(t *testing.T, s *ts, date string) string {
t.Helper()
var entries []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/schedule?from="+date+"&to="+date, nil), &entries)
if len(entries) == 0 {
return ""
}
return entries[0]["username"].(string)
}
// Taking a day somebody else holds is possible, but only by asking for it.
func TestSchedule_ReplaceTakesAnAssignedDate(t *testing.T) {
s := newTS(t)
addUser(t, s, "alex")
s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
resp := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}, "replace": true})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected replace to succeed, got %d", resp.StatusCode)
}
resp.Body.Close()
if got := scheduleHolder(t, s, "2026-06-01"); got != "alex" {
t.Errorf("expected alex to hold the day, got %q", got)
}
// One row, not two: two entries for a date would mean two people believing
// they are on call for it.
var entries []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
if len(entries) != 1 {
t.Errorf("expected exactly one entry for the date, got %d", len(entries))
}
}
// A week where only some days are taken is the case that was impossible before:
// the free days and the taken ones have to land together.
func TestSchedule_ReplaceMixedWeek(t *testing.T) {
s := newTS(t)
addUser(t, s, "alex")
s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-02", "2026-06-04"}}).Body.Close()
week := []string{"2026-06-01", "2026-06-02", "2026-06-03", "2026-06-04", "2026-06-05"}
resp := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 2, "dates": week, "replace": true})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected the mixed week to succeed, got %d", resp.StatusCode)
}
resp.Body.Close()
for _, d := range week {
if got := scheduleHolder(t, s, d); got != "alex" {
t.Errorf("%s: expected alex, got %q", d, got)
}
}
}
// Without replace the guard stands: nobody loses a shift by accident.
func TestSchedule_ReplaceDefaultsOff(t *testing.T) {
s := newTS(t)
addUser(t, s, "alex")
s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
resp := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}})
if resp.StatusCode != http.StatusConflict {
t.Fatalf("expected 409 without replace, got %d", resp.StatusCode)
}
resp.Body.Close()
if got := scheduleHolder(t, s, "2026-06-01"); got != "admin" {
t.Errorf("expected the original holder untouched, got %q", got)
}
}
// Replace makes a repeated date idempotent rather than a conflict: the second
// pass clears what the first wrote and rewrites it. Worth pinning down, because
// the same input without replace is a 409.
func TestSchedule_ReplaceCollapsesRepeatedDates(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodPost, "/api/schedule",
map[string]any{"user_id": 1, "dates": []string{"2026-06-01", "2026-06-01"}, "replace": true})
if resp.StatusCode != http.StatusCreated {
t.Fatalf("expected a repeated date to be accepted under replace, got %d", resp.StatusCode)
}
resp.Body.Close()
var entries []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
if len(entries) != 1 {
t.Errorf("expected one entry for the repeated date, got %d", len(entries))
}
}
// ---------------------------------------------------------------------------
// Stats
// ---------------------------------------------------------------------------
func TestStats_Totals(t *testing.T) {
s := newTS(t)
alerts := []map[string]any{
{"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"},
{"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"},
}
postWebhook(t, s, alerts)
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
var stats map[string]any
decode(t, resp, &stats)
if int(stats["total"].(float64)) != 2 {
t.Errorf("expected total=2, got %v", stats["total"])
}
if int(stats["firing"].(float64)) != 1 {
t.Errorf("expected firing=1, got %v", stats["firing"])
}
if int(stats["resolved"].(float64)) != 1 {
t.Errorf("expected resolved=1, got %v", stats["resolved"])
}
}
func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil)
var slots []any
decode(t, resp, &slots)
if len(slots) != 24 {
t.Errorf("expected 24 hour slots, got %d", len(slots))
}
}
// ---------------------------------------------------------------------------
// Archive
// ---------------------------------------------------------------------------
// Alert archiving is sweeper-only housekeeping now — nobody archives an alert by
// hand — but the list filter it drives is still part of the API.
func TestArchive_AlertListFilter(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{{
"status": "resolved", "fingerprint": "arch1",
"labels": map[string]string{"alertname": "Archivable"},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z",
"endsAt": "2026-05-20T11:00:00Z",
"generatorURL": "",
}})
// 1. Alert appears in the default list.
var alerts []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 {
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
}
// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.deadman, s.notify)
// 3. Default list excludes it.
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 0 {
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
}
// 4. archived=true shows it.
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
if len(alerts) != 1 {
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
}
}
// ---------------------------------------------------------------------------
// Stale-alert expiry
// ---------------------------------------------------------------------------
// noArchive is long enough that archiving never interferes with expiry tests.
const noArchive = 365 * 24 * time.Hour
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
const zeroTime = "0001-01-01T00:00:00Z"
// postAlert sends a single-alert webhook.
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
t.Helper()
postWebhook(t, s, []map[string]any{{
"status": status,
"labels": map[string]string{"alertname": "Stale"},
"annotations": map[string]string{},
"startsAt": startsAt,
"endsAt": endsAt,
"generatorURL": "",
"fingerprint": fingerprint,
}})
}
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
t.Helper()
api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.deadman, s.notify)
}
// A firing alert Alertmanager stopped refreshing is resolved via the
// received_at heartbeat, even with no ends_at watermark to go on.
func TestExpiry_StaleFiringAlert(t *testing.T) {
s := newTS(t)
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
// Age the last-seen timestamp past the staleness window.
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'stale1'",
time.Now().Add(-10*time.Hour).Unix())
sweep(t, s, 6*time.Hour)
status, source, _ := s.alertRow(t, "stale1")
if status != "resolved" {
t.Errorf("expected status resolved, got %q", status)
}
if source == nil || *source != "expiry" {
t.Errorf("expected resolution_source=expiry, got %v", source)
}
}
// A fresh webhook whose ends_at watermark has already passed is expired without
// waiting out the full staleness window.
func TestExpiry_PastEndsAt(t *testing.T) {
s := newTS(t)
postAlert(t, s, "stale2", "firing",
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
status, source, _ := s.alertRow(t, "stale2")
if status != "resolved" {
t.Errorf("expected status resolved, got %q", status)
}
if source == nil || *source != "expiry" {
t.Errorf("expected resolution_source=expiry, got %v", source)
}
}
// The regression that matters most: a genuinely firing alert must survive a
// sweep untouched.
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
s := newTS(t)
postAlert(t, s, "fresh1", "firing",
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
time.Now().Add(1*time.Hour).Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
status, source, _ := s.alertRow(t, "fresh1")
if status != "firing" {
t.Errorf("expected fresh alert to stay firing, got %q", status)
}
if source != nil {
t.Errorf("expected no resolution_source, got %q", *source)
}
}
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
s := newTS(t)
postAlert(t, s, "grace1", "firing",
time.Now().Add(-time.Hour).Format(time.RFC3339),
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
t.Errorf("expected alert within grace period to stay firing, got %q", status)
}
}
// ---------------------------------------------------------------------------
// Webhook resolution bookkeeping
// ---------------------------------------------------------------------------
func TestWebhook_ResolvedSetsSource(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
postAlert(t, s, "src1", "firing", start, zeroTime)
if _, source, _ := s.alertRow(t, "src1"); source != nil {
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
}
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
status, source, _ := s.alertRow(t, "src1")
if status != "resolved" {
t.Errorf("expected status resolved, got %q", status)
}
if source == nil || *source != "alertmanager" {
t.Errorf("expected resolution_source=alertmanager, got %v", source)
}
}
// A re-fire under the same fingerprint must leave the archive and clear the
// stale expiry marker, otherwise the alert stays invisible in the default list.
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
s := newTS(t)
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
// Expire it, then archive it.
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'refire1'",
time.Now().Add(-10*time.Hour).Unix())
sweep(t, s, 6*time.Hour)
s.exec(t, "UPDATE alerts SET archived_at = FLOOR(EXTRACT(EPOCH FROM now()))::bigint WHERE fingerprint = 'refire1'")
var alerts []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 0 {
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
}
// Fires again: a new alert instance, so a newer startsAt.
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
status, source, archivedAt := s.alertRow(t, "refire1")
if status != "firing" {
t.Errorf("expected status firing after re-fire, got %q", status)
}
if source != nil {
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
}
if archivedAt != nil {
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
}
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 {
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
}
}
// Alertmanager retries failed notifications, so a firing payload for an
// already-resolved instance can arrive late. It must not resurrect the alert.
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
end := time.Now().Format(time.RFC3339)
postAlert(t, s, "ooo1", "firing", start, zeroTime)
postAlert(t, s, "ooo1", "resolved", start, end)
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
status, source, _ := s.alertRow(t, "ooo1")
if status != "resolved" {
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
}
if source == nil || *source != "alertmanager" {
t.Errorf("expected resolution_source=alertmanager, got %v", source)
}
}
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
// Alertmanager watermark already on the row is preserved, and a row that never
// carried one is stamped at sweep time. Clients are told to read it that way —
// see "resolution_source says how much to trust ends_at" in the README.
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
s := newTS(t)
// No watermark: expires on the received_at heartbeat, so the sweeper has
// nothing to go on but its own clock.
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'ub-none'",
time.Now().Add(-10*time.Hour).Unix())
// Stale watermark: expires on the ends_at branch, and that reported time
// must survive the resolve rather than be overwritten with sweep time.
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
postAlert(t, s, "ub-mark", "firing",
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
}
stamped := s.alertEndsAt(t, "ub-none")
if stamped == nil {
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
}
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
}
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
}
switch kept := s.alertEndsAt(t, "ub-mark"); {
case kept == nil:
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
case *kept != watermark.Unix():
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
}
}
// ---------------------------------------------------------------------------
// received_at heartbeat contract
//
// received_at is documented as a public liveness signal, so these lock the
// behaviour clients are told they may rely on. See "received_at is a liveness
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
// ---------------------------------------------------------------------------
// The heartbeat itself: an unchanged firing notification — what Alertmanager
// re-sends every repeat_interval — must advance received_at, while leaving
// starts_at, which identifies the alert instance, untouched.
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
postAlert(t, s, "beat1", "firing", start, zeroTime)
startsBefore, _ := s.alertTimes(t, "beat1")
// received_at has one-second granularity, so back-date it to make the bump
// observable instead of sleeping out a second.
aged := time.Now().Add(-2 * time.Hour).Unix()
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat1'", aged)
// Identical re-send: same fingerprint, same startsAt, still firing.
postAlert(t, s, "beat1", "firing", start, zeroTime)
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
if receivedAfter <= aged {
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
}
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
}
if startsAfter != startsBefore {
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
}
}
// received_at tracks accepted payloads, not delivery attempts: a retry
// describing an already-resolved instance is discarded, so it must not register
// as a heartbeat and revive the alert's apparent liveness.
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
postAlert(t, s, "beat2", "firing", start, zeroTime)
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
aged := time.Now().Add(-2 * time.Hour).Unix()
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat2'", aged)
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
}
}
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
var slots []any
decode(t, resp, &slots)
if len(slots) != 7 {
t.Errorf("expected 7 day slots, got %d", len(slots))
}
}