74359c72ab
The rest of #4. Two halves that belong together because they are the same sentence from opposite ends: a team decides which of its alerts are heartbeats, and the UI has to be able to say which team it is talking about. Switches were three environment variables, which made them one setting for the whole install. That was the last piece of the alerting path a team could not control: it could take its own alerts on its own key and still not say which of them were heartbeats, or how long a silence had to last. They are a row per team now, edited by an owner through PUT /api/teams/{teamID}/deadman, and the sweeper runs each team against its own matchers, timeout and severity. The environment variables become the starting point rather than the setting. Every team without a configuration is seeded from them at startup, so an upgrade keeps watching exactly what it was watching, and SeedDeadmanConfigs never overwrites -- a redeploy must not put the environment's value back over an owner's edit. A team created later watches nothing until somebody says otherwise: inheriting an install-wide heartbeat would page a new team about a source it has never heard of, and a switch nobody chose is the kind that gets muted rather than fixed. A matcher string with no alertname in it is refused at the door instead of stored. Storing it would produce a switch that watches nothing silently, which is the exact failure the feature exists to prevent. NewRouter and Sweep lose their DeadmanConfig parameter -- there is no longer one answer to hand them. The type stays, because parsing a matcher string is still parsing a matcher string. The UI side: rows in the queue carry a team badge, the filter row gains a team chip per team, and "on call now" shows one card per team. All three appear only when the viewer is in more than one team -- otherwise they are the same word repeated down a list, which is noise rather than information, and the single-team install reads exactly as it did before teams existed. Verified against a live two-team server as well as in tests: the combined queue labelled by team, the team_id filter, a heartbeat that is a heartbeat in one team and an ordinary alert in another, and a new team's switches starting empty while the upgraded team keeps the environment's. Claude-Session: https://claude.ai/code/session_01RHPj4ggeFdEjKKfm4SHbD7
857 lines
30 KiB
Go
857 lines
30 KiB
Go
package api_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
|
)
|
|
|
|
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
|
|
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
|
|
type ts struct {
|
|
*httptest.Server
|
|
key string
|
|
db *sql.DB
|
|
notify api.NotifyConfig
|
|
deadman api.DeadmanConfig
|
|
}
|
|
|
|
// newTS builds a server over a fresh database. Notifications are off
|
|
// unless a NotifyConfig is passed, so tests that predate them are unaffected.
|
|
// Dead man's switches are off too — see newDeadmanTS.
|
|
func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
|
|
t.Helper()
|
|
var cfg api.NotifyConfig
|
|
if len(notify) > 0 {
|
|
cfg = notify[0]
|
|
}
|
|
return newDeadmanTS(t, api.DeadmanConfig{}, cfg)
|
|
}
|
|
|
|
// newDeadmanTS is newTS with the default team's dead man's switches configured.
|
|
func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts {
|
|
t.Helper()
|
|
var cfg api.NotifyConfig
|
|
if len(notify) > 0 {
|
|
cfg = notify[0]
|
|
}
|
|
|
|
database := newTestDB(t)
|
|
srv := httptest.NewServer(api.NewRouter(database, cfg))
|
|
t.Cleanup(srv.Close)
|
|
|
|
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
|
|
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if err != nil {
|
|
t.Fatalf("bootstrap: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("bootstrap returned %d", resp.StatusCode)
|
|
}
|
|
var result map[string]any
|
|
json.NewDecoder(resp.Body).Decode(&result)
|
|
key := result["api_key"].(map[string]any)["key"].(string)
|
|
|
|
s := &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman}
|
|
|
|
// Dead man's switches belong to a team now, so a test that wants them
|
|
// configures the default team the way an owner would.
|
|
if deadman.Timeout > 0 {
|
|
setTeamDeadman(t, s, deadman)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// setTeamDeadman configures the default team's switches over the API, rendering
|
|
// the matchers back into the string form the endpoint takes.
|
|
func setTeamDeadman(t *testing.T, s *ts, cfg api.DeadmanConfig) {
|
|
t.Helper()
|
|
matchers := make([]string, 0, len(cfg.Matchers))
|
|
for _, m := range cfg.Matchers {
|
|
parts := []string{"alertname=" + m.Name}
|
|
for k, v := range m.Labels {
|
|
parts = append(parts, k+"="+v)
|
|
}
|
|
sort.Strings(parts[1:])
|
|
matchers = append(matchers, strings.Join(parts, ","))
|
|
}
|
|
resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{
|
|
"matchers": strings.Join(matchers, "; "),
|
|
"timeout_seconds": int64(cfg.Timeout.Seconds()),
|
|
"severity": cfg.Severity,
|
|
})
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("configure the team's dead man's switches: %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// exec runs a statement against the test database.
|
|
func (s *ts) exec(t *testing.T, query string, args ...any) {
|
|
t.Helper()
|
|
if _, err := s.db.Exec(query, args...); err != nil {
|
|
t.Fatalf("exec %q: %v", query, err)
|
|
}
|
|
}
|
|
|
|
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
|
|
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = $1",
|
|
fingerprint).Scan(&status, &source, &archivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert %s: %v", fingerprint, err)
|
|
}
|
|
return status, source, archivedAt
|
|
}
|
|
|
|
// alertTimes reads the timestamp columns that make up the received_at contract.
|
|
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = $1",
|
|
fingerprint).Scan(&startsAt, &receivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert times %s: %v", fingerprint, err)
|
|
}
|
|
return startsAt, receivedAt
|
|
}
|
|
|
|
// alertEndsAt reads the nullable ends_at column of one alert.
|
|
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
|
|
t.Helper()
|
|
var endsAt *int64
|
|
if err := s.db.QueryRow(
|
|
"SELECT ends_at FROM alerts WHERE fingerprint = $1", fingerprint).Scan(&endsAt); err != nil {
|
|
t.Fatalf("read ends_at %s: %v", fingerprint, err)
|
|
}
|
|
return endsAt
|
|
}
|
|
|
|
// req sends an authenticated request, optionally with a JSON body.
|
|
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
|
|
t.Helper()
|
|
var r io.Reader
|
|
if body != nil {
|
|
data, _ := json.Marshal(body)
|
|
r = bytes.NewReader(data)
|
|
}
|
|
req, _ := http.NewRequest(method, s.URL+path, r)
|
|
req.Header.Set("Authorization", "Bearer "+s.key)
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
t.Fatalf("%s %s: %v", method, path, err)
|
|
}
|
|
return resp
|
|
}
|
|
|
|
func decode(t *testing.T, resp *http.Response, v any) {
|
|
t.Helper()
|
|
defer resp.Body.Close()
|
|
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Auth middleware
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestAuthMiddleware_MissingToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_InvalidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
req.Header.Set("Authorization", "Bearer notavalidkey")
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_ValidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/users", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Errorf("expected 200, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Bootstrap idempotency
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestBootstrap_SecondCallForbidden(t *testing.T) {
|
|
s := newTS(t) // already bootstrapped
|
|
body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"})
|
|
resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if resp.StatusCode != http.StatusForbidden {
|
|
t.Errorf("expected 403, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Alert upsert by fingerprint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting
|
|
// it exercises the fallback for senders that do not group, which is what most of
|
|
// these tests want.
|
|
func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) {
|
|
t.Helper()
|
|
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
|
|
if len(groupKey) > 0 {
|
|
payload["groupKey"] = groupKey[0]
|
|
// Alertmanager groups by alertname by default, so the group labels echo
|
|
// the first alert's name.
|
|
if len(alerts) > 0 {
|
|
if labels, ok := alerts[0]["labels"].(map[string]string); ok {
|
|
payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]}
|
|
}
|
|
}
|
|
}
|
|
data, _ := json.Marshal(payload)
|
|
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
t.Fatalf("post webhook: %v", err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("webhook returned %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": "HighCPU"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": "fp-upsert",
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
// Same fingerprint, now resolved.
|
|
alert["status"] = "resolved"
|
|
alert["endsAt"] = "2026-05-20T11:00:00Z"
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
|
|
if len(list) != 1 {
|
|
t.Fatalf("expected 1 alert (upsert), got %d", len(list))
|
|
}
|
|
if list[0]["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %s", list[0]["status"])
|
|
}
|
|
if list[0]["ends_at"] == nil {
|
|
t.Error("expected ends_at to be set after resolve")
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
for i := range 3 {
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": fmt.Sprintf("fp-%d", i),
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
}
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
if len(list) != 3 {
|
|
t.Errorf("expected 3 alerts, got %d", len(list))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Schedule conflict
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestSchedule_ConflictOnSameDate(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
first := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if first.StatusCode != http.StatusCreated {
|
|
t.Fatalf("first assignment returned %d", first.StatusCode)
|
|
}
|
|
first.Body.Close()
|
|
|
|
second := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if second.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode)
|
|
}
|
|
second.Body.Close()
|
|
}
|
|
|
|
func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// Claim 2026-06-10 first.
|
|
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close()
|
|
|
|
// Try to assign two dates in one request where the second conflicts.
|
|
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}})
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Fatalf("expected 409, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 2026-06-09 must NOT have been committed (transaction rolled back).
|
|
listResp := s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-09&to=2026-06-09", nil)
|
|
var entries []any
|
|
decode(t, listResp, &entries)
|
|
if len(entries) != 0 {
|
|
t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Schedule reassignment
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// addUser creates a second person to hand a shift to. The bootstrap user is
|
|
// admin, id 1.
|
|
// addUser creates a user and puts them in the default team, because a user who
|
|
// is in no team can be paged by nobody and take no shift — which is the rule
|
|
// these tests exercise around, not the one they are testing.
|
|
func addUser(t *testing.T, s *ts, username string) {
|
|
t.Helper()
|
|
resp := s.req(t, http.MethodPost, "/api/users",
|
|
map[string]any{"username": username, "email": username + "@test.com"})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
resp.Body.Close()
|
|
t.Fatalf("create user returned %d", resp.StatusCode)
|
|
}
|
|
var user struct {
|
|
ID int64 `json:"id"`
|
|
}
|
|
decode(t, resp, &user)
|
|
|
|
member := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members",
|
|
map[string]any{"user_id": user.ID, "role": "member"})
|
|
defer member.Body.Close()
|
|
if member.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("add %s to the team returned %d", username, member.StatusCode)
|
|
}
|
|
}
|
|
|
|
// scheduleHolder reports who is on call for one date, or "" for nobody.
|
|
func scheduleHolder(t *testing.T, s *ts, date string) string {
|
|
t.Helper()
|
|
var entries []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from="+date+"&to="+date, nil), &entries)
|
|
if len(entries) == 0 {
|
|
return ""
|
|
}
|
|
return entries[0]["username"].(string)
|
|
}
|
|
|
|
// Taking a day somebody else holds is possible, but only by asking for it.
|
|
func TestSchedule_ReplaceTakesAnAssignedDate(t *testing.T) {
|
|
s := newTS(t)
|
|
addUser(t, s, "alex")
|
|
|
|
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}, "replace": true})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("expected replace to succeed, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
if got := scheduleHolder(t, s, "2026-06-01"); got != "alex" {
|
|
t.Errorf("expected alex to hold the day, got %q", got)
|
|
}
|
|
|
|
// One row, not two: two entries for a date would mean two people believing
|
|
// they are on call for it.
|
|
var entries []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
|
|
if len(entries) != 1 {
|
|
t.Errorf("expected exactly one entry for the date, got %d", len(entries))
|
|
}
|
|
}
|
|
|
|
// A week where only some days are taken is the case that was impossible before:
|
|
// the free days and the taken ones have to land together.
|
|
func TestSchedule_ReplaceMixedWeek(t *testing.T) {
|
|
s := newTS(t)
|
|
addUser(t, s, "alex")
|
|
|
|
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-02", "2026-06-04"}}).Body.Close()
|
|
|
|
week := []string{"2026-06-01", "2026-06-02", "2026-06-03", "2026-06-04", "2026-06-05"}
|
|
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 2, "dates": week, "replace": true})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("expected the mixed week to succeed, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
for _, d := range week {
|
|
if got := scheduleHolder(t, s, d); got != "alex" {
|
|
t.Errorf("%s: expected alex, got %q", d, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Without replace the guard stands: nobody loses a shift by accident.
|
|
func TestSchedule_ReplaceDefaultsOff(t *testing.T) {
|
|
s := newTS(t)
|
|
addUser(t, s, "alex")
|
|
|
|
s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}})
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Fatalf("expected 409 without replace, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
if got := scheduleHolder(t, s, "2026-06-01"); got != "admin" {
|
|
t.Errorf("expected the original holder untouched, got %q", got)
|
|
}
|
|
}
|
|
|
|
// Replace makes a repeated date idempotent rather than a conflict: the second
|
|
// pass clears what the first wrote and rewrites it. Worth pinning down, because
|
|
// the same input without replace is a 409.
|
|
func TestSchedule_ReplaceCollapsesRepeatedDates(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01", "2026-06-01"}, "replace": true})
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("expected a repeated date to be accepted under replace, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
var entries []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/teams/"+defaultTeam+"/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
|
|
if len(entries) != 1 {
|
|
t.Errorf("expected one entry for the repeated date, got %d", len(entries))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Totals(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alerts := []map[string]any{
|
|
{"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"},
|
|
{"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"},
|
|
}
|
|
postWebhook(t, s, alerts)
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
|
|
if int(stats["total"].(float64)) != 2 {
|
|
t.Errorf("expected total=2, got %v", stats["total"])
|
|
}
|
|
if int(stats["firing"].(float64)) != 1 {
|
|
t.Errorf("expected firing=1, got %v", stats["firing"])
|
|
}
|
|
if int(stats["resolved"].(float64)) != 1 {
|
|
t.Errorf("expected resolved=1, got %v", stats["resolved"])
|
|
}
|
|
}
|
|
|
|
func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 24 {
|
|
t.Errorf("expected 24 hour slots, got %d", len(slots))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Alert archiving is sweeper-only housekeeping now — nobody archives an alert by
|
|
// hand — but the list filter it drives is still part of the API.
|
|
func TestArchive_AlertListFilter(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "resolved", "fingerprint": "arch1",
|
|
"labels": map[string]string{"alertname": "Archivable"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "2026-05-20T11:00:00Z",
|
|
"generatorURL": "",
|
|
}})
|
|
|
|
// 1. Alert appears in the default list.
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
|
}
|
|
|
|
// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
|
|
api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.notify)
|
|
|
|
// 3. Default list excludes it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
|
|
}
|
|
|
|
// 4. archived=true shows it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stale-alert expiry
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// noArchive is long enough that archiving never interferes with expiry tests.
|
|
const noArchive = 365 * 24 * time.Hour
|
|
|
|
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
|
|
const zeroTime = "0001-01-01T00:00:00Z"
|
|
|
|
// postAlert sends a single-alert webhook.
|
|
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
|
|
t.Helper()
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": status,
|
|
"labels": map[string]string{"alertname": "Stale"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}})
|
|
}
|
|
|
|
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
|
|
t.Helper()
|
|
api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.notify)
|
|
}
|
|
|
|
// A firing alert Alertmanager stopped refreshing is resolved via the
|
|
// received_at heartbeat, even with no ends_at watermark to go on.
|
|
func TestExpiry_StaleFiringAlert(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Age the last-seen timestamp past the staleness window.
|
|
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'stale1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "stale1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A fresh webhook whose ends_at watermark has already passed is expired without
|
|
// waiting out the full staleness window.
|
|
func TestExpiry_PastEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale2", "firing",
|
|
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
|
|
|
|
status, source, _ := s.alertRow(t, "stale2")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// The regression that matters most: a genuinely firing alert must survive a
|
|
// sweep untouched.
|
|
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fresh1", "firing",
|
|
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
|
|
time.Now().Add(1*time.Hour).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "fresh1")
|
|
if status != "firing" {
|
|
t.Errorf("expected fresh alert to stay firing, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected no resolution_source, got %q", *source)
|
|
}
|
|
}
|
|
|
|
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
|
|
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "grace1", "firing",
|
|
time.Now().Add(-time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
|
|
t.Errorf("expected alert within grace period to stay firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Webhook resolution bookkeeping
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_ResolvedSetsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "src1", "firing", start, zeroTime)
|
|
|
|
if _, source, _ := s.alertRow(t, "src1"); source != nil {
|
|
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
|
|
}
|
|
|
|
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
status, source, _ := s.alertRow(t, "src1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A re-fire under the same fingerprint must leave the archive and clear the
|
|
// stale expiry marker, otherwise the alert stays invisible in the default list.
|
|
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Expire it, then archive it.
|
|
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'refire1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
s.exec(t, "UPDATE alerts SET archived_at = FLOOR(EXTRACT(EPOCH FROM now()))::bigint WHERE fingerprint = 'refire1'")
|
|
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
|
|
}
|
|
|
|
// Fires again: a new alert instance, so a newer startsAt.
|
|
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
|
|
|
|
status, source, archivedAt := s.alertRow(t, "refire1")
|
|
if status != "firing" {
|
|
t.Errorf("expected status firing after re-fire, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
|
|
}
|
|
if archivedAt != nil {
|
|
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
|
|
}
|
|
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
// Alertmanager retries failed notifications, so a firing payload for an
|
|
// already-resolved instance can arrive late. It must not resurrect the alert.
|
|
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
end := time.Now().Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime)
|
|
postAlert(t, s, "ooo1", "resolved", start, end)
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
|
|
|
|
status, source, _ := s.alertRow(t, "ooo1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
|
|
// Alertmanager watermark already on the row is preserved, and a row that never
|
|
// carried one is stamped at sweep time. Clients are told to read it that way —
|
|
// see "resolution_source says how much to trust ends_at" in the README.
|
|
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// No watermark: expires on the received_at heartbeat, so the sweeper has
|
|
// nothing to go on but its own clock.
|
|
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'ub-none'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
// Stale watermark: expires on the ends_at branch, and that reported time
|
|
// must survive the resolve rather than be overwritten with sweep time.
|
|
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
|
|
postAlert(t, s, "ub-mark", "firing",
|
|
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
|
|
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
|
|
}
|
|
stamped := s.alertEndsAt(t, "ub-none")
|
|
if stamped == nil {
|
|
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
|
|
}
|
|
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
|
|
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
|
|
}
|
|
|
|
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
|
|
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
|
|
}
|
|
switch kept := s.alertEndsAt(t, "ub-mark"); {
|
|
case kept == nil:
|
|
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
|
|
case *kept != watermark.Unix():
|
|
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// received_at heartbeat contract
|
|
//
|
|
// received_at is documented as a public liveness signal, so these lock the
|
|
// behaviour clients are told they may rely on. See "received_at is a liveness
|
|
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// The heartbeat itself: an unchanged firing notification — what Alertmanager
|
|
// re-sends every repeat_interval — must advance received_at, while leaving
|
|
// starts_at, which identifies the alert instance, untouched.
|
|
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
|
|
|
startsBefore, _ := s.alertTimes(t, "beat1")
|
|
|
|
// received_at has one-second granularity, so back-date it to make the bump
|
|
// observable instead of sleeping out a second.
|
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
|
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat1'", aged)
|
|
|
|
// Identical re-send: same fingerprint, same startsAt, still firing.
|
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
|
|
|
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
|
|
if receivedAfter <= aged {
|
|
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
|
|
}
|
|
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
|
|
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
|
|
}
|
|
if startsAfter != startsBefore {
|
|
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
|
|
}
|
|
}
|
|
|
|
// received_at tracks accepted payloads, not delivery attempts: a retry
|
|
// describing an already-resolved instance is discarded, so it must not register
|
|
// as a heartbeat and revive the alert's apparent liveness.
|
|
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "beat2", "firing", start, zeroTime)
|
|
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
|
s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = 'beat2'", aged)
|
|
|
|
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
|
|
|
|
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
|
|
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
|
|
}
|
|
}
|
|
|
|
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 7 {
|
|
t.Errorf("expected 7 day slots, got %d", len(slots))
|
|
}
|
|
}
|