bc285799d1
An incident opened, got assigned to whoever held today's schedule entry,
and then sat there silently until somebody thought to look. The schedule
and the incident model were both built; nothing reached the person
holding the pager.
Notifications go out through ntfy, over plain HTTP with no new
dependencies. Delivery is an outbox rather than an inline call: the pool
is limited to a single connection, so a POST made while holding the
webhook's transaction would stall every other request behind it. The
webhook inserts a row and a notifier goroutine sends it within a tick,
retrying with exponential backoff.
Only opening an incident has to resolve a topic from scratch. Reminders
and all-clears reuse whatever that first notification chose, which keeps
configuration out of resolveIfSettled and gives the right rule for free:
you only hear that something resolved if you were told it started.
Each push carries an Acknowledge button, because the useful thing to do
at 3am is stop the pager without unlocking anything. It POSTs to an
unauthenticated /api/notify/ack/{token} — a notification body lives on
the ntfy server and in the device cache, so a real API key must never
appear in one. The token is minted per delivery, scoped to one incident
and one action, and expires in a day.
Reminders repeat until the incident stops being untouched. The stop
conditions are the states that already mean somebody has it: acknowledged,
snoozed, resolved, archived. Snooze is the mute button, so there is no
separate reminder cap.
Notifications sent to the fallback topic carry no Acknowledge button. The
topic is shared, and a button on it would let any subscriber acknowledge
as somebody else.
686 lines
24 KiB
Go
686 lines
24 KiB
Go
package api_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/yeniklas/terdut-server/internal/api"
|
|
"github.com/yeniklas/terdut-server/internal/db"
|
|
)
|
|
|
|
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
|
|
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
|
|
type ts struct {
|
|
*httptest.Server
|
|
key string
|
|
db *sql.DB
|
|
notify api.NotifyConfig
|
|
}
|
|
|
|
// newTS builds a server over a fresh in-memory database. Notifications are off
|
|
// unless a NotifyConfig is passed, so tests that predate them are unaffected.
|
|
func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
|
|
t.Helper()
|
|
var cfg api.NotifyConfig
|
|
if len(notify) > 0 {
|
|
cfg = notify[0]
|
|
}
|
|
|
|
database, err := db.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
if err := db.Migrate(database); err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
srv := httptest.NewServer(api.NewRouter(database, cfg))
|
|
t.Cleanup(func() { srv.Close(); database.Close() })
|
|
|
|
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
|
|
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if err != nil {
|
|
t.Fatalf("bootstrap: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("bootstrap returned %d", resp.StatusCode)
|
|
}
|
|
var result map[string]any
|
|
json.NewDecoder(resp.Body).Decode(&result)
|
|
key := result["api_key"].(map[string]any)["key"].(string)
|
|
|
|
return &ts{Server: srv, key: key, db: database, notify: cfg}
|
|
}
|
|
|
|
// exec runs a statement against the test database.
|
|
func (s *ts) exec(t *testing.T, query string, args ...any) {
|
|
t.Helper()
|
|
if _, err := s.db.Exec(query, args...); err != nil {
|
|
t.Fatalf("exec %q: %v", query, err)
|
|
}
|
|
}
|
|
|
|
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
|
|
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = ?",
|
|
fingerprint).Scan(&status, &source, &archivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert %s: %v", fingerprint, err)
|
|
}
|
|
return status, source, archivedAt
|
|
}
|
|
|
|
// alertTimes reads the timestamp columns that make up the received_at contract.
|
|
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = ?",
|
|
fingerprint).Scan(&startsAt, &receivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert times %s: %v", fingerprint, err)
|
|
}
|
|
return startsAt, receivedAt
|
|
}
|
|
|
|
// alertEndsAt reads the nullable ends_at column of one alert.
|
|
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
|
|
t.Helper()
|
|
var endsAt *int64
|
|
if err := s.db.QueryRow(
|
|
"SELECT ends_at FROM alerts WHERE fingerprint = ?", fingerprint).Scan(&endsAt); err != nil {
|
|
t.Fatalf("read ends_at %s: %v", fingerprint, err)
|
|
}
|
|
return endsAt
|
|
}
|
|
|
|
// req sends an authenticated request, optionally with a JSON body.
|
|
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
|
|
t.Helper()
|
|
var r io.Reader
|
|
if body != nil {
|
|
data, _ := json.Marshal(body)
|
|
r = bytes.NewReader(data)
|
|
}
|
|
req, _ := http.NewRequest(method, s.URL+path, r)
|
|
req.Header.Set("Authorization", "Bearer "+s.key)
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
t.Fatalf("%s %s: %v", method, path, err)
|
|
}
|
|
return resp
|
|
}
|
|
|
|
func decode(t *testing.T, resp *http.Response, v any) {
|
|
t.Helper()
|
|
defer resp.Body.Close()
|
|
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Auth middleware
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestAuthMiddleware_MissingToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_InvalidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
req.Header.Set("Authorization", "Bearer notavalidkey")
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_ValidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/users", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Errorf("expected 200, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Bootstrap idempotency
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestBootstrap_SecondCallForbidden(t *testing.T) {
|
|
s := newTS(t) // already bootstrapped
|
|
body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"})
|
|
resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if resp.StatusCode != http.StatusForbidden {
|
|
t.Errorf("expected 403, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Alert upsert by fingerprint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting
|
|
// it exercises the fallback for senders that do not group, which is what most of
|
|
// these tests want.
|
|
func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) {
|
|
t.Helper()
|
|
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
|
|
if len(groupKey) > 0 {
|
|
payload["groupKey"] = groupKey[0]
|
|
// Alertmanager groups by alertname by default, so the group labels echo
|
|
// the first alert's name.
|
|
if len(alerts) > 0 {
|
|
if labels, ok := alerts[0]["labels"].(map[string]string); ok {
|
|
payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]}
|
|
}
|
|
}
|
|
}
|
|
data, _ := json.Marshal(payload)
|
|
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
t.Fatalf("post webhook: %v", err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("webhook returned %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": "HighCPU"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": "fp-upsert",
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
// Same fingerprint, now resolved.
|
|
alert["status"] = "resolved"
|
|
alert["endsAt"] = "2026-05-20T11:00:00Z"
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
|
|
if len(list) != 1 {
|
|
t.Fatalf("expected 1 alert (upsert), got %d", len(list))
|
|
}
|
|
if list[0]["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %s", list[0]["status"])
|
|
}
|
|
if list[0]["ends_at"] == nil {
|
|
t.Error("expected ends_at to be set after resolve")
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
for i := range 3 {
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": fmt.Sprintf("fp-%d", i),
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
}
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
if len(list) != 3 {
|
|
t.Errorf("expected 3 alerts, got %d", len(list))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Schedule conflict
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestSchedule_ConflictOnSameDate(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
first := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if first.StatusCode != http.StatusCreated {
|
|
t.Fatalf("first assignment returned %d", first.StatusCode)
|
|
}
|
|
first.Body.Close()
|
|
|
|
second := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if second.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode)
|
|
}
|
|
second.Body.Close()
|
|
}
|
|
|
|
func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// Claim 2026-06-10 first.
|
|
s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close()
|
|
|
|
// Try to assign two dates in one request where the second conflicts.
|
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}})
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Fatalf("expected 409, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 2026-06-09 must NOT have been committed (transaction rolled back).
|
|
listResp := s.req(t, http.MethodGet, "/api/schedule?from=2026-06-09&to=2026-06-09", nil)
|
|
var entries []any
|
|
decode(t, listResp, &entries)
|
|
if len(entries) != 0 {
|
|
t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Totals(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alerts := []map[string]any{
|
|
{"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"},
|
|
{"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"},
|
|
}
|
|
postWebhook(t, s, alerts)
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
|
|
if int(stats["total"].(float64)) != 2 {
|
|
t.Errorf("expected total=2, got %v", stats["total"])
|
|
}
|
|
if int(stats["firing"].(float64)) != 1 {
|
|
t.Errorf("expected firing=1, got %v", stats["firing"])
|
|
}
|
|
if int(stats["resolved"].(float64)) != 1 {
|
|
t.Errorf("expected resolved=1, got %v", stats["resolved"])
|
|
}
|
|
}
|
|
|
|
func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 24 {
|
|
t.Errorf("expected 24 hour slots, got %d", len(slots))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Alert archiving is sweeper-only housekeeping now — nobody archives an alert by
|
|
// hand — but the list filter it drives is still part of the API.
|
|
func TestArchive_AlertListFilter(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "resolved", "fingerprint": "arch1",
|
|
"labels": map[string]string{"alertname": "Archivable"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "2026-05-20T11:00:00Z",
|
|
"generatorURL": "",
|
|
}})
|
|
|
|
// 1. Alert appears in the default list.
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
|
}
|
|
|
|
// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
|
|
api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour)
|
|
|
|
// 3. Default list excludes it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
|
|
}
|
|
|
|
// 4. archived=true shows it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stale-alert expiry
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// noArchive is long enough that archiving never interferes with expiry tests.
|
|
const noArchive = 365 * 24 * time.Hour
|
|
|
|
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
|
|
const zeroTime = "0001-01-01T00:00:00Z"
|
|
|
|
// postAlert sends a single-alert webhook.
|
|
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
|
|
t.Helper()
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": status,
|
|
"labels": map[string]string{"alertname": "Stale"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}})
|
|
}
|
|
|
|
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
|
|
t.Helper()
|
|
api.Sweep(context.Background(), s.db, noArchive, staleAfter)
|
|
}
|
|
|
|
// A firing alert Alertmanager stopped refreshing is resolved via the
|
|
// received_at heartbeat, even with no ends_at watermark to go on.
|
|
func TestExpiry_StaleFiringAlert(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Age the last-seen timestamp past the staleness window.
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'stale1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "stale1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A fresh webhook whose ends_at watermark has already passed is expired without
|
|
// waiting out the full staleness window.
|
|
func TestExpiry_PastEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale2", "firing",
|
|
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
|
|
|
|
status, source, _ := s.alertRow(t, "stale2")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// The regression that matters most: a genuinely firing alert must survive a
|
|
// sweep untouched.
|
|
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fresh1", "firing",
|
|
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
|
|
time.Now().Add(1*time.Hour).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "fresh1")
|
|
if status != "firing" {
|
|
t.Errorf("expected fresh alert to stay firing, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected no resolution_source, got %q", *source)
|
|
}
|
|
}
|
|
|
|
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
|
|
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "grace1", "firing",
|
|
time.Now().Add(-time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
|
|
t.Errorf("expected alert within grace period to stay firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Webhook resolution bookkeeping
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_ResolvedSetsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "src1", "firing", start, zeroTime)
|
|
|
|
if _, source, _ := s.alertRow(t, "src1"); source != nil {
|
|
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
|
|
}
|
|
|
|
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
status, source, _ := s.alertRow(t, "src1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A re-fire under the same fingerprint must leave the archive and clear the
|
|
// stale expiry marker, otherwise the alert stays invisible in the default list.
|
|
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Expire it, then archive it.
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'refire1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
s.exec(t, "UPDATE alerts SET archived_at = unixepoch() WHERE fingerprint = 'refire1'")
|
|
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
|
|
}
|
|
|
|
// Fires again: a new alert instance, so a newer startsAt.
|
|
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
|
|
|
|
status, source, archivedAt := s.alertRow(t, "refire1")
|
|
if status != "firing" {
|
|
t.Errorf("expected status firing after re-fire, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
|
|
}
|
|
if archivedAt != nil {
|
|
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
|
|
}
|
|
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
// Alertmanager retries failed notifications, so a firing payload for an
|
|
// already-resolved instance can arrive late. It must not resurrect the alert.
|
|
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
end := time.Now().Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime)
|
|
postAlert(t, s, "ooo1", "resolved", start, end)
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
|
|
|
|
status, source, _ := s.alertRow(t, "ooo1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
|
|
// Alertmanager watermark already on the row is preserved, and a row that never
|
|
// carried one is stamped at sweep time. Clients are told to read it that way —
|
|
// see "resolution_source says how much to trust ends_at" in the README.
|
|
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// No watermark: expires on the received_at heartbeat, so the sweeper has
|
|
// nothing to go on but its own clock.
|
|
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'ub-none'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
// Stale watermark: expires on the ends_at branch, and that reported time
|
|
// must survive the resolve rather than be overwritten with sweep time.
|
|
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
|
|
postAlert(t, s, "ub-mark", "firing",
|
|
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
|
|
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
|
|
}
|
|
stamped := s.alertEndsAt(t, "ub-none")
|
|
if stamped == nil {
|
|
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
|
|
}
|
|
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
|
|
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
|
|
}
|
|
|
|
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
|
|
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
|
|
}
|
|
switch kept := s.alertEndsAt(t, "ub-mark"); {
|
|
case kept == nil:
|
|
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
|
|
case *kept != watermark.Unix():
|
|
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// received_at heartbeat contract
|
|
//
|
|
// received_at is documented as a public liveness signal, so these lock the
|
|
// behaviour clients are told they may rely on. See "received_at is a liveness
|
|
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// The heartbeat itself: an unchanged firing notification — what Alertmanager
|
|
// re-sends every repeat_interval — must advance received_at, while leaving
|
|
// starts_at, which identifies the alert instance, untouched.
|
|
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
|
|
|
startsBefore, _ := s.alertTimes(t, "beat1")
|
|
|
|
// received_at has one-second granularity, so back-date it to make the bump
|
|
// observable instead of sleeping out a second.
|
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat1'", aged)
|
|
|
|
// Identical re-send: same fingerprint, same startsAt, still firing.
|
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
|
|
|
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
|
|
if receivedAfter <= aged {
|
|
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
|
|
}
|
|
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
|
|
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
|
|
}
|
|
if startsAfter != startsBefore {
|
|
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
|
|
}
|
|
}
|
|
|
|
// received_at tracks accepted payloads, not delivery attempts: a retry
|
|
// describing an already-resolved instance is discarded, so it must not register
|
|
// as a heartbeat and revive the alert's apparent liveness.
|
|
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "beat2", "firing", start, zeroTime)
|
|
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat2'", aged)
|
|
|
|
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
|
|
|
|
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
|
|
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
|
|
}
|
|
}
|
|
|
|
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 7 {
|
|
t.Errorf("expected 7 day slots, got %d", len(slots))
|
|
}
|
|
}
|