a602ff3efc
The API reference listed endpoints but never the alert object's fields, so
two of them were load-bearing for clients while being described nowhere.
received_at appeared only in passing, as a stats filter; resolution_source
only inside the stale-expiry prose.
Both carry meaning a client cannot derive on its own. starts_at comes from
Prometheus and never changes for an alert instance, so received_at is the
only signal that a firing alert is still being refreshed — it advances on
every accepted webhook, including the unchanged notifications Alertmanager
re-sends every repeat_interval. resolution_source then says how much to
trust ends_at: under 'alertmanager' it is an end time somebody reported,
but under 'expiry' nothing ever reported one, so it is either a stale
watermark or the sweep timestamp, and only an upper bound.
README gains an alert object field table plus a contract section for each,
including the nullability rules and the advice to tolerate unrecognised
resolution_source values. The field comments in models.Alert now say these
are public API rather than ingest details, and the upsert carries a note at
the received_at line, which is where a regression would be introduced.
Three tests lock the newly documented behaviour, none of which was covered
before — the whole suite passed with the received_at bump deleted from the
upsert, because the expiry tests only ever set that column via SQL:
- a re-send advances received_at and leaves starts_at alone
- a discarded out-of-order retry does not count as a heartbeat
- an expiry resolve preserves a reported ends_at watermark and stamps
sweep time only when none was known
762 lines
26 KiB
Go
762 lines
26 KiB
Go
package api_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/yeniklas/terdut-server/internal/api"
|
|
"github.com/yeniklas/terdut-server/internal/db"
|
|
)
|
|
|
|
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
|
|
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
|
|
type ts struct {
|
|
*httptest.Server
|
|
key string
|
|
db *sql.DB
|
|
}
|
|
|
|
func newTS(t *testing.T) *ts {
|
|
t.Helper()
|
|
database, err := db.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
if err := db.Migrate(database); err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
srv := httptest.NewServer(api.NewRouter(database))
|
|
t.Cleanup(func() { srv.Close(); database.Close() })
|
|
|
|
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
|
|
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if err != nil {
|
|
t.Fatalf("bootstrap: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("bootstrap returned %d", resp.StatusCode)
|
|
}
|
|
var result map[string]any
|
|
json.NewDecoder(resp.Body).Decode(&result)
|
|
key := result["api_key"].(map[string]any)["key"].(string)
|
|
|
|
return &ts{Server: srv, key: key, db: database}
|
|
}
|
|
|
|
// exec runs a statement against the test database.
|
|
func (s *ts) exec(t *testing.T, query string, args ...any) {
|
|
t.Helper()
|
|
if _, err := s.db.Exec(query, args...); err != nil {
|
|
t.Fatalf("exec %q: %v", query, err)
|
|
}
|
|
}
|
|
|
|
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
|
|
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = ?",
|
|
fingerprint).Scan(&status, &source, &archivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert %s: %v", fingerprint, err)
|
|
}
|
|
return status, source, archivedAt
|
|
}
|
|
|
|
// alertTimes reads the timestamp columns that make up the received_at contract.
|
|
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = ?",
|
|
fingerprint).Scan(&startsAt, &receivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert times %s: %v", fingerprint, err)
|
|
}
|
|
return startsAt, receivedAt
|
|
}
|
|
|
|
// alertEndsAt reads the nullable ends_at column of one alert.
|
|
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
|
|
t.Helper()
|
|
var endsAt *int64
|
|
if err := s.db.QueryRow(
|
|
"SELECT ends_at FROM alerts WHERE fingerprint = ?", fingerprint).Scan(&endsAt); err != nil {
|
|
t.Fatalf("read ends_at %s: %v", fingerprint, err)
|
|
}
|
|
return endsAt
|
|
}
|
|
|
|
// req sends an authenticated request, optionally with a JSON body.
|
|
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
|
|
t.Helper()
|
|
var r io.Reader
|
|
if body != nil {
|
|
data, _ := json.Marshal(body)
|
|
r = bytes.NewReader(data)
|
|
}
|
|
req, _ := http.NewRequest(method, s.URL+path, r)
|
|
req.Header.Set("Authorization", "Bearer "+s.key)
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
t.Fatalf("%s %s: %v", method, path, err)
|
|
}
|
|
return resp
|
|
}
|
|
|
|
func decode(t *testing.T, resp *http.Response, v any) {
|
|
t.Helper()
|
|
defer resp.Body.Close()
|
|
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Auth middleware
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestAuthMiddleware_MissingToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_InvalidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
req.Header.Set("Authorization", "Bearer notavalidkey")
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_ValidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/users", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Errorf("expected 200, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Bootstrap idempotency
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestBootstrap_SecondCallForbidden(t *testing.T) {
|
|
s := newTS(t) // already bootstrapped
|
|
body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"})
|
|
resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if resp.StatusCode != http.StatusForbidden {
|
|
t.Errorf("expected 403, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Alert upsert by fingerprint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func postWebhook(t *testing.T, s *ts, alerts []map[string]any) {
|
|
t.Helper()
|
|
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
|
|
data, _ := json.Marshal(payload)
|
|
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
t.Fatalf("post webhook: %v", err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("webhook returned %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": "HighCPU"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": "fp-upsert",
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
// Same fingerprint, now resolved.
|
|
alert["status"] = "resolved"
|
|
alert["endsAt"] = "2026-05-20T11:00:00Z"
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
|
|
if len(list) != 1 {
|
|
t.Fatalf("expected 1 alert (upsert), got %d", len(list))
|
|
}
|
|
if list[0]["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %s", list[0]["status"])
|
|
}
|
|
if list[0]["ends_at"] == nil {
|
|
t.Error("expected ends_at to be set after resolve")
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
for i := range 3 {
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": fmt.Sprintf("fp-%d", i),
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
}
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
if len(list) != 3 {
|
|
t.Errorf("expected 3 alerts, got %d", len(list))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Alert acknowledge
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestAcknowledge(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "firing", "labels": map[string]string{"alertname": "X"},
|
|
"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-ack",
|
|
}})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/alerts/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
|
}
|
|
var alert map[string]any
|
|
decode(t, resp, &alert)
|
|
if alert["acknowledged_by"] == nil {
|
|
t.Error("expected acknowledged_by to be set")
|
|
}
|
|
|
|
// Clear it.
|
|
resp = s.req(t, http.MethodDelete, "/api/alerts/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("unacknowledge returned %d", resp.StatusCode)
|
|
}
|
|
|
|
resp = s.req(t, http.MethodGet, "/api/alerts/1", nil)
|
|
var alert2 map[string]any
|
|
decode(t, resp, &alert2)
|
|
if alert2["acknowledged_by"] != nil {
|
|
t.Error("expected acknowledged_by to be cleared")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Comments — own-only deletion
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestComment_DeleteOwnOnly(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "firing", "labels": map[string]string{"alertname": "Y"},
|
|
"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-comment",
|
|
}})
|
|
|
|
// Create a second user and their own key.
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"})
|
|
keyResp := s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
|
map[string]string{"name": "alice-key"})
|
|
var keyData map[string]any
|
|
decode(t, keyResp, &keyData)
|
|
aliceKey := keyData["key"].(string)
|
|
|
|
// Admin posts a comment.
|
|
s.req(t, http.MethodPost, "/api/alerts/1/comments",
|
|
map[string]string{"content": "admin note"})
|
|
|
|
// Alice tries to delete admin's comment (should 404).
|
|
req, _ := http.NewRequest(http.MethodDelete, s.URL+"/api/alerts/1/comments/1", nil)
|
|
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 when deleting another user's comment, got %d", resp.StatusCode)
|
|
}
|
|
|
|
// Admin deletes own comment (should 204).
|
|
resp = s.req(t, http.MethodDelete, "/api/alerts/1/comments/1", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("expected 204 when deleting own comment, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Schedule conflict
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestSchedule_ConflictOnSameDate(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
first := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if first.StatusCode != http.StatusCreated {
|
|
t.Fatalf("first assignment returned %d", first.StatusCode)
|
|
}
|
|
first.Body.Close()
|
|
|
|
second := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if second.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode)
|
|
}
|
|
second.Body.Close()
|
|
}
|
|
|
|
func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// Claim 2026-06-10 first.
|
|
s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close()
|
|
|
|
// Try to assign two dates in one request where the second conflicts.
|
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}})
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Fatalf("expected 409, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 2026-06-09 must NOT have been committed (transaction rolled back).
|
|
listResp := s.req(t, http.MethodGet, "/api/schedule?from=2026-06-09&to=2026-06-09", nil)
|
|
var entries []any
|
|
decode(t, listResp, &entries)
|
|
if len(entries) != 0 {
|
|
t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Totals(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alerts := []map[string]any{
|
|
{"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"},
|
|
{"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"},
|
|
}
|
|
postWebhook(t, s, alerts)
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
|
|
if int(stats["total"].(float64)) != 2 {
|
|
t.Errorf("expected total=2, got %v", stats["total"])
|
|
}
|
|
if int(stats["firing"].(float64)) != 1 {
|
|
t.Errorf("expected firing=1, got %v", stats["firing"])
|
|
}
|
|
if int(stats["resolved"].(float64)) != 1 {
|
|
t.Errorf("expected resolved=1, got %v", stats["resolved"])
|
|
}
|
|
}
|
|
|
|
func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 24 {
|
|
t.Errorf("expected 24 hour slots, got %d", len(slots))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestArchive_RoundTrip(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "resolved", "fingerprint": "arch1",
|
|
"labels": map[string]string{"alertname": "Archivable"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z",
|
|
"generatorURL": "",
|
|
}})
|
|
|
|
// 1. Alert appears in default list (not archived).
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
|
}
|
|
id := int(alerts[0]["id"].(float64))
|
|
|
|
// 2. Archive it.
|
|
resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("archive: expected 200, got %d", resp.StatusCode)
|
|
}
|
|
var archived map[string]any
|
|
decode(t, resp, &archived)
|
|
if archived["archived_at"] == nil {
|
|
t.Error("expected archived_at to be set in response")
|
|
}
|
|
|
|
// 3. Default list excludes it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
|
|
}
|
|
|
|
// 4. archived=true shows it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
|
}
|
|
|
|
// 5. Un-archive.
|
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unarchive: expected 204, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 6. Back in default list.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Errorf("expected unarchived alert to reappear, got %d results", len(alerts))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stale-alert expiry
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// noArchive is long enough that archiving never interferes with expiry tests.
|
|
const noArchive = 365 * 24 * time.Hour
|
|
|
|
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
|
|
const zeroTime = "0001-01-01T00:00:00Z"
|
|
|
|
// postAlert sends a single-alert webhook.
|
|
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
|
|
t.Helper()
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": status,
|
|
"labels": map[string]string{"alertname": "Stale"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}})
|
|
}
|
|
|
|
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
|
|
t.Helper()
|
|
api.Sweep(context.Background(), s.db, noArchive, staleAfter)
|
|
}
|
|
|
|
// A firing alert Alertmanager stopped refreshing is resolved via the
|
|
// received_at heartbeat, even with no ends_at watermark to go on.
|
|
func TestExpiry_StaleFiringAlert(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Age the last-seen timestamp past the staleness window.
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'stale1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "stale1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A fresh webhook whose ends_at watermark has already passed is expired without
|
|
// waiting out the full staleness window.
|
|
func TestExpiry_PastEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale2", "firing",
|
|
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
|
|
|
|
status, source, _ := s.alertRow(t, "stale2")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// The regression that matters most: a genuinely firing alert must survive a
|
|
// sweep untouched.
|
|
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fresh1", "firing",
|
|
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
|
|
time.Now().Add(1*time.Hour).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "fresh1")
|
|
if status != "firing" {
|
|
t.Errorf("expected fresh alert to stay firing, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected no resolution_source, got %q", *source)
|
|
}
|
|
}
|
|
|
|
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
|
|
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "grace1", "firing",
|
|
time.Now().Add(-time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
|
|
t.Errorf("expected alert within grace period to stay firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Webhook resolution bookkeeping
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_ResolvedSetsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "src1", "firing", start, zeroTime)
|
|
|
|
if _, source, _ := s.alertRow(t, "src1"); source != nil {
|
|
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
|
|
}
|
|
|
|
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
status, source, _ := s.alertRow(t, "src1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A re-fire under the same fingerprint must leave the archive and clear the
|
|
// stale expiry marker, otherwise the alert stays invisible in the default list.
|
|
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Expire it, then archive it.
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'refire1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
s.exec(t, "UPDATE alerts SET archived_at = unixepoch() WHERE fingerprint = 'refire1'")
|
|
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
|
|
}
|
|
|
|
// Fires again: a new alert instance, so a newer startsAt.
|
|
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
|
|
|
|
status, source, archivedAt := s.alertRow(t, "refire1")
|
|
if status != "firing" {
|
|
t.Errorf("expected status firing after re-fire, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
|
|
}
|
|
if archivedAt != nil {
|
|
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
|
|
}
|
|
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
// Alertmanager retries failed notifications, so a firing payload for an
|
|
// already-resolved instance can arrive late. It must not resurrect the alert.
|
|
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
end := time.Now().Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime)
|
|
postAlert(t, s, "ooo1", "resolved", start, end)
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
|
|
|
|
status, source, _ := s.alertRow(t, "ooo1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
|
|
// Alertmanager watermark already on the row is preserved, and a row that never
|
|
// carried one is stamped at sweep time. Clients are told to read it that way —
|
|
// see "resolution_source says how much to trust ends_at" in the README.
|
|
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// No watermark: expires on the received_at heartbeat, so the sweeper has
|
|
// nothing to go on but its own clock.
|
|
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'ub-none'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
// Stale watermark: expires on the ends_at branch, and that reported time
|
|
// must survive the resolve rather than be overwritten with sweep time.
|
|
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
|
|
postAlert(t, s, "ub-mark", "firing",
|
|
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
|
|
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
|
|
}
|
|
stamped := s.alertEndsAt(t, "ub-none")
|
|
if stamped == nil {
|
|
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
|
|
}
|
|
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
|
|
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
|
|
}
|
|
|
|
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
|
|
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
|
|
}
|
|
switch kept := s.alertEndsAt(t, "ub-mark"); {
|
|
case kept == nil:
|
|
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
|
|
case *kept != watermark.Unix():
|
|
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// received_at heartbeat contract
|
|
//
|
|
// received_at is documented as a public liveness signal, so these lock the
|
|
// behaviour clients are told they may rely on. See "received_at is a liveness
|
|
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// The heartbeat itself: an unchanged firing notification — what Alertmanager
|
|
// re-sends every repeat_interval — must advance received_at, while leaving
|
|
// starts_at, which identifies the alert instance, untouched.
|
|
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
|
|
|
startsBefore, _ := s.alertTimes(t, "beat1")
|
|
|
|
// received_at has one-second granularity, so back-date it to make the bump
|
|
// observable instead of sleeping out a second.
|
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat1'", aged)
|
|
|
|
// Identical re-send: same fingerprint, same startsAt, still firing.
|
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
|
|
|
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
|
|
if receivedAfter <= aged {
|
|
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
|
|
}
|
|
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
|
|
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
|
|
}
|
|
if startsAfter != startsBefore {
|
|
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
|
|
}
|
|
}
|
|
|
|
// received_at tracks accepted payloads, not delivery attempts: a retry
|
|
// describing an already-resolved instance is discarded, so it must not register
|
|
// as a heartbeat and revive the alert's apparent liveness.
|
|
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "beat2", "firing", start, zeroTime)
|
|
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat2'", aged)
|
|
|
|
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
|
|
|
|
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
|
|
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
|
|
}
|
|
}
|
|
|
|
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 7 {
|
|
t.Errorf("expected 7 day slots, got %d", len(slots))
|
|
}
|
|
}
|