42e846f876
Release / build (amd64, darwin) (push) Failing after 12s
Release / build (arm64, darwin) (push) Failing after 11s
Release / build (arm64, linux) (push) Failing after 11s
Release / release (push) Has been skipped
Release / docker (push) Failing after 19s
Release / build (amd64, linux) (push) Failing after 12s
Release / chart (push) Failing after 9s
A resolved webhook was the only path out of the firing state, so a
notification that was dropped, silenced, or lost to a restart pinned an
alert as firing forever — Prometheus showed it resolved while
terdut-server kept listing it. The archiver only ever touched resolved
alerts, and both the list and stats queries compared status with plain
equality, so a stale row was indistinguishable from a live one.
A sweeper pass now resolves firing alerts on either of two signals: the
ends_at watermark Alertmanager sets on outgoing firing notifications has
passed (plus a grace period for clock skew), or no webhook has refreshed
the alert within TERDUT_STALE_AFTER (default 6h, above Alertmanager's 4h
repeat_interval). Such alerts get resolution_source = 'expiry',
distinguishing them from a real 'alertmanager' resolve.
Two related webhook bugs fixed alongside:
- The upsert had no ordering guard, so a retried firing notification
arriving after the resolved one resurrected the alert. Payloads for
an older alert instance are now discarded: a stale retry carries the
same startsAt, a genuine re-fire a newer one.
- archived_at was never cleared on re-fire, leaving a re-fired alert
archived and invisible in the default list.
Stats now exclude archived alerts to match the default list view; this
lowers historical firing/resolved totals.
The chart exposes both sweeper durations via sweeper.staleAfter and
sweeper.archiveAfter.
638 lines
21 KiB
Go
638 lines
21 KiB
Go
package api_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/yeniklas/terdut-server/internal/api"
|
|
"github.com/yeniklas/terdut-server/internal/db"
|
|
)
|
|
|
|
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
|
|
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
|
|
type ts struct {
|
|
*httptest.Server
|
|
key string
|
|
db *sql.DB
|
|
}
|
|
|
|
func newTS(t *testing.T) *ts {
|
|
t.Helper()
|
|
database, err := db.Open(":memory:")
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
if err := db.Migrate(database); err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
srv := httptest.NewServer(api.NewRouter(database))
|
|
t.Cleanup(func() { srv.Close(); database.Close() })
|
|
|
|
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
|
|
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if err != nil {
|
|
t.Fatalf("bootstrap: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusCreated {
|
|
t.Fatalf("bootstrap returned %d", resp.StatusCode)
|
|
}
|
|
var result map[string]any
|
|
json.NewDecoder(resp.Body).Decode(&result)
|
|
key := result["api_key"].(map[string]any)["key"].(string)
|
|
|
|
return &ts{Server: srv, key: key, db: database}
|
|
}
|
|
|
|
// exec runs a statement against the test database.
|
|
func (s *ts) exec(t *testing.T, query string, args ...any) {
|
|
t.Helper()
|
|
if _, err := s.db.Exec(query, args...); err != nil {
|
|
t.Fatalf("exec %q: %v", query, err)
|
|
}
|
|
}
|
|
|
|
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
|
|
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
|
|
t.Helper()
|
|
err := s.db.QueryRow(
|
|
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = ?",
|
|
fingerprint).Scan(&status, &source, &archivedAt)
|
|
if err != nil {
|
|
t.Fatalf("read alert %s: %v", fingerprint, err)
|
|
}
|
|
return status, source, archivedAt
|
|
}
|
|
|
|
// req sends an authenticated request, optionally with a JSON body.
|
|
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
|
|
t.Helper()
|
|
var r io.Reader
|
|
if body != nil {
|
|
data, _ := json.Marshal(body)
|
|
r = bytes.NewReader(data)
|
|
}
|
|
req, _ := http.NewRequest(method, s.URL+path, r)
|
|
req.Header.Set("Authorization", "Bearer "+s.key)
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
t.Fatalf("%s %s: %v", method, path, err)
|
|
}
|
|
return resp
|
|
}
|
|
|
|
func decode(t *testing.T, resp *http.Response, v any) {
|
|
t.Helper()
|
|
defer resp.Body.Close()
|
|
if err := json.NewDecoder(resp.Body).Decode(v); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Auth middleware
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestAuthMiddleware_MissingToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_InvalidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
req, _ := http.NewRequest(http.MethodGet, s.URL+"/api/users", nil)
|
|
req.Header.Set("Authorization", "Bearer notavalidkey")
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
if resp.StatusCode != http.StatusUnauthorized {
|
|
t.Errorf("expected 401, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAuthMiddleware_ValidToken(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/users", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Errorf("expected 200, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Bootstrap idempotency
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestBootstrap_SecondCallForbidden(t *testing.T) {
|
|
s := newTS(t) // already bootstrapped
|
|
body, _ := json.Marshal(map[string]string{"username": "x", "email": "x@x.com"})
|
|
resp, _ := http.Post(s.URL+"/api/bootstrap", "application/json", bytes.NewReader(body))
|
|
if resp.StatusCode != http.StatusForbidden {
|
|
t.Errorf("expected 403, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Alert upsert by fingerprint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func postWebhook(t *testing.T, s *ts, alerts []map[string]any) {
|
|
t.Helper()
|
|
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
|
|
data, _ := json.Marshal(payload)
|
|
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
t.Fatalf("post webhook: %v", err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("webhook returned %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_SameFingerprintUpdates(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": "HighCPU"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": "fp-upsert",
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
// Same fingerprint, now resolved.
|
|
alert["status"] = "resolved"
|
|
alert["endsAt"] = "2026-05-20T11:00:00Z"
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
|
|
if len(list) != 1 {
|
|
t.Fatalf("expected 1 alert (upsert), got %d", len(list))
|
|
}
|
|
if list[0]["status"] != "resolved" {
|
|
t.Errorf("expected status resolved, got %s", list[0]["status"])
|
|
}
|
|
if list[0]["ends_at"] == nil {
|
|
t.Error("expected ends_at to be set after resolve")
|
|
}
|
|
}
|
|
|
|
func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
for i := range 3 {
|
|
alert := map[string]any{
|
|
"status": "firing",
|
|
"labels": map[string]string{"alertname": fmt.Sprintf("Alert%d", i)},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": "",
|
|
"fingerprint": fmt.Sprintf("fp-%d", i),
|
|
}
|
|
postWebhook(t, s, []map[string]any{alert})
|
|
}
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/alerts", nil)
|
|
var list []map[string]any
|
|
decode(t, resp, &list)
|
|
if len(list) != 3 {
|
|
t.Errorf("expected 3 alerts, got %d", len(list))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Alert acknowledge
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestAcknowledge(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "firing", "labels": map[string]string{"alertname": "X"},
|
|
"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-ack",
|
|
}})
|
|
|
|
resp := s.req(t, http.MethodPost, "/api/alerts/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
|
}
|
|
var alert map[string]any
|
|
decode(t, resp, &alert)
|
|
if alert["acknowledged_by"] == nil {
|
|
t.Error("expected acknowledged_by to be set")
|
|
}
|
|
|
|
// Clear it.
|
|
resp = s.req(t, http.MethodDelete, "/api/alerts/1/acknowledge", nil)
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("unacknowledge returned %d", resp.StatusCode)
|
|
}
|
|
|
|
resp = s.req(t, http.MethodGet, "/api/alerts/1", nil)
|
|
var alert2 map[string]any
|
|
decode(t, resp, &alert2)
|
|
if alert2["acknowledged_by"] != nil {
|
|
t.Error("expected acknowledged_by to be cleared")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Comments — own-only deletion
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestComment_DeleteOwnOnly(t *testing.T) {
|
|
s := newTS(t)
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "firing", "labels": map[string]string{"alertname": "Y"},
|
|
"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
|
|
"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-comment",
|
|
}})
|
|
|
|
// Create a second user and their own key.
|
|
s.req(t, http.MethodPost, "/api/users",
|
|
map[string]string{"username": "alice", "email": "alice@test.com"})
|
|
keyResp := s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
|
map[string]string{"name": "alice-key"})
|
|
var keyData map[string]any
|
|
decode(t, keyResp, &keyData)
|
|
aliceKey := keyData["key"].(string)
|
|
|
|
// Admin posts a comment.
|
|
s.req(t, http.MethodPost, "/api/alerts/1/comments",
|
|
map[string]string{"content": "admin note"})
|
|
|
|
// Alice tries to delete admin's comment (should 404).
|
|
req, _ := http.NewRequest(http.MethodDelete, s.URL+"/api/alerts/1/comments/1", nil)
|
|
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
|
resp, _ := http.DefaultClient.Do(req)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Errorf("expected 404 when deleting another user's comment, got %d", resp.StatusCode)
|
|
}
|
|
|
|
// Admin deletes own comment (should 204).
|
|
resp = s.req(t, http.MethodDelete, "/api/alerts/1/comments/1", nil)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Errorf("expected 204 when deleting own comment, got %d", resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Schedule conflict
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestSchedule_ConflictOnSameDate(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
first := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if first.StatusCode != http.StatusCreated {
|
|
t.Fatalf("first assignment returned %d", first.StatusCode)
|
|
}
|
|
first.Body.Close()
|
|
|
|
second := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}})
|
|
if second.StatusCode != http.StatusConflict {
|
|
t.Errorf("expected 409 on duplicate date, got %d", second.StatusCode)
|
|
}
|
|
second.Body.Close()
|
|
}
|
|
|
|
func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
// Claim 2026-06-10 first.
|
|
s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-10"}}).Body.Close()
|
|
|
|
// Try to assign two dates in one request where the second conflicts.
|
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-09", "2026-06-10"}})
|
|
if resp.StatusCode != http.StatusConflict {
|
|
t.Fatalf("expected 409, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 2026-06-09 must NOT have been committed (transaction rolled back).
|
|
listResp := s.req(t, http.MethodGet, "/api/schedule?from=2026-06-09&to=2026-06-09", nil)
|
|
var entries []any
|
|
decode(t, listResp, &entries)
|
|
if len(entries) != 0 {
|
|
t.Errorf("expected rollback to leave 2026-06-09 unassigned, got %d entries", len(entries))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stats
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestStats_Totals(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
alerts := []map[string]any{
|
|
{"status": "firing", "labels": map[string]string{"alertname": "A"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "s1"},
|
|
{"status": "resolved", "labels": map[string]string{"alertname": "B"}, "annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z", "generatorURL": "", "fingerprint": "s2"},
|
|
}
|
|
postWebhook(t, s, alerts)
|
|
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
|
var stats map[string]any
|
|
decode(t, resp, &stats)
|
|
|
|
if int(stats["total"].(float64)) != 2 {
|
|
t.Errorf("expected total=2, got %v", stats["total"])
|
|
}
|
|
if int(stats["firing"].(float64)) != 1 {
|
|
t.Errorf("expected firing=1, got %v", stats["firing"])
|
|
}
|
|
if int(stats["resolved"].(float64)) != 1 {
|
|
t.Errorf("expected resolved=1, got %v", stats["resolved"])
|
|
}
|
|
}
|
|
|
|
func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-hour", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 24 {
|
|
t.Errorf("expected 24 hour slots, got %d", len(slots))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Archive
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestArchive_RoundTrip(t *testing.T) {
|
|
s := newTS(t)
|
|
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": "resolved", "fingerprint": "arch1",
|
|
"labels": map[string]string{"alertname": "Archivable"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z",
|
|
"generatorURL": "",
|
|
}})
|
|
|
|
// 1. Alert appears in default list (not archived).
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
|
}
|
|
id := int(alerts[0]["id"].(float64))
|
|
|
|
// 2. Archive it.
|
|
resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("archive: expected 200, got %d", resp.StatusCode)
|
|
}
|
|
var archived map[string]any
|
|
decode(t, resp, &archived)
|
|
if archived["archived_at"] == nil {
|
|
t.Error("expected archived_at to be set in response")
|
|
}
|
|
|
|
// 3. Default list excludes it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
|
|
}
|
|
|
|
// 4. archived=true shows it.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
|
}
|
|
|
|
// 5. Un-archive.
|
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
t.Fatalf("unarchive: expected 204, got %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
|
|
// 6. Back in default list.
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Errorf("expected unarchived alert to reappear, got %d results", len(alerts))
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stale-alert expiry
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// noArchive is long enough that archiving never interferes with expiry tests.
|
|
const noArchive = 365 * 24 * time.Hour
|
|
|
|
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
|
|
const zeroTime = "0001-01-01T00:00:00Z"
|
|
|
|
// postAlert sends a single-alert webhook.
|
|
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
|
|
t.Helper()
|
|
postWebhook(t, s, []map[string]any{{
|
|
"status": status,
|
|
"labels": map[string]string{"alertname": "Stale"},
|
|
"annotations": map[string]string{},
|
|
"startsAt": startsAt,
|
|
"endsAt": endsAt,
|
|
"generatorURL": "",
|
|
"fingerprint": fingerprint,
|
|
}})
|
|
}
|
|
|
|
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
|
|
t.Helper()
|
|
api.Sweep(context.Background(), s.db, noArchive, staleAfter)
|
|
}
|
|
|
|
// A firing alert Alertmanager stopped refreshing is resolved via the
|
|
// received_at heartbeat, even with no ends_at watermark to go on.
|
|
func TestExpiry_StaleFiringAlert(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Age the last-seen timestamp past the staleness window.
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'stale1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "stale1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A fresh webhook whose ends_at watermark has already passed is expired without
|
|
// waiting out the full staleness window.
|
|
func TestExpiry_PastEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "stale2", "firing",
|
|
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
|
|
|
|
status, source, _ := s.alertRow(t, "stale2")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "expiry" {
|
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
|
}
|
|
}
|
|
|
|
// The regression that matters most: a genuinely firing alert must survive a
|
|
// sweep untouched.
|
|
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "fresh1", "firing",
|
|
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
|
|
time.Now().Add(1*time.Hour).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
status, source, _ := s.alertRow(t, "fresh1")
|
|
if status != "firing" {
|
|
t.Errorf("expected fresh alert to stay firing, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected no resolution_source, got %q", *source)
|
|
}
|
|
}
|
|
|
|
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
|
|
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "grace1", "firing",
|
|
time.Now().Add(-time.Hour).Format(time.RFC3339),
|
|
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
|
|
|
|
sweep(t, s, 6*time.Hour)
|
|
|
|
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
|
|
t.Errorf("expected alert within grace period to stay firing, got %q", status)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Webhook resolution bookkeeping
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func TestWebhook_ResolvedSetsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
postAlert(t, s, "src1", "firing", start, zeroTime)
|
|
|
|
if _, source, _ := s.alertRow(t, "src1"); source != nil {
|
|
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
|
|
}
|
|
|
|
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
|
|
|
|
status, source, _ := s.alertRow(t, "src1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected status resolved, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
// A re-fire under the same fingerprint must leave the archive and clear the
|
|
// stale expiry marker, otherwise the alert stays invisible in the default list.
|
|
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
|
|
s := newTS(t)
|
|
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
|
|
|
// Expire it, then archive it.
|
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'refire1'",
|
|
time.Now().Add(-10*time.Hour).Unix())
|
|
sweep(t, s, 6*time.Hour)
|
|
s.exec(t, "UPDATE alerts SET archived_at = unixepoch() WHERE fingerprint = 'refire1'")
|
|
|
|
var alerts []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 0 {
|
|
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
|
|
}
|
|
|
|
// Fires again: a new alert instance, so a newer startsAt.
|
|
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
|
|
|
|
status, source, archivedAt := s.alertRow(t, "refire1")
|
|
if status != "firing" {
|
|
t.Errorf("expected status firing after re-fire, got %q", status)
|
|
}
|
|
if source != nil {
|
|
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
|
|
}
|
|
if archivedAt != nil {
|
|
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
|
|
}
|
|
|
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
|
if len(alerts) != 1 {
|
|
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
|
|
}
|
|
}
|
|
|
|
// Alertmanager retries failed notifications, so a firing payload for an
|
|
// already-resolved instance can arrive late. It must not resurrect the alert.
|
|
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
|
|
s := newTS(t)
|
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
|
end := time.Now().Format(time.RFC3339)
|
|
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime)
|
|
postAlert(t, s, "ooo1", "resolved", start, end)
|
|
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
|
|
|
|
status, source, _ := s.alertRow(t, "ooo1")
|
|
if status != "resolved" {
|
|
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
|
|
}
|
|
if source == nil || *source != "alertmanager" {
|
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
|
}
|
|
}
|
|
|
|
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
|
|
s := newTS(t)
|
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
|
|
var slots []any
|
|
decode(t, resp, &slots)
|
|
if len(slots) != 7 {
|
|
t.Errorf("expected 7 day slots, got %d", len(slots))
|
|
}
|
|
}
|