diff --git a/README.md b/README.md index b089419..ebf85d0 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,86 @@ passed. Resolved alerts carry `resolution_source`: `"alertmanager"` for a real resolved webhook, `"expiry"` when the sweeper inferred it (see [Stale alert expiry](#stale-alert-expiry)). +#### The alert object + +Returned by `GET /api/alerts` (as an array) and `GET /api/alerts/{id}`. +Timestamps are RFC 3339 in UTC. Fields marked *optional* are omitted entirely +when unset, so clients must treat them as nullable. + +| Field | Type | Notes | +|---|---|---| +| `id` | integer | Server-assigned; stable for the life of the row | +| `fingerprint` | string | Alertmanager's fingerprint — the upsert key | +| `name` | string | From the `alertname` label | +| `status` | string | `"firing"` or `"resolved"` | +| `labels` | object | String→string, as sent by Alertmanager | +| `annotations` | object | String→string, as sent by Alertmanager | +| `starts_at` | timestamp | When the alert instance began, **per Prometheus** | +| `ends_at` | timestamp | *optional* — absent while no end is known | +| `generator_url` | string | Link back to the originating Prometheus | +| `received_at` | timestamp | When the server last accepted a webhook for this alert — see below | +| `acknowledged_by_id` | integer | *optional* — user id | +| `acknowledged_by` | string | *optional* — username | +| `acknowledged_at` | timestamp | *optional* | +| `resolution_source` | string | *optional* — `"alertmanager"` or `"expiry"` | +| `archived_at` | timestamp | *optional* — set while archived | + +##### `received_at` is a liveness heartbeat + +`starts_at` comes from Prometheus and **never changes** for the lifetime of an +alert instance. It says when the problem began, not whether it is still +happening — an alert that started twelve days ago looks identical whether +Alertmanager refreshed it a minute ago or went silent a week ago. + +`received_at` is the field that answers "is this still live". It is set to the +server's clock on **every accepted webhook** for that fingerprint, including the +unchanged firing notifications Alertmanager re-sends every `repeat_interval`. +Clients may rely on this: + +- **A firing alert whose `received_at` is advancing is still being refreshed.** + Stale-dating it against `repeat_interval` is a valid liveness check, and it is + what the built-in sweeper does (see + [Stale alert expiry](#stale-alert-expiry)). +- **`received_at` tracks accepted payloads, not delivery attempts.** A retry + that describes an older instance than the stored one is discarded, and a + discarded payload does not move `received_at`. +- **It stops advancing once the alert resolves,** because Alertmanager stops + re-sending. On an alert resolved by the sweeper + (`"resolution_source": "expiry"`) it therefore marks the last time + Alertmanager was actually heard from, which is earlier than `ends_at`. + +`GET /api/alerts` is ordered by `received_at` descending — most recently +refreshed first — and the `?from=` / `?to=` filters on both the alert and stats +endpoints select on `received_at`, not `starts_at`. + +##### `resolution_source` says how much to trust `ends_at` + +An alert can leave the firing state two ways, and `resolution_source` records +which happened. Clients may rely on this: + +- **Absent while firing.** It is set only on resolve, and a re-fire under the + same fingerprint clears it again, so its presence always agrees with + `"status": "resolved"`. +- **`"alertmanager"` — a real resolved webhook arrived.** `ends_at` is the end + time Alertmanager reported. It is an observed value and can be displayed as + fact. +- **`"expiry"` — the sweeper inferred the resolve** because Alertmanager stopped + refreshing the alert (see [Stale alert expiry](#stale-alert-expiry)). Nothing + ever reported an end, so **`ends_at` is approximate**: it is either the stale + `endsAt` watermark from the last notification, or — when that notification + carried none — the time the sweep ran, which lags the last real contact by up + to `TERDUT_STALE_AFTER` plus a sweep interval. Treat it as "no later than", + not as when the problem stopped. + + On these alerts `received_at` is the more truthful signal: it marks the last + time Alertmanager was actually heard from. Surfacing the distinction is + worthwhile, since `"expiry"` can also mean the alert is still firing and the + notification path broke. + +Treat the value as an open set and tolerate ones you do not recognise — new +sources may be added, and unknown values should degrade to "resolved, reason +unknown" rather than being rejected. + ### On-call schedule | Method | Path | Description | diff --git a/internal/api/alertmanager.go b/internal/api/alertmanager.go index 37aa4b1..dc40505 100644 --- a/internal/api/alertmanager.go +++ b/internal/api/alertmanager.go @@ -79,6 +79,10 @@ func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc { starts_at = excluded.starts_at, ends_at = excluded.ends_at, generator_url = excluded.generator_url, + -- Load-bearing: advancing received_at on every accepted + -- payload, re-sends included, is the documented liveness + -- heartbeat clients and the sweeper both read. Removing it + -- is a breaking API change — see models.Alert.ReceivedAt. received_at = excluded.received_at, resolution_source = excluded.resolution_source, -- A re-fire makes the alert current again, so it leaves the archive. diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 87f4b15..57b7987 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -72,6 +72,29 @@ func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source * return status, source, archivedAt } +// alertTimes reads the timestamp columns that make up the received_at contract. +func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) { + t.Helper() + err := s.db.QueryRow( + "SELECT starts_at, received_at FROM alerts WHERE fingerprint = ?", + fingerprint).Scan(&startsAt, &receivedAt) + if err != nil { + t.Fatalf("read alert times %s: %v", fingerprint, err) + } + return startsAt, receivedAt +} + +// alertEndsAt reads the nullable ends_at column of one alert. +func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 { + t.Helper() + var endsAt *int64 + if err := s.db.QueryRow( + "SELECT ends_at FROM alerts WHERE fingerprint = ?", fingerprint).Scan(&endsAt); err != nil { + t.Fatalf("read ends_at %s: %v", fingerprint, err) + } + return endsAt +} + // req sends an authenticated request, optionally with a JSON body. func (s *ts) req(t *testing.T, method, path string, body any) *http.Response { t.Helper() @@ -626,6 +649,107 @@ func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) { } } +// An expiry resolve writes ends_at as an upper bound, not an observed end: an +// Alertmanager watermark already on the row is preserved, and a row that never +// carried one is stamped at sweep time. Clients are told to read it that way — +// see "resolution_source says how much to trust ends_at" in the README. +func TestExpiry_EndsAtIsUpperBound(t *testing.T) { + s := newTS(t) + + // No watermark: expires on the received_at heartbeat, so the sweeper has + // nothing to go on but its own clock. + postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime) + s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'ub-none'", + time.Now().Add(-10*time.Hour).Unix()) + + // Stale watermark: expires on the ends_at branch, and that reported time + // must survive the resolve rather than be overwritten with sweep time. + watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second) + postAlert(t, s, "ub-mark", "firing", + time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339)) + + sweep(t, s, 6*time.Hour) + + if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" { + t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source) + } + stamped := s.alertEndsAt(t, "ub-none") + if stamped == nil { + t.Fatal("expected expiry to stamp ends_at when no watermark was known") + } + if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 { + t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew) + } + + if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" { + t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source) + } + switch kept := s.alertEndsAt(t, "ub-mark"); { + case kept == nil: + t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix()) + case *kept != watermark.Unix(): + t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept) + } +} + +// --------------------------------------------------------------------------- +// received_at heartbeat contract +// +// received_at is documented as a public liveness signal, so these lock the +// behaviour clients are told they may rely on. See "received_at is a liveness +// heartbeat" in the README and the comment on models.Alert.ReceivedAt. +// --------------------------------------------------------------------------- + +// The heartbeat itself: an unchanged firing notification — what Alertmanager +// re-sends every repeat_interval — must advance received_at, while leaving +// starts_at, which identifies the alert instance, untouched. +func TestWebhook_ResendBumpsReceivedAt(t *testing.T) { + s := newTS(t) + start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + postAlert(t, s, "beat1", "firing", start, zeroTime) + + startsBefore, _ := s.alertTimes(t, "beat1") + + // received_at has one-second granularity, so back-date it to make the bump + // observable instead of sleeping out a second. + aged := time.Now().Add(-2 * time.Hour).Unix() + s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat1'", aged) + + // Identical re-send: same fingerprint, same startsAt, still firing. + postAlert(t, s, "beat1", "firing", start, zeroTime) + + startsAfter, receivedAfter := s.alertTimes(t, "beat1") + if receivedAfter <= aged { + t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter) + } + if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 { + t.Errorf("expected received_at to track the server clock, off by %ds", skew) + } + if startsAfter != startsBefore { + t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore) + } +} + +// received_at tracks accepted payloads, not delivery attempts: a retry +// describing an already-resolved instance is discarded, so it must not register +// as a heartbeat and revive the alert's apparent liveness. +func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) { + s := newTS(t) + start := time.Now().Add(-time.Hour).Format(time.RFC3339) + + postAlert(t, s, "beat2", "firing", start, zeroTime) + postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339)) + + aged := time.Now().Add(-2 * time.Hour).Unix() + s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat2'", aged) + + postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded + + if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged { + t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter) + } +} + func TestStats_ByDayReturnsSevenSlots(t *testing.T) { s := newTS(t) resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil) diff --git a/internal/models/alert.go b/internal/models/alert.go index bfb3633..0edadd1 100644 --- a/internal/models/alert.go +++ b/internal/models/alert.go @@ -12,7 +12,20 @@ type Alert struct { StartsAt time.Time `json:"starts_at"` EndsAt *time.Time `json:"ends_at,omitempty"` GeneratorURL string `json:"generator_url"` - ReceivedAt time.Time `json:"received_at"` + + // ReceivedAt is when the server last accepted a webhook for this + // fingerprint, including the unchanged firing notifications Alertmanager + // re-sends every repeat_interval. + // + // This is a documented part of the public API, not an internal ingest + // detail: StartsAt never changes for an alert instance, so ReceivedAt is + // the only signal a client has that a firing alert is still being + // refreshed. The sweeper stale-dates against it (see expireStale), API + // clients render it, and GET /api/alerts is ordered by it. Anything that + // stops the webhook handler from advancing it on a re-send is a breaking + // change — see "received_at is a liveness heartbeat" in the README and + // TestWebhook_ResendBumpsReceivedAt. + ReceivedAt time.Time `json:"received_at"` // Populated when the alert has been acknowledged. AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"` @@ -21,7 +34,15 @@ type Alert struct { // ResolutionSource records why a resolved alert left the firing state: // "alertmanager" for a real resolved webhook, "expiry" when the sweeper - // inferred it after the alert stopped being refreshed. + // inferred it after the alert stopped being refreshed. Nil while firing, and + // cleared again by a re-fire under the same fingerprint. + // + // Also public API: it is how a client knows whether EndsAt was observed or + // inferred. Under "expiry" nothing ever reported an end, so EndsAt is only + // an upper bound (see expireStale) and ReceivedAt is the more truthful + // signal. Treat the value set as open — see "resolution_source says how much + // to trust ends_at" in the README, and TestWebhook_ResolvedSetsSource / + // TestExpiry_StaleFiringAlert. ResolutionSource *string `json:"resolution_source,omitempty"` ArchivedAt *time.Time `json:"archived_at,omitempty"`