Document received_at and resolution_source as public contract

The API reference listed endpoints but never the alert object's fields, so
two of them were load-bearing for clients while being described nowhere.
received_at appeared only in passing, as a stats filter; resolution_source
only inside the stale-expiry prose.

Both carry meaning a client cannot derive on its own. starts_at comes from
Prometheus and never changes for an alert instance, so received_at is the
only signal that a firing alert is still being refreshed — it advances on
every accepted webhook, including the unchanged notifications Alertmanager
re-sends every repeat_interval. resolution_source then says how much to
trust ends_at: under 'alertmanager' it is an end time somebody reported,
but under 'expiry' nothing ever reported one, so it is either a stale
watermark or the sweep timestamp, and only an upper bound.

README gains an alert object field table plus a contract section for each,
including the nullability rules and the advice to tolerate unrecognised
resolution_source values. The field comments in models.Alert now say these
are public API rather than ingest details, and the upsert carries a note at
the received_at line, which is where a regression would be introduced.

Three tests lock the newly documented behaviour, none of which was covered
before — the whole suite passed with the received_at bump deleted from the
upsert, because the expiry tests only ever set that column via SQL:

  - a re-send advances received_at and leaves starts_at alone
  - a discarded out-of-order retry does not count as a heartbeat
  - an expiry resolve preserves a reported ends_at watermark and stamps
    sweep time only when none was known
This commit is contained in:
Niklas Ye
2026-07-30 09:02:44 +02:00
parent 79afd05ea5
commit a602ff3efc
4 changed files with 231 additions and 2 deletions
+124
View File
@@ -72,6 +72,29 @@ func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *
return status, source, archivedAt
}
// alertTimes reads the timestamp columns that make up the received_at contract.
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
t.Helper()
err := s.db.QueryRow(
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = ?",
fingerprint).Scan(&startsAt, &receivedAt)
if err != nil {
t.Fatalf("read alert times %s: %v", fingerprint, err)
}
return startsAt, receivedAt
}
// alertEndsAt reads the nullable ends_at column of one alert.
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
t.Helper()
var endsAt *int64
if err := s.db.QueryRow(
"SELECT ends_at FROM alerts WHERE fingerprint = ?", fingerprint).Scan(&endsAt); err != nil {
t.Fatalf("read ends_at %s: %v", fingerprint, err)
}
return endsAt
}
// req sends an authenticated request, optionally with a JSON body.
func (s *ts) req(t *testing.T, method, path string, body any) *http.Response {
t.Helper()
@@ -626,6 +649,107 @@ func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
}
}
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
// Alertmanager watermark already on the row is preserved, and a row that never
// carried one is stamped at sweep time. Clients are told to read it that way —
// see "resolution_source says how much to trust ends_at" in the README.
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
s := newTS(t)
// No watermark: expires on the received_at heartbeat, so the sweeper has
// nothing to go on but its own clock.
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'ub-none'",
time.Now().Add(-10*time.Hour).Unix())
// Stale watermark: expires on the ends_at branch, and that reported time
// must survive the resolve rather than be overwritten with sweep time.
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
postAlert(t, s, "ub-mark", "firing",
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
sweep(t, s, 6*time.Hour)
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
}
stamped := s.alertEndsAt(t, "ub-none")
if stamped == nil {
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
}
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
}
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
}
switch kept := s.alertEndsAt(t, "ub-mark"); {
case kept == nil:
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
case *kept != watermark.Unix():
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
}
}
// ---------------------------------------------------------------------------
// received_at heartbeat contract
//
// received_at is documented as a public liveness signal, so these lock the
// behaviour clients are told they may rely on. See "received_at is a liveness
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
// ---------------------------------------------------------------------------
// The heartbeat itself: an unchanged firing notification — what Alertmanager
// re-sends every repeat_interval — must advance received_at, while leaving
// starts_at, which identifies the alert instance, untouched.
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
postAlert(t, s, "beat1", "firing", start, zeroTime)
startsBefore, _ := s.alertTimes(t, "beat1")
// received_at has one-second granularity, so back-date it to make the bump
// observable instead of sleeping out a second.
aged := time.Now().Add(-2 * time.Hour).Unix()
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat1'", aged)
// Identical re-send: same fingerprint, same startsAt, still firing.
postAlert(t, s, "beat1", "firing", start, zeroTime)
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
if receivedAfter <= aged {
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
}
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
}
if startsAfter != startsBefore {
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
}
}
// received_at tracks accepted payloads, not delivery attempts: a retry
// describing an already-resolved instance is discarded, so it must not register
// as a heartbeat and revive the alert's apparent liveness.
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
s := newTS(t)
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
postAlert(t, s, "beat2", "firing", start, zeroTime)
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
aged := time.Now().Add(-2 * time.Hour).Unix()
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat2'", aged)
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
}
}
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)