Expire stale firing alerts

A resolved webhook was the only path out of the firing state, so a
notification that was dropped, silenced, or lost to a restart pinned an
alert as firing forever — Prometheus showed it resolved while
terdut-server kept listing it. The archiver only ever touched resolved
alerts, and both the list and stats queries compared status with plain
equality, so a stale row was indistinguishable from a live one.

A sweeper pass now resolves firing alerts on either of two signals: the
ends_at watermark Alertmanager sets on outgoing firing notifications has
passed (plus a grace period for clock skew), or no webhook has refreshed
the alert within TERDUT_STALE_AFTER (default 6h, above Alertmanager's 4h
repeat_interval). Such alerts get resolution_source = 'expiry',
distinguishing them from a real 'alertmanager' resolve.

Two related webhook bugs fixed alongside:

  - The upsert had no ordering guard, so a retried firing notification
    arriving after the resolved one resurrected the alert. Payloads for
    an older alert instance are now discarded: a stale retry carries the
    same startsAt, a genuine re-fire a newer one.
  - archived_at was never cleared on re-fire, leaving a re-fired alert
    archived and invisible in the default list.

Stats now exclude archived alerts to match the default list view; this
lowers historical firing/resolved totals.

The chart exposes both sweeper durations via sweeper.staleAfter and
sweeper.archiveAfter.

internal/api/archiver.go

@@ -7,33 +7,85 @@ import (
 	"time"
 )
 
-func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
-	ticker := time.NewTicker(15 * time.Minute)
+const (
+	// sweepInterval is how often the background sweeper runs.
+	sweepInterval = 15 * time.Minute
+
+	// expiryGrace absorbs clock skew and notification latency before an alert
+	// whose ends_at watermark has passed is treated as stale.
+	expiryGrace = 5 * time.Minute
+)
+
+// StartArchiver runs the alert sweeper until ctx is cancelled, starting with an
+// immediate pass so a restart reconciles state right away.
+func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) {
+	ticker := time.NewTicker(sweepInterval)
 	defer ticker.Stop()
 
-	run := func() {
-		cutoff := time.Now().Add(-archiveAfter).Unix()
-		res, err := db.ExecContext(ctx,
-			`UPDATE alerts SET archived_at = unixepoch()
-			 WHERE status = 'resolved'
-			   AND archived_at IS NULL
-			   AND COALESCE(ends_at, received_at) < ?`, cutoff)
-		if err != nil {
-			log.Printf("archiver: %v", err)
-			return
-		}
-		if n, _ := res.RowsAffected(); n > 0 {
-			log.Printf("archiver: archived %d resolved alert(s)", n)
-		}
-	}
-
-	run()
+	Sweep(ctx, db, archiveAfter, staleAfter)
 	for {
 		select {
 		case <-ticker.C:
-			run()
+			Sweep(ctx, db, archiveAfter, staleAfter)
 		case <-ctx.Done():
 			return
 		}
 	}
 }
+
+// Sweep runs a single pass: expire stale firing alerts, then archive resolved
+// ones. Expiry runs first so an alert can expire and be archived in one pass.
+// Exported so tests can drive a pass without waiting on the ticker.
+func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) {
+	expireStale(ctx, db, staleAfter)
+	archiveResolved(ctx, db, archiveAfter)
+}
+
+// expireStale resolves firing alerts that Alertmanager has stopped refreshing.
+//
+// A resolved webhook is otherwise the only way out of the firing state, so a
+// notification that is dropped, silenced, or lost to a restart would pin the
+// alert as firing forever. Two independent signals mark an alert stale:
+//
+//   - ends_at, the "valid until" watermark Alertmanager sets on outgoing firing
+//     notifications, has passed (plus expiryGrace for clock skew). Absent on
+//     rows whose payload carried no ends_at, hence the second signal.
+//   - received_at is older than staleAfter. Alertmanager re-sends firing
+//     notifications every repeat_interval, making received_at a liveness
+//     heartbeat — provided staleAfter exceeds that interval.
+func expireStale(ctx context.Context, db *sql.DB, staleAfter time.Duration) {
+	now := time.Now()
+	res, err := db.ExecContext(ctx, `
+		UPDATE alerts
+		   SET status            = 'resolved',
+		       resolution_source = ?,
+		       ends_at           = COALESCE(ends_at, unixepoch())
+		 WHERE status = 'firing'
+		   AND archived_at IS NULL
+		   AND ((ends_at IS NOT NULL AND ends_at < ?) OR received_at < ?)`,
+		resolutionExpiry, now.Add(-expiryGrace).Unix(), now.Add(-staleAfter).Unix())
+	if err != nil {
+		log.Printf("sweeper: expire stale: %v", err)
+		return
+	}
+	if n, _ := res.RowsAffected(); n > 0 {
+		log.Printf("sweeper: expired %d stale firing alert(s)", n)
+	}
+}
+
+// archiveResolved hides resolved alerts that have been settled for archiveAfter.
+func archiveResolved(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
+	cutoff := time.Now().Add(-archiveAfter).Unix()
+	res, err := db.ExecContext(ctx,
+		`UPDATE alerts SET archived_at = unixepoch()
+		 WHERE status = 'resolved'
+		   AND archived_at IS NULL
+		   AND COALESCE(ends_at, received_at) < ?`, cutoff)
+	if err != nil {
+		log.Printf("archiver: %v", err)
+		return
+	}
+	if n, _ := res.RowsAffected(); n > 0 {
+		log.Printf("archiver: archived %d resolved alert(s)", n)
+	}
+}