package api import ( "context" "database/sql" "log" "time" ) const ( // sweepInterval is how often the background sweeper runs. sweepInterval = 15 * time.Minute // expiryGrace absorbs clock skew and notification latency before an alert // whose ends_at watermark has passed is treated as stale. expiryGrace = 5 * time.Minute ) // StartArchiver runs the alert sweeper until ctx is cancelled, starting with an // immediate pass so a restart reconciles state right away. func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) { ticker := time.NewTicker(sweepInterval) defer ticker.Stop() Sweep(ctx, db, archiveAfter, staleAfter) for { select { case <-ticker.C: Sweep(ctx, db, archiveAfter, staleAfter) case <-ctx.Done(): return } } } // Sweep runs a single pass: expire stale firing alerts, then archive resolved // ones. Expiry runs first so an alert can expire and be archived in one pass. // Exported so tests can drive a pass without waiting on the ticker. func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) { expireStale(ctx, db, staleAfter) archiveResolved(ctx, db, archiveAfter) } // expireStale resolves firing alerts that Alertmanager has stopped refreshing. // // A resolved webhook is otherwise the only way out of the firing state, so a // notification that is dropped, silenced, or lost to a restart would pin the // alert as firing forever. Two independent signals mark an alert stale: // // - ends_at, the "valid until" watermark Alertmanager sets on outgoing firing // notifications, has passed (plus expiryGrace for clock skew). Absent on // rows whose payload carried no ends_at, hence the second signal. // - received_at is older than staleAfter. Alertmanager re-sends firing // notifications every repeat_interval, making received_at a liveness // heartbeat — provided staleAfter exceeds that interval. func expireStale(ctx context.Context, db *sql.DB, staleAfter time.Duration) { now := time.Now() res, err := db.ExecContext(ctx, ` UPDATE alerts SET status = 'resolved', resolution_source = ?, ends_at = COALESCE(ends_at, unixepoch()) WHERE status = 'firing' AND archived_at IS NULL AND ((ends_at IS NOT NULL AND ends_at < ?) OR received_at < ?)`, resolutionExpiry, now.Add(-expiryGrace).Unix(), now.Add(-staleAfter).Unix()) if err != nil { log.Printf("sweeper: expire stale: %v", err) return } if n, _ := res.RowsAffected(); n > 0 { log.Printf("sweeper: expired %d stale firing alert(s)", n) } } // archiveResolved hides resolved alerts that have been settled for archiveAfter. func archiveResolved(ctx context.Context, db *sql.DB, archiveAfter time.Duration) { cutoff := time.Now().Add(-archiveAfter).Unix() res, err := db.ExecContext(ctx, `UPDATE alerts SET archived_at = unixepoch() WHERE status = 'resolved' AND archived_at IS NULL AND COALESCE(ends_at, received_at) < ?`, cutoff) if err != nil { log.Printf("archiver: %v", err) return } if n, _ := res.RowsAffected(); n > 0 { log.Printf("archiver: archived %d resolved alert(s)", n) } }