terdut-server/internal/api/archiver.go

package api

import (
	"context"
	"database/sql"
	"log"
	"time"
)

const (
	// sweepInterval is how often the background sweeper runs.
	sweepInterval = 15 * time.Minute

	// expiryGrace absorbs clock skew and notification latency before an alert
	// whose ends_at watermark has passed is treated as stale.
	expiryGrace = 5 * time.Minute
)

// StartArchiver runs the alert sweeper until ctx is cancelled, starting with an
// immediate pass so a restart reconciles state right away.
func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) {
	ticker := time.NewTicker(sweepInterval)
	defer ticker.Stop()

	Sweep(ctx, db, archiveAfter, staleAfter)
	for {
		select {
		case <-ticker.C:
			Sweep(ctx, db, archiveAfter, staleAfter)
		case <-ctx.Done():
			return
		}
	}
}

// Sweep runs a single pass: expire stale firing alerts, then archive resolved
// ones. Expiry runs first so an alert can expire and be archived in one pass.
// Exported so tests can drive a pass without waiting on the ticker.
func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) {
	expireStale(ctx, db, staleAfter)
	archiveResolved(ctx, db, archiveAfter)
}

// expireStale resolves firing alerts that Alertmanager has stopped refreshing.
//
// A resolved webhook is otherwise the only way out of the firing state, so a
// notification that is dropped, silenced, or lost to a restart would pin the
// alert as firing forever. Two independent signals mark an alert stale:
//
//   - ends_at, the "valid until" watermark Alertmanager sets on outgoing firing
//     notifications, has passed (plus expiryGrace for clock skew). Absent on
//     rows whose payload carried no ends_at, hence the second signal.
//   - received_at is older than staleAfter. Alertmanager re-sends firing
//     notifications every repeat_interval, making received_at a liveness
//     heartbeat — provided staleAfter exceeds that interval.
func expireStale(ctx context.Context, db *sql.DB, staleAfter time.Duration) {
	now := time.Now()
	res, err := db.ExecContext(ctx, `
		UPDATE alerts
		   SET status            = 'resolved',
		       resolution_source = ?,
		       ends_at           = COALESCE(ends_at, unixepoch())
		 WHERE status = 'firing'
		   AND archived_at IS NULL
		   AND ((ends_at IS NOT NULL AND ends_at < ?) OR received_at < ?)`,
		resolutionExpiry, now.Add(-expiryGrace).Unix(), now.Add(-staleAfter).Unix())
	if err != nil {
		log.Printf("sweeper: expire stale: %v", err)
		return
	}
	if n, _ := res.RowsAffected(); n > 0 {
		log.Printf("sweeper: expired %d stale firing alert(s)", n)
	}
}

// archiveResolved hides resolved alerts that have been settled for archiveAfter.
func archiveResolved(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
	cutoff := time.Now().Add(-archiveAfter).Unix()
	res, err := db.ExecContext(ctx,
		`UPDATE alerts SET archived_at = unixepoch()
		 WHERE status = 'resolved'
		   AND archived_at IS NULL
		   AND COALESCE(ends_at, received_at) < ?`, cutoff)
	if err != nil {
		log.Printf("archiver: %v", err)
		return
	}
	if n, _ := res.RowsAffected(); n > 0 {
		log.Printf("archiver: archived %d resolved alert(s)", n)
	}
}