Graph the metric behind each alert on the incident page

Each alert carries a generatorURL pointing at the Prometheus rule that fired it. The server now parses that link, queries the same Prometheus for the rule's expression and draws it under the alert, with the current value beside it. The browser never talks to Prometheus, so the CSP stays at connect-src 'self'. The URL comes out of a webhook payload, so whoever holds an integration key chooses it. Querying it is therefore opt-in and bounded by an allowlist, TERDUT_PROMETHEUS_ALLOWED_HOSTS (names, "*" as one label, a leading "**" as several, IPs, CIDRs). The check runs on the address the dialer is about to use, a name that resolves to loopback or link-local is refused even when it matches, and redirects and proxy variables are ignored. A worker takes a snapshot when an incident opens and another when it closes, so the graph outlives Prometheus' retention. It finds its work in the database rather than being triggered from the webhook, so ingestion is untouched and a restart loses nothing. While an incident is open the metrics endpoint also queries live, cached for 15 seconds. The rule expression keeps its threshold, so Prometheus returns data only while the alert condition holds; "no data" for the current value means the condition cleared. Stripping the threshold needs a PromQL parser and is left out. Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN
2026-10-22 14:10:34 +02:00
parent 94c394e8f1
commit f344bb707b
22 changed files with 1909 additions and 2 deletions
@@ -1,8 +1,12 @@
 package api

 import (
+	"context"
+	"database/sql"
 	"strings"
 	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/config"
 )

 // DeadmanConfig is a test fixture only: a set of matchers with one timeout and
@@ -29,3 +33,9 @@ func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string)
 	}
 	return cfg
 }
+
+// MetricsSweep runs one pass of the metrics worker, so a test need not wait on
+// its ticker.
+func MetricsSweep(ctx context.Context, db *sql.DB, cfg config.Config) {
+	newMetricsService(cfg).sweep(ctx, db)
+}
@@ -0,0 +1,456 @@
+package api
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/config"
+	"git.ryuvia.com/niklas/terdut-server/internal/prom"
+)
+
+const (
+	// metricsInterval is how often the worker looks for incidents that need a
+	// snapshot. Short, because the "opened" snapshot is most useful when it is
+	// taken close to the moment the incident opened.
+	metricsInterval = 15 * time.Second
+
+	// metricsBatch caps one pass, and metricsConcurrency how many Prometheus
+	// requests run at once within it.
+	metricsBatch       = 50
+	metricsConcurrency = 4
+
+	// metricsMaxAttempts stops a failing capture being retried forever, and
+	// metricsRetryStep spaces the retries: attempt n waits n*metricsRetryStep.
+	metricsMaxAttempts = 4
+	metricsRetryStep   = time.Minute
+
+	// metricsLookback is how far back the worker goes for incidents that are
+	// already closed. Without it, switching the feature on would try to
+	// reconstruct the whole history from a Prometheus that has probably aged it
+	// out. Incidents that are still open are always eligible.
+	metricsLookback = 24 * time.Hour
+
+	// maxAlertsPerIncident bounds the Prometheus work one incident can cause.
+	maxAlertsPerIncident = 10
+
+	// metricsLiveTTL is how long a live answer is reused, so a room full of
+	// people watching one incident is one query per alert, not one each.
+	metricsLiveTTL = 15 * time.Second
+
+	// metricsLockKey is the advisory lock a pass holds, distinct from the
+	// archiver's and the notifier's.
+	metricsLockKey int64 = 7265_0004
+)
+
+// metricsService is the Prometheus side of the incident page: a client that only
+// reaches allowed hosts, and a short-lived cache of live answers.
+type metricsService struct {
+	client  *prom.Client
+	preroll time.Duration
+
+	mu    sync.Mutex
+	cache map[int64]liveEntry
+}
+
+type liveEntry struct {
+	at   time.Time
+	live *liveMetrics
+}
+
+func newMetricsService(cfg config.Config) *metricsService {
+	// Validate has already refused a bad list at startup; a failure here can only
+	// be a test building a Config by hand, and no allowlist is the safe reading.
+	allow, err := prom.ParseAllowlist(cfg.PrometheusAllowedHosts)
+	if err != nil {
+		allow = nil
+	}
+	preroll := cfg.PrometheusPreroll
+	if preroll <= 0 {
+		preroll = time.Hour
+	}
+	return &metricsService{
+		client:  prom.NewClient(allow, cfg.PrometheusTimeout),
+		preroll: preroll,
+		cache:   map[int64]liveEntry{},
+	}
+}
+
+func (m *metricsService) enabled() bool { return m.client.Enabled() }
+
+// StartMetricsWorker captures the opened and closed snapshots until ctx is
+// cancelled. With no allowed hosts configured it does nothing.
+func StartMetricsWorker(ctx context.Context, db *sql.DB, cfg config.Config) {
+	m := newMetricsService(cfg)
+	if !m.enabled() {
+		log.Print("metrics: disabled (TERDUT_PROMETHEUS_ALLOWED_HOSTS is empty)")
+		return
+	}
+	log.Printf("metrics: capturing from %d allowed Prometheus host entries", len(cfg.PrometheusAllowedHosts))
+
+	ticker := time.NewTicker(metricsInterval)
+	defer ticker.Stop()
+	sweep := func() {
+		withAdvisoryLock(ctx, db, metricsLockKey, "metrics", func() {
+			m.sweep(ctx, db)
+		})
+	}
+	sweep()
+	for {
+		select {
+		case <-ticker.C:
+			sweep()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// snapshotJob is one (incident, alert, kind) that still needs a snapshot.
+type snapshotJob struct {
+	incidentID   int64
+	alertID      int64
+	kind         string
+	resolvedAt   sql.NullInt64
+	startsAt     int64
+	labels       map[string]string
+	generatorURL string
+}
+
+// sweep finds the snapshots that are missing, or failed and are due a retry, and
+// takes them.
+func (m *metricsService) sweep(ctx context.Context, db *sql.DB) {
+	now := time.Now()
+	rows, err := db.QueryContext(ctx, `
+		SELECT i.id, a.id, k.kind, i.resolved_at, a.starts_at, a.labels, a.generator_url
+		FROM   incidents i
+		JOIN   incident_alerts ia ON ia.incident_id = i.id
+		JOIN   alerts a ON a.id = ia.alert_id
+		CROSS  JOIN (VALUES ('opened'), ('closed')) AS k(kind)
+		LEFT   JOIN incident_metric_snapshots s
+		       ON s.incident_id = i.id AND s.alert_id = a.id AND s.kind = k.kind
+		WHERE  a.generator_url <> ''
+		  AND  i.archived_at IS NULL
+		  AND  (i.resolved_at IS NULL OR i.resolved_at >= $1)
+		  AND  (k.kind = 'opened' OR i.resolved_at IS NOT NULL)
+		  AND  (s.id IS NULL
+		        OR (s.error IS NOT NULL AND s.attempts < $2
+		            AND s.captured_at + s.attempts * $3 <= $4))
+		ORDER  BY i.id, a.id, k.kind
+		LIMIT  $5`,
+		now.Add(-metricsLookback).Unix(), metricsMaxAttempts, int64(metricsRetryStep/time.Second), now.Unix(), metricsBatch)
+	if err != nil {
+		log.Printf("metrics: find work: %v", err)
+		return
+	}
+	var jobs []snapshotJob
+	for rows.Next() {
+		var j snapshotJob
+		var labels []byte
+		if err := rows.Scan(&j.incidentID, &j.alertID, &j.kind, &j.resolvedAt, &j.startsAt, &labels, &j.generatorURL); err != nil {
+			log.Printf("metrics: scan: %v", err)
+			rows.Close()
+			return
+		}
+		_ = json.Unmarshal(labels, &j.labels)
+		jobs = append(jobs, j)
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		log.Printf("metrics: find work: %v", err)
+		return
+	}
+
+	sem := make(chan struct{}, metricsConcurrency)
+	var wg sync.WaitGroup
+	for _, j := range jobs {
+		sem <- struct{}{}
+		wg.Add(1)
+		go func() {
+			defer func() { <-sem; wg.Done() }()
+			m.capture(ctx, db, j)
+		}()
+	}
+	wg.Wait()
+}
+
+// window is the span a snapshot covers: from the pre-roll before the alert
+// started to the end of the incident, or to now while it is open.
+func (m *metricsService) window(startsAt int64, resolvedAt sql.NullInt64, now time.Time) (time.Time, time.Time) {
+	start := time.Unix(startsAt, 0).Add(-m.preroll)
+	end := now
+	if resolvedAt.Valid {
+		end = time.Unix(resolvedAt.Int64, 0)
+	}
+	if !end.After(start) {
+		end = start.Add(time.Minute)
+	}
+	return start, end
+}
+
+func (m *metricsService) capture(ctx context.Context, db *sql.DB, j snapshotJob) {
+	now := time.Now()
+	start, end := m.window(j.startsAt, j.resolvedAt, now)
+	step := prom.StepFor(start, end)
+
+	series, expr, err := m.fetchRange(ctx, j.generatorURL, j.labels, start, end, step)
+
+	attempts := 1
+	var errText any
+	if err != nil {
+		errText = err.Error()
+		if errors.Is(err, prom.ErrNotAllowed) || errors.Is(err, errNotGraphLink) {
+			attempts = metricsMaxAttempts // retrying cannot change the answer
+		}
+	}
+	if series == nil {
+		series = []prom.Series{}
+	}
+	body, _ := json.Marshal(series)
+
+	_, dbErr := db.ExecContext(ctx, `
+		INSERT INTO incident_metric_snapshots
+		       (incident_id, alert_id, kind, expr, window_start, window_end, step_seconds, series, error, attempts, captured_at)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+		ON CONFLICT (incident_id, alert_id, kind) DO UPDATE SET
+		       expr = excluded.expr, window_start = excluded.window_start, window_end = excluded.window_end,
+		       step_seconds = excluded.step_seconds, series = excluded.series, error = excluded.error,
+		       attempts = CASE WHEN excluded.error IS NULL THEN 1
+		                       ELSE GREATEST(incident_metric_snapshots.attempts + 1, excluded.attempts) END,
+		       captured_at = excluded.captured_at`,
+		j.incidentID, j.alertID, j.kind, expr, start.Unix(), end.Unix(), int(step/time.Second),
+		string(body), errText, attempts, now.Unix())
+	if dbErr != nil {
+		log.Printf("metrics: store snapshot (incident %d, alert %d, %s): %v", j.incidentID, j.alertID, j.kind, dbErr)
+	}
+}
+
+var errNotGraphLink = errors.New("the alert's generator URL is not a Prometheus graph link")
+
+// fetchRange parses the generator URL and returns the series of the alert's own
+// expression over [start, end]. expr is returned even on error where known.
+func (m *metricsService) fetchRange(ctx context.Context, generatorURL string, labels map[string]string, start, end time.Time, step time.Duration) ([]prom.Series, string, error) {
+	base, expr, ok := prom.ParseGeneratorURL(generatorURL)
+	if !ok {
+		return nil, "", errNotGraphLink
+	}
+	series, err := m.client.QueryRange(ctx, base, expr, start, end, step)
+	if err != nil {
+		return nil, expr, err
+	}
+	return prom.SelectSeries(series, labels), expr, nil
+}
+
+// --- REST ---------------------------------------------------------------
+
+type metricsResponse struct {
+	// Enabled is false when no Prometheus host is allowed, so a client can hide
+	// the section rather than show every alert as unavailable.
+	Enabled bool           `json:"enabled"`
+	Alerts  []alertMetrics `json:"alerts"`
+}
+
+type alertMetrics struct {
+	AlertID   int64  `json:"alert_id"`
+	AlertName string `json:"alert_name"`
+	Expr      string `json:"expr,omitempty"`
+
+	// Available says whether this alert can be queried at all; Reason says why
+	// not when it cannot.
+	Available bool   `json:"available"`
+	Reason    string `json:"reason,omitempty"`
+
+	Snapshots map[string]*snapshotView `json:"snapshots"`
+	Live      *liveMetrics             `json:"live,omitempty"`
+}
+
+type snapshotView struct {
+	WindowStart int64           `json:"window_start"`
+	WindowEnd   int64           `json:"window_end"`
+	Step        int             `json:"step_seconds"`
+	Series      json.RawMessage `json:"series"`
+	Error       string          `json:"error,omitempty"`
+	CapturedAt  int64           `json:"captured_at"`
+}
+
+type liveMetrics struct {
+	WindowStart int64         `json:"window_start"`
+	WindowEnd   int64         `json:"window_end"`
+	Step        int           `json:"step_seconds"`
+	Series      []prom.Series `json:"series"`
+	// Current is the expression's value right now, one point per series. Empty
+	// means the condition that fires the alert no longer holds.
+	Current []prom.Series `json:"current"`
+	Error   string        `json:"error,omitempty"`
+}
+
+func handleIncidentMetrics(db *sql.DB, m *metricsService) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r, db)
+		if !ok {
+			return
+		}
+		resp := metricsResponse{Enabled: m.enabled(), Alerts: []alertMetrics{}}
+		if !resp.Enabled {
+			respond(w, http.StatusOK, resp)
+			return
+		}
+
+		var resolvedAt sql.NullInt64
+		if err := db.QueryRowContext(r.Context(), "SELECT resolved_at FROM incidents WHERE id = $1", id).Scan(&resolvedAt); err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		rows, err := db.QueryContext(r.Context(), `
+			SELECT a.id, a.name, a.labels, a.generator_url, a.starts_at
+			FROM   alerts a
+			JOIN   incident_alerts ia ON ia.alert_id = a.id
+			WHERE  ia.incident_id = $1
+			ORDER  BY a.starts_at, a.id
+			LIMIT  $2`, id, maxAlertsPerIncident)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		type alertRow struct {
+			am       alertMetrics
+			labels   map[string]string
+			url      string
+			startsAt int64
+		}
+		var alerts []*alertRow
+		for rows.Next() {
+			var a alertRow
+			var labels []byte
+			if err := rows.Scan(&a.am.AlertID, &a.am.AlertName, &labels, &a.url, &a.startsAt); err != nil {
+				rows.Close()
+				serverError(w, r, err)
+				return
+			}
+			_ = json.Unmarshal(labels, &a.labels)
+			a.am.Snapshots = map[string]*snapshotView{}
+			alerts = append(alerts, &a)
+		}
+		rows.Close()
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		snaps, err := loadSnapshots(r.Context(), db, id)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		var wg sync.WaitGroup
+		for _, a := range alerts {
+			a := a
+			if base, expr, ok := prom.ParseGeneratorURL(a.url); !ok {
+				a.am.Reason = "no Prometheus link on this alert"
+			} else if !m.client.Allowed(base) {
+				a.am.Reason = "Prometheus host is not allowed"
+				a.am.Expr = expr
+			} else {
+				a.am.Available = true
+				a.am.Expr = expr
+			}
+			for kind, s := range snaps[a.am.AlertID] {
+				a.am.Snapshots[kind] = s
+			}
+			if a.am.Available && !resolvedAt.Valid {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					a.am.Live = m.live(r.Context(), a.am.AlertID, a.url, a.labels, a.startsAt)
+				}()
+			}
+		}
+		wg.Wait()
+
+		for _, a := range alerts {
+			resp.Alerts = append(resp.Alerts, a.am)
+		}
+		respond(w, http.StatusOK, resp)
+	}
+}
+
+func loadSnapshots(ctx context.Context, db *sql.DB, incidentID int64) (map[int64]map[string]*snapshotView, error) {
+	rows, err := db.QueryContext(ctx, `
+		SELECT alert_id, kind, window_start, window_end, step_seconds, series, COALESCE(error, ''), captured_at
+		FROM   incident_metric_snapshots
+		WHERE  incident_id = $1`, incidentID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[int64]map[string]*snapshotView{}
+	for rows.Next() {
+		var alertID int64
+		var kind string
+		var s snapshotView
+		var series []byte
+		if err := rows.Scan(&alertID, &kind, &s.WindowStart, &s.WindowEnd, &s.Step, &series, &s.Error, &s.CapturedAt); err != nil {
+			return nil, err
+		}
+		s.Series = json.RawMessage(series)
+		if out[alertID] == nil {
+			out[alertID] = map[string]*snapshotView{}
+		}
+		out[alertID][kind] = &s
+	}
+	return out, rows.Err()
+}
+
+// live queries Prometheus now, reusing an answer younger than metricsLiveTTL.
+// The cache is per alert; the window start is fixed by the alert, only its end
+// moves, so one entry serves every viewer.
+func (m *metricsService) live(ctx context.Context, alertID int64, generatorURL string, labels map[string]string, startsAt int64) *liveMetrics {
+	m.mu.Lock()
+	if e, ok := m.cache[alertID]; ok && time.Since(e.at) < metricsLiveTTL {
+		m.mu.Unlock()
+		return e.live
+	}
+	m.mu.Unlock()
+
+	now := time.Now()
+	start, end := m.window(startsAt, sql.NullInt64{}, now)
+	step := prom.StepFor(start, end)
+	lm := &liveMetrics{WindowStart: start.Unix(), WindowEnd: end.Unix(), Step: int(step / time.Second),
+		Series: []prom.Series{}, Current: []prom.Series{}}
+
+	base, expr, ok := prom.ParseGeneratorURL(generatorURL)
+	if !ok {
+		lm.Error = errNotGraphLink.Error()
+		return lm
+	}
+	series, err := m.client.QueryRange(ctx, base, expr, start, end, step)
+	if err != nil {
+		lm.Error = err.Error()
+	} else {
+		lm.Series = prom.SelectSeries(series, labels)
+		if cur, err := m.client.Query(ctx, base, expr, now); err == nil {
+			lm.Current = prom.SelectSeries(cur, labels)
+		} else {
+			lm.Error = err.Error()
+		}
+	}
+
+	// A failed answer is cached too, for the same TTL: a Prometheus that is down
+	// should not be hit once per viewer per poll.
+	m.mu.Lock()
+	for k, e := range m.cache {
+		if time.Since(e.at) > 10*metricsLiveTTL {
+			delete(m.cache, k)
+		}
+	}
+	m.cache[alertID] = liveEntry{at: now, live: lm}
+	m.mu.Unlock()
+	return lm
+}
@@ -0,0 +1,189 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/api"
+)
+
+// fakePrometheus answers the two query endpoints and counts the requests.
+func fakePrometheus(t *testing.T) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var hits atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v1/query_range":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[
+				{"metric":{"instance":"web-1"},"values":[[100,"1"],[160,"2"]]},
+				{"metric":{"instance":"web-2"},"values":[[100,"9"]]}]}}`))
+		case "/api/v1/query":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[
+				{"metric":{"instance":"web-1"},"value":[170,"3"]}]}}`))
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hits
+}
+
+type metricsBody struct {
+	Enabled bool `json:"enabled"`
+	Alerts  []struct {
+		AlertID   int64  `json:"alert_id"`
+		Expr      string `json:"expr"`
+		Available bool   `json:"available"`
+		Reason    string `json:"reason"`
+		Snapshots map[string]struct {
+			Series json.RawMessage `json:"series"`
+			Error  string          `json:"error"`
+		} `json:"snapshots"`
+		Live *struct {
+			Series  []struct{ Labels map[string]string } `json:"series"`
+			Current []struct{ Labels map[string]string } `json:"current"`
+		} `json:"live"`
+	} `json:"alerts"`
+}
+
+func getMetrics(t *testing.T, s *ts, id int) metricsBody {
+	t.Helper()
+	var out metricsBody
+	decode(t, s.req(t, http.MethodGet, "/api/incidents/"+itoa(id)+"/metrics", nil), &out)
+	return out
+}
+
+func itoa(n int) string {
+	b, _ := json.Marshal(n)
+	return string(b)
+}
+
+func metricsAlert(promURL string) map[string]any {
+	a := amAlert("fp-metrics", "HighLatency", "firing", "2026-05-20T10:00:00Z", "0001-01-01T00:00:00Z",
+		map[string]string{"instance": "web-1"})
+	a["generatorURL"] = promURL + "/graph?g0.expr=latency%3E1&g0.tab=1"
+	return a
+}
+
+func TestMetrics_SnapshotsOpenedAndClosedAndLive(t *testing.T) {
+	prom, _ := fakePrometheus(t)
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"} // httptest listens on loopback
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-metrics")
+	incs := listIncidents(t, s, "")
+	if len(incs) != 1 {
+		t.Fatalf("want one incident, got %d", len(incs))
+	}
+	id := int(incs[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, conf)
+
+	m := getMetrics(t, s, id)
+	if !m.Enabled || len(m.Alerts) != 1 {
+		t.Fatalf("metrics = %+v", m)
+	}
+	a := m.Alerts[0]
+	if !a.Available || a.Expr != "latency>1" {
+		t.Fatalf("alert = %+v", a)
+	}
+	opened, ok := a.Snapshots["opened"]
+	if !ok || opened.Error != "" {
+		t.Fatalf("opened snapshot = %+v", a.Snapshots)
+	}
+	var series []struct{ Labels map[string]string }
+	json.Unmarshal(opened.Series, &series)
+	if len(series) != 1 || series[0].Labels["instance"] != "web-1" {
+		t.Errorf("snapshot should hold only this alert's series (web-1), got %+v", series)
+	}
+	if _, ok := a.Snapshots["closed"]; ok {
+		t.Error("closed snapshot exists while the incident is open")
+	}
+	if a.Live == nil || len(a.Live.Series) != 1 || len(a.Live.Current) != 1 {
+		t.Errorf("an open incident should have live data, got %+v", a.Live)
+	}
+
+	// A second pass must not capture again.
+	var n int
+	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n)
+	api.MetricsSweep(context.Background(), s.db, conf)
+	var n2 int
+	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n2)
+	if n != 1 || n2 != 1 {
+		t.Errorf("snapshot rows = %d then %d, want 1 and 1", n, n2)
+	}
+
+	// Resolving the incident gets the closed snapshot, and no more live data.
+	resp := s.req(t, http.MethodPost, "/api/incidents/"+itoa(id)+"/resolve", nil)
+	resp.Body.Close()
+	api.MetricsSweep(context.Background(), s.db, conf)
+	m = getMetrics(t, s, id)
+	a = m.Alerts[0]
+	if _, ok := a.Snapshots["closed"]; !ok {
+		t.Errorf("no closed snapshot after resolve: %+v", a.Snapshots)
+	}
+	if a.Live != nil {
+		t.Error("a resolved incident should not query live")
+	}
+}
+
+func TestMetrics_DisabledWithoutAllowedHosts(t *testing.T) {
+	prom, hits := fakePrometheus(t)
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-off")
+	id := int(listIncidents(t, s, "")[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, testConfig())
+	m := getMetrics(t, s, id)
+	if m.Enabled || len(m.Alerts) != 0 {
+		t.Errorf("metrics = %+v, want disabled and empty", m)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("Prometheus was contacted %d times", hits.Load())
+	}
+}
+
+func TestMetrics_UnlistedHostIsRecordedNotQueried(t *testing.T) {
+	prom, hits := fakePrometheus(t)
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"*.example.com"}
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-unlisted")
+	id := int(listIncidents(t, s, "")[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, conf)
+	api.MetricsSweep(context.Background(), s.db, conf) // must not retry a permanent failure
+
+	m := getMetrics(t, s, id)
+	a := m.Alerts[0]
+	if a.Available || a.Reason == "" {
+		t.Errorf("alert should be unavailable with a reason: %+v", a)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("an unlisted host was contacted %d times", hits.Load())
+	}
+	var attempts int
+	var errText *string
+	s.db.QueryRow("SELECT attempts, error FROM incident_metric_snapshots").Scan(&attempts, &errText)
+	if errText == nil || attempts < 4 {
+		t.Errorf("failure should be recorded as final: attempts=%d error=%v", attempts, errText)
+	}
+}
+
+func TestMetrics_IncidentScoping(t *testing.T) {
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"}
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+	resp := s.req(t, http.MethodGet, "/api/incidents/9999/metrics", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("unknown incident: %d, want 404", resp.StatusCode)
+	}
+}
@@ -28,6 +28,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 	signupLimiter := newLoginLimiter(db)
 	oidcLimit := newLoginLimiter(db)

+	metrics := newMetricsService(cfg)
+
 	r := chi.NewRouter()
 	r.Use(requestLogger)
 	r.Use(middleware.Recoverer)
@@ -149,6 +151,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Get("/api/incidents/clusters", handleListClusters(db))
 		r.Get("/api/incidents/{id}", handleGetIncident(db))
 		r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db))
+		r.Get("/api/incidents/{id}/metrics", handleIncidentMetrics(db, metrics))
 		r.Get("/api/incidents/{id}/similar", handleIncidentSimilar(db))
 		r.Post("/api/incidents/{id}/acknowledge", handleIncidentAcknowledge(db))
 		r.Delete("/api/incidents/{id}/acknowledge", handleIncidentUnacknowledge(db))