Graph the metric behind each alert on the incident page

Each alert carries a generatorURL pointing at the Prometheus rule that fired
it. The server now parses that link, queries the same Prometheus for the
rule's expression and draws it under the alert, with the current value
beside it. The browser never talks to Prometheus, so the CSP stays at
connect-src 'self'.

The URL comes out of a webhook payload, so whoever holds an integration key
chooses it. Querying it is therefore opt-in and bounded by an allowlist,
TERDUT_PROMETHEUS_ALLOWED_HOSTS (names, "*" as one label, a leading "**" as
several, IPs, CIDRs). The check runs on the address the dialer is about to
use, a name that resolves to loopback or link-local is refused even when it
matches, and redirects and proxy variables are ignored.

A worker takes a snapshot when an incident opens and another when it
closes, so the graph outlives Prometheus' retention. It finds its work in
the database rather than being triggered from the webhook, so ingestion is
untouched and a restart loses nothing. While an incident is open the
metrics endpoint also queries live, cached for 15 seconds.

The rule expression keeps its threshold, so Prometheus returns data only
while the alert condition holds; "no data" for the current value means the
condition cleared. Stripping the threshold needs a PromQL parser and is left
out.

Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN

README.md

@@ -92,6 +92,7 @@ The [documentation index](./docs/README.md) lists everything. The main pages:
 |---|---|
 | [Deployment](./docs/deployment.md) | Docker, Helm chart, database, backups, the operator |
 | [Configuration](./docs/configuration.md) | Environment variables and settings |
+| [Metrics](./docs/metrics.md) | Graphs from the alert's Prometheus on the incident page |
 | [Alertmanager configuration](./docs/alertmanager.md) | Routes, keys, webhooks |
 | [Alerts and incidents](./docs/incidents.md) | Correlation, lifecycle, on-call assignment |
 | [Single sign-on](./docs/single-sign-on.md) | OIDC and the terminal device flow |

charts/terdut-server/templates/deployment.yaml

@@ -120,6 +120,14 @@ spec:
               value: {{ .Values.trustedProxies | quote }}
             - name: TERDUT_OPERATOR_MODE
               value: {{ .Values.operatorMode | quote }}
+            {{- if .Values.prometheus.allowedHosts }}
+            - name: TERDUT_PROMETHEUS_ALLOWED_HOSTS
+              value: {{ join "," .Values.prometheus.allowedHosts | quote }}
+            - name: TERDUT_PROMETHEUS_PREROLL
+              value: {{ .Values.prometheus.preroll | quote }}
+            - name: TERDUT_PROMETHEUS_TIMEOUT
+              value: {{ .Values.prometheus.timeout | quote }}
+            {{- end }}
             {{- if .Values.oidc.enabled }}
             - name: TERDUT_OIDC_ISSUER
               value: {{ required "oidc.issuer is required when oidc.enabled" .Values.oidc.issuer | quote }}

charts/terdut-server/values.yaml

@@ -114,6 +114,18 @@ passwordLogin: true
 # configuration from here on.
 operatorMode: false
 
+# Metrics on incidents: the server queries the Prometheus an alert came from
+# (the host in its generatorURL) and graphs the alert's expression. The URL is
+# taken from a webhook payload, so only hosts listed here are ever contacted;
+# empty turns the feature off. Entries are host names, with "*" as one DNS label
+# and a leading "**" as one or more (e.g. "*.monitoring.svc.cluster.local",
+# "prometheus-*.monitoring"), IPs or CIDR ranges, each with an optional ":port".
+prometheus:
+  allowedHosts: []
+  # How far before an alert started its graph begins.
+  preroll: 1h
+  timeout: 10s
+
 # Single sign-on through an OpenID Connect provider such as Authentik.
 #
 # At the provider, create an OAuth2/OpenID application whose redirect URI is

cmd/terdut/main.go

@@ -64,6 +64,7 @@ func main() {
 
 	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notifyCfg)
 	go api.StartNotifier(ctx, database, notifyCfg)
+	go api.StartMetricsWorker(ctx, database, cfg)
 
 	go func() {
 		log.Printf("terdut-server %s listening on %s", version, cfg.Addr)

docs/api.md

@@ -236,6 +236,7 @@ the team gets the same `404` as anybody else.
 | `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline |
 | `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident |
 | `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological |
+| `GET` | `/api/incidents/{id}/metrics` | Metrics behind the incident's alerts, from their Prometheus — see [Metrics](./metrics.md) |
 | `POST` | `/api/incidents/{id}/acknowledge` | Acknowledge (stamps authed user + time) |
 | `DELETE` | `/api/incidents/{id}/acknowledge` | Clear acknowledgement, back to `triggered` |
 | `POST` | `/api/incidents/{id}/resolve` | Close by hand — **terminal**, see above |

docs/configuration.md

@@ -30,6 +30,9 @@ over an administrator's edit.
 | `TERDUT_TRUSTED_PROXIES` | `1` | How many reverse proxies in front of the server append to `X-Forwarded-For`; the per-address rate limits use the entry that many hops from the right. `0` ignores the header |
 | `TERDUT_OPERATOR_KEY` | — | At least 32 characters. When set, the instance-scoped service account `terdut-operator` is created if missing and its `seed` key replaced with this value at every start — how terdut-operator authenticates without a bootstrap handshake. An instance-scoped account acts as owner of every team (team configuration) but is not a member of any, so it reads no incidents |
 | `TERDUT_OPERATOR_MODE` | `false` | Declares this install gitops-managed: a session's or a user's own API key's writes to teams, escalation policies, dead man's switches and integrations are refused (`403 reason:"operator_managed"`); a [service account](./api.md#service-accounts)'s are not. Team membership and the schedule stay editable regardless |
+| `TERDUT_PROMETHEUS_ALLOWED_HOSTS` | — | Comma-separated Prometheus servers the [metrics on an incident](./metrics.md) may be fetched from, matched against the host in each alert's `generatorURL`. Host names (`prometheus.monitoring.svc`), wildcards (`*` is one DNS label, so `*.example.com` or `prom-*.monitoring.svc`; a leading `**` is one or more labels), IPs or CIDR ranges, each with an optional `:port`. Empty turns the feature off. A bare `*` is refused at startup |
+| `TERDUT_PROMETHEUS_PREROLL` | `1h` | How far before an alert started its metrics graph begins |
+| `TERDUT_PROMETHEUS_TIMEOUT` | `10s` | Per-request timeout for a Prometheus query |
 | `TERDUT_OIDC_ISSUER` | — | Turns single sign-on on. The provider's issuer URL; discovery is read from `<issuer>/.well-known/openid-configuration`. See [Single sign-on](./single-sign-on.md#single-sign-on-oidc) |
 | `TERDUT_OIDC_CLIENT_ID` / `TERDUT_OIDC_CLIENT_SECRET` | — | **Required with an issuer.** The confidential client registered at the provider. Keep the secret in a Secret, not in values |
 | `TERDUT_OIDC_NAME` | `SSO` | What the sign-in button calls the provider |

docs/metrics.md

@@ -0,0 +1,75 @@
+# Metrics on incidents
+
+Alertmanager sends each alert with a `generatorURL`: the link to the rule's graph in
+the Prometheus that fired it. With the feature on, terdut-server parses that link,
+queries the same Prometheus for the rule's expression, and shows a graph and the
+current value under each alert on the incident page. The browser never talks to
+Prometheus; the server does.
+
+## Turning it on
+
+Set `TERDUT_PROMETHEUS_ALLOWED_HOSTS` (Helm: `prometheus.allowedHosts`). The URL arrives in a
+webhook payload, so whoever holds an integration key chooses it; without an allowlist the
+server would fetch whatever it was told to. Empty, the default, means off.
+
+| Entry | Matches |
+|---|---|
+| `prometheus.monitoring.svc` | that host, any port |
+| `prometheus.monitoring.svc:9090` | that host and port |
+| `*.example.com` | `a.example.com`; not `example.com`, not `a.b.example.com` |
+| `prom-*.monitoring.svc` | `prom-0.monitoring.svc` |
+| `**.example.com` | `a.example.com` and `a.b.example.com` |
+| `10.0.0.0/8`, `[fd00::]/8:9090` | addresses in the range |
+
+`*` stands for one DNS label (or part of one); `**` is allowed only as the first label and means
+one or more. Matching ignores case. A bare `*` or `**`, or any entry whose last label is only a
+wildcard, is refused at startup.
+
+The check is made on the address the connection is about to use, not just on the name, so a
+name that resolves somewhere else (DNS rebinding) is caught. A host admitted by name is still
+refused if it resolves to a loopback, link-local (which includes the cloud metadata address),
+unspecified or multicast address; to allow one deliberately, list it as an IP or CIDR. Redirects
+and proxy environment variables are ignored.
+
+## What is captured
+
+- **Snapshots.** A background worker stores each alert's series when the incident opens
+  (`opened`) and when it closes (`closed`), in Postgres, so the graph outlives Prometheus'
+  own retention. Incidents closed more than 24 hours before the feature was switched on are not
+  backfilled. A failed capture is retried a few times; a host that is not allowed, or a link that is
+  not a Prometheus graph link, is recorded as failed once.
+- **Live data.** While an incident is open, `GET /api/incidents/{id}/metrics` also queries
+  Prometheus (answers are reused for 15 seconds) and returns the current value.
+- **Window.** From `TERDUT_PROMETHEUS_PREROLL` (default 1h) before the alert started until now,
+  or until the incident was resolved.
+
+## Reading the graph
+
+The expression is the alert rule's own, threshold included (`latency > 1`), so Prometheus only
+returns data while the condition is true. The graph therefore shows the firing period, and **"no
+data" for the current value means the condition no longer holds**, not that Prometheus is down.
+A rule expression matches every series, not just the one that fired; only series that agree with the
+alert's labels are kept, at most five per alert.
+
+## Endpoint
+
+`GET /api/incidents/{id}/metrics` — same team scoping as the other incident routes.
+
+```json
+{
+  "enabled": true,
+  "alerts": [{
+    "alert_id": 12, "alert_name": "HighLatency", "expr": "latency>1",
+    "available": true,
+    "snapshots": { "opened": { "window_start": 1, "window_end": 2, "step_seconds": 15,
+                               "series": [{"labels": {"instance": "web-1"}, "points": [[100, 1.5]]}],
+                               "captured_at": 3 } },
+    "live": { "window_start": 1, "window_end": 2, "step_seconds": 15,
+              "series": [], "current": [] }
+  }]
+}
+```
+
+`enabled` is `false` when no host is allowed. `available` is `false`, with a `reason`, when this
+alert has no Prometheus link or its host is not allowed. A snapshot that failed carries `error`.
+`live` is absent once the incident is resolved. Points are `[unix seconds, value]`.

internal/api/export_test.go

@@ -1,8 +1,12 @@
 package api
 
 import (
+	"context"
+	"database/sql"
 	"strings"
 	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/config"
 )
 
 // DeadmanConfig is a test fixture only: a set of matchers with one timeout and
@@ -29,3 +33,9 @@ func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string)
 	}
 	return cfg
 }
+
+// MetricsSweep runs one pass of the metrics worker, so a test need not wait on
+// its ticker.
+func MetricsSweep(ctx context.Context, db *sql.DB, cfg config.Config) {
+	newMetricsService(cfg).sweep(ctx, db)
+}

internal/api/metrics.go

@@ -0,0 +1,456 @@
+package api
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/config"
+	"git.ryuvia.com/niklas/terdut-server/internal/prom"
+)
+
+const (
+	// metricsInterval is how often the worker looks for incidents that need a
+	// snapshot. Short, because the "opened" snapshot is most useful when it is
+	// taken close to the moment the incident opened.
+	metricsInterval = 15 * time.Second
+
+	// metricsBatch caps one pass, and metricsConcurrency how many Prometheus
+	// requests run at once within it.
+	metricsBatch       = 50
+	metricsConcurrency = 4
+
+	// metricsMaxAttempts stops a failing capture being retried forever, and
+	// metricsRetryStep spaces the retries: attempt n waits n*metricsRetryStep.
+	metricsMaxAttempts = 4
+	metricsRetryStep   = time.Minute
+
+	// metricsLookback is how far back the worker goes for incidents that are
+	// already closed. Without it, switching the feature on would try to
+	// reconstruct the whole history from a Prometheus that has probably aged it
+	// out. Incidents that are still open are always eligible.
+	metricsLookback = 24 * time.Hour
+
+	// maxAlertsPerIncident bounds the Prometheus work one incident can cause.
+	maxAlertsPerIncident = 10
+
+	// metricsLiveTTL is how long a live answer is reused, so a room full of
+	// people watching one incident is one query per alert, not one each.
+	metricsLiveTTL = 15 * time.Second
+
+	// metricsLockKey is the advisory lock a pass holds, distinct from the
+	// archiver's and the notifier's.
+	metricsLockKey int64 = 7265_0004
+)
+
+// metricsService is the Prometheus side of the incident page: a client that only
+// reaches allowed hosts, and a short-lived cache of live answers.
+type metricsService struct {
+	client  *prom.Client
+	preroll time.Duration
+
+	mu    sync.Mutex
+	cache map[int64]liveEntry
+}
+
+type liveEntry struct {
+	at   time.Time
+	live *liveMetrics
+}
+
+func newMetricsService(cfg config.Config) *metricsService {
+	// Validate has already refused a bad list at startup; a failure here can only
+	// be a test building a Config by hand, and no allowlist is the safe reading.
+	allow, err := prom.ParseAllowlist(cfg.PrometheusAllowedHosts)
+	if err != nil {
+		allow = nil
+	}
+	preroll := cfg.PrometheusPreroll
+	if preroll <= 0 {
+		preroll = time.Hour
+	}
+	return &metricsService{
+		client:  prom.NewClient(allow, cfg.PrometheusTimeout),
+		preroll: preroll,
+		cache:   map[int64]liveEntry{},
+	}
+}
+
+func (m *metricsService) enabled() bool { return m.client.Enabled() }
+
+// StartMetricsWorker captures the opened and closed snapshots until ctx is
+// cancelled. With no allowed hosts configured it does nothing.
+func StartMetricsWorker(ctx context.Context, db *sql.DB, cfg config.Config) {
+	m := newMetricsService(cfg)
+	if !m.enabled() {
+		log.Print("metrics: disabled (TERDUT_PROMETHEUS_ALLOWED_HOSTS is empty)")
+		return
+	}
+	log.Printf("metrics: capturing from %d allowed Prometheus host entries", len(cfg.PrometheusAllowedHosts))
+
+	ticker := time.NewTicker(metricsInterval)
+	defer ticker.Stop()
+	sweep := func() {
+		withAdvisoryLock(ctx, db, metricsLockKey, "metrics", func() {
+			m.sweep(ctx, db)
+		})
+	}
+	sweep()
+	for {
+		select {
+		case <-ticker.C:
+			sweep()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// snapshotJob is one (incident, alert, kind) that still needs a snapshot.
+type snapshotJob struct {
+	incidentID   int64
+	alertID      int64
+	kind         string
+	resolvedAt   sql.NullInt64
+	startsAt     int64
+	labels       map[string]string
+	generatorURL string
+}
+
+// sweep finds the snapshots that are missing, or failed and are due a retry, and
+// takes them.
+func (m *metricsService) sweep(ctx context.Context, db *sql.DB) {
+	now := time.Now()
+	rows, err := db.QueryContext(ctx, `
+		SELECT i.id, a.id, k.kind, i.resolved_at, a.starts_at, a.labels, a.generator_url
+		FROM   incidents i
+		JOIN   incident_alerts ia ON ia.incident_id = i.id
+		JOIN   alerts a ON a.id = ia.alert_id
+		CROSS  JOIN (VALUES ('opened'), ('closed')) AS k(kind)
+		LEFT   JOIN incident_metric_snapshots s
+		       ON s.incident_id = i.id AND s.alert_id = a.id AND s.kind = k.kind
+		WHERE  a.generator_url <> ''
+		  AND  i.archived_at IS NULL
+		  AND  (i.resolved_at IS NULL OR i.resolved_at >= $1)
+		  AND  (k.kind = 'opened' OR i.resolved_at IS NOT NULL)
+		  AND  (s.id IS NULL
+		        OR (s.error IS NOT NULL AND s.attempts < $2
+		            AND s.captured_at + s.attempts * $3 <= $4))
+		ORDER  BY i.id, a.id, k.kind
+		LIMIT  $5`,
+		now.Add(-metricsLookback).Unix(), metricsMaxAttempts, int64(metricsRetryStep/time.Second), now.Unix(), metricsBatch)
+	if err != nil {
+		log.Printf("metrics: find work: %v", err)
+		return
+	}
+	var jobs []snapshotJob
+	for rows.Next() {
+		var j snapshotJob
+		var labels []byte
+		if err := rows.Scan(&j.incidentID, &j.alertID, &j.kind, &j.resolvedAt, &j.startsAt, &labels, &j.generatorURL); err != nil {
+			log.Printf("metrics: scan: %v", err)
+			rows.Close()
+			return
+		}
+		_ = json.Unmarshal(labels, &j.labels)
+		jobs = append(jobs, j)
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		log.Printf("metrics: find work: %v", err)
+		return
+	}
+
+	sem := make(chan struct{}, metricsConcurrency)
+	var wg sync.WaitGroup
+	for _, j := range jobs {
+		sem <- struct{}{}
+		wg.Add(1)
+		go func() {
+			defer func() { <-sem; wg.Done() }()
+			m.capture(ctx, db, j)
+		}()
+	}
+	wg.Wait()
+}
+
+// window is the span a snapshot covers: from the pre-roll before the alert
+// started to the end of the incident, or to now while it is open.
+func (m *metricsService) window(startsAt int64, resolvedAt sql.NullInt64, now time.Time) (time.Time, time.Time) {
+	start := time.Unix(startsAt, 0).Add(-m.preroll)
+	end := now
+	if resolvedAt.Valid {
+		end = time.Unix(resolvedAt.Int64, 0)
+	}
+	if !end.After(start) {
+		end = start.Add(time.Minute)
+	}
+	return start, end
+}
+
+func (m *metricsService) capture(ctx context.Context, db *sql.DB, j snapshotJob) {
+	now := time.Now()
+	start, end := m.window(j.startsAt, j.resolvedAt, now)
+	step := prom.StepFor(start, end)
+
+	series, expr, err := m.fetchRange(ctx, j.generatorURL, j.labels, start, end, step)
+
+	attempts := 1
+	var errText any
+	if err != nil {
+		errText = err.Error()
+		if errors.Is(err, prom.ErrNotAllowed) || errors.Is(err, errNotGraphLink) {
+			attempts = metricsMaxAttempts // retrying cannot change the answer
+		}
+	}
+	if series == nil {
+		series = []prom.Series{}
+	}
+	body, _ := json.Marshal(series)
+
+	_, dbErr := db.ExecContext(ctx, `
+		INSERT INTO incident_metric_snapshots
+		       (incident_id, alert_id, kind, expr, window_start, window_end, step_seconds, series, error, attempts, captured_at)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+		ON CONFLICT (incident_id, alert_id, kind) DO UPDATE SET
+		       expr = excluded.expr, window_start = excluded.window_start, window_end = excluded.window_end,
+		       step_seconds = excluded.step_seconds, series = excluded.series, error = excluded.error,
+		       attempts = CASE WHEN excluded.error IS NULL THEN 1
+		                       ELSE GREATEST(incident_metric_snapshots.attempts + 1, excluded.attempts) END,
+		       captured_at = excluded.captured_at`,
+		j.incidentID, j.alertID, j.kind, expr, start.Unix(), end.Unix(), int(step/time.Second),
+		string(body), errText, attempts, now.Unix())
+	if dbErr != nil {
+		log.Printf("metrics: store snapshot (incident %d, alert %d, %s): %v", j.incidentID, j.alertID, j.kind, dbErr)
+	}
+}
+
+var errNotGraphLink = errors.New("the alert's generator URL is not a Prometheus graph link")
+
+// fetchRange parses the generator URL and returns the series of the alert's own
+// expression over [start, end]. expr is returned even on error where known.
+func (m *metricsService) fetchRange(ctx context.Context, generatorURL string, labels map[string]string, start, end time.Time, step time.Duration) ([]prom.Series, string, error) {
+	base, expr, ok := prom.ParseGeneratorURL(generatorURL)
+	if !ok {
+		return nil, "", errNotGraphLink
+	}
+	series, err := m.client.QueryRange(ctx, base, expr, start, end, step)
+	if err != nil {
+		return nil, expr, err
+	}
+	return prom.SelectSeries(series, labels), expr, nil
+}
+
+// --- REST ---------------------------------------------------------------
+
+type metricsResponse struct {
+	// Enabled is false when no Prometheus host is allowed, so a client can hide
+	// the section rather than show every alert as unavailable.
+	Enabled bool           `json:"enabled"`
+	Alerts  []alertMetrics `json:"alerts"`
+}
+
+type alertMetrics struct {
+	AlertID   int64  `json:"alert_id"`
+	AlertName string `json:"alert_name"`
+	Expr      string `json:"expr,omitempty"`
+
+	// Available says whether this alert can be queried at all; Reason says why
+	// not when it cannot.
+	Available bool   `json:"available"`
+	Reason    string `json:"reason,omitempty"`
+
+	Snapshots map[string]*snapshotView `json:"snapshots"`
+	Live      *liveMetrics             `json:"live,omitempty"`
+}
+
+type snapshotView struct {
+	WindowStart int64           `json:"window_start"`
+	WindowEnd   int64           `json:"window_end"`
+	Step        int             `json:"step_seconds"`
+	Series      json.RawMessage `json:"series"`
+	Error       string          `json:"error,omitempty"`
+	CapturedAt  int64           `json:"captured_at"`
+}
+
+type liveMetrics struct {
+	WindowStart int64         `json:"window_start"`
+	WindowEnd   int64         `json:"window_end"`
+	Step        int           `json:"step_seconds"`
+	Series      []prom.Series `json:"series"`
+	// Current is the expression's value right now, one point per series. Empty
+	// means the condition that fires the alert no longer holds.
+	Current []prom.Series `json:"current"`
+	Error   string        `json:"error,omitempty"`
+}
+
+func handleIncidentMetrics(db *sql.DB, m *metricsService) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r, db)
+		if !ok {
+			return
+		}
+		resp := metricsResponse{Enabled: m.enabled(), Alerts: []alertMetrics{}}
+		if !resp.Enabled {
+			respond(w, http.StatusOK, resp)
+			return
+		}
+
+		var resolvedAt sql.NullInt64
+		if err := db.QueryRowContext(r.Context(), "SELECT resolved_at FROM incidents WHERE id = $1", id).Scan(&resolvedAt); err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		rows, err := db.QueryContext(r.Context(), `
+			SELECT a.id, a.name, a.labels, a.generator_url, a.starts_at
+			FROM   alerts a
+			JOIN   incident_alerts ia ON ia.alert_id = a.id
+			WHERE  ia.incident_id = $1
+			ORDER  BY a.starts_at, a.id
+			LIMIT  $2`, id, maxAlertsPerIncident)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		type alertRow struct {
+			am       alertMetrics
+			labels   map[string]string
+			url      string
+			startsAt int64
+		}
+		var alerts []*alertRow
+		for rows.Next() {
+			var a alertRow
+			var labels []byte
+			if err := rows.Scan(&a.am.AlertID, &a.am.AlertName, &labels, &a.url, &a.startsAt); err != nil {
+				rows.Close()
+				serverError(w, r, err)
+				return
+			}
+			_ = json.Unmarshal(labels, &a.labels)
+			a.am.Snapshots = map[string]*snapshotView{}
+			alerts = append(alerts, &a)
+		}
+		rows.Close()
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		snaps, err := loadSnapshots(r.Context(), db, id)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		var wg sync.WaitGroup
+		for _, a := range alerts {
+			a := a
+			if base, expr, ok := prom.ParseGeneratorURL(a.url); !ok {
+				a.am.Reason = "no Prometheus link on this alert"
+			} else if !m.client.Allowed(base) {
+				a.am.Reason = "Prometheus host is not allowed"
+				a.am.Expr = expr
+			} else {
+				a.am.Available = true
+				a.am.Expr = expr
+			}
+			for kind, s := range snaps[a.am.AlertID] {
+				a.am.Snapshots[kind] = s
+			}
+			if a.am.Available && !resolvedAt.Valid {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					a.am.Live = m.live(r.Context(), a.am.AlertID, a.url, a.labels, a.startsAt)
+				}()
+			}
+		}
+		wg.Wait()
+
+		for _, a := range alerts {
+			resp.Alerts = append(resp.Alerts, a.am)
+		}
+		respond(w, http.StatusOK, resp)
+	}
+}
+
+func loadSnapshots(ctx context.Context, db *sql.DB, incidentID int64) (map[int64]map[string]*snapshotView, error) {
+	rows, err := db.QueryContext(ctx, `
+		SELECT alert_id, kind, window_start, window_end, step_seconds, series, COALESCE(error, ''), captured_at
+		FROM   incident_metric_snapshots
+		WHERE  incident_id = $1`, incidentID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[int64]map[string]*snapshotView{}
+	for rows.Next() {
+		var alertID int64
+		var kind string
+		var s snapshotView
+		var series []byte
+		if err := rows.Scan(&alertID, &kind, &s.WindowStart, &s.WindowEnd, &s.Step, &series, &s.Error, &s.CapturedAt); err != nil {
+			return nil, err
+		}
+		s.Series = json.RawMessage(series)
+		if out[alertID] == nil {
+			out[alertID] = map[string]*snapshotView{}
+		}
+		out[alertID][kind] = &s
+	}
+	return out, rows.Err()
+}
+
+// live queries Prometheus now, reusing an answer younger than metricsLiveTTL.
+// The cache is per alert; the window start is fixed by the alert, only its end
+// moves, so one entry serves every viewer.
+func (m *metricsService) live(ctx context.Context, alertID int64, generatorURL string, labels map[string]string, startsAt int64) *liveMetrics {
+	m.mu.Lock()
+	if e, ok := m.cache[alertID]; ok && time.Since(e.at) < metricsLiveTTL {
+		m.mu.Unlock()
+		return e.live
+	}
+	m.mu.Unlock()
+
+	now := time.Now()
+	start, end := m.window(startsAt, sql.NullInt64{}, now)
+	step := prom.StepFor(start, end)
+	lm := &liveMetrics{WindowStart: start.Unix(), WindowEnd: end.Unix(), Step: int(step / time.Second),
+		Series: []prom.Series{}, Current: []prom.Series{}}
+
+	base, expr, ok := prom.ParseGeneratorURL(generatorURL)
+	if !ok {
+		lm.Error = errNotGraphLink.Error()
+		return lm
+	}
+	series, err := m.client.QueryRange(ctx, base, expr, start, end, step)
+	if err != nil {
+		lm.Error = err.Error()
+	} else {
+		lm.Series = prom.SelectSeries(series, labels)
+		if cur, err := m.client.Query(ctx, base, expr, now); err == nil {
+			lm.Current = prom.SelectSeries(cur, labels)
+		} else {
+			lm.Error = err.Error()
+		}
+	}
+
+	// A failed answer is cached too, for the same TTL: a Prometheus that is down
+	// should not be hit once per viewer per poll.
+	m.mu.Lock()
+	for k, e := range m.cache {
+		if time.Since(e.at) > 10*metricsLiveTTL {
+			delete(m.cache, k)
+		}
+	}
+	m.cache[alertID] = liveEntry{at: now, live: lm}
+	m.mu.Unlock()
+	return lm
+}

internal/api/metrics_test.go

@@ -0,0 +1,189 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/api"
+)
+
+// fakePrometheus answers the two query endpoints and counts the requests.
+func fakePrometheus(t *testing.T) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var hits atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v1/query_range":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[
+				{"metric":{"instance":"web-1"},"values":[[100,"1"],[160,"2"]]},
+				{"metric":{"instance":"web-2"},"values":[[100,"9"]]}]}}`))
+		case "/api/v1/query":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[
+				{"metric":{"instance":"web-1"},"value":[170,"3"]}]}}`))
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hits
+}
+
+type metricsBody struct {
+	Enabled bool `json:"enabled"`
+	Alerts  []struct {
+		AlertID   int64  `json:"alert_id"`
+		Expr      string `json:"expr"`
+		Available bool   `json:"available"`
+		Reason    string `json:"reason"`
+		Snapshots map[string]struct {
+			Series json.RawMessage `json:"series"`
+			Error  string          `json:"error"`
+		} `json:"snapshots"`
+		Live *struct {
+			Series  []struct{ Labels map[string]string } `json:"series"`
+			Current []struct{ Labels map[string]string } `json:"current"`
+		} `json:"live"`
+	} `json:"alerts"`
+}
+
+func getMetrics(t *testing.T, s *ts, id int) metricsBody {
+	t.Helper()
+	var out metricsBody
+	decode(t, s.req(t, http.MethodGet, "/api/incidents/"+itoa(id)+"/metrics", nil), &out)
+	return out
+}
+
+func itoa(n int) string {
+	b, _ := json.Marshal(n)
+	return string(b)
+}
+
+func metricsAlert(promURL string) map[string]any {
+	a := amAlert("fp-metrics", "HighLatency", "firing", "2026-05-20T10:00:00Z", "0001-01-01T00:00:00Z",
+		map[string]string{"instance": "web-1"})
+	a["generatorURL"] = promURL + "/graph?g0.expr=latency%3E1&g0.tab=1"
+	return a
+}
+
+func TestMetrics_SnapshotsOpenedAndClosedAndLive(t *testing.T) {
+	prom, _ := fakePrometheus(t)
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"} // httptest listens on loopback
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-metrics")
+	incs := listIncidents(t, s, "")
+	if len(incs) != 1 {
+		t.Fatalf("want one incident, got %d", len(incs))
+	}
+	id := int(incs[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, conf)
+
+	m := getMetrics(t, s, id)
+	if !m.Enabled || len(m.Alerts) != 1 {
+		t.Fatalf("metrics = %+v", m)
+	}
+	a := m.Alerts[0]
+	if !a.Available || a.Expr != "latency>1" {
+		t.Fatalf("alert = %+v", a)
+	}
+	opened, ok := a.Snapshots["opened"]
+	if !ok || opened.Error != "" {
+		t.Fatalf("opened snapshot = %+v", a.Snapshots)
+	}
+	var series []struct{ Labels map[string]string }
+	json.Unmarshal(opened.Series, &series)
+	if len(series) != 1 || series[0].Labels["instance"] != "web-1" {
+		t.Errorf("snapshot should hold only this alert's series (web-1), got %+v", series)
+	}
+	if _, ok := a.Snapshots["closed"]; ok {
+		t.Error("closed snapshot exists while the incident is open")
+	}
+	if a.Live == nil || len(a.Live.Series) != 1 || len(a.Live.Current) != 1 {
+		t.Errorf("an open incident should have live data, got %+v", a.Live)
+	}
+
+	// A second pass must not capture again.
+	var n int
+	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n)
+	api.MetricsSweep(context.Background(), s.db, conf)
+	var n2 int
+	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n2)
+	if n != 1 || n2 != 1 {
+		t.Errorf("snapshot rows = %d then %d, want 1 and 1", n, n2)
+	}
+
+	// Resolving the incident gets the closed snapshot, and no more live data.
+	resp := s.req(t, http.MethodPost, "/api/incidents/"+itoa(id)+"/resolve", nil)
+	resp.Body.Close()
+	api.MetricsSweep(context.Background(), s.db, conf)
+	m = getMetrics(t, s, id)
+	a = m.Alerts[0]
+	if _, ok := a.Snapshots["closed"]; !ok {
+		t.Errorf("no closed snapshot after resolve: %+v", a.Snapshots)
+	}
+	if a.Live != nil {
+		t.Error("a resolved incident should not query live")
+	}
+}
+
+func TestMetrics_DisabledWithoutAllowedHosts(t *testing.T) {
+	prom, hits := fakePrometheus(t)
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-off")
+	id := int(listIncidents(t, s, "")[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, testConfig())
+	m := getMetrics(t, s, id)
+	if m.Enabled || len(m.Alerts) != 0 {
+		t.Errorf("metrics = %+v, want disabled and empty", m)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("Prometheus was contacted %d times", hits.Load())
+	}
+}
+
+func TestMetrics_UnlistedHostIsRecordedNotQueried(t *testing.T) {
+	prom, hits := fakePrometheus(t)
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"*.example.com"}
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-unlisted")
+	id := int(listIncidents(t, s, "")[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, conf)
+	api.MetricsSweep(context.Background(), s.db, conf) // must not retry a permanent failure
+
+	m := getMetrics(t, s, id)
+	a := m.Alerts[0]
+	if a.Available || a.Reason == "" {
+		t.Errorf("alert should be unavailable with a reason: %+v", a)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("an unlisted host was contacted %d times", hits.Load())
+	}
+	var attempts int
+	var errText *string
+	s.db.QueryRow("SELECT attempts, error FROM incident_metric_snapshots").Scan(&attempts, &errText)
+	if errText == nil || attempts < 4 {
+		t.Errorf("failure should be recorded as final: attempts=%d error=%v", attempts, errText)
+	}
+}
+
+func TestMetrics_IncidentScoping(t *testing.T) {
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"}
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+	resp := s.req(t, http.MethodGet, "/api/incidents/9999/metrics", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("unknown incident: %d, want 404", resp.StatusCode)
+	}
+}

internal/api/router.go

@@ -28,6 +28,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 	signupLimiter := newLoginLimiter(db)
 	oidcLimit := newLoginLimiter(db)
 
+	metrics := newMetricsService(cfg)
+
 	r := chi.NewRouter()
 	r.Use(requestLogger)
 	r.Use(middleware.Recoverer)
@@ -149,6 +151,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Get("/api/incidents/clusters", handleListClusters(db))
 		r.Get("/api/incidents/{id}", handleGetIncident(db))
 		r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db))
+		r.Get("/api/incidents/{id}/metrics", handleIncidentMetrics(db, metrics))
 		r.Get("/api/incidents/{id}/similar", handleIncidentSimilar(db))
 		r.Post("/api/incidents/{id}/acknowledge", handleIncidentAcknowledge(db))
 		r.Delete("/api/incidents/{id}/acknowledge", handleIncidentUnacknowledge(db))

internal/config/config.go

@@ -8,6 +8,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/prom"
 )
 
 // MinOperatorKeyLength is the shortest TERDUT_OPERATOR_KEY accepted: it is a
@@ -79,6 +81,20 @@ type Config struct {
 	// "where this server is plugged in" — it is a statement about who owns
 	// this install's configuration, not a per-request toggle.
 	OperatorMode bool
+
+	// PrometheusAllowedHosts lists the Prometheus servers this install may query
+	// for the metrics behind an alert, matched against the host in each alert's
+	// generatorURL. Entries are host names (with "*" and a leading "**" as
+	// wildcards), IPs or CIDR ranges, optionally with a port; see prom.Allowlist.
+	// Empty turns the feature off, which is the default: the URL comes from a
+	// webhook payload, so querying it is opt-in.
+	PrometheusAllowedHosts []string
+
+	// PrometheusPreroll is how far before an alert started its graph begins.
+	PrometheusPreroll time.Duration
+
+	// PrometheusTimeout bounds one request to a Prometheus.
+	PrometheusTimeout time.Duration
 }
 
 // OIDC is the single sign-on configuration. Groups from the provider decide
@@ -151,6 +167,10 @@ func Load() Config {
 		OIDC:           loadOIDC(),
 
 		OperatorMode: boolean("TERDUT_OPERATOR_MODE", false),
+
+		PrometheusAllowedHosts: list("TERDUT_PROMETHEUS_ALLOWED_HOSTS", ""),
+		PrometheusPreroll:      duration("TERDUT_PROMETHEUS_PREROLL", time.Hour),
+		PrometheusTimeout:      duration("TERDUT_PROMETHEUS_TIMEOUT", 10*time.Second),
 	}
 }
 
@@ -180,6 +200,9 @@ func (c Config) Validate() error {
 	if c.OperatorKey != "" && len(c.OperatorKey) < MinOperatorKeyLength {
 		return fmt.Errorf("TERDUT_OPERATOR_KEY must be at least %d characters", MinOperatorKeyLength)
 	}
+	if _, err := prom.ParseAllowlist(c.PrometheusAllowedHosts); err != nil {
+		return fmt.Errorf("TERDUT_PROMETHEUS_ALLOWED_HOSTS: %w", err)
+	}
 	o := c.OIDC
 	if !o.Enabled() {
 		if c.DisablePasswordLogin {

internal/db/migrations/002_incident_metrics.sql

@@ -0,0 +1,25 @@
+-- Metrics captured from the Prometheus an alert came from (its generatorURL),
+-- once when the incident opens and once when it closes, so the graph outlives
+-- Prometheus' own retention. One row per incident, alert and kind.
+--
+-- A row exists whether or not the capture worked: a failure is recorded in
+-- error with the attempt count, which is what stops the worker from retrying a
+-- host that is not on the allowlist (or is gone) forever.
+
+CREATE TABLE incident_metric_snapshots (
+    id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    incident_id bigint NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+    alert_id bigint NOT NULL REFERENCES alerts(id) ON DELETE CASCADE,
+    kind text NOT NULL CHECK (kind IN ('opened', 'closed')),
+    expr text NOT NULL DEFAULT '',
+    window_start bigint NOT NULL,
+    window_end bigint NOT NULL,
+    step_seconds integer NOT NULL DEFAULT 0,
+    series jsonb NOT NULL DEFAULT '[]'::jsonb,
+    error text,
+    attempts integer NOT NULL DEFAULT 1,
+    captured_at bigint NOT NULL,
+    UNIQUE (incident_id, alert_id, kind)
+);
+
+CREATE INDEX incident_metric_snapshots_alert_id_idx ON incident_metric_snapshots (alert_id);

internal/prom/allow.go

@@ -0,0 +1,268 @@
+// Package prom fetches metrics from the Prometheus an alert came from.
+//
+// The address to query is the alert's generatorURL, which arrives in a webhook
+// payload and is therefore chosen by whoever holds an integration key. Left
+// unchecked, that would let the key holder make this server issue requests to
+// any host it can reach. Everything here is built around the Allowlist so that
+// cannot happen: a host is queried only if the operator listed it.
+package prom
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"net/url"
+	"path"
+	"strings"
+)
+
+// Allowlist is the set of Prometheus servers this install may query. The zero
+// value, and an empty list, permit nothing, which is what keeps the feature off
+// until an operator opts in.
+//
+// An entry is one of:
+//
+//   - a host name, optionally with a port: prometheus.monitoring.svc, prom:9090
+//   - a wildcard host name: "*" stands for exactly one DNS label (or part of
+//     one), so *.example.com matches a.example.com but neither example.com nor
+//     a.b.example.com, and prometheus-*.monitoring.svc matches prometheus-0.
+//     A leading "**" label matches one or more labels, so **.example.com also
+//     matches a.b.example.com.
+//   - an IP address or a CIDR range, optionally with a port: 10.0.0.0/8,
+//     [fd00::]:9090
+//
+// Without a port an entry matches any port. Matching is case-insensitive.
+type Allowlist struct {
+	hosts []hostPattern
+	nets  []netEntry
+}
+
+type hostPattern struct {
+	labels []string // lower-cased; a leading "**" label is kept as is
+	port   string   // empty matches any port
+}
+
+type netEntry struct {
+	prefix netip.Prefix
+	port   string
+}
+
+// ParseAllowlist validates and compiles the configured entries. It refuses an
+// entry that would admit every host (a bare "*", "*.*"): a typo there would
+// quietly turn the guard off.
+func ParseAllowlist(entries []string) (*Allowlist, error) {
+	a := &Allowlist{}
+	for _, raw := range entries {
+		e := strings.ToLower(strings.TrimSpace(raw))
+		if e == "" {
+			continue
+		}
+		if err := a.add(e); err != nil {
+			return nil, fmt.Errorf("allowed host %q: %w", raw, err)
+		}
+	}
+	return a, nil
+}
+
+func (a *Allowlist) add(e string) error {
+	// CIDR first: its colons (IPv6) and slash would confuse the host:port split.
+	if strings.Contains(e, "/") {
+		host, port := e, ""
+		if strings.HasPrefix(e, "[") {
+			end := strings.Index(e, "]")
+			if end < 0 {
+				return fmt.Errorf("unterminated [")
+			}
+			host = e[1:end]
+			rest := e[end+1:]
+			if strings.HasPrefix(rest, ":") {
+				port = rest[1:]
+			}
+		}
+		p, err := netip.ParsePrefix(host)
+		if err != nil {
+			return fmt.Errorf("not a CIDR range: %w", err)
+		}
+		if err := checkPort(port); err != nil {
+			return err
+		}
+		a.nets = append(a.nets, netEntry{prefix: p.Masked(), port: port})
+		return nil
+	}
+
+	// A bare IP address, IPv6 included.
+	if ip, err := netip.ParseAddr(e); err == nil {
+		a.nets = append(a.nets, netEntry{prefix: netip.PrefixFrom(ip.Unmap(), ip.Unmap().BitLen())})
+		return nil
+	}
+
+	host, port := e, ""
+	if strings.HasPrefix(e, "[") || strings.Count(e, ":") == 1 {
+		h, p, err := net.SplitHostPort(e)
+		if err != nil {
+			return fmt.Errorf("bad host:port: %w", err)
+		}
+		host, port = h, p
+		if ip, err := netip.ParseAddr(host); err == nil {
+			if err := checkPort(port); err != nil {
+				return err
+			}
+			a.nets = append(a.nets, netEntry{prefix: netip.PrefixFrom(ip.Unmap(), ip.Unmap().BitLen()), port: port})
+			return nil
+		}
+	}
+	if err := checkPort(port); err != nil {
+		return err
+	}
+
+	host = strings.TrimSuffix(host, ".")
+	if host == "" {
+		return fmt.Errorf("empty host")
+	}
+	for _, r := range host {
+		ok := r == '.' || r == '-' || r == '_' || r == '*' || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
+		if !ok {
+			return fmt.Errorf("unexpected character %q", r)
+		}
+	}
+	labels := strings.Split(host, ".")
+	for i, l := range labels {
+		if l == "" {
+			return fmt.Errorf("empty label")
+		}
+		if l == "**" && i != 0 {
+			return fmt.Errorf("** is only allowed as the first label")
+		}
+		if strings.Contains(l, "**") && l != "**" {
+			return fmt.Errorf("** must be a whole label")
+		}
+	}
+	// The last label has to say something, or the entry is allow-all.
+	if strings.Trim(labels[len(labels)-1], "*") == "" {
+		return fmt.Errorf("would match every host: the last label cannot be a wildcard alone")
+	}
+	a.hosts = append(a.hosts, hostPattern{labels: labels, port: port})
+	return nil
+}
+
+func checkPort(p string) error {
+	if p == "" {
+		return nil
+	}
+	for _, r := range p {
+		if r < '0' || r > '9' {
+			return fmt.Errorf("bad port %q", p)
+		}
+	}
+	return nil
+}
+
+// Empty reports whether nothing is allowed, i.e. the feature is off.
+func (a *Allowlist) Empty() bool {
+	return a == nil || (len(a.hosts) == 0 && len(a.nets) == 0)
+}
+
+// Port returns the effective port of u: the explicit one, else the scheme's.
+func Port(u *url.URL) string {
+	if p := u.Port(); p != "" {
+		return p
+	}
+	if u.Scheme == "https" {
+		return "443"
+	}
+	return "80"
+}
+
+// permits decides whether host:port may be contacted at ip. host is what the
+// URL said (a name or an IP literal), ip is where it actually resolved to; the
+// check runs on the resolved address at dial time so that a name that resolves
+// somewhere else than it did when it was listed (DNS rebinding) is caught.
+//
+//   - An address inside a listed CIDR range or equal to a listed IP is allowed.
+//   - A host name that matches a listed pattern is allowed, unless it resolved
+//     to an address that is never a Prometheus (loopback, link-local, which
+//     includes the cloud metadata address, unspecified, multicast). A wildcard
+//     over a domain somebody else controls must not be a way to reach internal
+//     services. To allow such an address deliberately, list it as a CIDR.
+func (a *Allowlist) permits(host, port string, ip netip.Addr) bool {
+	if a == nil {
+		return false
+	}
+	ip = ip.Unmap()
+	for _, n := range a.nets {
+		if n.prefix.Contains(ip) && (n.port == "" || n.port == port) {
+			return true
+		}
+	}
+	if forbiddenAddr(ip) {
+		return false
+	}
+	if _, isIP := parseHostIP(host); isIP {
+		return false // an IP literal is allowed only through a CIDR entry
+	}
+	return a.matchesName(host, port)
+}
+
+func parseHostIP(host string) (netip.Addr, bool) {
+	ip, err := netip.ParseAddr(strings.Trim(host, "[]"))
+	return ip, err == nil
+}
+
+// MatchesHost reports whether the URL's host is worth dialing at all: a listed
+// name, or an IP that falls in a listed range. It is the cheap early answer;
+// the dial-time check in permits is the one that is authoritative for names.
+func (a *Allowlist) MatchesHost(u *url.URL) bool {
+	if a.Empty() {
+		return false
+	}
+	host := strings.ToLower(strings.TrimSuffix(u.Hostname(), "."))
+	port := Port(u)
+	if ip, ok := parseHostIP(host); ok {
+		return a.permits(host, port, ip)
+	}
+	if a.matchesName(host, port) {
+		return true
+	}
+	// A name outside the patterns can still be admitted by a CIDR entry once it
+	// resolves; leave that to the dialer.
+	return len(a.nets) > 0
+}
+
+func (a *Allowlist) matchesName(host, port string) bool {
+	labels := strings.Split(strings.TrimSuffix(strings.ToLower(host), "."), ".")
+	for _, p := range a.hosts {
+		if (p.port == "" || p.port == port) && matchLabels(p.labels, labels) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchLabels(pattern, host []string) bool {
+	if len(pattern) > 0 && pattern[0] == "**" {
+		rest := pattern[1:]
+		if len(host) <= len(rest) {
+			return false
+		}
+		return matchLabels(rest, host[len(host)-len(rest):])
+	}
+	if len(pattern) != len(host) {
+		return false
+	}
+	for i := range pattern {
+		// The pattern was restricted to [a-z0-9_.-*] when parsed, so path.Match
+		// sees nothing but literals and "*" here.
+		if ok, _ := path.Match(pattern[i], host[i]); !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// forbiddenAddr is the set of addresses a Prometheus is never reached at and
+// that are dangerous to reach by mistake. Private ranges are deliberately not
+// in it: a Prometheus inside the cluster lives there.
+func forbiddenAddr(ip netip.Addr) bool {
+	return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified()
+}

internal/prom/client.go

@@ -0,0 +1,274 @@
+package prom
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	// maxResponseBytes bounds what one query may make us read.
+	maxResponseBytes = 5 << 20
+
+	// maxSeries bounds how many series one query may return before we give up on
+	// it; a rule expression without aggregation can match thousands.
+	maxSeries = 2000
+
+	// TargetPoints is how many points a graph is asked for.
+	TargetPoints = 300
+)
+
+// ErrNotAllowed means the target host is not on the allowlist.
+var ErrNotAllowed = errors.New("host is not in the allowed Prometheus hosts")
+
+// Series is one time series, ready to chart. Points are [unix seconds, value].
+type Series struct {
+	Labels map[string]string `json:"labels"`
+	Points [][2]float64      `json:"points"`
+}
+
+// Client queries Prometheus servers that the Allowlist permits.
+type Client struct {
+	http  *http.Client
+	allow *Allowlist
+}
+
+// NewClient returns a client that only ever connects to allowed addresses. The
+// check is made on the address the dialer is about to use, after name
+// resolution, and redirects and environment proxies are off: either would let a
+// permitted host send the request somewhere that is not.
+func NewClient(allow *Allowlist, timeout time.Duration) *Client {
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+	dialer := &net.Dialer{Timeout: timeout}
+	tr := &http.Transport{
+		Proxy:               nil,
+		DialContext:         guardedDial(allow, dialer),
+		TLSHandshakeTimeout: timeout,
+		MaxIdleConns:        4,
+		IdleConnTimeout:     30 * time.Second,
+	}
+	return &Client{
+		allow: allow,
+		http: &http.Client{
+			Timeout:   timeout,
+			Transport: tr,
+			CheckRedirect: func(*http.Request, []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		},
+	}
+}
+
+// Enabled reports whether any host is allowed at all.
+func (c *Client) Enabled() bool { return c != nil && !c.allow.Empty() }
+
+// Allowed is the early, resolution-free answer: whether base could be queried.
+func (c *Client) Allowed(base *url.URL) bool {
+	return c != nil && c.allow.MatchesHost(base)
+}
+
+func guardedDial(allow *Allowlist, d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var ips []netip.Addr
+		if ip, err := netip.ParseAddr(host); err == nil {
+			ips = []netip.Addr{ip}
+		} else {
+			ips, err = net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+			if err != nil {
+				return nil, err
+			}
+		}
+		var lastErr error = ErrNotAllowed
+		for _, ip := range ips {
+			if !allow.permits(host, port, ip) {
+				continue
+			}
+			// Dial the checked address itself, so a second lookup cannot return
+			// something different.
+			conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+			if err == nil {
+				return conn, nil
+			}
+			lastErr = err
+		}
+		return nil, lastErr
+	}
+}
+
+// QueryRange runs expr over [start, end] at the given step.
+func (c *Client) QueryRange(ctx context.Context, base *url.URL, expr string, start, end time.Time, step time.Duration) ([]Series, error) {
+	form := url.Values{
+		"query": {expr},
+		"start": {strconv.FormatInt(start.Unix(), 10)},
+		"end":   {strconv.FormatInt(end.Unix(), 10)},
+		"step":  {strconv.FormatInt(int64(step/time.Second), 10)},
+	}
+	res, err := c.do(ctx, base, "/api/v1/query_range", form)
+	if err != nil {
+		return nil, err
+	}
+	if res.ResultType != "matrix" {
+		return nil, fmt.Errorf("unexpected result type %q", res.ResultType)
+	}
+	var raw []struct {
+		Metric map[string]string    `json:"metric"`
+		Values [][2]json.RawMessage `json:"values"`
+	}
+	if err := json.Unmarshal(res.Result, &raw); err != nil {
+		return nil, fmt.Errorf("decode matrix: %w", err)
+	}
+	out := make([]Series, 0, len(raw))
+	for _, r := range raw {
+		s := Series{Labels: r.Metric, Points: make([][2]float64, 0, len(r.Values))}
+		for _, v := range r.Values {
+			if p, ok := point(v); ok {
+				s.Points = append(s.Points, p)
+			}
+		}
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+// Query runs expr at a single instant and returns one point per series.
+func (c *Client) Query(ctx context.Context, base *url.URL, expr string, at time.Time) ([]Series, error) {
+	form := url.Values{
+		"query": {expr},
+		"time":  {strconv.FormatInt(at.Unix(), 10)},
+	}
+	res, err := c.do(ctx, base, "/api/v1/query", form)
+	if err != nil {
+		return nil, err
+	}
+	if res.ResultType != "vector" {
+		return nil, fmt.Errorf("unexpected result type %q", res.ResultType)
+	}
+	var raw []struct {
+		Metric map[string]string  `json:"metric"`
+		Value  [2]json.RawMessage `json:"value"`
+	}
+	if err := json.Unmarshal(res.Result, &raw); err != nil {
+		return nil, fmt.Errorf("decode vector: %w", err)
+	}
+	out := make([]Series, 0, len(raw))
+	for _, r := range raw {
+		s := Series{Labels: r.Metric}
+		if p, ok := point(r.Value); ok {
+			s.Points = [][2]float64{p}
+		}
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+type apiResult struct {
+	ResultType string          `json:"resultType"`
+	Result     json.RawMessage `json:"result"`
+}
+
+func (c *Client) do(ctx context.Context, base *url.URL, apiPath string, form url.Values) (*apiResult, error) {
+	if !c.Allowed(base) {
+		return nil, ErrNotAllowed
+	}
+	u := *base
+	u.Path = strings.TrimSuffix(base.Path, "/") + apiPath
+	// POST, so a long expression is not limited by URL length.
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(form.Encode()))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(body) > maxResponseBytes {
+		return nil, errors.New("response too large")
+	}
+	var env struct {
+		Status    string     `json:"status"`
+		Data      *apiResult `json:"data"`
+		ErrorType string     `json:"errorType"`
+		Error     string     `json:"error"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		return nil, fmt.Errorf("prometheus answered %d with a body that is not JSON", resp.StatusCode)
+	}
+	if resp.StatusCode/100 != 2 || env.Status != "success" || env.Data == nil {
+		msg := env.Error
+		if msg == "" {
+			msg = "status " + strconv.Itoa(resp.StatusCode)
+		}
+		return nil, fmt.Errorf("prometheus: %s", truncate(msg, 200))
+	}
+	if n := countResults(env.Data.Result); n > maxSeries {
+		return nil, fmt.Errorf("query matches %d series, over the limit of %d", n, maxSeries)
+	}
+	return env.Data, nil
+}
+
+func countResults(raw json.RawMessage) int {
+	var items []json.RawMessage
+	if json.Unmarshal(raw, &items) != nil {
+		return 0
+	}
+	return len(items)
+}
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}
+
+// point decodes a [timestamp, "value"] pair. Prometheus writes values as
+// strings so that NaN and ±Inf survive; JSON cannot carry them, so those points
+// are dropped.
+func point(v [2]json.RawMessage) ([2]float64, bool) {
+	var ts float64
+	var s string
+	if json.Unmarshal(v[0], &ts) != nil || json.Unmarshal(v[1], &s) != nil {
+		return [2]float64{}, false
+	}
+	f, err := strconv.ParseFloat(s, 64)
+	if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
+		return [2]float64{}, false
+	}
+	return [2]float64{ts, f}, true
+}
+
+// StepFor picks a query step that yields about TargetPoints over the span, never
+// finer than a second.
+func StepFor(start, end time.Time) time.Duration {
+	span := end.Sub(start)
+	step := span / TargetPoints
+	if step < time.Second {
+		return time.Second
+	}
+	return step.Round(time.Second)
+}

internal/prom/filter.go

@@ -0,0 +1,51 @@
+package prom
+
+import "sort"
+
+// MaxSeriesPerAlert caps how many series are kept for one alert.
+const MaxSeriesPerAlert = 5
+
+// SelectSeries keeps the series that belong to this alert. The rule expression
+// returns every series it matches, not just the one that fired, so a series is
+// kept only if it agrees with the alert's labels on every label name they both
+// carry. Labels only one side has (the rule's severity, the series' __name__)
+// say nothing either way, and alertname is the rule's, not the series'.
+//
+// The result is capped at MaxSeriesPerAlert, best-labelled first: the more
+// labels a series shares with the alert, the likelier it is the one that fired.
+func SelectSeries(series []Series, alertLabels map[string]string) []Series {
+	type scored struct {
+		s     Series
+		score int
+	}
+	var kept []scored
+	for _, s := range series {
+		score, ok := 0, true
+		for k, v := range s.Labels {
+			if k == "__name__" || k == "alertname" {
+				continue
+			}
+			av, has := alertLabels[k]
+			if !has {
+				continue
+			}
+			if av != v {
+				ok = false
+				break
+			}
+			score++
+		}
+		if ok && len(s.Points) > 0 {
+			kept = append(kept, scored{s, score})
+		}
+	}
+	sort.SliceStable(kept, func(i, j int) bool { return kept[i].score > kept[j].score })
+	if len(kept) > MaxSeriesPerAlert {
+		kept = kept[:MaxSeriesPerAlert]
+	}
+	out := make([]Series, len(kept))
+	for i, k := range kept {
+		out[i] = k.s
+	}
+	return out
+}

internal/prom/parse.go

@@ -0,0 +1,38 @@
+package prom
+
+import (
+	"net/url"
+	"strings"
+)
+
+// ParseGeneratorURL splits an Alertmanager generatorURL into the Prometheus it
+// points at and the expression of the rule that fired. Prometheus builds it as
+// <external-url>/graph?g0.expr=<expr>&g0.tab=1 (/query on Prometheus 3), and the
+// external URL may carry a path prefix, which is kept in base.
+//
+// ok is false for anything else: a non-HTTP scheme, credentials in the URL, a
+// page that is not the graph page, or no expression. The caller then has
+// nothing to query and falls back to showing the link.
+func ParseGeneratorURL(raw string) (base *url.URL, expr string, ok bool) {
+	u, err := url.Parse(strings.TrimSpace(raw))
+	if err != nil || u.Host == "" || u.User != nil {
+		return nil, "", false
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, "", false
+	}
+	prefix := ""
+	switch {
+	case strings.HasSuffix(u.Path, "/graph"):
+		prefix = strings.TrimSuffix(u.Path, "/graph")
+	case strings.HasSuffix(u.Path, "/query"):
+		prefix = strings.TrimSuffix(u.Path, "/query")
+	default:
+		return nil, "", false
+	}
+	expr = strings.TrimSpace(u.Query().Get("g0.expr"))
+	if expr == "" {
+		return nil, "", false
+	}
+	return &url.URL{Scheme: u.Scheme, Host: u.Host, Path: prefix}, expr, true
+}

internal/prom/prom_test.go

@@ -0,0 +1,237 @@
+package prom
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"testing"
+	"time"
+)
+
+func TestParseAllowlistRejectsAllowAll(t *testing.T) {
+	for _, e := range []string{"*", "**", "*.*", "**.*", "a.**", "pro**.example.com", "ex ample.com", "host:abc", "10.0.0.0/33"} {
+		if _, err := ParseAllowlist([]string{e}); err == nil {
+			t.Errorf("%q was accepted", e)
+		}
+	}
+}
+
+func TestAllowlistNames(t *testing.T) {
+	a, err := ParseAllowlist([]string{
+		"prometheus.monitoring.svc",
+		"prom-*.monitoring.svc",
+		"*.example.com",
+		"**.corp.example",
+		"fixed.example.org:9090",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	pub := netip.MustParseAddr("10.1.2.3")
+	cases := []struct {
+		host, port string
+		want       bool
+	}{
+		{"prometheus.monitoring.svc", "9090", true},
+		{"PROMETHEUS.monitoring.svc", "80", true},
+		{"prom-0.monitoring.svc", "9090", true},
+		{"prom-0.other.svc", "9090", false},
+		{"a.example.com", "443", true},
+		{"example.com", "443", false},
+		{"a.b.example.com", "443", false},
+		{"a.corp.example", "80", true},
+		{"a.b.c.corp.example", "80", true},
+		{"corp.example", "80", false},
+		{"fixed.example.org", "9090", true},
+		{"fixed.example.org", "80", false},
+		{"evilexample.com", "80", false},
+		{"a.example.com.evil.net", "80", false},
+	}
+	for _, c := range cases {
+		if got := a.permits(c.host, c.port, pub); got != c.want {
+			t.Errorf("permits(%s:%s) = %v, want %v", c.host, c.port, got, c.want)
+		}
+	}
+}
+
+func TestAllowlistForbiddenAddressesAndCIDR(t *testing.T) {
+	a, _ := ParseAllowlist([]string{"*.example.com", "192.168.0.0/16", "[fd00::]:9090", "127.0.0.9"})
+	cases := []struct {
+		name, host, port, ip string
+		want                 bool
+	}{
+		{"wildcard to private is fine", "a.example.com", "80", "10.0.0.5", true},
+		{"wildcard rebinding to loopback", "a.example.com", "80", "127.0.0.1", false},
+		{"wildcard to metadata", "a.example.com", "80", "169.254.169.254", false},
+		{"wildcard to unspecified", "a.example.com", "80", "0.0.0.0", false},
+		{"wildcard to v4-mapped loopback", "a.example.com", "80", "::ffff:127.0.0.1", false},
+		{"cidr literal", "192.168.4.5", "80", "192.168.4.5", true},
+		{"outside cidr", "10.9.9.9", "80", "10.9.9.9", false},
+		{"cidr admits unlisted name", "anything.test", "80", "192.168.1.1", true},
+		{"explicit loopback entry", "127.0.0.9", "80", "127.0.0.9", true},
+		{"other loopback", "127.0.0.1", "80", "127.0.0.1", false},
+		{"ipv6 with port ok", "fd00::1", "9090", "fd00::1", false}, // entry is a single address fd00::, not the range
+		{"ipv6 exact", "fd00::", "9090", "fd00::", true},
+		{"ipv6 wrong port", "fd00::", "80", "fd00::", false},
+	}
+	for _, c := range cases {
+		if got := a.permits(c.host, c.port, netip.MustParseAddr(c.ip)); got != c.want {
+			t.Errorf("%s: permits = %v, want %v", c.name, got, c.want)
+		}
+	}
+}
+
+func TestEmptyAllowlistPermitsNothing(t *testing.T) {
+	var nilList *Allowlist
+	empty, _ := ParseAllowlist(nil)
+	for _, a := range []*Allowlist{nilList, empty} {
+		if !a.Empty() || a.permits("x.example.com", "80", netip.MustParseAddr("10.0.0.1")) {
+			t.Error("empty allowlist permitted something")
+		}
+	}
+}
+
+func TestParseGeneratorURL(t *testing.T) {
+	cases := []struct {
+		raw      string
+		wantBase string
+		wantExpr string
+		ok       bool
+	}{
+		{"http://prom:9090/graph?g0.expr=up%3D%3D0&g0.tab=1", "http://prom:9090", "up==0", true},
+		{"https://p.example.com/prom/graph?g0.expr=rate%28x%5B5m%5D%29%3E1", "https://p.example.com/prom", "rate(x[5m])>1", true},
+		{"https://p.example.com/query?g0.expr=up", "https://p.example.com", "up", true},
+		{"http://prom:9090/graph?g0.tab=1", "", "", false},
+		{"http://prom:9090/other?g0.expr=up", "", "", false},
+		{"ftp://prom/graph?g0.expr=up", "", "", false},
+		{"http://user:pw@prom/graph?g0.expr=up", "", "", false},
+		{"", "", "", false},
+		{"not a url", "", "", false},
+	}
+	for _, c := range cases {
+		base, expr, ok := ParseGeneratorURL(c.raw)
+		if ok != c.ok || expr != c.wantExpr || (ok && base.String() != c.wantBase) {
+			t.Errorf("%q = (%v, %q, %v), want (%q, %q, %v)", c.raw, base, expr, ok, c.wantBase, c.wantExpr, c.ok)
+		}
+	}
+}
+
+func TestSelectSeries(t *testing.T) {
+	mk := func(l map[string]string) Series { return Series{Labels: l, Points: [][2]float64{{1, 1}}} }
+	series := []Series{
+		mk(map[string]string{"__name__": "up", "instance": "a", "job": "x"}),
+		mk(map[string]string{"__name__": "up", "instance": "b", "job": "x"}),
+		mk(map[string]string{"instance": "a", "job": "y"}),
+		{Labels: map[string]string{"instance": "a"}}, // no points
+	}
+	got := SelectSeries(series, map[string]string{"alertname": "Down", "instance": "a", "job": "x", "severity": "page"})
+	if len(got) != 1 || got[0].Labels["job"] != "x" || got[0].Labels["instance"] != "a" {
+		t.Fatalf("got %+v", got)
+	}
+	if n := len(SelectSeries(series, nil)); n != 3 {
+		t.Errorf("no alert labels should keep every series with points, got %d", n)
+	}
+}
+
+func TestStepFor(t *testing.T) {
+	now := time.Now()
+	if s := StepFor(now.Add(-time.Minute), now); s != time.Second {
+		t.Errorf("short span step = %v", s)
+	}
+	if s := StepFor(now.Add(-5*time.Hour), now); s != time.Minute {
+		t.Errorf("5h step = %v, want 1m", s)
+	}
+}
+
+func fakeProm(t *testing.T) (*httptest.Server, *int) {
+	t.Helper()
+	hits := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits++
+		r.ParseForm()
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v1/query_range":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[
+				{"metric":{"instance":"a"},"values":[[100,"1"],[110,"NaN"],[120,"2.5"]]}]}}`))
+		case "/api/v1/query":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[
+				{"metric":{"instance":"a"},"value":[130,"3"]}]}}`))
+		case "/redirect/api/v1/query":
+			http.Redirect(w, r, "http://127.0.0.1:1/", http.StatusFound)
+		default:
+			w.WriteHeader(400)
+			w.Write([]byte(`{"status":"error","errorType":"bad_data","error":"nope"}`))
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hits
+}
+
+func TestClientQueries(t *testing.T) {
+	srv, _ := fakeProm(t)
+	u, _ := url.Parse(srv.URL)
+	// httptest listens on loopback, which is refused unless listed as a CIDR.
+	allow, _ := ParseAllowlist([]string{"127.0.0.0/8"})
+	c := NewClient(allow, 5*time.Second)
+
+	got, err := c.QueryRange(context.Background(), u, "up", time.Unix(100, 0), time.Unix(120, 0), 10*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || len(got[0].Points) != 2 || got[0].Points[1] != [2]float64{120, 2.5} {
+		t.Fatalf("range = %+v (NaN point should be dropped)", got)
+	}
+	inst, err := c.Query(context.Background(), u, "up", time.Unix(130, 0))
+	if err != nil || len(inst) != 1 || inst[0].Points[0][1] != 3 {
+		t.Fatalf("instant = %+v, %v", inst, err)
+	}
+}
+
+func TestClientRefusesUnlistedAndLoopback(t *testing.T) {
+	srv, hits := fakeProm(t)
+	u, _ := url.Parse(srv.URL)
+
+	// Listed by name, but the name resolves to loopback: refused at dial time.
+	local, _ := url.Parse("http://localhost:" + u.Port())
+	wild, _ := ParseAllowlist([]string{"local*"})
+	if _, err := NewClient(wild, time.Second).Query(context.Background(), local, "up", time.Now()); err == nil {
+		t.Error("wildcard-matched name resolving to loopback was queried")
+	}
+
+	// Not listed at all.
+	other, _ := ParseAllowlist([]string{"*.example.com"})
+	if _, err := NewClient(other, time.Second).Query(context.Background(), u, "up", time.Now()); !errors.Is(err, ErrNotAllowed) {
+		t.Errorf("unlisted host: err = %v", err)
+	}
+
+	// Disabled.
+	if _, err := NewClient(nil, time.Second).Query(context.Background(), u, "up", time.Now()); !errors.Is(err, ErrNotAllowed) {
+		t.Errorf("disabled: err = %v", err)
+	}
+	if *hits != 0 {
+		t.Errorf("server was contacted %d times", *hits)
+	}
+}
+
+func TestClientDoesNotFollowRedirects(t *testing.T) {
+	srv, _ := fakeProm(t)
+	u, _ := url.Parse(srv.URL + "/redirect")
+	allow, _ := ParseAllowlist([]string{"127.0.0.0/8"})
+	if _, err := NewClient(allow, time.Second).Query(context.Background(), u, "up", time.Now()); err == nil {
+		t.Error("redirect response was treated as success")
+	}
+}
+
+func TestClientSurfacesPrometheusError(t *testing.T) {
+	srv, _ := fakeProm(t)
+	u, _ := url.Parse(srv.URL + "/missing")
+	allow, _ := ParseAllowlist([]string{"127.0.0.0/8"})
+	_, err := NewClient(allow, time.Second).Query(context.Background(), u, "up", time.Now())
+	if err == nil {
+		t.Fatal("expected an error")
+	}
+}

internal/web/static/app.css

@@ -600,6 +600,33 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
 .alert-item-summary { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
 .alert-item-foot { display: flex; flex-wrap: wrap; gap: 4px 12px; font-size: 13px; color: var(--faint); }
 .alert-item-foot a { color: var(--accent); font-weight: 600; }
+
+/* Metrics under an alert */
+.metrics { display: grid; gap: 6px; }
+.metrics-note { color: var(--faint); font-size: 13px; }
+.metrics-now { display: flex; align-items: baseline; gap: 8px; font-size: 14px; }
+.metrics-now strong { font-size: 18px; font-variant-numeric: tabular-nums; }
+.metrics-now.none { color: var(--muted); }
+.metrics-now-label { color: var(--faint); font-size: 12px; font-weight: 650; text-transform: uppercase; letter-spacing: .04em; }
+.metric-chart { width: 100%; height: auto; display: block; }
+.mc-grid { stroke: var(--border); stroke-width: 1; }
+.mc-label { fill: var(--faint); font-size: 10px; }
+.mc-marker { stroke-width: 1; stroke-dasharray: 3 3; }
+.mc-marker.start { stroke: var(--crit); }
+.mc-marker.end { stroke: var(--ok); }
+.mc-line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; }
+.mc-line.s0, .mc-swatch.s0 { stroke: var(--accent); background: var(--accent); }
+.mc-line.s1, .mc-swatch.s1 { stroke: var(--warn); background: var(--warn); }
+.mc-line.s2, .mc-swatch.s2 { stroke: var(--ok); background: var(--ok); }
+.mc-line.s3, .mc-swatch.s3 { stroke: var(--crit); background: var(--crit); }
+.mc-line.s4, .mc-swatch.s4 { stroke: var(--muted); background: var(--muted); }
+.mc-dot.s0 { fill: var(--accent); } .mc-dot.s1 { fill: var(--warn); } .mc-dot.s2 { fill: var(--ok); }
+.mc-dot.s3 { fill: var(--crit); } .mc-dot.s4 { fill: var(--muted); }
+.mc-swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; flex: none; }
+.metrics-legend { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 2px 14px; font-size: 12px; color: var(--muted); }
+.metrics-legend li { display: flex; align-items: center; gap: 6px; overflow-wrap: anywhere; }
+.metrics-expr { font-size: 12px; color: var(--faint); overflow-wrap: anywhere; }
+
 /* A disclosure is a button: a chevron that turns, and a 44px target on a
    touch screen (the bare triangle it replaces was a few pixels wide). */
 details > summary {

internal/web/static/js/api.js

@@ -108,6 +108,7 @@ export const unsnooze = (id) => call('DELETE', `/incidents/${id}/snooze`);
 export const archive = (id) => call('POST', `/incidents/${id}/archive`);
 export const unarchive = (id) => call('DELETE', `/incidents/${id}/archive`);
 export const addNote = (id, content, pinned = false) => call('POST', `/incidents/${id}/notes`, { body: { content, pinned } });
+export const metrics = (id) => call('GET', `/incidents/${id}/metrics`);
 export const similar = (id) => call('GET', `/incidents/${id}/similar`);
 export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/notes/${eventID}`);
 

internal/web/static/js/incident.js

@@ -11,6 +11,7 @@ import {
 } from './format.js';
 import { myID, users } from './state.js';
 import { back } from './app.js';
+import { alertMetrics } from './metricchart.js';
 
 const pane = () => document.getElementById('detail');
 
@@ -18,6 +19,7 @@ let currentID = null;
 let inc = null;
 let events = [];
 let similarList = [];
+let metricsByAlert = {};
 let error = null;
 let busy = false;
 
@@ -26,6 +28,7 @@ export function show(id) {
   currentID = id;
   inc = null;
   events = [];
+  metricsByAlert = {};
   error = null;
   if (id == null) {
     renderPlaceholder();
@@ -41,13 +44,17 @@ export async function refresh() {
   try {
     // Similar incidents are a courtesy: an older server answers 404 and a
     // failure here must not hide the incident itself.
-    const [i, t, sim] = await Promise.all([
-      api.incident(id), api.timeline(id), api.similar(id).catch(() => []),
+    // Metrics are a courtesy too: an older server has no such endpoint, and a
+    // Prometheus that is slow or down must not hold up the incident.
+    const [i, t, sim, met] = await Promise.all([
+      api.incident(id), api.timeline(id), api.similar(id).catch(() => []), api.metrics(id).catch(() => null),
     ]);
     if (id !== currentID) return;
     inc = i;
     events = t;
     similarList = sim;
+    metricsByAlert = {};
+    if (met && met.enabled) for (const m of met.alerts) metricsByAlert[m.alert_id] = m;
     error = null;
   } catch (err) {
     if (id !== currentID) return;
@@ -203,6 +210,7 @@ function alertItem(a) {
       h('span', { class: 'alert-item-name', text: a.name }),
       badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`)),
     summary && h('div', { class: 'alert-item-summary', text: summary }),
+    alertMetrics(metricsByAlert[a.id], a.starts_at, inc.resolved_at),
     h('div', { class: 'alert-item-foot' },
       h('span', { text: `Started ${ago(a.starts_at)}` }),
       h('span', { text: `Last seen ${ago(a.received_at)}` }),

internal/web/static/js/metricchart.js

@@ -0,0 +1,196 @@
+// The metrics under one alert: a line chart of the expression that fired it, and
+// what that expression reads right now. The data comes from this server, which
+// asked the alert's own Prometheus (see GET /api/incidents/{id}/metrics); the
+// browser never talks to Prometheus.
+
+import { h } from './ui.js';
+import { when } from './format.js';
+
+const SVG_NS = 'http://www.w3.org/2000/svg';
+const MAX_SERIES = 5;
+
+function svg(tag, attrs = {}, text) {
+  const el = document.createElementNS(SVG_NS, tag);
+  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
+  if (text != null) el.textContent = text;
+  return el;
+}
+
+const iso = (sec) => new Date(sec * 1000).toISOString();
+
+// Compact number for an axis or a badge: 1.2k, 0.034, 12.
+export function fmtValue(v) {
+  if (v == null || Number.isNaN(v)) return '–';
+  const a = Math.abs(v);
+  if (a >= 1e9) return `${+(v / 1e9).toPrecision(3)}G`;
+  if (a >= 1e6) return `${+(v / 1e6).toPrecision(3)}M`;
+  if (a >= 1e4) return `${+(v / 1e3).toPrecision(3)}k`;
+  if (a === 0) return '0';
+  if (a >= 100) return String(Math.round(v));
+  return String(+v.toPrecision(3));
+}
+
+const keyOf = (labels) => JSON.stringify(Object.entries(labels || {}).sort());
+
+// seriesName is the labels that tell the series apart, e.g. {instance="web-1"}.
+function seriesName(labels) {
+  const parts = Object.entries(labels || {}).filter(([k]) => k !== '__name__').map(([k, v]) => `${k}="${v}"`);
+  return parts.length ? `{${parts.join(', ')}}` : 'value';
+}
+
+// merge combines the snapshot taken when the incident opened with the live
+// window, so the chart keeps history Prometheus may since have dropped. On the
+// same timestamp the later source wins.
+function merge(...sources) {
+  const byKey = new Map();
+  for (const src of sources) {
+    for (const s of src || []) {
+      const k = keyOf(s.labels);
+      const cur = byKey.get(k) || { labels: s.labels, points: new Map() };
+      for (const [t, v] of s.points || []) cur.points.set(t, v);
+      byKey.set(k, cur);
+    }
+  }
+  return [...byKey.values()].map((s) => ({
+    labels: s.labels,
+    points: [...s.points.entries()].sort((a, b) => a[0] - b[0]),
+  }));
+}
+
+function parseSeries(raw) {
+  if (Array.isArray(raw)) return raw;
+  return [];
+}
+
+// pick chooses what to draw: while the incident is open, the opened snapshot
+// plus live data; once it is closed, the closed snapshot (the whole incident),
+// falling back to the opened one.
+function pick(entry) {
+  const snaps = entry.snapshots || {};
+  const live = entry.live;
+  if (live && !live.error) {
+    return {
+      series: merge(parseSeries(snaps.opened && snaps.opened.series), live.series),
+      from: Math.min(live.window_start, snaps.opened ? snaps.opened.window_start : Infinity),
+      to: live.window_end,
+      current: live.current || [],
+      isLive: true,
+    };
+  }
+  const snap = snaps.closed || snaps.opened;
+  if (snap && !snap.error) {
+    return {
+      series: merge(parseSeries(snap.series)), from: snap.window_start, to: snap.window_end, current: [], isLive: false,
+    };
+  }
+  return null;
+}
+
+// alertMetrics renders the metrics block for one alert, or null when there is
+// nothing worth saying (the feature off, or no Prometheus link on the alert).
+export function alertMetrics(entry, startsAtISO, resolvedISO) {
+  if (!entry) return null;
+  if (!entry.available) {
+    return entry.reason && entry.expr
+      ? h('div', { class: 'metrics-note', text: `Metrics unavailable: ${entry.reason}.` })
+      : null;
+  }
+  const data = pick(entry);
+  if (!data) {
+    const err = (entry.live && entry.live.error)
+      || (entry.snapshots && ((entry.snapshots.closed && entry.snapshots.closed.error) || (entry.snapshots.opened && entry.snapshots.opened.error)));
+    return h('div', { class: 'metrics-note', text: err ? `Metrics could not be fetched: ${err}` : 'Metrics are being collected…' });
+  }
+  const startSec = startsAtISO ? Date.parse(startsAtISO) / 1000 : null;
+  const endSec = resolvedISO ? Date.parse(resolvedISO) / 1000 : null;
+  return h('div', { class: 'metrics' },
+    currentLine(entry, data),
+    data.series.length ? chart(data, startSec, endSec) : h('div', { class: 'metrics-note', text: 'No samples in this window.' }),
+    legend(data.series),
+    entry.expr && h('code', { class: 'metrics-expr', text: entry.expr }),
+  );
+}
+
+function currentLine(entry, data) {
+  if (!data.isLive) return null;
+  const vals = data.current.map((s) => s.points && s.points[0] && s.points[0][1]).filter((v) => v != null);
+  if (!vals.length) {
+    return h('div', { class: 'metrics-now none' },
+      h('span', { class: 'metrics-now-label', text: 'Now' }),
+      h('span', { text: 'no data — the alert condition no longer holds' }));
+  }
+  return h('div', { class: 'metrics-now' },
+    h('span', { class: 'metrics-now-label', text: 'Now' }),
+    h('strong', { text: vals.length === 1 ? fmtValue(vals[0]) : vals.map(fmtValue).join(' · ') }));
+}
+
+function legend(series) {
+  if (series.length < 2 && !(series[0] && Object.keys(series[0].labels || {}).length)) return null;
+  return h('ul', { class: 'metrics-legend' }, series.slice(0, MAX_SERIES).map((s, i) =>
+    h('li', {}, h('span', { class: `mc-swatch s${i}` }), h('span', { text: seriesName(s.labels) }))));
+}
+
+function chart(data, startSec, endSec) {
+  const W = 480;
+  const H = 130;
+  const padL = 40;
+  const padR = 8;
+  const padT = 8;
+  const padB = 18;
+  const series = data.series.slice(0, MAX_SERIES);
+
+  let t0 = Infinity; let t1 = -Infinity; let lo = Infinity; let hi = -Infinity;
+  for (const s of series) {
+    for (const [t, v] of s.points) {
+      if (t < t0) t0 = t;
+      if (t > t1) t1 = t;
+      if (v < lo) lo = v;
+      if (v > hi) hi = v;
+    }
+  }
+  if (!Number.isFinite(t0)) { t0 = data.from; t1 = data.to; lo = 0; hi = 1; }
+  // Show the whole window, not just where there are samples, so the shape of
+  // "it only exists while firing" is visible.
+  t0 = Math.min(t0, data.from);
+  t1 = Math.max(t1, data.to);
+  if (t1 === t0) t1 = t0 + 1;
+  if (lo === hi) { lo -= 1; hi += 1; }
+  if (lo > 0 && lo < (hi - lo) * 0.5) lo = 0;
+
+  const x = (t) => padL + ((t - t0) / (t1 - t0)) * (W - padL - padR);
+  const y = (v) => padT + (1 - (v - lo) / (hi - lo)) * (H - padT - padB);
+
+  const root = svg('svg', {
+    class: 'metric-chart', viewBox: `0 0 ${W} ${H}`, role: 'img',
+    'aria-label': `Metric over time, ${fmtValue(lo)} to ${fmtValue(hi)}, from ${when(iso(t0))} to ${when(iso(t1))}`,
+  });
+  // Gridlines and y labels at the extremes and the middle.
+  for (const v of [lo, (lo + hi) / 2, hi]) {
+    root.appendChild(svg('line', { class: 'mc-grid', x1: padL, x2: W - padR, y1: y(v), y2: y(v) }));
+    root.appendChild(svg('text', { class: 'mc-label', x: padL - 4, y: y(v) + 3, 'text-anchor': 'end' }, fmtValue(v)));
+  }
+  root.appendChild(svg('text', { class: 'mc-label', x: padL, y: H - 4, 'text-anchor': 'start' }, when(iso(t0))));
+  root.appendChild(svg('text', { class: 'mc-label', x: W - padR, y: H - 4, 'text-anchor': 'end' }, when(iso(t1))));
+
+  const marker = (t, cls, text) => {
+    if (t == null || t < t0 || t > t1) return;
+    const m = svg('line', { class: `mc-marker ${cls}`, x1: x(t), x2: x(t), y1: padT, y2: H - padB });
+    m.appendChild(svg('title', {}, `${text} ${when(iso(t))}`));
+    root.appendChild(m);
+  };
+  marker(startSec, 'start', 'Alert started');
+  marker(endSec, 'end', 'Incident resolved');
+
+  series.forEach((s, i) => {
+    if (!s.points.length) return;
+    const d = s.points.map(([t, v], n) => `${n ? 'L' : 'M'}${x(t).toFixed(1)},${y(v).toFixed(1)}`).join(' ');
+    const line = svg('path', { class: `mc-line s${i}`, d });
+    line.appendChild(svg('title', {}, `${seriesName(s.labels)}: ${fmtValue(s.points[s.points.length - 1][1])} at ${when(iso(s.points[s.points.length - 1][0]))}`));
+    root.appendChild(line);
+    // A single sample has no line to draw; show it as a dot.
+    if (s.points.length === 1) {
+      root.appendChild(svg('circle', { class: `mc-dot s${i}`, cx: x(s.points[0][0]), cy: y(s.points[0][1]), r: 3 }));
+    }
+  });
+  return root;
+}