Set the chart's placeholder version to 0.49.0

Cosmetic: make helm-package passes --version and --app-version from the
tag, so these fields decide nothing about what is published. They are kept
in step so the tree heading for v0.49.0 does not say 0.48.0, as in
12535e0 and 220eb97.

Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN

.gitea/workflows/ci.yaml

@@ -35,7 +35,7 @@ jobs:
       # Runs inside the toolchain image rather than installing Go per job. Note this puts
       # the job on the dind bridge, which cannot reach github.com or get.helm.sh --
       # proxy.golang.org and git.ryuvia.com are reachable, which is all this job needs.
-      image: golang:1.26.9-bookworm
+      image: golang:1.26.10-bookworm
       # act_runner destroys a job's own volumes when it finishes, so without these every
       # run re-downloads the whole module graph. The names must appear in the runner's
       # container.valid_volumes allowlist (charts/act-runner in the k8s repo); unlisted
@@ -101,7 +101,7 @@ jobs:
   security:
     runs-on: ubuntu-latest
     container:
-      image: golang:1.26.9-bookworm
+      image: golang:1.26.10-bookworm
       volumes:
         - go-mod-cache:/go/pkg/mod
         - go-build-cache:/root/.cache/go-build

.gitea/workflows/release.yaml

@@ -30,7 +30,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     container:
-      image: golang:1.26.9-bookworm
+      image: golang:1.26.10-bookworm
       volumes:
         - go-mod-cache:/go/pkg/mod
         - go-build-cache:/root/.cache/go-build
@@ -71,7 +71,7 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     container:
-      image: golang:1.26.9-bookworm
+      image: golang:1.26.10-bookworm
       volumes:
         - go-mod-cache:/go/pkg/mod
         - go-build-cache:/root/.cache/go-build

Dockerfile

@@ -4,7 +4,7 @@
 # in per platform. The CI runner has no binfmt registration and no way to get one (the
 # JS action that used to install it cannot run there), so this is not just an
 # optimisation -- it is what makes the arm64 image buildable at all.
-FROM --platform=$BUILDPLATFORM golang:1.26.9-alpine AS builder
+FROM --platform=$BUILDPLATFORM golang:1.26.10-alpine AS builder
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download

README.md

@@ -92,6 +92,7 @@ The [documentation index](./docs/README.md) lists everything. The main pages:
 |---|---|
 | [Deployment](./docs/deployment.md) | Docker, Helm chart, database, backups, the operator |
 | [Configuration](./docs/configuration.md) | Environment variables and settings |
+| [Metrics](./docs/metrics.md) | Graphs from the alert's Prometheus on the incident page |
 | [Alertmanager configuration](./docs/alertmanager.md) | Routes, keys, webhooks |
 | [Alerts and incidents](./docs/incidents.md) | Correlation, lifecycle, on-call assignment |
 | [Single sign-on](./docs/single-sign-on.md) | OIDC and the terminal device flow |

charts/terdut-server/Chart.yaml

@@ -15,5 +15,5 @@ type: application
 # appVersion and image.tag in values.yaml no longer agree, and that is not an oversight:
 # image.tag stays "latest", which is what a local install actually pulls. appVersion is
 # metadata and drives nothing.
-version: 0.46.1
-appVersion: "v0.46.1"
+version: 0.49.0
+appVersion: "v0.49.0"

charts/terdut-server/templates/deployment.yaml

@@ -120,6 +120,14 @@ spec:
               value: {{ .Values.trustedProxies | quote }}
             - name: TERDUT_OPERATOR_MODE
               value: {{ .Values.operatorMode | quote }}
+            {{- if .Values.prometheus.allowedHosts }}
+            - name: TERDUT_PROMETHEUS_ALLOWED_HOSTS
+              value: {{ join "," .Values.prometheus.allowedHosts | quote }}
+            - name: TERDUT_PROMETHEUS_PREROLL
+              value: {{ .Values.prometheus.preroll | quote }}
+            - name: TERDUT_PROMETHEUS_TIMEOUT
+              value: {{ .Values.prometheus.timeout | quote }}
+            {{- end }}
             {{- if .Values.oidc.enabled }}
             - name: TERDUT_OIDC_ISSUER
               value: {{ required "oidc.issuer is required when oidc.enabled" .Values.oidc.issuer | quote }}

charts/terdut-server/values.yaml

@@ -114,6 +114,18 @@ passwordLogin: true
 # configuration from here on.
 operatorMode: false
 
+# Metrics on incidents: the server queries the Prometheus an alert came from
+# (the host in its generatorURL) and graphs the alert's expression. The URL is
+# taken from a webhook payload, so only hosts listed here are ever contacted;
+# empty turns the feature off. Entries are host names, with "*" as one DNS label
+# and a leading "**" as one or more (e.g. "*.monitoring.svc.cluster.local",
+# "prometheus-*.monitoring"), IPs or CIDR ranges, each with an optional ":port".
+prometheus:
+  allowedHosts: []
+  # How far before an alert started its graph begins.
+  preroll: 1h
+  timeout: 10s
+
 # Single sign-on through an OpenID Connect provider such as Authentik.
 #
 # At the provider, create an OAuth2/OpenID application whose redirect URI is

cmd/terdut/main.go

@@ -64,6 +64,7 @@ func main() {
 
 	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notifyCfg)
 	go api.StartNotifier(ctx, database, notifyCfg)
+	go api.StartMetricsWorker(ctx, database, cfg)
 
 	go func() {
 		log.Printf("terdut-server %s listening on %s", version, cfg.Addr)

docs/api.md

@@ -236,6 +236,7 @@ the team gets the same `404` as anybody else.
 | `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline |
 | `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident |
 | `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological |
+| `GET` | `/api/incidents/{id}/metrics` | Metrics behind the incident's alerts, from their Prometheus — see [Metrics](./metrics.md) |
 | `POST` | `/api/incidents/{id}/acknowledge` | Acknowledge (stamps authed user + time) |
 | `DELETE` | `/api/incidents/{id}/acknowledge` | Clear acknowledgement, back to `triggered` |
 | `POST` | `/api/incidents/{id}/resolve` | Close by hand — **terminal**, see above |
@@ -427,13 +428,15 @@ Every figure counts the caller's own teams only: a report that counted other
 teams' incidents would leak their volume, and their alert names through the
 top-alerts list, and would not be a number about the reader's work anyway.
 
-All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats filter on `triggered_at`.
+All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats, and the top, heatmap and per-alert endpoints (an occurrence is an incident), filter on `triggered_at`.
 
 | Method | Path | Description |
 |---|---|---|
 | `GET` | `/api/stats/incidents` | `{total, triggered, acknowledged, resolved, mtta_seconds, mttr_seconds}` |
 | `GET` | `/api/stats/alerts` | `{total, firing, resolved}` counts |
-| `GET` | `/api/stats/alerts/top` | Most frequent alert names. `?limit=` (default 10, max 100) |
+| `GET` | `/api/stats/alerts/top` | Alert names ranked by incidents opened (filters on `triggered_at`). Each entry: `{name, count, last_seen, mean_resolve_seconds, daily}`; `daily` is 30 per-day counts ending at `?to` (or today), regardless of `?from`. `?limit=` (default 10, max 100) |
+| `GET` | `/api/stats/alerts/heatmap` | `{grid}`: incidents by UTC weekday (0=Sunday) × hour, always 7×24 |
+| `GET` | `/api/stats/alerts/{name}` | One alert's drilldown over its incidents: `{name, count, first_seen, last_seen, mean_gap_seconds, mean/median_ack_seconds, mean/median_resolve_seconds, longest_open_seconds, by_hour[24], by_weekday[7], occurrences[]}` (newest 200; means are `null` when unmeasurable). 404 when the alert opened no incident in scope |
 | `GET` | `/api/stats/alerts/by-hour` | Count per hour-of-day (UTC), all 24 slots returned |
 | `GET` | `/api/stats/alerts/by-day` | Count per day-of-week, all 7 slots with names returned |
 

docs/configuration.md

@@ -30,6 +30,9 @@ over an administrator's edit.
 | `TERDUT_TRUSTED_PROXIES` | `1` | How many reverse proxies in front of the server append to `X-Forwarded-For`; the per-address rate limits use the entry that many hops from the right. `0` ignores the header |
 | `TERDUT_OPERATOR_KEY` | — | At least 32 characters. When set, the instance-scoped service account `terdut-operator` is created if missing and its `seed` key replaced with this value at every start — how terdut-operator authenticates without a bootstrap handshake. An instance-scoped account acts as owner of every team (team configuration) but is not a member of any, so it reads no incidents |
 | `TERDUT_OPERATOR_MODE` | `false` | Declares this install gitops-managed: a session's or a user's own API key's writes to teams, escalation policies, dead man's switches and integrations are refused (`403 reason:"operator_managed"`); a [service account](./api.md#service-accounts)'s are not. Team membership and the schedule stay editable regardless |
+| `TERDUT_PROMETHEUS_ALLOWED_HOSTS` | — | Comma-separated Prometheus servers the [metrics on an incident](./metrics.md) may be fetched from, matched against the host in each alert's `generatorURL`. Host names (`prometheus.monitoring.svc`), wildcards (`*` is one DNS label, so `*.example.com` or `prom-*.monitoring.svc`; a leading `**` is one or more labels), IPs or CIDR ranges, each with an optional `:port`. Empty turns the feature off. A bare `*` is refused at startup |
+| `TERDUT_PROMETHEUS_PREROLL` | `1h` | How far before an alert started its metrics graph begins |
+| `TERDUT_PROMETHEUS_TIMEOUT` | `10s` | Per-request timeout for a Prometheus query |
 | `TERDUT_OIDC_ISSUER` | — | Turns single sign-on on. The provider's issuer URL; discovery is read from `<issuer>/.well-known/openid-configuration`. See [Single sign-on](./single-sign-on.md#single-sign-on-oidc) |
 | `TERDUT_OIDC_CLIENT_ID` / `TERDUT_OIDC_CLIENT_SECRET` | — | **Required with an issuer.** The confidential client registered at the provider. Keep the secret in a Secret, not in values |
 | `TERDUT_OIDC_NAME` | `SSO` | What the sign-in button calls the provider |

docs/metrics.md

@@ -0,0 +1,75 @@
+# Metrics on incidents
+
+Alertmanager sends each alert with a `generatorURL`: the link to the rule's graph in
+the Prometheus that fired it. With the feature on, terdut-server parses that link,
+queries the same Prometheus for the rule's expression, and shows a graph and the
+current value under each alert on the incident page. The browser never talks to
+Prometheus; the server does.
+
+## Turning it on
+
+Set `TERDUT_PROMETHEUS_ALLOWED_HOSTS` (Helm: `prometheus.allowedHosts`). The URL arrives in a
+webhook payload, so whoever holds an integration key chooses it; without an allowlist the
+server would fetch whatever it was told to. Empty, the default, means off.
+
+| Entry | Matches |
+|---|---|
+| `prometheus.monitoring.svc` | that host, any port |
+| `prometheus.monitoring.svc:9090` | that host and port |
+| `*.example.com` | `a.example.com`; not `example.com`, not `a.b.example.com` |
+| `prom-*.monitoring.svc` | `prom-0.monitoring.svc` |
+| `**.example.com` | `a.example.com` and `a.b.example.com` |
+| `10.0.0.0/8`, `[fd00::]/8:9090` | addresses in the range |
+
+`*` stands for one DNS label (or part of one); `**` is allowed only as the first label and means
+one or more. Matching ignores case. A bare `*` or `**`, or any entry whose last label is only a
+wildcard, is refused at startup.
+
+The check is made on the address the connection is about to use, not just on the name, so a
+name that resolves somewhere else (DNS rebinding) is caught. A host admitted by name is still
+refused if it resolves to a loopback, link-local (which includes the cloud metadata address),
+unspecified or multicast address; to allow one deliberately, list it as an IP or CIDR. Redirects
+and proxy environment variables are ignored.
+
+## What is captured
+
+- **Snapshots.** A background worker stores each alert's series when the incident opens
+  (`opened`) and when it closes (`closed`), in Postgres, so the graph outlives Prometheus'
+  own retention. Incidents closed more than 24 hours before the feature was switched on are not
+  backfilled. A failed capture is retried a few times; a host that is not allowed, or a link that is
+  not a Prometheus graph link, is recorded as failed once.
+- **Live data.** While an incident is open, `GET /api/incidents/{id}/metrics` also queries
+  Prometheus (answers are reused for 15 seconds) and returns the current value.
+- **Window.** From `TERDUT_PROMETHEUS_PREROLL` (default 1h) before the alert started until now,
+  or until the incident was resolved.
+
+## Reading the graph
+
+The expression is the alert rule's own, threshold included (`latency > 1`), so Prometheus only
+returns data while the condition is true. The graph therefore shows the firing period, and **"no
+data" for the current value means the condition no longer holds**, not that Prometheus is down.
+A rule expression matches every series, not just the one that fired; only series that agree with the
+alert's labels are kept, at most five per alert.
+
+## Endpoint
+
+`GET /api/incidents/{id}/metrics` — same team scoping as the other incident routes.
+
+```json
+{
+  "enabled": true,
+  "alerts": [{
+    "alert_id": 12, "alert_name": "HighLatency", "expr": "latency>1",
+    "available": true,
+    "snapshots": { "opened": { "window_start": 1, "window_end": 2, "step_seconds": 15,
+                               "series": [{"labels": {"instance": "web-1"}, "points": [[100, 1.5]]}],
+                               "captured_at": 3 } },
+    "live": { "window_start": 1, "window_end": 2, "step_seconds": 15,
+              "series": [], "current": [] }
+  }]
+}
+```
+
+`enabled` is `false` when no host is allowed. `available` is `false`, with a `reason`, when this
+alert has no Prometheus link or its host is not allowed. A snapshot that failed carries `error`.
+`live` is absent once the incident is resolved. Points are `[unix seconds, value]`.

internal/api/export_test.go

@@ -1,8 +1,12 @@
 package api
 
 import (
+	"context"
+	"database/sql"
 	"strings"
 	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/config"
 )
 
 // DeadmanConfig is a test fixture only: a set of matchers with one timeout and
@@ -29,3 +33,9 @@ func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string)
 	}
 	return cfg
 }
+
+// MetricsSweep runs one pass of the metrics worker, so a test need not wait on
+// its ticker.
+func MetricsSweep(ctx context.Context, db *sql.DB, cfg config.Config) {
+	newMetricsService(cfg).sweep(ctx, db)
+}

internal/api/metrics.go

@@ -0,0 +1,456 @@
+package api
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/config"
+	"git.ryuvia.com/niklas/terdut-server/internal/prom"
+)
+
+const (
+	// metricsInterval is how often the worker looks for incidents that need a
+	// snapshot. Short, because the "opened" snapshot is most useful when it is
+	// taken close to the moment the incident opened.
+	metricsInterval = 15 * time.Second
+
+	// metricsBatch caps one pass, and metricsConcurrency how many Prometheus
+	// requests run at once within it.
+	metricsBatch       = 50
+	metricsConcurrency = 4
+
+	// metricsMaxAttempts stops a failing capture being retried forever, and
+	// metricsRetryStep spaces the retries: attempt n waits n*metricsRetryStep.
+	metricsMaxAttempts = 4
+	metricsRetryStep   = time.Minute
+
+	// metricsLookback is how far back the worker goes for incidents that are
+	// already closed. Without it, switching the feature on would try to
+	// reconstruct the whole history from a Prometheus that has probably aged it
+	// out. Incidents that are still open are always eligible.
+	metricsLookback = 24 * time.Hour
+
+	// maxAlertsPerIncident bounds the Prometheus work one incident can cause.
+	maxAlertsPerIncident = 10
+
+	// metricsLiveTTL is how long a live answer is reused, so a room full of
+	// people watching one incident is one query per alert, not one each.
+	metricsLiveTTL = 15 * time.Second
+
+	// metricsLockKey is the advisory lock a pass holds, distinct from the
+	// archiver's and the notifier's.
+	metricsLockKey int64 = 7265_0004
+)
+
+// metricsService is the Prometheus side of the incident page: a client that only
+// reaches allowed hosts, and a short-lived cache of live answers.
+type metricsService struct {
+	client  *prom.Client
+	preroll time.Duration
+
+	mu    sync.Mutex
+	cache map[int64]liveEntry
+}
+
+type liveEntry struct {
+	at   time.Time
+	live *liveMetrics
+}
+
+func newMetricsService(cfg config.Config) *metricsService {
+	// Validate has already refused a bad list at startup; a failure here can only
+	// be a test building a Config by hand, and no allowlist is the safe reading.
+	allow, err := prom.ParseAllowlist(cfg.PrometheusAllowedHosts)
+	if err != nil {
+		allow = nil
+	}
+	preroll := cfg.PrometheusPreroll
+	if preroll <= 0 {
+		preroll = time.Hour
+	}
+	return &metricsService{
+		client:  prom.NewClient(allow, cfg.PrometheusTimeout),
+		preroll: preroll,
+		cache:   map[int64]liveEntry{},
+	}
+}
+
+func (m *metricsService) enabled() bool { return m.client.Enabled() }
+
+// StartMetricsWorker captures the opened and closed snapshots until ctx is
+// cancelled. With no allowed hosts configured it does nothing.
+func StartMetricsWorker(ctx context.Context, db *sql.DB, cfg config.Config) {
+	m := newMetricsService(cfg)
+	if !m.enabled() {
+		log.Print("metrics: disabled (TERDUT_PROMETHEUS_ALLOWED_HOSTS is empty)")
+		return
+	}
+	log.Printf("metrics: capturing from %d allowed Prometheus host entries", len(cfg.PrometheusAllowedHosts))
+
+	ticker := time.NewTicker(metricsInterval)
+	defer ticker.Stop()
+	sweep := func() {
+		withAdvisoryLock(ctx, db, metricsLockKey, "metrics", func() {
+			m.sweep(ctx, db)
+		})
+	}
+	sweep()
+	for {
+		select {
+		case <-ticker.C:
+			sweep()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// snapshotJob is one (incident, alert, kind) that still needs a snapshot.
+type snapshotJob struct {
+	incidentID   int64
+	alertID      int64
+	kind         string
+	resolvedAt   sql.NullInt64
+	startsAt     int64
+	labels       map[string]string
+	generatorURL string
+}
+
+// sweep finds the snapshots that are missing, or failed and are due a retry, and
+// takes them.
+func (m *metricsService) sweep(ctx context.Context, db *sql.DB) {
+	now := time.Now()
+	rows, err := db.QueryContext(ctx, `
+		SELECT i.id, a.id, k.kind, i.resolved_at, a.starts_at, a.labels, a.generator_url
+		FROM   incidents i
+		JOIN   incident_alerts ia ON ia.incident_id = i.id
+		JOIN   alerts a ON a.id = ia.alert_id
+		CROSS  JOIN (VALUES ('opened'), ('closed')) AS k(kind)
+		LEFT   JOIN incident_metric_snapshots s
+		       ON s.incident_id = i.id AND s.alert_id = a.id AND s.kind = k.kind
+		WHERE  a.generator_url <> ''
+		  AND  i.archived_at IS NULL
+		  AND  (i.resolved_at IS NULL OR i.resolved_at >= $1)
+		  AND  (k.kind = 'opened' OR i.resolved_at IS NOT NULL)
+		  AND  (s.id IS NULL
+		        OR (s.error IS NOT NULL AND s.attempts < $2
+		            AND s.captured_at + s.attempts * $3 <= $4))
+		ORDER  BY i.id, a.id, k.kind
+		LIMIT  $5`,
+		now.Add(-metricsLookback).Unix(), metricsMaxAttempts, int64(metricsRetryStep/time.Second), now.Unix(), metricsBatch)
+	if err != nil {
+		log.Printf("metrics: find work: %v", err)
+		return
+	}
+	var jobs []snapshotJob
+	for rows.Next() {
+		var j snapshotJob
+		var labels []byte
+		if err := rows.Scan(&j.incidentID, &j.alertID, &j.kind, &j.resolvedAt, &j.startsAt, &labels, &j.generatorURL); err != nil {
+			log.Printf("metrics: scan: %v", err)
+			rows.Close()
+			return
+		}
+		_ = json.Unmarshal(labels, &j.labels)
+		jobs = append(jobs, j)
+	}
+	rows.Close()
+	if err := rows.Err(); err != nil {
+		log.Printf("metrics: find work: %v", err)
+		return
+	}
+
+	sem := make(chan struct{}, metricsConcurrency)
+	var wg sync.WaitGroup
+	for _, j := range jobs {
+		sem <- struct{}{}
+		wg.Add(1)
+		go func() {
+			defer func() { <-sem; wg.Done() }()
+			m.capture(ctx, db, j)
+		}()
+	}
+	wg.Wait()
+}
+
+// window is the span a snapshot covers: from the pre-roll before the alert
+// started to the end of the incident, or to now while it is open.
+func (m *metricsService) window(startsAt int64, resolvedAt sql.NullInt64, now time.Time) (time.Time, time.Time) {
+	start := time.Unix(startsAt, 0).Add(-m.preroll)
+	end := now
+	if resolvedAt.Valid {
+		end = time.Unix(resolvedAt.Int64, 0)
+	}
+	if !end.After(start) {
+		end = start.Add(time.Minute)
+	}
+	return start, end
+}
+
+func (m *metricsService) capture(ctx context.Context, db *sql.DB, j snapshotJob) {
+	now := time.Now()
+	start, end := m.window(j.startsAt, j.resolvedAt, now)
+	step := prom.StepFor(start, end)
+
+	series, expr, err := m.fetchRange(ctx, j.generatorURL, j.labels, start, end, step)
+
+	attempts := 1
+	var errText any
+	if err != nil {
+		errText = err.Error()
+		if errors.Is(err, prom.ErrNotAllowed) || errors.Is(err, errNotGraphLink) {
+			attempts = metricsMaxAttempts // retrying cannot change the answer
+		}
+	}
+	if series == nil {
+		series = []prom.Series{}
+	}
+	body, _ := json.Marshal(series)
+
+	_, dbErr := db.ExecContext(ctx, `
+		INSERT INTO incident_metric_snapshots
+		       (incident_id, alert_id, kind, expr, window_start, window_end, step_seconds, series, error, attempts, captured_at)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+		ON CONFLICT (incident_id, alert_id, kind) DO UPDATE SET
+		       expr = excluded.expr, window_start = excluded.window_start, window_end = excluded.window_end,
+		       step_seconds = excluded.step_seconds, series = excluded.series, error = excluded.error,
+		       attempts = CASE WHEN excluded.error IS NULL THEN 1
+		                       ELSE GREATEST(incident_metric_snapshots.attempts + 1, excluded.attempts) END,
+		       captured_at = excluded.captured_at`,
+		j.incidentID, j.alertID, j.kind, expr, start.Unix(), end.Unix(), int(step/time.Second),
+		string(body), errText, attempts, now.Unix())
+	if dbErr != nil {
+		log.Printf("metrics: store snapshot (incident %d, alert %d, %s): %v", j.incidentID, j.alertID, j.kind, dbErr)
+	}
+}
+
+var errNotGraphLink = errors.New("the alert's generator URL is not a Prometheus graph link")
+
+// fetchRange parses the generator URL and returns the series of the alert's own
+// expression over [start, end]. expr is returned even on error where known.
+func (m *metricsService) fetchRange(ctx context.Context, generatorURL string, labels map[string]string, start, end time.Time, step time.Duration) ([]prom.Series, string, error) {
+	base, expr, ok := prom.ParseGeneratorURL(generatorURL)
+	if !ok {
+		return nil, "", errNotGraphLink
+	}
+	series, err := m.client.QueryRange(ctx, base, expr, start, end, step)
+	if err != nil {
+		return nil, expr, err
+	}
+	return prom.SelectSeries(series, labels), expr, nil
+}
+
+// --- REST ---------------------------------------------------------------
+
+type metricsResponse struct {
+	// Enabled is false when no Prometheus host is allowed, so a client can hide
+	// the section rather than show every alert as unavailable.
+	Enabled bool           `json:"enabled"`
+	Alerts  []alertMetrics `json:"alerts"`
+}
+
+type alertMetrics struct {
+	AlertID   int64  `json:"alert_id"`
+	AlertName string `json:"alert_name"`
+	Expr      string `json:"expr,omitempty"`
+
+	// Available says whether this alert can be queried at all; Reason says why
+	// not when it cannot.
+	Available bool   `json:"available"`
+	Reason    string `json:"reason,omitempty"`
+
+	Snapshots map[string]*snapshotView `json:"snapshots"`
+	Live      *liveMetrics             `json:"live,omitempty"`
+}
+
+type snapshotView struct {
+	WindowStart int64           `json:"window_start"`
+	WindowEnd   int64           `json:"window_end"`
+	Step        int             `json:"step_seconds"`
+	Series      json.RawMessage `json:"series"`
+	Error       string          `json:"error,omitempty"`
+	CapturedAt  int64           `json:"captured_at"`
+}
+
+type liveMetrics struct {
+	WindowStart int64         `json:"window_start"`
+	WindowEnd   int64         `json:"window_end"`
+	Step        int           `json:"step_seconds"`
+	Series      []prom.Series `json:"series"`
+	// Current is the expression's value right now, one point per series. Empty
+	// means the condition that fires the alert no longer holds.
+	Current []prom.Series `json:"current"`
+	Error   string        `json:"error,omitempty"`
+}
+
+func handleIncidentMetrics(db *sql.DB, m *metricsService) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r, db)
+		if !ok {
+			return
+		}
+		resp := metricsResponse{Enabled: m.enabled(), Alerts: []alertMetrics{}}
+		if !resp.Enabled {
+			respond(w, http.StatusOK, resp)
+			return
+		}
+
+		var resolvedAt sql.NullInt64
+		if err := db.QueryRowContext(r.Context(), "SELECT resolved_at FROM incidents WHERE id = $1", id).Scan(&resolvedAt); err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		rows, err := db.QueryContext(r.Context(), `
+			SELECT a.id, a.name, a.labels, a.generator_url, a.starts_at
+			FROM   alerts a
+			JOIN   incident_alerts ia ON ia.alert_id = a.id
+			WHERE  ia.incident_id = $1
+			ORDER  BY a.starts_at, a.id
+			LIMIT  $2`, id, maxAlertsPerIncident)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		type alertRow struct {
+			am       alertMetrics
+			labels   map[string]string
+			url      string
+			startsAt int64
+		}
+		var alerts []*alertRow
+		for rows.Next() {
+			var a alertRow
+			var labels []byte
+			if err := rows.Scan(&a.am.AlertID, &a.am.AlertName, &labels, &a.url, &a.startsAt); err != nil {
+				rows.Close()
+				serverError(w, r, err)
+				return
+			}
+			_ = json.Unmarshal(labels, &a.labels)
+			a.am.Snapshots = map[string]*snapshotView{}
+			alerts = append(alerts, &a)
+		}
+		rows.Close()
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		snaps, err := loadSnapshots(r.Context(), db, id)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		var wg sync.WaitGroup
+		for _, a := range alerts {
+			a := a
+			if base, expr, ok := prom.ParseGeneratorURL(a.url); !ok {
+				a.am.Reason = "no Prometheus link on this alert"
+			} else if !m.client.Allowed(base) {
+				a.am.Reason = "Prometheus host is not allowed"
+				a.am.Expr = expr
+			} else {
+				a.am.Available = true
+				a.am.Expr = expr
+			}
+			for kind, s := range snaps[a.am.AlertID] {
+				a.am.Snapshots[kind] = s
+			}
+			if a.am.Available && !resolvedAt.Valid {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					a.am.Live = m.live(r.Context(), a.am.AlertID, a.url, a.labels, a.startsAt)
+				}()
+			}
+		}
+		wg.Wait()
+
+		for _, a := range alerts {
+			resp.Alerts = append(resp.Alerts, a.am)
+		}
+		respond(w, http.StatusOK, resp)
+	}
+}
+
+func loadSnapshots(ctx context.Context, db *sql.DB, incidentID int64) (map[int64]map[string]*snapshotView, error) {
+	rows, err := db.QueryContext(ctx, `
+		SELECT alert_id, kind, window_start, window_end, step_seconds, series, COALESCE(error, ''), captured_at
+		FROM   incident_metric_snapshots
+		WHERE  incident_id = $1`, incidentID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[int64]map[string]*snapshotView{}
+	for rows.Next() {
+		var alertID int64
+		var kind string
+		var s snapshotView
+		var series []byte
+		if err := rows.Scan(&alertID, &kind, &s.WindowStart, &s.WindowEnd, &s.Step, &series, &s.Error, &s.CapturedAt); err != nil {
+			return nil, err
+		}
+		s.Series = json.RawMessage(series)
+		if out[alertID] == nil {
+			out[alertID] = map[string]*snapshotView{}
+		}
+		out[alertID][kind] = &s
+	}
+	return out, rows.Err()
+}
+
+// live queries Prometheus now, reusing an answer younger than metricsLiveTTL.
+// The cache is per alert; the window start is fixed by the alert, only its end
+// moves, so one entry serves every viewer.
+func (m *metricsService) live(ctx context.Context, alertID int64, generatorURL string, labels map[string]string, startsAt int64) *liveMetrics {
+	m.mu.Lock()
+	if e, ok := m.cache[alertID]; ok && time.Since(e.at) < metricsLiveTTL {
+		m.mu.Unlock()
+		return e.live
+	}
+	m.mu.Unlock()
+
+	now := time.Now()
+	start, end := m.window(startsAt, sql.NullInt64{}, now)
+	step := prom.StepFor(start, end)
+	lm := &liveMetrics{WindowStart: start.Unix(), WindowEnd: end.Unix(), Step: int(step / time.Second),
+		Series: []prom.Series{}, Current: []prom.Series{}}
+
+	base, expr, ok := prom.ParseGeneratorURL(generatorURL)
+	if !ok {
+		lm.Error = errNotGraphLink.Error()
+		return lm
+	}
+	series, err := m.client.QueryRange(ctx, base, expr, start, end, step)
+	if err != nil {
+		lm.Error = err.Error()
+	} else {
+		lm.Series = prom.SelectSeries(series, labels)
+		if cur, err := m.client.Query(ctx, base, expr, now); err == nil {
+			lm.Current = prom.SelectSeries(cur, labels)
+		} else {
+			lm.Error = err.Error()
+		}
+	}
+
+	// A failed answer is cached too, for the same TTL: a Prometheus that is down
+	// should not be hit once per viewer per poll.
+	m.mu.Lock()
+	for k, e := range m.cache {
+		if time.Since(e.at) > 10*metricsLiveTTL {
+			delete(m.cache, k)
+		}
+	}
+	m.cache[alertID] = liveEntry{at: now, live: lm}
+	m.mu.Unlock()
+	return lm
+}

internal/api/metrics_test.go

@@ -0,0 +1,189 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/api"
+)
+
+// fakePrometheus answers the two query endpoints and counts the requests.
+func fakePrometheus(t *testing.T) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var hits atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v1/query_range":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[
+				{"metric":{"instance":"web-1"},"values":[[100,"1"],[160,"2"]]},
+				{"metric":{"instance":"web-2"},"values":[[100,"9"]]}]}}`))
+		case "/api/v1/query":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[
+				{"metric":{"instance":"web-1"},"value":[170,"3"]}]}}`))
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hits
+}
+
+type metricsBody struct {
+	Enabled bool `json:"enabled"`
+	Alerts  []struct {
+		AlertID   int64  `json:"alert_id"`
+		Expr      string `json:"expr"`
+		Available bool   `json:"available"`
+		Reason    string `json:"reason"`
+		Snapshots map[string]struct {
+			Series json.RawMessage `json:"series"`
+			Error  string          `json:"error"`
+		} `json:"snapshots"`
+		Live *struct {
+			Series  []struct{ Labels map[string]string } `json:"series"`
+			Current []struct{ Labels map[string]string } `json:"current"`
+		} `json:"live"`
+	} `json:"alerts"`
+}
+
+func getMetrics(t *testing.T, s *ts, id int) metricsBody {
+	t.Helper()
+	var out metricsBody
+	decode(t, s.req(t, http.MethodGet, "/api/incidents/"+itoa(id)+"/metrics", nil), &out)
+	return out
+}
+
+func itoa(n int) string {
+	b, _ := json.Marshal(n)
+	return string(b)
+}
+
+func metricsAlert(promURL string) map[string]any {
+	a := amAlert("fp-metrics", "HighLatency", "firing", "2026-05-20T10:00:00Z", "0001-01-01T00:00:00Z",
+		map[string]string{"instance": "web-1"})
+	a["generatorURL"] = promURL + "/graph?g0.expr=latency%3E1&g0.tab=1"
+	return a
+}
+
+func TestMetrics_SnapshotsOpenedAndClosedAndLive(t *testing.T) {
+	prom, _ := fakePrometheus(t)
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"} // httptest listens on loopback
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-metrics")
+	incs := listIncidents(t, s, "")
+	if len(incs) != 1 {
+		t.Fatalf("want one incident, got %d", len(incs))
+	}
+	id := int(incs[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, conf)
+
+	m := getMetrics(t, s, id)
+	if !m.Enabled || len(m.Alerts) != 1 {
+		t.Fatalf("metrics = %+v", m)
+	}
+	a := m.Alerts[0]
+	if !a.Available || a.Expr != "latency>1" {
+		t.Fatalf("alert = %+v", a)
+	}
+	opened, ok := a.Snapshots["opened"]
+	if !ok || opened.Error != "" {
+		t.Fatalf("opened snapshot = %+v", a.Snapshots)
+	}
+	var series []struct{ Labels map[string]string }
+	json.Unmarshal(opened.Series, &series)
+	if len(series) != 1 || series[0].Labels["instance"] != "web-1" {
+		t.Errorf("snapshot should hold only this alert's series (web-1), got %+v", series)
+	}
+	if _, ok := a.Snapshots["closed"]; ok {
+		t.Error("closed snapshot exists while the incident is open")
+	}
+	if a.Live == nil || len(a.Live.Series) != 1 || len(a.Live.Current) != 1 {
+		t.Errorf("an open incident should have live data, got %+v", a.Live)
+	}
+
+	// A second pass must not capture again.
+	var n int
+	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n)
+	api.MetricsSweep(context.Background(), s.db, conf)
+	var n2 int
+	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n2)
+	if n != 1 || n2 != 1 {
+		t.Errorf("snapshot rows = %d then %d, want 1 and 1", n, n2)
+	}
+
+	// Resolving the incident gets the closed snapshot, and no more live data.
+	resp := s.req(t, http.MethodPost, "/api/incidents/"+itoa(id)+"/resolve", nil)
+	resp.Body.Close()
+	api.MetricsSweep(context.Background(), s.db, conf)
+	m = getMetrics(t, s, id)
+	a = m.Alerts[0]
+	if _, ok := a.Snapshots["closed"]; !ok {
+		t.Errorf("no closed snapshot after resolve: %+v", a.Snapshots)
+	}
+	if a.Live != nil {
+		t.Error("a resolved incident should not query live")
+	}
+}
+
+func TestMetrics_DisabledWithoutAllowedHosts(t *testing.T) {
+	prom, hits := fakePrometheus(t)
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-off")
+	id := int(listIncidents(t, s, "")[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, testConfig())
+	m := getMetrics(t, s, id)
+	if m.Enabled || len(m.Alerts) != 0 {
+		t.Errorf("metrics = %+v, want disabled and empty", m)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("Prometheus was contacted %d times", hits.Load())
+	}
+}
+
+func TestMetrics_UnlistedHostIsRecordedNotQueried(t *testing.T) {
+	prom, hits := fakePrometheus(t)
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"*.example.com"}
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-unlisted")
+	id := int(listIncidents(t, s, "")[0]["id"].(float64))
+
+	api.MetricsSweep(context.Background(), s.db, conf)
+	api.MetricsSweep(context.Background(), s.db, conf) // must not retry a permanent failure
+
+	m := getMetrics(t, s, id)
+	a := m.Alerts[0]
+	if a.Available || a.Reason == "" {
+		t.Errorf("alert should be unavailable with a reason: %+v", a)
+	}
+	if hits.Load() != 0 {
+		t.Errorf("an unlisted host was contacted %d times", hits.Load())
+	}
+	var attempts int
+	var errText *string
+	s.db.QueryRow("SELECT attempts, error FROM incident_metric_snapshots").Scan(&attempts, &errText)
+	if errText == nil || attempts < 4 {
+		t.Errorf("failure should be recorded as final: attempts=%d error=%v", attempts, errText)
+	}
+}
+
+func TestMetrics_IncidentScoping(t *testing.T) {
+	conf := testConfig()
+	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"}
+	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
+	resp := s.req(t, http.MethodGet, "/api/incidents/9999/metrics", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("unknown incident: %d, want 404", resp.StatusCode)
+	}
+}

internal/api/router.go

@@ -28,6 +28,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 	signupLimiter := newLoginLimiter(db)
 	oidcLimit := newLoginLimiter(db)
 
+	metrics := newMetricsService(cfg)
+
 	r := chi.NewRouter()
 	r.Use(requestLogger)
 	r.Use(middleware.Recoverer)
@@ -149,6 +151,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Get("/api/incidents/clusters", handleListClusters(db))
 		r.Get("/api/incidents/{id}", handleGetIncident(db))
 		r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db))
+		r.Get("/api/incidents/{id}/metrics", handleIncidentMetrics(db, metrics))
 		r.Get("/api/incidents/{id}/similar", handleIncidentSimilar(db))
 		r.Post("/api/incidents/{id}/acknowledge", handleIncidentAcknowledge(db))
 		r.Delete("/api/incidents/{id}/acknowledge", handleIncidentUnacknowledge(db))
@@ -227,6 +230,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Get("/api/stats/alerts/top", handleStatsTop(db))
 		r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db))
 		r.Get("/api/stats/alerts/by-day", handleStatsByDay(db))
+		r.Get("/api/stats/alerts/heatmap", handleStatsHeatmap(db))
+		r.Get("/api/stats/alerts/{name}", handleStatsAlert(db))
 	})
 
 	// Anything else under /api is a mistake in a client, and should say so in

internal/api/service_accounts_test.go

@@ -490,3 +490,31 @@ func TestDeadman_UpdateInPlacePreservesID(t *testing.T) {
 		t.Errorf("expected the update to replace in place, not add a row, got %d switches", len(list))
 	}
 }
+
+// The operator reconciles a team's dead man's switches by listing them, and its
+// instance-scoped account is not a member of any team it manages.
+func TestServiceAccount_InstanceScopeListsATeamsDeadmanSwitches(t *testing.T) {
+	s := newTS(t)
+	instanceKey := createServiceAccount(t, s, s.key, "terdut-operator", models.ServiceAccountScopeInstance, 0)
+	team := createTeamAs(t, s, instanceKey, "team-a")
+	path := "/api/teams/" + id64(team) + "/deadman/switches"
+
+	resp := s.reqAs(t, instanceKey, http.MethodPost, path,
+		map[string]any{"name": "watchdog", "matcher": "alertname=Watchdog", "timeout_seconds": 900, "severity": "critical"})
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("create switch: %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	resp = s.reqAs(t, instanceKey, http.MethodGet, path, nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("list switches as the instance account: %d", resp.StatusCode)
+	}
+	var list []struct {
+		Name string `json:"name"`
+	}
+	decode(t, resp, &list)
+	if len(list) != 1 || list[0].Name != "watchdog" {
+		t.Errorf("expected the one switch just created, got %+v", list)
+	}
+}

internal/api/stats.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strings"
 	"time"
+
+	"github.com/go-chi/chi/v5"
 )
 
 func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
@@ -35,12 +38,24 @@ func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
 	}
 }
 
+// alertNameSQL is the alert-name half of an incident's signature (see
+// incidentSignature: "name|stable,labels"). An occurrence of an alert is an
+// incident, because alert rows are reused per fingerprint and carry no history.
+const alertNameSQL = "split_part(signature, '|', 1)"
+
+const sparkDays = 30
+
+// handleStatsTop ranks alert names by how many incidents they opened in the
+// window, with what a list row needs besides the count: when it last fired, how
+// long it takes to fix, and a per-day series for the last sparkDays days.
 func handleStatsTop(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))
+		q := r.URL.Query()
+		teams := callerTeamIDs(r.Context())
+		where, args := statsFilter(q, "triggered_at", teams)
 
 		limit := 10
-		if l := r.URL.Query().Get("limit"); l != "" {
+		if l := q.Get("limit"); l != "" {
 			var n int
 			fmt.Sscanf(l, "%d", &n)
 			if n > 0 && n <= 100 {
@@ -48,12 +63,13 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 			}
 		}
 		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
-			SELECT name, COUNT(*) AS cnt
-			FROM   alerts
-			WHERE  %s
+			SELECT %[1]s AS name, COUNT(*) AS cnt, MAX(triggered_at),
+			       AVG(CASE WHEN resolved_at IS NOT NULL THEN resolved_at - triggered_at END)
+			FROM   incidents
+			WHERE  %[2]s
 			GROUP  BY name
-			ORDER  BY cnt DESC
-			LIMIT  %s`, where, args.add(limit)), args.all()...)
+			ORDER  BY cnt DESC, name ASC
+			LIMIT  %[3]s`, alertNameSQL, where, args.add(limit)), args.all()...)
 		if err != nil {
 			serverError(w, r, err)
 			return
@@ -61,26 +77,247 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 		defer rows.Close()
 
 		type entry struct {
-			Name  string `json:"name"`
-			Count int64  `json:"count"`
+			Name        string   `json:"name"`
+			Count       int64    `json:"count"`
+			LastSeen    int64    `json:"last_seen"`
+			MeanResolve *float64 `json:"mean_resolve_seconds"`
+			Daily       []int64  `json:"daily"`
 		}
 		result := []entry{}
+		names := []string{}
 		for rows.Next() {
 			var e entry
-			if err := rows.Scan(&e.Name, &e.Count); err != nil {
+			if err := rows.Scan(&e.Name, &e.Count, &e.LastSeen, &e.MeanResolve); err != nil {
 				serverError(w, r, err)
 				return
 			}
+			e.Daily = make([]int64, sparkDays)
 			result = append(result, e)
+			names = append(names, e.Name)
 		}
 		if err := rows.Err(); err != nil {
 			serverError(w, r, err)
 			return
 		}
+
+		// The sparkline always covers the last sparkDays days ending at ?to (or
+		// today), whatever ?from says, so the rows stay comparable.
+		end := time.Now().UTC().Truncate(24 * time.Hour)
+		if to, err := time.Parse("2006-01-02", q.Get("to")); err == nil {
+			end = to.UTC()
+		}
+		first := end.AddDate(0, 0, -(sparkDays - 1))
+		sq := url.Values{}
+		sq.Set("from", first.Format("2006-01-02"))
+		sq.Set("to", end.Format("2006-01-02"))
+		swhere, sargs := statsFilter(sq, "triggered_at", teams)
+		if len(names) > 0 {
+			swhere += " AND " + alertNameSQL + " = ANY(" + sargs.add(names) + ")"
+			srows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+				SELECT %s AS name, ((triggered_at - %s) / 86400)::int AS day, COUNT(*)
+				FROM   incidents
+				WHERE  %s
+				GROUP  BY name, day`, alertNameSQL, sargs.add(first.Unix()), swhere), sargs.all()...)
+			if err != nil {
+				serverError(w, r, err)
+				return
+			}
+			defer srows.Close()
+			idx := map[string]int{}
+			for i, e := range result {
+				idx[e.Name] = i
+			}
+			for srows.Next() {
+				var name string
+				var day int
+				var cnt int64
+				if err := srows.Scan(&name, &day, &cnt); err != nil {
+					serverError(w, r, err)
+					return
+				}
+				if day >= 0 && day < sparkDays {
+					result[idx[name]].Daily[day] = cnt
+				}
+			}
+			if err := srows.Err(); err != nil {
+				serverError(w, r, err)
+				return
+			}
+		}
 		respond(w, http.StatusOK, result)
 	}
 }
 
+// handleStatsHeatmap counts incidents by UTC weekday (0=Sunday) and hour:
+// always 7 rows of 24 cells, so a client never has to fill gaps.
+func handleStatsHeatmap(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		where, args := statsFilter(r.URL.Query(), "triggered_at", callerTeamIDs(r.Context()))
+		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+			SELECT EXTRACT(DOW  FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
+			       EXTRACT(HOUR FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
+			       COUNT(*)
+			FROM   incidents
+			WHERE  %s
+			GROUP  BY 1, 2`, where), args.all()...)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		defer rows.Close()
+		grid := make([][]int64, 7)
+		for d := range grid {
+			grid[d] = make([]int64, 24)
+		}
+		for rows.Next() {
+			var d, h int
+			var c int64
+			if err := rows.Scan(&d, &h, &c); err != nil {
+				serverError(w, r, err)
+				return
+			}
+			grid[d][h] = c
+		}
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		respond(w, http.StatusOK, map[string]any{"grid": grid})
+	}
+}
+
+// maxOccurrences caps the drilldown's table; the aggregates still cover all.
+const maxOccurrences = 200
+
+// handleStatsAlert is the per-alert drilldown. Every figure is computed over
+// the incidents that alert opened in the window, so it agrees with the top list.
+func handleStatsAlert(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// chi hands back the still-escaped segment when the path has %2F in it.
+		name, err := url.PathUnescape(chi.URLParam(r, "name"))
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid alert name"))
+			return
+		}
+		where, args := statsFilter(r.URL.Query(), "i.triggered_at", callerTeamIDs(r.Context()))
+		where = strings.ReplaceAll(where, "archived_at", "i.archived_at")
+		where = strings.ReplaceAll(where, "team_id =", "i.team_id =")
+		where += " AND split_part(i.signature, '|', 1) = " + args.add(name)
+
+		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+			SELECT i.id, i.triggered_at, i.acknowledged_at, i.resolved_at, i.status, i.severity,
+			       COALESCE(u.username, sa.name)
+			FROM   incidents i
+			LEFT   JOIN users u ON u.id = i.acknowledged_by
+			LEFT   JOIN service_accounts sa ON sa.id = i.acknowledged_by_service_account_id
+			WHERE  %s
+			ORDER  BY i.triggered_at DESC`, where), args.all()...)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		defer rows.Close()
+
+		type occurrence struct {
+			IncidentID     int64   `json:"incident_id"`
+			TriggeredAt    int64   `json:"triggered_at"`
+			AckSeconds     *int64  `json:"ack_seconds"`
+			ResolveSeconds *int64  `json:"resolve_seconds"`
+			Status         string  `json:"status"`
+			Severity       *string `json:"severity,omitempty"`
+			AcknowledgedBy *string `json:"acknowledged_by,omitempty"`
+		}
+		all := []occurrence{}
+		for rows.Next() {
+			var o occurrence
+			var ack, res *int64
+			if err := rows.Scan(&o.IncidentID, &o.TriggeredAt, &ack, &res, &o.Status, &o.Severity, &o.AcknowledgedBy); err != nil {
+				serverError(w, r, err)
+				return
+			}
+			if ack != nil {
+				v := *ack - o.TriggeredAt
+				o.AckSeconds = &v
+			}
+			if res != nil {
+				v := *res - o.TriggeredAt
+				o.ResolveSeconds = &v
+			}
+			all = append(all, o)
+		}
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		if len(all) == 0 {
+			respond(w, http.StatusNotFound, errResp("no incidents for this alert"))
+			return
+		}
+
+		byHour := make([]int64, 24)
+		byDay := make([]int64, 7)
+		var acks, fixes []int64
+		var longest int64
+		for _, o := range all {
+			t := time.Unix(o.TriggeredAt, 0).UTC()
+			byHour[t.Hour()]++
+			byDay[int(t.Weekday())]++
+			if o.AckSeconds != nil {
+				acks = append(acks, *o.AckSeconds)
+			}
+			if o.ResolveSeconds != nil {
+				fixes = append(fixes, *o.ResolveSeconds)
+				longest = max(longest, *o.ResolveSeconds)
+			}
+		}
+		// all is newest first, so the gap runs from the last row back to the first.
+		var meanGap *float64
+		if n := len(all); n > 1 {
+			g := float64(all[0].TriggeredAt-all[n-1].TriggeredAt) / float64(n-1)
+			meanGap = &g
+		}
+		stat := func(xs []int64) (mean, median *float64) {
+			if len(xs) == 0 {
+				return nil, nil
+			}
+			s := append([]int64(nil), xs...)
+			sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
+			var sum int64
+			for _, x := range s {
+				sum += x
+			}
+			m, md := float64(sum)/float64(len(s)), float64(s[len(s)/2])
+			return &m, &md
+		}
+		meanAck, medAck := stat(acks)
+		meanFix, medFix := stat(fixes)
+
+		occ := all
+		if len(occ) > maxOccurrences {
+			occ = occ[:maxOccurrences]
+		}
+		resp := map[string]any{
+			"name":                   name,
+			"count":                  len(all),
+			"first_seen":             all[len(all)-1].TriggeredAt,
+			"last_seen":              all[0].TriggeredAt,
+			"mean_gap_seconds":       meanGap,
+			"mean_ack_seconds":       meanAck,
+			"median_ack_seconds":     medAck,
+			"mean_resolve_seconds":   meanFix,
+			"median_resolve_seconds": medFix,
+			"longest_open_seconds":   nil,
+			"by_hour":                byHour,
+			"by_weekday":             byDay,
+			"occurrences":            occ,
+		}
+		if len(fixes) > 0 {
+			resp["longest_open_seconds"] = longest
+		}
+		respond(w, http.StatusOK, resp)
+	}
+}
+
 func handleStatsByHour(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))

internal/api/stats_alert_test.go

@@ -0,0 +1,88 @@
+package api_test
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestStatsTop_CountsIncidentsWithSparkAndMeans(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	postToIntegration(t, s, red.key, "fp-1", "DiskFull")
+	postToIntegration(t, s, red.key, "fp-2", "DiskFull") // same group key: one incident
+	postToIntegration(t, s, red.key, "fp-3", "Memory")
+
+	top := list(t, red.call(http.MethodGet, "/api/stats/alerts/top", nil))
+	if len(top) != 2 {
+		t.Fatalf("expected 2 names, got %d", len(top))
+	}
+	for _, row := range top {
+		if row["count"].(float64) != 1 {
+			t.Errorf("%v: count %v, want 1 incident", row["name"], row["count"])
+		}
+		daily := row["daily"].([]any)
+		if len(daily) != 30 {
+			t.Fatalf("daily has %d days, want 30", len(daily))
+		}
+		if row["mean_resolve_seconds"] != nil {
+			t.Errorf("unresolved incident must have null mean, got %v", row["mean_resolve_seconds"])
+		}
+	}
+}
+
+func TestStatsHeatmap_ShapeAndScope(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	blue := newTeam(t, s, "blue")
+	postToIntegration(t, s, red.key, "fp-r", "RedOnly")
+	postToIntegration(t, s, blue.key, "fp-b", "BlueOnly")
+
+	var out struct {
+		Grid [][]int64 `json:"grid"`
+	}
+	decode(t, red.call(http.MethodGet, "/api/stats/alerts/heatmap", nil), &out)
+	if len(out.Grid) != 7 || len(out.Grid[0]) != 24 {
+		t.Fatalf("grid is %dx?, want 7x24", len(out.Grid))
+	}
+	var sum int64
+	for _, row := range out.Grid {
+		for _, c := range row {
+			sum += c
+		}
+	}
+	if sum != 1 {
+		t.Errorf("red's heatmap counted %d incidents, want 1", sum)
+	}
+}
+
+func TestStatsAlert_DrilldownAndScope(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	blue := newTeam(t, s, "blue")
+	postToIntegration(t, s, red.key, "fp-r", "DiskFull")
+	postToIntegration(t, s, blue.key, "fp-b", "BlueOnly")
+
+	var d map[string]any
+	decode(t, red.call(http.MethodGet, "/api/stats/alerts/DiskFull", nil), &d)
+	if d["count"].(float64) != 1 {
+		t.Errorf("count = %v, want 1", d["count"])
+	}
+	if len(d["by_hour"].([]any)) != 24 || len(d["by_weekday"].([]any)) != 7 {
+		t.Errorf("by_hour/by_weekday have wrong lengths")
+	}
+	if occ := d["occurrences"].([]any); len(occ) != 1 {
+		t.Errorf("occurrences = %d, want 1", len(occ))
+	}
+	if d["mean_ack_seconds"] != nil || d["longest_open_seconds"] != nil {
+		t.Errorf("nothing acked or resolved yet, means must be null: %v", d)
+	}
+
+	// Another team's alert, and a name nobody has, are both a plain 404.
+	for _, name := range []string{"BlueOnly", "Nope"} {
+		resp := red.call(http.MethodGet, "/api/stats/alerts/"+name, nil)
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusNotFound {
+			t.Errorf("%s: expected 404, got %d", name, resp.StatusCode)
+		}
+	}
+}

internal/api/teams.go

@@ -826,7 +826,10 @@ func handleListTeamDeadman(db *sql.DB) http.HandlerFunc {
 		if !ok {
 			return
 		}
-		if !requireTeamMember(w, r, teamID) {
+		// The operator's instance-scoped account reconciles the switches by name,
+		// so it has to read them without being a member: that is owner reach,
+		// the same as the writes below, and still reads no incidents.
+		if !isInstanceServiceAccount(r.Context()) && !requireTeamMember(w, r, teamID) {
 			return
 		}
 

internal/config/config.go

@@ -8,6 +8,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/prom"
 )
 
 // MinOperatorKeyLength is the shortest TERDUT_OPERATOR_KEY accepted: it is a
@@ -79,6 +81,20 @@ type Config struct {
 	// "where this server is plugged in" — it is a statement about who owns
 	// this install's configuration, not a per-request toggle.
 	OperatorMode bool
+
+	// PrometheusAllowedHosts lists the Prometheus servers this install may query
+	// for the metrics behind an alert, matched against the host in each alert's
+	// generatorURL. Entries are host names (with "*" and a leading "**" as
+	// wildcards), IPs or CIDR ranges, optionally with a port; see prom.Allowlist.
+	// Empty turns the feature off, which is the default: the URL comes from a
+	// webhook payload, so querying it is opt-in.
+	PrometheusAllowedHosts []string
+
+	// PrometheusPreroll is how far before an alert started its graph begins.
+	PrometheusPreroll time.Duration
+
+	// PrometheusTimeout bounds one request to a Prometheus.
+	PrometheusTimeout time.Duration
 }
 
 // OIDC is the single sign-on configuration. Groups from the provider decide
@@ -151,6 +167,10 @@ func Load() Config {
 		OIDC:           loadOIDC(),
 
 		OperatorMode: boolean("TERDUT_OPERATOR_MODE", false),
+
+		PrometheusAllowedHosts: list("TERDUT_PROMETHEUS_ALLOWED_HOSTS", ""),
+		PrometheusPreroll:      duration("TERDUT_PROMETHEUS_PREROLL", time.Hour),
+		PrometheusTimeout:      duration("TERDUT_PROMETHEUS_TIMEOUT", 10*time.Second),
 	}
 }
 
@@ -180,6 +200,9 @@ func (c Config) Validate() error {
 	if c.OperatorKey != "" && len(c.OperatorKey) < MinOperatorKeyLength {
 		return fmt.Errorf("TERDUT_OPERATOR_KEY must be at least %d characters", MinOperatorKeyLength)
 	}
+	if _, err := prom.ParseAllowlist(c.PrometheusAllowedHosts); err != nil {
+		return fmt.Errorf("TERDUT_PROMETHEUS_ALLOWED_HOSTS: %w", err)
+	}
 	o := c.OIDC
 	if !o.Enabled() {
 		if c.DisablePasswordLogin {

internal/db/migrations/002_incident_metrics.sql

@@ -0,0 +1,25 @@
+-- Metrics captured from the Prometheus an alert came from (its generatorURL),
+-- once when the incident opens and once when it closes, so the graph outlives
+-- Prometheus' own retention. One row per incident, alert and kind.
+--
+-- A row exists whether or not the capture worked: a failure is recorded in
+-- error with the attempt count, which is what stops the worker from retrying a
+-- host that is not on the allowlist (or is gone) forever.
+
+CREATE TABLE incident_metric_snapshots (
+    id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    incident_id bigint NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+    alert_id bigint NOT NULL REFERENCES alerts(id) ON DELETE CASCADE,
+    kind text NOT NULL CHECK (kind IN ('opened', 'closed')),
+    expr text NOT NULL DEFAULT '',
+    window_start bigint NOT NULL,
+    window_end bigint NOT NULL,
+    step_seconds integer NOT NULL DEFAULT 0,
+    series jsonb NOT NULL DEFAULT '[]'::jsonb,
+    error text,
+    attempts integer NOT NULL DEFAULT 1,
+    captured_at bigint NOT NULL,
+    UNIQUE (incident_id, alert_id, kind)
+);
+
+CREATE INDEX incident_metric_snapshots_alert_id_idx ON incident_metric_snapshots (alert_id);

internal/prom/allow.go

@@ -0,0 +1,268 @@
+// Package prom fetches metrics from the Prometheus an alert came from.
+//
+// The address to query is the alert's generatorURL, which arrives in a webhook
+// payload and is therefore chosen by whoever holds an integration key. Left
+// unchecked, that would let the key holder make this server issue requests to
+// any host it can reach. Everything here is built around the Allowlist so that
+// cannot happen: a host is queried only if the operator listed it.
+package prom
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+	"net/url"
+	"path"
+	"strings"
+)
+
+// Allowlist is the set of Prometheus servers this install may query. The zero
+// value, and an empty list, permit nothing, which is what keeps the feature off
+// until an operator opts in.
+//
+// An entry is one of:
+//
+//   - a host name, optionally with a port: prometheus.monitoring.svc, prom:9090
+//   - a wildcard host name: "*" stands for exactly one DNS label (or part of
+//     one), so *.example.com matches a.example.com but neither example.com nor
+//     a.b.example.com, and prometheus-*.monitoring.svc matches prometheus-0.
+//     A leading "**" label matches one or more labels, so **.example.com also
+//     matches a.b.example.com.
+//   - an IP address or a CIDR range, optionally with a port: 10.0.0.0/8,
+//     [fd00::]:9090
+//
+// Without a port an entry matches any port. Matching is case-insensitive.
+type Allowlist struct {
+	hosts []hostPattern
+	nets  []netEntry
+}
+
+type hostPattern struct {
+	labels []string // lower-cased; a leading "**" label is kept as is
+	port   string   // empty matches any port
+}
+
+type netEntry struct {
+	prefix netip.Prefix
+	port   string
+}
+
+// ParseAllowlist validates and compiles the configured entries. It refuses an
+// entry that would admit every host (a bare "*", "*.*"): a typo there would
+// quietly turn the guard off.
+func ParseAllowlist(entries []string) (*Allowlist, error) {
+	a := &Allowlist{}
+	for _, raw := range entries {
+		e := strings.ToLower(strings.TrimSpace(raw))
+		if e == "" {
+			continue
+		}
+		if err := a.add(e); err != nil {
+			return nil, fmt.Errorf("allowed host %q: %w", raw, err)
+		}
+	}
+	return a, nil
+}
+
+func (a *Allowlist) add(e string) error {
+	// CIDR first: its colons (IPv6) and slash would confuse the host:port split.
+	if strings.Contains(e, "/") {
+		host, port := e, ""
+		if strings.HasPrefix(e, "[") {
+			end := strings.Index(e, "]")
+			if end < 0 {
+				return fmt.Errorf("unterminated [")
+			}
+			host = e[1:end]
+			rest := e[end+1:]
+			if strings.HasPrefix(rest, ":") {
+				port = rest[1:]
+			}
+		}
+		p, err := netip.ParsePrefix(host)
+		if err != nil {
+			return fmt.Errorf("not a CIDR range: %w", err)
+		}
+		if err := checkPort(port); err != nil {
+			return err
+		}
+		a.nets = append(a.nets, netEntry{prefix: p.Masked(), port: port})
+		return nil
+	}
+
+	// A bare IP address, IPv6 included.
+	if ip, err := netip.ParseAddr(e); err == nil {
+		a.nets = append(a.nets, netEntry{prefix: netip.PrefixFrom(ip.Unmap(), ip.Unmap().BitLen())})
+		return nil
+	}
+
+	host, port := e, ""
+	if strings.HasPrefix(e, "[") || strings.Count(e, ":") == 1 {
+		h, p, err := net.SplitHostPort(e)
+		if err != nil {
+			return fmt.Errorf("bad host:port: %w", err)
+		}
+		host, port = h, p
+		if ip, err := netip.ParseAddr(host); err == nil {
+			if err := checkPort(port); err != nil {
+				return err
+			}
+			a.nets = append(a.nets, netEntry{prefix: netip.PrefixFrom(ip.Unmap(), ip.Unmap().BitLen()), port: port})
+			return nil
+		}
+	}
+	if err := checkPort(port); err != nil {
+		return err
+	}
+
+	host = strings.TrimSuffix(host, ".")
+	if host == "" {
+		return fmt.Errorf("empty host")
+	}
+	for _, r := range host {
+		ok := r == '.' || r == '-' || r == '_' || r == '*' || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
+		if !ok {
+			return fmt.Errorf("unexpected character %q", r)
+		}
+	}
+	labels := strings.Split(host, ".")
+	for i, l := range labels {
+		if l == "" {
+			return fmt.Errorf("empty label")
+		}
+		if l == "**" && i != 0 {
+			return fmt.Errorf("** is only allowed as the first label")
+		}
+		if strings.Contains(l, "**") && l != "**" {
+			return fmt.Errorf("** must be a whole label")
+		}
+	}
+	// The last label has to say something, or the entry is allow-all.
+	if strings.Trim(labels[len(labels)-1], "*") == "" {
+		return fmt.Errorf("would match every host: the last label cannot be a wildcard alone")
+	}
+	a.hosts = append(a.hosts, hostPattern{labels: labels, port: port})
+	return nil
+}
+
+func checkPort(p string) error {
+	if p == "" {
+		return nil
+	}
+	for _, r := range p {
+		if r < '0' || r > '9' {
+			return fmt.Errorf("bad port %q", p)
+		}
+	}
+	return nil
+}
+
+// Empty reports whether nothing is allowed, i.e. the feature is off.
+func (a *Allowlist) Empty() bool {
+	return a == nil || (len(a.hosts) == 0 && len(a.nets) == 0)
+}
+
+// Port returns the effective port of u: the explicit one, else the scheme's.
+func Port(u *url.URL) string {
+	if p := u.Port(); p != "" {
+		return p
+	}
+	if u.Scheme == "https" {
+		return "443"
+	}
+	return "80"
+}
+
+// permits decides whether host:port may be contacted at ip. host is what the
+// URL said (a name or an IP literal), ip is where it actually resolved to; the
+// check runs on the resolved address at dial time so that a name that resolves
+// somewhere else than it did when it was listed (DNS rebinding) is caught.
+//
+//   - An address inside a listed CIDR range or equal to a listed IP is allowed.
+//   - A host name that matches a listed pattern is allowed, unless it resolved
+//     to an address that is never a Prometheus (loopback, link-local, which
+//     includes the cloud metadata address, unspecified, multicast). A wildcard
+//     over a domain somebody else controls must not be a way to reach internal
+//     services. To allow such an address deliberately, list it as a CIDR.
+func (a *Allowlist) permits(host, port string, ip netip.Addr) bool {
+	if a == nil {
+		return false
+	}
+	ip = ip.Unmap()
+	for _, n := range a.nets {
+		if n.prefix.Contains(ip) && (n.port == "" || n.port == port) {
+			return true
+		}
+	}
+	if forbiddenAddr(ip) {
+		return false
+	}
+	if _, isIP := parseHostIP(host); isIP {
+		return false // an IP literal is allowed only through a CIDR entry
+	}
+	return a.matchesName(host, port)
+}
+
+func parseHostIP(host string) (netip.Addr, bool) {
+	ip, err := netip.ParseAddr(strings.Trim(host, "[]"))
+	return ip, err == nil
+}
+
+// MatchesHost reports whether the URL's host is worth dialing at all: a listed
+// name, or an IP that falls in a listed range. It is the cheap early answer;
+// the dial-time check in permits is the one that is authoritative for names.
+func (a *Allowlist) MatchesHost(u *url.URL) bool {
+	if a.Empty() {
+		return false
+	}
+	host := strings.ToLower(strings.TrimSuffix(u.Hostname(), "."))
+	port := Port(u)
+	if ip, ok := parseHostIP(host); ok {
+		return a.permits(host, port, ip)
+	}
+	if a.matchesName(host, port) {
+		return true
+	}
+	// A name outside the patterns can still be admitted by a CIDR entry once it
+	// resolves; leave that to the dialer.
+	return len(a.nets) > 0
+}
+
+func (a *Allowlist) matchesName(host, port string) bool {
+	labels := strings.Split(strings.TrimSuffix(strings.ToLower(host), "."), ".")
+	for _, p := range a.hosts {
+		if (p.port == "" || p.port == port) && matchLabels(p.labels, labels) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchLabels(pattern, host []string) bool {
+	if len(pattern) > 0 && pattern[0] == "**" {
+		rest := pattern[1:]
+		if len(host) <= len(rest) {
+			return false
+		}
+		return matchLabels(rest, host[len(host)-len(rest):])
+	}
+	if len(pattern) != len(host) {
+		return false
+	}
+	for i := range pattern {
+		// The pattern was restricted to [a-z0-9_.-*] when parsed, so path.Match
+		// sees nothing but literals and "*" here.
+		if ok, _ := path.Match(pattern[i], host[i]); !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// forbiddenAddr is the set of addresses a Prometheus is never reached at and
+// that are dangerous to reach by mistake. Private ranges are deliberately not
+// in it: a Prometheus inside the cluster lives there.
+func forbiddenAddr(ip netip.Addr) bool {
+	return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified()
+}

internal/prom/client.go

@@ -0,0 +1,274 @@
+package prom
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	// maxResponseBytes bounds what one query may make us read.
+	maxResponseBytes = 5 << 20
+
+	// maxSeries bounds how many series one query may return before we give up on
+	// it; a rule expression without aggregation can match thousands.
+	maxSeries = 2000
+
+	// TargetPoints is how many points a graph is asked for.
+	TargetPoints = 300
+)
+
+// ErrNotAllowed means the target host is not on the allowlist.
+var ErrNotAllowed = errors.New("host is not in the allowed Prometheus hosts")
+
+// Series is one time series, ready to chart. Points are [unix seconds, value].
+type Series struct {
+	Labels map[string]string `json:"labels"`
+	Points [][2]float64      `json:"points"`
+}
+
+// Client queries Prometheus servers that the Allowlist permits.
+type Client struct {
+	http  *http.Client
+	allow *Allowlist
+}
+
+// NewClient returns a client that only ever connects to allowed addresses. The
+// check is made on the address the dialer is about to use, after name
+// resolution, and redirects and environment proxies are off: either would let a
+// permitted host send the request somewhere that is not.
+func NewClient(allow *Allowlist, timeout time.Duration) *Client {
+	if timeout <= 0 {
+		timeout = 10 * time.Second
+	}
+	dialer := &net.Dialer{Timeout: timeout}
+	tr := &http.Transport{
+		Proxy:               nil,
+		DialContext:         guardedDial(allow, dialer),
+		TLSHandshakeTimeout: timeout,
+		MaxIdleConns:        4,
+		IdleConnTimeout:     30 * time.Second,
+	}
+	return &Client{
+		allow: allow,
+		http: &http.Client{
+			Timeout:   timeout,
+			Transport: tr,
+			CheckRedirect: func(*http.Request, []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		},
+	}
+}
+
+// Enabled reports whether any host is allowed at all.
+func (c *Client) Enabled() bool { return c != nil && !c.allow.Empty() }
+
+// Allowed is the early, resolution-free answer: whether base could be queried.
+func (c *Client) Allowed(base *url.URL) bool {
+	return c != nil && c.allow.MatchesHost(base)
+}
+
+func guardedDial(allow *Allowlist, d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		var ips []netip.Addr
+		if ip, err := netip.ParseAddr(host); err == nil {
+			ips = []netip.Addr{ip}
+		} else {
+			ips, err = net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+			if err != nil {
+				return nil, err
+			}
+		}
+		var lastErr error = ErrNotAllowed
+		for _, ip := range ips {
+			if !allow.permits(host, port, ip) {
+				continue
+			}
+			// Dial the checked address itself, so a second lookup cannot return
+			// something different.
+			conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+			if err == nil {
+				return conn, nil
+			}
+			lastErr = err
+		}
+		return nil, lastErr
+	}
+}
+
+// QueryRange runs expr over [start, end] at the given step.
+func (c *Client) QueryRange(ctx context.Context, base *url.URL, expr string, start, end time.Time, step time.Duration) ([]Series, error) {
+	form := url.Values{
+		"query": {expr},
+		"start": {strconv.FormatInt(start.Unix(), 10)},
+		"end":   {strconv.FormatInt(end.Unix(), 10)},
+		"step":  {strconv.FormatInt(int64(step/time.Second), 10)},
+	}
+	res, err := c.do(ctx, base, "/api/v1/query_range", form)
+	if err != nil {
+		return nil, err
+	}
+	if res.ResultType != "matrix" {
+		return nil, fmt.Errorf("unexpected result type %q", res.ResultType)
+	}
+	var raw []struct {
+		Metric map[string]string    `json:"metric"`
+		Values [][2]json.RawMessage `json:"values"`
+	}
+	if err := json.Unmarshal(res.Result, &raw); err != nil {
+		return nil, fmt.Errorf("decode matrix: %w", err)
+	}
+	out := make([]Series, 0, len(raw))
+	for _, r := range raw {
+		s := Series{Labels: r.Metric, Points: make([][2]float64, 0, len(r.Values))}
+		for _, v := range r.Values {
+			if p, ok := point(v); ok {
+				s.Points = append(s.Points, p)
+			}
+		}
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+// Query runs expr at a single instant and returns one point per series.
+func (c *Client) Query(ctx context.Context, base *url.URL, expr string, at time.Time) ([]Series, error) {
+	form := url.Values{
+		"query": {expr},
+		"time":  {strconv.FormatInt(at.Unix(), 10)},
+	}
+	res, err := c.do(ctx, base, "/api/v1/query", form)
+	if err != nil {
+		return nil, err
+	}
+	if res.ResultType != "vector" {
+		return nil, fmt.Errorf("unexpected result type %q", res.ResultType)
+	}
+	var raw []struct {
+		Metric map[string]string  `json:"metric"`
+		Value  [2]json.RawMessage `json:"value"`
+	}
+	if err := json.Unmarshal(res.Result, &raw); err != nil {
+		return nil, fmt.Errorf("decode vector: %w", err)
+	}
+	out := make([]Series, 0, len(raw))
+	for _, r := range raw {
+		s := Series{Labels: r.Metric}
+		if p, ok := point(r.Value); ok {
+			s.Points = [][2]float64{p}
+		}
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+type apiResult struct {
+	ResultType string          `json:"resultType"`
+	Result     json.RawMessage `json:"result"`
+}
+
+func (c *Client) do(ctx context.Context, base *url.URL, apiPath string, form url.Values) (*apiResult, error) {
+	if !c.Allowed(base) {
+		return nil, ErrNotAllowed
+	}
+	u := *base
+	u.Path = strings.TrimSuffix(base.Path, "/") + apiPath
+	// POST, so a long expression is not limited by URL length.
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(form.Encode()))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(body) > maxResponseBytes {
+		return nil, errors.New("response too large")
+	}
+	var env struct {
+		Status    string     `json:"status"`
+		Data      *apiResult `json:"data"`
+		ErrorType string     `json:"errorType"`
+		Error     string     `json:"error"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		return nil, fmt.Errorf("prometheus answered %d with a body that is not JSON", resp.StatusCode)
+	}
+	if resp.StatusCode/100 != 2 || env.Status != "success" || env.Data == nil {
+		msg := env.Error
+		if msg == "" {
+			msg = "status " + strconv.Itoa(resp.StatusCode)
+		}
+		return nil, fmt.Errorf("prometheus: %s", truncate(msg, 200))
+	}
+	if n := countResults(env.Data.Result); n > maxSeries {
+		return nil, fmt.Errorf("query matches %d series, over the limit of %d", n, maxSeries)
+	}
+	return env.Data, nil
+}
+
+func countResults(raw json.RawMessage) int {
+	var items []json.RawMessage
+	if json.Unmarshal(raw, &items) != nil {
+		return 0
+	}
+	return len(items)
+}
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}
+
+// point decodes a [timestamp, "value"] pair. Prometheus writes values as
+// strings so that NaN and ±Inf survive; JSON cannot carry them, so those points
+// are dropped.
+func point(v [2]json.RawMessage) ([2]float64, bool) {
+	var ts float64
+	var s string
+	if json.Unmarshal(v[0], &ts) != nil || json.Unmarshal(v[1], &s) != nil {
+		return [2]float64{}, false
+	}
+	f, err := strconv.ParseFloat(s, 64)
+	if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
+		return [2]float64{}, false
+	}
+	return [2]float64{ts, f}, true
+}
+
+// StepFor picks a query step that yields about TargetPoints over the span, never
+// finer than a second.
+func StepFor(start, end time.Time) time.Duration {
+	span := end.Sub(start)
+	step := span / TargetPoints
+	if step < time.Second {
+		return time.Second
+	}
+	return step.Round(time.Second)
+}

internal/prom/filter.go

@@ -0,0 +1,51 @@
+package prom
+
+import "sort"
+
+// MaxSeriesPerAlert caps how many series are kept for one alert.
+const MaxSeriesPerAlert = 5
+
+// SelectSeries keeps the series that belong to this alert. The rule expression
+// returns every series it matches, not just the one that fired, so a series is
+// kept only if it agrees with the alert's labels on every label name they both
+// carry. Labels only one side has (the rule's severity, the series' __name__)
+// say nothing either way, and alertname is the rule's, not the series'.
+//
+// The result is capped at MaxSeriesPerAlert, best-labelled first: the more
+// labels a series shares with the alert, the likelier it is the one that fired.
+func SelectSeries(series []Series, alertLabels map[string]string) []Series {
+	type scored struct {
+		s     Series
+		score int
+	}
+	var kept []scored
+	for _, s := range series {
+		score, ok := 0, true
+		for k, v := range s.Labels {
+			if k == "__name__" || k == "alertname" {
+				continue
+			}
+			av, has := alertLabels[k]
+			if !has {
+				continue
+			}
+			if av != v {
+				ok = false
+				break
+			}
+			score++
+		}
+		if ok && len(s.Points) > 0 {
+			kept = append(kept, scored{s, score})
+		}
+	}
+	sort.SliceStable(kept, func(i, j int) bool { return kept[i].score > kept[j].score })
+	if len(kept) > MaxSeriesPerAlert {
+		kept = kept[:MaxSeriesPerAlert]
+	}
+	out := make([]Series, len(kept))
+	for i, k := range kept {
+		out[i] = k.s
+	}
+	return out
+}

internal/prom/parse.go

@@ -0,0 +1,38 @@
+package prom
+
+import (
+	"net/url"
+	"strings"
+)
+
+// ParseGeneratorURL splits an Alertmanager generatorURL into the Prometheus it
+// points at and the expression of the rule that fired. Prometheus builds it as
+// <external-url>/graph?g0.expr=<expr>&g0.tab=1 (/query on Prometheus 3), and the
+// external URL may carry a path prefix, which is kept in base.
+//
+// ok is false for anything else: a non-HTTP scheme, credentials in the URL, a
+// page that is not the graph page, or no expression. The caller then has
+// nothing to query and falls back to showing the link.
+func ParseGeneratorURL(raw string) (base *url.URL, expr string, ok bool) {
+	u, err := url.Parse(strings.TrimSpace(raw))
+	if err != nil || u.Host == "" || u.User != nil {
+		return nil, "", false
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil, "", false
+	}
+	prefix := ""
+	switch {
+	case strings.HasSuffix(u.Path, "/graph"):
+		prefix = strings.TrimSuffix(u.Path, "/graph")
+	case strings.HasSuffix(u.Path, "/query"):
+		prefix = strings.TrimSuffix(u.Path, "/query")
+	default:
+		return nil, "", false
+	}
+	expr = strings.TrimSpace(u.Query().Get("g0.expr"))
+	if expr == "" {
+		return nil, "", false
+	}
+	return &url.URL{Scheme: u.Scheme, Host: u.Host, Path: prefix}, expr, true
+}

internal/prom/prom_test.go

@@ -0,0 +1,237 @@
+package prom
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"testing"
+	"time"
+)
+
+func TestParseAllowlistRejectsAllowAll(t *testing.T) {
+	for _, e := range []string{"*", "**", "*.*", "**.*", "a.**", "pro**.example.com", "ex ample.com", "host:abc", "10.0.0.0/33"} {
+		if _, err := ParseAllowlist([]string{e}); err == nil {
+			t.Errorf("%q was accepted", e)
+		}
+	}
+}
+
+func TestAllowlistNames(t *testing.T) {
+	a, err := ParseAllowlist([]string{
+		"prometheus.monitoring.svc",
+		"prom-*.monitoring.svc",
+		"*.example.com",
+		"**.corp.example",
+		"fixed.example.org:9090",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	pub := netip.MustParseAddr("10.1.2.3")
+	cases := []struct {
+		host, port string
+		want       bool
+	}{
+		{"prometheus.monitoring.svc", "9090", true},
+		{"PROMETHEUS.monitoring.svc", "80", true},
+		{"prom-0.monitoring.svc", "9090", true},
+		{"prom-0.other.svc", "9090", false},
+		{"a.example.com", "443", true},
+		{"example.com", "443", false},
+		{"a.b.example.com", "443", false},
+		{"a.corp.example", "80", true},
+		{"a.b.c.corp.example", "80", true},
+		{"corp.example", "80", false},
+		{"fixed.example.org", "9090", true},
+		{"fixed.example.org", "80", false},
+		{"evilexample.com", "80", false},
+		{"a.example.com.evil.net", "80", false},
+	}
+	for _, c := range cases {
+		if got := a.permits(c.host, c.port, pub); got != c.want {
+			t.Errorf("permits(%s:%s) = %v, want %v", c.host, c.port, got, c.want)
+		}
+	}
+}
+
+func TestAllowlistForbiddenAddressesAndCIDR(t *testing.T) {
+	a, _ := ParseAllowlist([]string{"*.example.com", "192.168.0.0/16", "[fd00::]:9090", "127.0.0.9"})
+	cases := []struct {
+		name, host, port, ip string
+		want                 bool
+	}{
+		{"wildcard to private is fine", "a.example.com", "80", "10.0.0.5", true},
+		{"wildcard rebinding to loopback", "a.example.com", "80", "127.0.0.1", false},
+		{"wildcard to metadata", "a.example.com", "80", "169.254.169.254", false},
+		{"wildcard to unspecified", "a.example.com", "80", "0.0.0.0", false},
+		{"wildcard to v4-mapped loopback", "a.example.com", "80", "::ffff:127.0.0.1", false},
+		{"cidr literal", "192.168.4.5", "80", "192.168.4.5", true},
+		{"outside cidr", "10.9.9.9", "80", "10.9.9.9", false},
+		{"cidr admits unlisted name", "anything.test", "80", "192.168.1.1", true},
+		{"explicit loopback entry", "127.0.0.9", "80", "127.0.0.9", true},
+		{"other loopback", "127.0.0.1", "80", "127.0.0.1", false},
+		{"ipv6 with port ok", "fd00::1", "9090", "fd00::1", false}, // entry is a single address fd00::, not the range
+		{"ipv6 exact", "fd00::", "9090", "fd00::", true},
+		{"ipv6 wrong port", "fd00::", "80", "fd00::", false},
+	}
+	for _, c := range cases {
+		if got := a.permits(c.host, c.port, netip.MustParseAddr(c.ip)); got != c.want {
+			t.Errorf("%s: permits = %v, want %v", c.name, got, c.want)
+		}
+	}
+}
+
+func TestEmptyAllowlistPermitsNothing(t *testing.T) {
+	var nilList *Allowlist
+	empty, _ := ParseAllowlist(nil)
+	for _, a := range []*Allowlist{nilList, empty} {
+		if !a.Empty() || a.permits("x.example.com", "80", netip.MustParseAddr("10.0.0.1")) {
+			t.Error("empty allowlist permitted something")
+		}
+	}
+}
+
+func TestParseGeneratorURL(t *testing.T) {
+	cases := []struct {
+		raw      string
+		wantBase string
+		wantExpr string
+		ok       bool
+	}{
+		{"http://prom:9090/graph?g0.expr=up%3D%3D0&g0.tab=1", "http://prom:9090", "up==0", true},
+		{"https://p.example.com/prom/graph?g0.expr=rate%28x%5B5m%5D%29%3E1", "https://p.example.com/prom", "rate(x[5m])>1", true},
+		{"https://p.example.com/query?g0.expr=up", "https://p.example.com", "up", true},
+		{"http://prom:9090/graph?g0.tab=1", "", "", false},
+		{"http://prom:9090/other?g0.expr=up", "", "", false},
+		{"ftp://prom/graph?g0.expr=up", "", "", false},
+		{"http://user:pw@prom/graph?g0.expr=up", "", "", false},
+		{"", "", "", false},
+		{"not a url", "", "", false},
+	}
+	for _, c := range cases {
+		base, expr, ok := ParseGeneratorURL(c.raw)
+		if ok != c.ok || expr != c.wantExpr || (ok && base.String() != c.wantBase) {
+			t.Errorf("%q = (%v, %q, %v), want (%q, %q, %v)", c.raw, base, expr, ok, c.wantBase, c.wantExpr, c.ok)
+		}
+	}
+}
+
+func TestSelectSeries(t *testing.T) {
+	mk := func(l map[string]string) Series { return Series{Labels: l, Points: [][2]float64{{1, 1}}} }
+	series := []Series{
+		mk(map[string]string{"__name__": "up", "instance": "a", "job": "x"}),
+		mk(map[string]string{"__name__": "up", "instance": "b", "job": "x"}),
+		mk(map[string]string{"instance": "a", "job": "y"}),
+		{Labels: map[string]string{"instance": "a"}}, // no points
+	}
+	got := SelectSeries(series, map[string]string{"alertname": "Down", "instance": "a", "job": "x", "severity": "page"})
+	if len(got) != 1 || got[0].Labels["job"] != "x" || got[0].Labels["instance"] != "a" {
+		t.Fatalf("got %+v", got)
+	}
+	if n := len(SelectSeries(series, nil)); n != 3 {
+		t.Errorf("no alert labels should keep every series with points, got %d", n)
+	}
+}
+
+func TestStepFor(t *testing.T) {
+	now := time.Now()
+	if s := StepFor(now.Add(-time.Minute), now); s != time.Second {
+		t.Errorf("short span step = %v", s)
+	}
+	if s := StepFor(now.Add(-5*time.Hour), now); s != time.Minute {
+		t.Errorf("5h step = %v, want 1m", s)
+	}
+}
+
+func fakeProm(t *testing.T) (*httptest.Server, *int) {
+	t.Helper()
+	hits := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits++
+		r.ParseForm()
+		w.Header().Set("Content-Type", "application/json")
+		switch r.URL.Path {
+		case "/api/v1/query_range":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[
+				{"metric":{"instance":"a"},"values":[[100,"1"],[110,"NaN"],[120,"2.5"]]}]}}`))
+		case "/api/v1/query":
+			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[
+				{"metric":{"instance":"a"},"value":[130,"3"]}]}}`))
+		case "/redirect/api/v1/query":
+			http.Redirect(w, r, "http://127.0.0.1:1/", http.StatusFound)
+		default:
+			w.WriteHeader(400)
+			w.Write([]byte(`{"status":"error","errorType":"bad_data","error":"nope"}`))
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hits
+}
+
+func TestClientQueries(t *testing.T) {
+	srv, _ := fakeProm(t)
+	u, _ := url.Parse(srv.URL)
+	// httptest listens on loopback, which is refused unless listed as a CIDR.
+	allow, _ := ParseAllowlist([]string{"127.0.0.0/8"})
+	c := NewClient(allow, 5*time.Second)
+
+	got, err := c.QueryRange(context.Background(), u, "up", time.Unix(100, 0), time.Unix(120, 0), 10*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || len(got[0].Points) != 2 || got[0].Points[1] != [2]float64{120, 2.5} {
+		t.Fatalf("range = %+v (NaN point should be dropped)", got)
+	}
+	inst, err := c.Query(context.Background(), u, "up", time.Unix(130, 0))
+	if err != nil || len(inst) != 1 || inst[0].Points[0][1] != 3 {
+		t.Fatalf("instant = %+v, %v", inst, err)
+	}
+}
+
+func TestClientRefusesUnlistedAndLoopback(t *testing.T) {
+	srv, hits := fakeProm(t)
+	u, _ := url.Parse(srv.URL)
+
+	// Listed by name, but the name resolves to loopback: refused at dial time.
+	local, _ := url.Parse("http://localhost:" + u.Port())
+	wild, _ := ParseAllowlist([]string{"local*"})
+	if _, err := NewClient(wild, time.Second).Query(context.Background(), local, "up", time.Now()); err == nil {
+		t.Error("wildcard-matched name resolving to loopback was queried")
+	}
+
+	// Not listed at all.
+	other, _ := ParseAllowlist([]string{"*.example.com"})
+	if _, err := NewClient(other, time.Second).Query(context.Background(), u, "up", time.Now()); !errors.Is(err, ErrNotAllowed) {
+		t.Errorf("unlisted host: err = %v", err)
+	}
+
+	// Disabled.
+	if _, err := NewClient(nil, time.Second).Query(context.Background(), u, "up", time.Now()); !errors.Is(err, ErrNotAllowed) {
+		t.Errorf("disabled: err = %v", err)
+	}
+	if *hits != 0 {
+		t.Errorf("server was contacted %d times", *hits)
+	}
+}
+
+func TestClientDoesNotFollowRedirects(t *testing.T) {
+	srv, _ := fakeProm(t)
+	u, _ := url.Parse(srv.URL + "/redirect")
+	allow, _ := ParseAllowlist([]string{"127.0.0.0/8"})
+	if _, err := NewClient(allow, time.Second).Query(context.Background(), u, "up", time.Now()); err == nil {
+		t.Error("redirect response was treated as success")
+	}
+}
+
+func TestClientSurfacesPrometheusError(t *testing.T) {
+	srv, _ := fakeProm(t)
+	u, _ := url.Parse(srv.URL + "/missing")
+	allow, _ := ParseAllowlist([]string{"127.0.0.0/8"})
+	_, err := NewClient(allow, time.Second).Query(context.Background(), u, "up", time.Now())
+	if err == nil {
+		t.Fatal("expected an error")
+	}
+}

internal/web/static/app.css

@@ -7,6 +7,7 @@
 
 :root {
   --bg: #f5f6f8;
+  --dot: rgba(60, 70, 95, 0.16);
   --surface: #ffffff;
   --surface-2: #eff1f4;
   --surface-hover: #f7f8fa;
@@ -79,6 +80,7 @@
 @media (prefers-color-scheme: dark) {
   :root:not([data-theme="light"]) {
     --bg: #0f1115;
+    --dot: rgba(148, 156, 180, 0.26);
     --surface: #1a1e26;
     --surface-2: #232834;
     --surface-hover: #20252e;
@@ -114,6 +116,7 @@
 }
 :root[data-theme="dark"] {
   --bg: #0f1115;
+  --dot: rgba(148, 156, 180, 0.26);
   --surface: #1a1e26;
   --surface-2: #232834;
   --surface-hover: #20252e;
@@ -163,6 +166,17 @@ body {
   -webkit-tap-highlight-color: transparent;
 }
 
+/* Dot grid behind everything, fading out down the screen. z-index -1 sits above
+   the canvas background but below all in-flow content, so no wrapper element
+   is needed. */
+body::before {
+  content: ""; position: fixed; inset: 0; z-index: -1; pointer-events: none;
+  background-image: radial-gradient(circle, var(--dot) 1.25px, transparent 1.75px);
+  background-size: 28px 28px;
+  -webkit-mask-image: linear-gradient(to bottom, #000 0%, #000 25%, transparent 88%);
+  mask-image: linear-gradient(to bottom, #000 0%, #000 25%, transparent 88%);
+}
+
 a { color: inherit; text-decoration: none; }
 button, input, textarea, select { font: inherit; color: inherit; }
 code { font-family: var(--mono); font-size: 0.9em; }
@@ -586,6 +600,33 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
 .alert-item-summary { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
 .alert-item-foot { display: flex; flex-wrap: wrap; gap: 4px 12px; font-size: 13px; color: var(--faint); }
 .alert-item-foot a { color: var(--accent); font-weight: 600; }
+
+/* Metrics under an alert */
+.metrics { display: grid; gap: 6px; }
+.metrics-note { color: var(--faint); font-size: 13px; }
+.metrics-now { display: flex; align-items: baseline; gap: 8px; font-size: 14px; }
+.metrics-now strong { font-size: 18px; font-variant-numeric: tabular-nums; }
+.metrics-now.none { color: var(--muted); }
+.metrics-now-label { color: var(--faint); font-size: 12px; font-weight: 650; text-transform: uppercase; letter-spacing: .04em; }
+.metric-chart { width: 100%; height: auto; display: block; }
+.mc-grid { stroke: var(--border); stroke-width: 1; }
+.mc-label { fill: var(--faint); font-size: 10px; }
+.mc-marker { stroke-width: 1; stroke-dasharray: 3 3; }
+.mc-marker.start { stroke: var(--crit); }
+.mc-marker.end { stroke: var(--ok); }
+.mc-line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; }
+.mc-line.s0, .mc-swatch.s0 { stroke: var(--accent); background: var(--accent); }
+.mc-line.s1, .mc-swatch.s1 { stroke: var(--warn); background: var(--warn); }
+.mc-line.s2, .mc-swatch.s2 { stroke: var(--ok); background: var(--ok); }
+.mc-line.s3, .mc-swatch.s3 { stroke: var(--crit); background: var(--crit); }
+.mc-line.s4, .mc-swatch.s4 { stroke: var(--muted); background: var(--muted); }
+.mc-dot.s0 { fill: var(--accent); } .mc-dot.s1 { fill: var(--warn); } .mc-dot.s2 { fill: var(--ok); }
+.mc-dot.s3 { fill: var(--crit); } .mc-dot.s4 { fill: var(--muted); }
+.mc-swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; flex: none; }
+.metrics-legend { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 2px 14px; font-size: 12px; color: var(--muted); }
+.metrics-legend li { display: flex; align-items: center; gap: 6px; overflow-wrap: anywhere; }
+.metrics-expr { font-size: 12px; color: var(--faint); overflow-wrap: anywhere; }
+
 /* A disclosure is a button: a chevron that turns, and a 44px target on a
    touch screen (the bare triangle it replaces was a few pixels wide). */
 details > summary {
@@ -1320,9 +1361,111 @@ button.rota-week:hover { background: var(--surface-2); color: var(--text); }
 .columns .col-bar { fill: var(--accent); }
 .columns .col:hover .col-bar { opacity: 0.75; }
 .columns .col-label { fill: var(--faint); font-size: 10px; font-family: var(--font); }
+.chart-head { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: baseline; gap: 4px 8px; margin-bottom: 10px; }
+.chart-head .chart-title { margin-bottom: 0; }
+.chart-note { font-size: 12px; color: var(--muted); }
+.stat-note { margin-top: 2px; font-size: 12px; color: var(--muted); }
+.stats-head { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 12px; }
+.stats-head .chips { padding-left: 0; padding-right: 0; flex: 1 1 100%; }
+.stats-alert-name { font-family: var(--mono, ui-monospace, monospace); font-size: 20px; overflow-wrap: anywhere; }
+.stats-split { display: grid; gap: 14px; }
+.pattern-strip { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 20px; font-size: 14px; }
+.pattern-strip .chart-title { margin-bottom: 0; }
+.patterns { display: grid; gap: 14px; }
+.pattern-title { font-size: 15px; font-weight: 600; }
+.pattern-sub { margin-top: 2px; font-size: 13px; color: var(--muted); }
+
+/* Top alerts: each row is a link to the alert's drilldown. */
+.toplist { display: grid; overflow-x: auto; }
+.top-row {
+  display: grid; align-items: center; gap: 12px; min-width: 640px; min-height: 44px; padding: 6px 4px;
+  grid-template-columns: minmax(140px, 2.4fr) minmax(130px, 1.4fr) 100px 100px 70px 18px;
+  border-top: 1px solid var(--border); color: inherit; text-decoration: none; font-size: 13px;
+}
+a.top-row:hover { background: var(--surface-2); }
+.top-head { min-height: 0; border-top: 0; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); grid-template-columns: minmax(140px, 2.4fr) minmax(130px, 1.4fr) 100px 100px 70px 18px; }
+.top-count { display: flex; align-items: center; gap: 8px; }
+.top-count .hbar-track { flex: 1; height: 6px; }
+.top-count .hbar-count { order: -1; min-width: 2ch; color: var(--text); font-weight: 600; }
+.top-pct, .top-cell { color: var(--muted); font-variant-numeric: tabular-nums; }
+.spark { display: flex; align-items: flex-end; gap: 1px; height: 24px; }
+.spark-bar { width: 2px; border-radius: 1px; background: var(--border-strong); }
+.spark-bar.on { background: var(--accent); }
+
+/* Heatmap: weekday × hour. */
+.heatmap { display: grid; gap: 3px; overflow-x: auto; min-width: 0; }
+.heatmap > * { min-width: 520px; }
+.heat-row { display: flex; align-items: center; gap: 8px; }
+.heat-day { width: 30px; flex: none; font-size: 12px; color: var(--muted); }
+.heat-cells { flex: 1; display: grid; grid-template-columns: repeat(24, minmax(0, 1fr)); gap: 3px; }
+.heat-cell { height: 24px; border-radius: 4px; background: var(--surface-2); }
+.heat-cell.lv1 { background: color-mix(in srgb, var(--accent) 30%, var(--surface-2)); }
+.heat-cell.lv2 { background: color-mix(in srgb, var(--accent) 55%, var(--surface-2)); }
+.heat-cell.lv3 { background: color-mix(in srgb, var(--accent) 80%, var(--surface-2)); }
+.heat-cell.lv4 { background: var(--accent); }
+.heat-hour { font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; }
+
+/* Drilldown: timeline, resolve bars, occurrences. */
+.tl { position: relative; height: 56px; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; background: var(--surface-2); }
+.tl-mark { position: absolute; height: 18px; min-width: 8px; border-radius: 5px; background: var(--accent); }
+.tl-axis { display: flex; justify-content: space-between; margin-top: 6px; font-size: 11px; color: var(--muted); }
+.fixbars { display: grid; gap: 8px; }
+.fix-row { display: flex; align-items: center; gap: 10px; font-size: 12.5px; }
+.fix-label { width: 104px; flex: none; color: var(--muted); font-variant-numeric: tabular-nums; }
+.fix-track { flex: 1; min-width: 0; display: flex; height: 16px; border-radius: 4px; background: var(--surface-2); overflow: hidden; }
+.fix-ack { background: var(--warn); }
+.fix-work { background: var(--accent); }
+.fix-total { width: 56px; flex: none; text-align: right; font-weight: 600; font-variant-numeric: tabular-nums; }
+.occ-wrap { overflow-x: auto; }
+.occ { width: 100%; min-width: 560px; border-collapse: collapse; font-size: 13px; font-variant-numeric: tabular-nums; }
+.occ th { text-align: left; padding: 6px 8px; font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
+.occ td { padding: 10px 8px; border-top: 1px solid var(--border); }
 @media (min-width: 900px) {
   .stat-tiles { grid-template-columns: repeat(3, 1fr); }
 }
+@media (min-width: 900px) {
+  .stats-split { grid-template-columns: 3fr 1fr; align-items: stretch; }
+}
+
+/* Phones: nothing on the stats pages may be wider than the screen, so the
+   wide rows fold into two or three lines instead of scrolling sideways. */
+.stats > *, .stats-split > * { min-width: 0; }
+@media (max-width: 599px) {
+  .top-head { display: none; }
+  .toplist { overflow-x: visible; }
+  .top-row {
+    min-width: 0; grid-template-columns: minmax(0, 1fr) auto; row-gap: 4px; padding: 10px 4px;
+  }
+  .top-row > :nth-child(1) { grid-column: 1; grid-row: 1; font-weight: 600; }
+  .top-row > :nth-child(6) { grid-column: 2; grid-row: 1; }
+  .top-row > :nth-child(2) { grid-column: 1; grid-row: 2; }
+  .top-row > :nth-child(3) { grid-column: 2; grid-row: 2; }
+  .top-row > :nth-child(4) { grid-column: 1; grid-row: 3; font-size: 12px; }
+  .top-row > :nth-child(5) { grid-column: 2; grid-row: 3; font-size: 12px; text-align: right; }
+  .top-row > :nth-child(4)::before { content: "Last seen "; }
+  .top-row > :nth-child(5)::before { content: "Fix "; }
+
+  .heatmap > * { min-width: 0; }
+  .heat-row, .heat-cells { gap: 2px; }
+  .heat-day { width: 26px; }
+  .heat-cell { height: 18px; border-radius: 3px; }
+  .heat-hour { font-size: 9px; overflow: visible; white-space: nowrap; }
+  .heat-hour:not(:nth-child(6n+1)) { visibility: hidden; }
+
+  .fix-row { flex-wrap: wrap; row-gap: 2px; }
+  .fix-label { width: auto; flex: 1; }
+  .fix-total { width: auto; }
+  .fix-track { flex: 1 1 100%; order: 3; }
+
+  .occ-wrap { overflow-x: visible; }
+  .occ { min-width: 0; }
+  .occ thead { display: none; }
+  .occ, .occ tbody { display: block; }
+  .occ tr { display: grid; grid-template-columns: 1fr 1fr; gap: 2px 12px; padding: 10px 0; border-top: 1px solid var(--border); }
+  .occ td { border-top: 0; padding: 0; overflow-wrap: anywhere; }
+  .occ td:first-child { grid-column: 1 / -1; font-weight: 600; }
+  .occ td[data-label]::before { content: attr(data-label) ": "; color: var(--muted); font-size: 12px; }
+}
 
 /* On-call, wide: the hero and the week on the left, your shifts on the right.
    .view-page > * caps its children at 760px, so the grid lifts that. */

internal/web/static/js/api.js

@@ -108,6 +108,7 @@ export const unsnooze = (id) => call('DELETE', `/incidents/${id}/snooze`);
 export const archive = (id) => call('POST', `/incidents/${id}/archive`);
 export const unarchive = (id) => call('DELETE', `/incidents/${id}/archive`);
 export const addNote = (id, content, pinned = false) => call('POST', `/incidents/${id}/notes`, { body: { content, pinned } });
+export const metrics = (id) => call('GET', `/incidents/${id}/metrics`);
 export const similar = (id) => call('GET', `/incidents/${id}/similar`);
 export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/notes/${eventID}`);
 
@@ -115,6 +116,8 @@ export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/note
 export const statsIncidents = (query) => call('GET', '/stats/incidents', { query });
 export const statsTop = (query) => call('GET', '/stats/alerts/top', { query });
 export const statsByHour = (query) => call('GET', '/stats/alerts/by-hour', { query });
+export const statsHeatmap = (query) => call('GET', '/stats/alerts/heatmap', { query });
+export const statsAlert = (name, query) => call('GET', `/stats/alerts/${encodeURIComponent(name)}`, { query });
 export const statsByDay = (query) => call('GET', '/stats/alerts/by-day', { query });
 
 // alerts

internal/web/static/js/app.js

@@ -59,6 +59,8 @@ function parseRoute(pathname) {
   // /admin/teams/7 fall through to the queue.
   const g = pathname.match(/^\/admin\/teams\/(\d+)\/?$/);
   if (g) return { section: 'adminteam', team: Number(g[1]) };
+  const sa = pathname.match(/^\/stats\/alert\/([^/]+)\/?$/);
+  if (sa) return { section: 'stats', alert: decodeURIComponent(sa[1]) };
   const name = pathname.replace(/^\/|\/$/g, '');
   // The Admin and Team tabs' sub-sections are routes of their own. Each view
   // owns the table of its own, since each also builds the strip that links to
@@ -147,7 +149,7 @@ function render() {
   // the settings should not land you halfway down them. So does a changed
   // subject — one team to the next is two pages, not one scrolled page.
   else if (prev.section !== route.section || prev.tab !== route.tab
-    || prev.user !== route.user || prev.team !== route.team) window.scrollTo(0, 0);
+    || prev.user !== route.user || prev.team !== route.team || prev.alert !== route.alert) window.scrollTo(0, 0);
 
   updateTitle();
 }

internal/web/static/js/charts.js

@@ -0,0 +1,117 @@
+// Shared by the stats overview and the per-alert drilldown: the time-range
+// chips, the chart cards and the SVG column chart.
+
+import { h } from './ui.js';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+// `days` counts back from today, inclusive; the server reads from/to as UTC
+// dates, so these are too.
+export const RANGES = [
+  { id: 'today', label: 'Today', days: 1 },
+  { id: '7d', label: '7d', days: 7 },
+  { id: '30d', label: '30d', days: 30 },
+  { id: '90d', label: '90d', days: 90 },
+  { id: 'all', label: 'All', days: null },
+];
+
+// The chosen range outlives navigating between the overview and a drilldown.
+let range = '30d';
+export const getRange = () => range;
+export const setRangeId = (id) => { range = id; };
+
+const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10);
+
+export function rangeQuery() {
+  const r = RANGES.find((x) => x.id === range);
+  if (!r.days) return {};
+  const now = Date.now();
+  return { from: utcDate(now - (r.days - 1) * DAY_MS), to: utcDate(now) };
+}
+
+export function chips(onPick, ranges = RANGES) {
+  return h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Time range' },
+    ranges.map((r) => h('button', {
+      class: 'chip',
+      type: 'button',
+      role: 'tab',
+      'aria-selected': String(r.id === range),
+      onclick: () => onPick(r.id),
+      text: r.label,
+    })));
+}
+
+export function card(title, content, note) {
+  return h('section', { class: 'chart-card card card-pad' },
+    h('div', { class: 'chart-head' },
+      h('h3', { class: 'chart-title', text: title }),
+      note && h('span', { class: 'chart-note', text: note })),
+    content);
+}
+
+export const pad2 = (n) => String(n).padStart(2, '0');
+export const WEEKDAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
+// Monday first, as the mockup has it; server weekdays are 0=Sunday.
+export const MONDAY_FIRST = [1, 2, 3, 4, 5, 6, 0];
+
+const SVG_NS = 'http://www.w3.org/2000/svg';
+
+function svg(tag, attrs = {}, text) {
+  const el = document.createElementNS(SVG_NS, tag);
+  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
+  if (text != null) el.textContent = text;
+  return el;
+}
+
+// A column chart: one bar per item, the value in a tooltip, and a label under
+// the items marked `tick`.
+export function columns(items, label) {
+  const W = 480;
+  const H = 140;
+  const base = H - 18;
+  const step = W / items.length;
+  const max = Math.max(...items.map((x) => x.value), 1);
+  const root = svg('svg', {
+    class: 'columns', viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': label,
+  });
+  root.appendChild(svg('line', { class: 'axis', x1: 0, x2: W, y1: base, y2: base }));
+  items.forEach((it, i) => {
+    const bh = it.value ? Math.max(2, (it.value / max) * (base - 6)) : 0;
+    const x = i * step + step * 0.15;
+    const g = svg('g', { class: 'col' });
+    g.appendChild(svg('title', {}, `${it.label}: ${it.value}`));
+    // A full-height transparent hit area, so a tiny bar is still hoverable.
+    g.appendChild(svg('rect', { class: 'col-hit', x: i * step, y: 0, width: step, height: base }));
+    if (bh) g.appendChild(svg('rect', { class: 'col-bar', x, y: base - bh, width: step * 0.7, height: bh, rx: 2 }));
+    root.appendChild(g);
+    if (it.tick) {
+      root.appendChild(svg('text', { class: 'col-label', x: i * step + step / 2, y: H - 4, 'text-anchor': 'middle' }, it.label));
+    }
+  });
+  return root;
+}
+
+// A 7×24 weekday-by-hour grid from `grid[weekday][hour]` (0=Sunday); cell
+// opacity steps with the count relative to the busiest cell.
+export function heatmap(grid) {
+  const max = Math.max(...grid.flat(), 1);
+  const level = (c) => (c === 0 ? 0 : Math.min(4, Math.ceil((c / max) * 4)));
+  const rows = MONDAY_FIRST.map((w) => h('div', { class: 'heat-row' },
+    h('span', { class: 'heat-day', text: WEEKDAYS[w] }),
+    h('div', { class: 'heat-cells' }, grid[w].map((c, hr) => h('div', {
+      class: `heat-cell lv${level(c)}`,
+      title: `${WEEKDAYS[w]} ${pad2(hr)}:00 UTC: ${c}`,
+    })))));
+  const axis = h('div', { class: 'heat-row' },
+    h('span', { class: 'heat-day' }),
+    h('div', { class: 'heat-cells' }, Array.from({ length: 24 }, (_, hr) =>
+      h('span', { class: 'heat-hour', text: hr % 3 === 0 ? pad2(hr) : '' }))));
+  const peak = grid.flatMap((row, d) => row.map((c, hr) => ({ d, hr, c }))).reduce((a, b) => (b.c > a.c ? b : a));
+  return h('div', {
+    class: 'heatmap',
+    role: 'img',
+    'aria-label': peak.c
+      ? `Heatmap of incidents by weekday and hour in UTC. The busiest slot is ${WEEKDAYS[peak.d]} at ${pad2(peak.hr)}:00 with ${peak.c}.`
+      : 'Heatmap of incidents by weekday and hour in UTC. Nothing in this range.',
+  }, rows, axis);
+}

internal/web/static/js/incident.js

@@ -11,6 +11,7 @@ import {
 } from './format.js';
 import { myID, users } from './state.js';
 import { back } from './app.js';
+import { alertMetrics } from './metricchart.js';
 
 const pane = () => document.getElementById('detail');
 
@@ -18,6 +19,7 @@ let currentID = null;
 let inc = null;
 let events = [];
 let similarList = [];
+let metricsByAlert = {};
 let error = null;
 let busy = false;
 
@@ -26,6 +28,7 @@ export function show(id) {
   currentID = id;
   inc = null;
   events = [];
+  metricsByAlert = {};
   error = null;
   if (id == null) {
     renderPlaceholder();
@@ -41,13 +44,17 @@ export async function refresh() {
   try {
     // Similar incidents are a courtesy: an older server answers 404 and a
     // failure here must not hide the incident itself.
-    const [i, t, sim] = await Promise.all([
-      api.incident(id), api.timeline(id), api.similar(id).catch(() => []),
+    // Metrics are a courtesy too: an older server has no such endpoint, and a
+    // Prometheus that is slow or down must not hold up the incident.
+    const [i, t, sim, met] = await Promise.all([
+      api.incident(id), api.timeline(id), api.similar(id).catch(() => []), api.metrics(id).catch(() => null),
     ]);
     if (id !== currentID) return;
     inc = i;
     events = t;
     similarList = sim;
+    metricsByAlert = {};
+    if (met && met.enabled) for (const m of met.alerts) metricsByAlert[m.alert_id] = m;
     error = null;
   } catch (err) {
     if (id !== currentID) return;
@@ -203,6 +210,7 @@ function alertItem(a) {
       h('span', { class: 'alert-item-name', text: a.name }),
       badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`)),
     summary && h('div', { class: 'alert-item-summary', text: summary }),
+    alertMetrics(metricsByAlert[a.id], a.starts_at, inc.resolved_at),
     h('div', { class: 'alert-item-foot' },
       h('span', { text: `Started ${ago(a.starts_at)}` }),
       h('span', { text: `Last seen ${ago(a.received_at)}` }),

internal/web/static/js/metricchart.js

@@ -0,0 +1,196 @@
+// The metrics under one alert: a line chart of the expression that fired it, and
+// what that expression reads right now. The data comes from this server, which
+// asked the alert's own Prometheus (see GET /api/incidents/{id}/metrics); the
+// browser never talks to Prometheus.
+
+import { h } from './ui.js';
+import { when } from './format.js';
+
+const SVG_NS = 'http://www.w3.org/2000/svg';
+const MAX_SERIES = 5;
+
+function svg(tag, attrs = {}, text) {
+  const el = document.createElementNS(SVG_NS, tag);
+  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
+  if (text != null) el.textContent = text;
+  return el;
+}
+
+const iso = (sec) => new Date(sec * 1000).toISOString();
+
+// Compact number for an axis or a badge: 1.2k, 0.034, 12.
+export function fmtValue(v) {
+  if (v == null || Number.isNaN(v)) return '–';
+  const a = Math.abs(v);
+  if (a >= 1e9) return `${+(v / 1e9).toPrecision(3)}G`;
+  if (a >= 1e6) return `${+(v / 1e6).toPrecision(3)}M`;
+  if (a >= 1e4) return `${+(v / 1e3).toPrecision(3)}k`;
+  if (a === 0) return '0';
+  if (a >= 100) return String(Math.round(v));
+  return String(+v.toPrecision(3));
+}
+
+const keyOf = (labels) => JSON.stringify(Object.entries(labels || {}).sort());
+
+// seriesName is the labels that tell the series apart, e.g. {instance="web-1"}.
+function seriesName(labels) {
+  const parts = Object.entries(labels || {}).filter(([k]) => k !== '__name__').map(([k, v]) => `${k}="${v}"`);
+  return parts.length ? `{${parts.join(', ')}}` : 'value';
+}
+
+// merge combines the snapshot taken when the incident opened with the live
+// window, so the chart keeps history Prometheus may since have dropped. On the
+// same timestamp the later source wins.
+function merge(...sources) {
+  const byKey = new Map();
+  for (const src of sources) {
+    for (const s of src || []) {
+      const k = keyOf(s.labels);
+      const cur = byKey.get(k) || { labels: s.labels, points: new Map() };
+      for (const [t, v] of s.points || []) cur.points.set(t, v);
+      byKey.set(k, cur);
+    }
+  }
+  return [...byKey.values()].map((s) => ({
+    labels: s.labels,
+    points: [...s.points.entries()].sort((a, b) => a[0] - b[0]),
+  }));
+}
+
+function parseSeries(raw) {
+  if (Array.isArray(raw)) return raw;
+  return [];
+}
+
+// pick chooses what to draw: while the incident is open, the opened snapshot
+// plus live data; once it is closed, the closed snapshot (the whole incident),
+// falling back to the opened one.
+function pick(entry) {
+  const snaps = entry.snapshots || {};
+  const live = entry.live;
+  if (live && !live.error) {
+    return {
+      series: merge(parseSeries(snaps.opened && snaps.opened.series), live.series),
+      from: Math.min(live.window_start, snaps.opened ? snaps.opened.window_start : Infinity),
+      to: live.window_end,
+      current: live.current || [],
+      isLive: true,
+    };
+  }
+  const snap = snaps.closed || snaps.opened;
+  if (snap && !snap.error) {
+    return {
+      series: merge(parseSeries(snap.series)), from: snap.window_start, to: snap.window_end, current: [], isLive: false,
+    };
+  }
+  return null;
+}
+
+// alertMetrics renders the metrics block for one alert, or null when there is
+// nothing worth saying (the feature off, or no Prometheus link on the alert).
+export function alertMetrics(entry, startsAtISO, resolvedISO) {
+  if (!entry) return null;
+  if (!entry.available) {
+    return entry.reason && entry.expr
+      ? h('div', { class: 'metrics-note', text: `Metrics unavailable: ${entry.reason}.` })
+      : null;
+  }
+  const data = pick(entry);
+  if (!data) {
+    const err = (entry.live && entry.live.error)
+      || (entry.snapshots && ((entry.snapshots.closed && entry.snapshots.closed.error) || (entry.snapshots.opened && entry.snapshots.opened.error)));
+    return h('div', { class: 'metrics-note', text: err ? `Metrics could not be fetched: ${err}` : 'Metrics are being collected…' });
+  }
+  const startSec = startsAtISO ? Date.parse(startsAtISO) / 1000 : null;
+  const endSec = resolvedISO ? Date.parse(resolvedISO) / 1000 : null;
+  return h('div', { class: 'metrics' },
+    currentLine(entry, data),
+    data.series.length ? chart(data, startSec, endSec) : h('div', { class: 'metrics-note', text: 'No samples in this window.' }),
+    legend(data.series),
+    entry.expr && h('code', { class: 'metrics-expr', text: entry.expr }),
+  );
+}
+
+function currentLine(entry, data) {
+  if (!data.isLive) return null;
+  const vals = data.current.map((s) => s.points && s.points[0] && s.points[0][1]).filter((v) => v != null);
+  if (!vals.length) {
+    return h('div', { class: 'metrics-now none' },
+      h('span', { class: 'metrics-now-label', text: 'Now' }),
+      h('span', { text: 'no data — the alert condition no longer holds' }));
+  }
+  return h('div', { class: 'metrics-now' },
+    h('span', { class: 'metrics-now-label', text: 'Now' }),
+    h('strong', { text: vals.length === 1 ? fmtValue(vals[0]) : vals.map(fmtValue).join(' · ') }));
+}
+
+function legend(series) {
+  if (series.length < 2 && !(series[0] && Object.keys(series[0].labels || {}).length)) return null;
+  return h('ul', { class: 'metrics-legend' }, series.slice(0, MAX_SERIES).map((s, i) =>
+    h('li', {}, h('span', { class: `mc-swatch s${i}` }), h('span', { text: seriesName(s.labels) }))));
+}
+
+function chart(data, startSec, endSec) {
+  const W = 480;
+  const H = 130;
+  const padL = 40;
+  const padR = 8;
+  const padT = 8;
+  const padB = 18;
+  const series = data.series.slice(0, MAX_SERIES);
+
+  let t0 = Infinity; let t1 = -Infinity; let lo = Infinity; let hi = -Infinity;
+  for (const s of series) {
+    for (const [t, v] of s.points) {
+      if (t < t0) t0 = t;
+      if (t > t1) t1 = t;
+      if (v < lo) lo = v;
+      if (v > hi) hi = v;
+    }
+  }
+  if (!Number.isFinite(t0)) { t0 = data.from; t1 = data.to; lo = 0; hi = 1; }
+  // Show the whole window, not just where there are samples, so the shape of
+  // "it only exists while firing" is visible.
+  t0 = Math.min(t0, data.from);
+  t1 = Math.max(t1, data.to);
+  if (t1 === t0) t1 = t0 + 1;
+  if (lo === hi) { lo -= 1; hi += 1; }
+  if (lo > 0 && lo < (hi - lo) * 0.5) lo = 0;
+
+  const x = (t) => padL + ((t - t0) / (t1 - t0)) * (W - padL - padR);
+  const y = (v) => padT + (1 - (v - lo) / (hi - lo)) * (H - padT - padB);
+
+  const root = svg('svg', {
+    class: 'metric-chart', viewBox: `0 0 ${W} ${H}`, role: 'img',
+    'aria-label': `Metric over time, ${fmtValue(lo)} to ${fmtValue(hi)}, from ${when(iso(t0))} to ${when(iso(t1))}`,
+  });
+  // Gridlines and y labels at the extremes and the middle.
+  for (const v of [lo, (lo + hi) / 2, hi]) {
+    root.appendChild(svg('line', { class: 'mc-grid', x1: padL, x2: W - padR, y1: y(v), y2: y(v) }));
+    root.appendChild(svg('text', { class: 'mc-label', x: padL - 4, y: y(v) + 3, 'text-anchor': 'end' }, fmtValue(v)));
+  }
+  root.appendChild(svg('text', { class: 'mc-label', x: padL, y: H - 4, 'text-anchor': 'start' }, when(iso(t0))));
+  root.appendChild(svg('text', { class: 'mc-label', x: W - padR, y: H - 4, 'text-anchor': 'end' }, when(iso(t1))));
+
+  const marker = (t, cls, text) => {
+    if (t == null || t < t0 || t > t1) return;
+    const m = svg('line', { class: `mc-marker ${cls}`, x1: x(t), x2: x(t), y1: padT, y2: H - padB });
+    m.appendChild(svg('title', {}, `${text} ${when(iso(t))}`));
+    root.appendChild(m);
+  };
+  marker(startSec, 'start', 'Alert started');
+  marker(endSec, 'end', 'Incident resolved');
+
+  series.forEach((s, i) => {
+    if (!s.points.length) return;
+    const d = s.points.map(([t, v], n) => `${n ? 'L' : 'M'}${x(t).toFixed(1)},${y(v).toFixed(1)}`).join(' ');
+    const line = svg('path', { class: `mc-line s${i}`, d });
+    line.appendChild(svg('title', {}, `${seriesName(s.labels)}: ${fmtValue(s.points[s.points.length - 1][1])} at ${when(iso(s.points[s.points.length - 1][0]))}`));
+    root.appendChild(line);
+    // A single sample has no line to draw; show it as a dot.
+    if (s.points.length === 1) {
+      root.appendChild(svg('circle', { class: `mc-dot s${i}`, cx: x(s.points[0][0]), cy: y(s.points[0][1]), r: 3 }));
+    }
+  });
+  return root;
+}

internal/web/static/js/stats.js

@@ -1,99 +1,87 @@
 // Statistics: how many incidents, how fast they are answered, and when and
 // what the alerts are. The same figures the TUI's Stats tab shows, over a
 // range picked with the chips. The server scopes them to the caller's teams.
+// /stats/alert/{name} is the drilldown behind a row of the top-alerts list.
 
 import * as api from './api.js';
-import { h, clear, emptyState, spinner } from './ui.js';
+import { h, clear, emptyState, spinner, icon } from './ui.js';
 import { duration } from './format.js';
-
-const DAY_MS = 24 * 60 * 60 * 1000;
-
-// `days` counts back from today, inclusive; the server reads from/to as UTC
-// dates, so these are too.
-const RANGES = [
-  { id: 'today', label: 'Today', days: 1 },
-  { id: '7d', label: '7d', days: 7 },
-  { id: '30d', label: '30d', days: 30 },
-  { id: '90d', label: '90d', days: 90 },
-  { id: 'all', label: 'All', days: null },
-];
+import { chips, card, heatmap, rangeQuery, getRange, setRangeId, pad2 } from './charts.js';
+import * as drill from './statsalert.js';
 
 const view = () => document.getElementById('view-stats');
 
-let range = '30d';
+// What the page shows: the overview (null) or one alert's drilldown.
+let alertName = null;
 let data = null;
 let error = null;
 
-const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10);
-
-function query(r) {
-  if (!r.days) return {};
-  const now = Date.now();
-  return { from: utcDate(now - (r.days - 1) * DAY_MS), to: utcDate(now) };
-}
-
-export function show() {
+export function show(route) {
+  const next = route?.alert ?? null;
+  if (next !== alertName) data = null;
+  alertName = next;
+  error = null;
   render();
   refresh();
 }
 
 export async function refresh() {
-  const requested = range;
-  const q = query(RANGES.find((x) => x.id === range));
+  const requested = getRange();
+  const name = alertName;
+  const q = rangeQuery();
   try {
-    const [incidents, top, byHour, byDay] = await Promise.all([
-      api.statsIncidents(q),
-      api.statsTop({ ...q, limit: 10 }),
-      api.statsByHour(q),
-      api.statsByDay(q),
-    ]);
-    if (requested !== range) return;
-    data = { incidents, top, byHour, byDay };
+    if (name != null) {
+      try {
+        data = await drill.load(name);
+      } catch (err) {
+        if (err.status !== 404) throw err;
+        data = 'none';
+      }
+    } else {
+      const [incidents, top, heat] = await Promise.all([
+        api.statsIncidents(q),
+        api.statsTop({ ...q, limit: 10 }),
+        api.statsHeatmap(q),
+      ]);
+      data = { incidents, top, heat };
+    }
+    if (requested !== getRange() || name !== alertName) return;
     error = null;
   } catch (err) {
-    if (requested !== range) return;
+    if (requested !== getRange() || name !== alertName) return;
     error = err.message;
   }
   render();
 }
 
 function setRange(id) {
-  if (id === range) return;
-  range = id;
+  if (id === getRange()) return;
+  setRangeId(id);
   data = null;
   render();
   refresh();
 }
 
 function render() {
-  const chips = h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Time range' },
-    RANGES.map((r) => h('button', {
-      class: 'chip',
-      type: 'button',
-      role: 'tab',
-      'aria-selected': String(r.id === range),
-      onclick: () => setRange(r.id),
-      text: r.label,
-    })));
-
+  if (alertName != null) {
+    clear(view(), drill.body(alertName, data, error, setRange));
+    return;
+  }
   let body;
   if (error && !data) body = h('div', { class: 'load-error', text: error });
   else if (!data) body = spinner();
-  else if (!data.incidents.total && !data.byHour.some((x) => x.count)) {
+  else if (!data.incidents.total) {
     body = emptyState('No data in this range', 'Nothing happened in this window.', 'chart');
   } else {
     body = h('div', { class: 'stats' },
       error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
       tiles(data.incidents),
-      data.top.length > 0 && card('Top alerts', topAlerts(data.top)),
-      card('Alerts by hour (UTC)', columns(
-        data.byHour.map((x) => ({ label: String(x.hour), value: x.count, tick: x.hour % 6 === 0 })),
-        'Alerts by hour of day')),
-      card('Alerts by day', columns(
-        data.byDay.map((x) => ({ label: x.day_name.slice(0, 3), value: x.count, tick: true })),
-        'Alerts by day of week')));
+      data.top.length > 0 && card('Top alerts', topAlerts(data.top), 'Select an alert to see when and how often it fires'),
+      h('div', { class: 'stats-split' },
+        card('When incidents open', heatmap(data.heat.grid), 'Weekday × hour (UTC)'),
+        patternsCard(data)));
   }
-  clear(view(), h('div', {}, chips, body));
+  clear(view(), h('div', {}, chips(setRange), body));
 }
 
 // A missing mean means nothing has been acknowledged or resolved yet.
@@ -112,57 +100,65 @@ function tiles(s) {
     tile('Mean time to resolve', mean(s.mttr_seconds)));
 }
 
-function card(title, content) {
-  return h('section', { class: 'chart-card card card-pad' },
-    h('h3', { class: 'chart-title', text: title }), content);
-}
-
-// Ranked names with a bar scaled to the busiest one.
-function topAlerts(items) {
-  const max = Math.max(...items.map((x) => x.count), 1);
-  return h('ol', { class: 'hbars' }, items.map((x) => {
-    const fill = h('span', { class: 'hbar-fill' });
-    fill.style.width = `${Math.max(2, (x.count / max) * 100)}%`;
-    return h('li', { class: 'hbar' },
-      h('span', { class: 'hbar-name', title: x.name, text: x.name }),
-      h('span', { class: 'hbar-track' }, fill),
-      h('span', { class: 'hbar-count', text: String(x.count) }));
+function spark(daily) {
+  const max = Math.max(...daily, 1);
+  return h('span', { class: 'spark', 'aria-hidden': 'true' }, daily.map((c) => {
+    const b = h('span', { class: c ? 'spark-bar on' : 'spark-bar' });
+    b.style.height = `${c ? 6 + (c / max) * 16 : 2}px`;
+    return b;
   }));
 }
 
-const SVG_NS = 'http://www.w3.org/2000/svg';
+const lastSeen = (unix) => {
+  const d = new Date(unix * 1000);
+  return `${d.toLocaleString('en', { month: 'short', timeZone: 'UTC' })} ${d.getUTCDate()}, ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}`;
+};
 
-function svg(tag, attrs = {}, text) {
-  const el = document.createElementNS(SVG_NS, tag);
-  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
-  if (text != null) el.textContent = text;
-  return el;
+// Ranked names, each a link to its drilldown: a bar scaled to the busiest, the
+// last 30 days as a sparkline, when it last fired and how long it takes to fix.
+function topAlerts(items) {
+  const max = Math.max(...items.map((x) => x.count), 1);
+  const total = items.reduce((n, x) => n + x.count, 0);
+  return h('div', { class: 'toplist' },
+    h('div', { class: 'top-row top-head', 'aria-hidden': 'true' },
+      ['Alert', 'Count', 'Last 30 days', 'Last seen', 'Mean fix'].map((t) => h('span', { text: t }))),
+    items.map((x) => {
+      const fill = h('span', { class: 'hbar-fill' });
+      fill.style.width = `${Math.max(2, (x.count / max) * 100)}%`;
+      return h('a', { class: 'top-row', href: drill.path(x.name) },
+        h('span', { class: 'hbar-name', title: x.name, text: x.name }),
+        h('span', { class: 'top-count' },
+          h('span', { class: 'hbar-count', text: String(x.count) }),
+          h('span', { class: 'hbar-track' }, fill),
+          h('span', { class: 'top-pct', text: `${Math.round((x.count / total) * 100)}%` })),
+        spark(x.daily),
+        h('span', { class: 'top-cell', text: lastSeen(x.last_seen) }),
+        h('span', { class: 'top-cell', text: mean(x.mean_resolve_seconds) }),
+        icon('chevronRight'));
+    }));
 }
 
-// A column chart: one bar per item, the value in a tooltip, and a label under
-// the items marked `tick`.
-function columns(items, label) {
-  const W = 480;
-  const H = 140;
-  const base = H - 18;
-  const step = W / items.length;
-  const max = Math.max(...items.map((x) => x.value), 1);
-  const root = svg('svg', {
-    class: 'columns', viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': label,
-  });
-  root.appendChild(svg('line', { class: 'axis', x1: 0, x2: W, y1: base, y2: base }));
-  items.forEach((it, i) => {
-    const bh = it.value ? Math.max(2, (it.value / max) * (base - 6)) : 0;
-    const x = i * step + step * 0.15;
-    const g = svg('g', { class: 'col' });
-    g.appendChild(svg('title', {}, `${it.label}: ${it.value}`));
-    // A full-height transparent hit area, so a tiny bar is still hoverable.
-    g.appendChild(svg('rect', { class: 'col-hit', x: i * step, y: 0, width: step, height: base }));
-    if (bh) g.appendChild(svg('rect', { class: 'col-bar', x, y: base - bh, width: step * 0.7, height: bh, rx: 2 }));
-    root.appendChild(g);
-    if (it.tick) {
-      root.appendChild(svg('text', { class: 'col-label', x: i * step + step / 2, y: H - 4, 'text-anchor': 'middle' }, it.label));
-    }
-  });
-  return root;
+// Observations drawn from the figures already on the page.
+function patternsCard(d) {
+  const flat = d.heat.grid;
+  const total = flat.flat().reduce((a, b) => a + b, 0);
+  const days = flat.map((row) => row.reduce((a, b) => a + b, 0));
+  const hours = Array.from({ length: 24 }, (_, hr) => flat.reduce((a, row) => a + row[hr], 0));
+  const topDay = days.indexOf(Math.max(...days));
+  const topHour = hours.indexOf(Math.max(...hours));
+  const topAlert = d.top[0];
+  const full = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
+  const items = [];
+  if (total > 1) {
+    items.push([`${full[topDay]}s are the noisiest day`, `${days[topDay]} of ${total} incidents opened on a ${full[topDay]}.`]);
+    items.push([`Peak hour is ${pad2(topHour)}:00 UTC`, `${hours[topHour]} incidents opened in that single hour.`]);
+  }
+  if (topAlert && total > 1 && topAlert.count / total >= 0.4) {
+    items.push(['One alert drives most of the volume', `${topAlert.name} is ${Math.round((topAlert.count / total) * 100)}% of everything that fired.`]);
+  }
+  const body = items.length
+    ? items.map(([t, s]) => h('div', { class: 'pattern' }, h('div', { class: 'pattern-title', text: t }), h('div', { class: 'pattern-sub', text: s })))
+    : h('p', { class: 'chart-note', text: 'Not enough incidents yet to see a pattern.' });
+  return card('Patterns', h('div', { class: 'patterns' }, body,
+    topAlert && h('a', { class: 'btn btn-ghost', href: drill.path(topAlert.name), text: `Open ${topAlert.name} →` })));
 }

internal/web/static/js/statsalert.js

@@ -0,0 +1,139 @@
+// The drilldown behind a row of the stats page's top-alerts list: when one
+// alert fired, how fast it was picked up and fixed. An occurrence is an
+// incident that alert opened (alert rows carry no history of their own).
+
+import * as api from './api.js';
+import { h, emptyState, spinner } from './ui.js';
+import { duration } from './format.js';
+import { chips, card, columns, rangeQuery, pad2, WEEKDAYS, MONDAY_FIRST } from './charts.js';
+
+const sec = (s) => (s == null ? '—' : duration(s * 1000));
+const fmtTime = (unix) => {
+  const d = new Date(unix * 1000);
+  const mon = d.toLocaleString('en', { month: 'short', timeZone: 'UTC' });
+  return `${mon} ${d.getUTCDate()}, ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}`;
+};
+
+export const path = (name) => `/stats/alert/${encodeURIComponent(name)}`;
+
+export async function load(name) {
+  return api.statsAlert(name, rangeQuery());
+}
+
+function tiles(d) {
+  const tile = (label, value, note) => h('div', { class: 'stat-tile' },
+    h('div', { class: 'stat-value', text: String(value) }),
+    h('div', { class: 'stat-label', text: label }),
+    note && h('div', { class: 'stat-note', text: note }));
+  return h('div', { class: 'stat-tiles' },
+    tile('Occurrences', d.count),
+    tile('Last seen', fmtTime(d.last_seen), 'UTC'),
+    tile('Average gap', d.mean_gap_seconds == null ? '—' : sec(d.mean_gap_seconds), 'between occurrences'),
+    tile('Mean time to acknowledge', sec(d.mean_ack_seconds), d.median_ack_seconds == null ? '' : `median ${sec(d.median_ack_seconds)}`),
+    tile('Mean time to resolve', sec(d.mean_resolve_seconds), d.median_resolve_seconds == null ? '' : `median ${sec(d.median_resolve_seconds)}`),
+    tile('Longest open', sec(d.longest_open_seconds)));
+}
+
+// Plain-language observations, derived from the same occurrences.
+function patterns(d) {
+  const n = d.count;
+  const out = [];
+  const sunday = d.by_weekday[0];
+  const top = d.by_weekday.indexOf(Math.max(...d.by_weekday));
+  if (n > 1 && d.by_weekday[top] > n / 2) out.push(`${d.by_weekday[top]} of ${n} fired on a ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][top]}`);
+  else if (sunday > 0 && n > 1) out.push(`${sunday} of ${n} fired on a Sunday`);
+  const peak = d.by_hour.indexOf(Math.max(...d.by_hour));
+  if (n > 1 && d.by_hour[peak] > 1) out.push(`Peak hour is ${pad2(peak)}:00 UTC (${d.by_hour[peak]} of ${n})`);
+  const times = d.occurrences.map((o) => o.triggered_at).sort((a, b) => a - b);
+  const repeats = times.filter((t, i) => i > 0 && t - times[i - 1] < 86400).length;
+  if (repeats) out.push(`${repeats} of ${n} fired again within 24 hours of the previous one`);
+  return out;
+}
+
+function patternStrip(d) {
+  const items = patterns(d);
+  if (!items.length) return null;
+  return h('div', { class: 'pattern-strip card card-pad' },
+    h('span', { class: 'chart-title', text: 'Pattern' }),
+    items.map((t) => h('span', { text: t })));
+}
+
+// One block per occurrence along a time axis from the first to the last, as wide
+// as the incident stayed open. Overlaps drop to a second lane.
+function timeline(d) {
+  const occ = [...d.occurrences].sort((a, b) => a.triggered_at - b.triggered_at);
+  const start = occ[0].triggered_at;
+  const end = Math.max(...occ.map((o) => o.triggered_at + (o.resolve_seconds || 0)), start + 3600);
+  const span = end - start;
+  let prev = -100;
+  const marks = occ.map((o) => {
+    const left = ((o.triggered_at - start) / span) * 100;
+    const lane = left - prev < 1.5 ? 1 : 0;
+    prev = left;
+    const m = h('div', {
+      class: 'tl-mark',
+      title: `${fmtTime(o.triggered_at)} UTC · ${o.resolve_seconds == null ? 'still open' : `open for ${sec(o.resolve_seconds)}`}`,
+    });
+    m.style.left = `${left}%`;
+    m.style.width = `${((o.resolve_seconds || 0) / span) * 100}%`;
+    m.style.top = `${8 + lane * 22}px`;
+    return m;
+  });
+  return h('div', {},
+    h('div', { class: 'tl', role: 'img', 'aria-label': `Timeline of ${occ.length} occurrences between ${fmtTime(start)} and ${fmtTime(end)} UTC.` }, marks),
+    h('div', { class: 'tl-axis' },
+      h('span', { text: fmtTime(start) }), h('span', { text: fmtTime(end) })));
+}
+
+// Per occurrence: the wait for an acknowledgement, then the time being worked.
+function fixBars(d) {
+  const rows = d.occurrences.filter((o) => o.resolve_seconds != null).slice(0, 12);
+  if (!rows.length) return h('p', { class: 'chart-note', text: 'Nothing resolved in this range.' });
+  const max = Math.max(...rows.map((o) => o.resolve_seconds), 1);
+  return h('div', { class: 'fixbars' }, rows.map((o) => {
+    const ack = Math.min(o.ack_seconds ?? 0, o.resolve_seconds);
+    const a = h('span', { class: 'fix-ack', title: `Waiting for acknowledgement: ${sec(ack)}` });
+    const w = h('span', { class: 'fix-work', title: `Being worked: ${sec(o.resolve_seconds - ack)}` });
+    a.style.width = `${(ack / max) * 100}%`;
+    w.style.width = `${((o.resolve_seconds - ack) / max) * 100}%`;
+    return h('div', { class: 'fix-row' },
+      h('span', { class: 'fix-label', text: fmtTime(o.triggered_at) }),
+      h('span', { class: 'fix-track' }, a, w),
+      h('span', { class: 'fix-total', text: sec(o.resolve_seconds) }));
+  }));
+}
+
+// On a phone each row folds into a card; the labels come from data-label.
+const OCC_COLS = ['Fired (UTC)', 'Time to ack', 'Duration', 'Acknowledged by', 'Status'];
+
+function table(d) {
+  return h('div', { class: 'occ-wrap' },
+    h('table', { class: 'occ' },
+      h('thead', {}, h('tr', {}, OCC_COLS.map((t) => h('th', { text: t })))),
+      h('tbody', {}, d.occurrences.map((o) => h('tr', {},
+        h('td', {}, h('a', { href: `/incidents/${o.incident_id}`, text: fmtTime(o.triggered_at) }),
+          ` ${WEEKDAYS[new Date(o.triggered_at * 1000).getUTCDay()]}`),
+        h('td', { 'data-label': OCC_COLS[1], text: sec(o.ack_seconds) }),
+        h('td', { 'data-label': OCC_COLS[2], text: sec(o.resolve_seconds) }),
+        h('td', { 'data-label': OCC_COLS[3], text: o.acknowledged_by || '—' }),
+        h('td', { 'data-label': OCC_COLS[4], text: o.status }))))));
+}
+
+export function body(name, d, error, onRange) {
+  const head = h('div', { class: 'stats-head' },
+    h('a', { class: 'btn btn-ghost', href: '/stats', text: '← Stats' }),
+    h('h2', { class: 'stats-alert-name', text: name }),
+    chips(onRange));
+  if (error && !d) return h('div', {}, head, h('div', { class: 'load-error', text: error }));
+  if (!d) return h('div', {}, head, spinner());
+  if (d === 'none') return h('div', {}, head, emptyState('No occurrences in this range', 'This alert did not open an incident in this window.', 'chart'));
+  return h('div', {}, head, h('div', { class: 'stats' },
+    error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
+    tiles(d),
+    patternStrip(d),
+    card('When it fired', timeline(d), 'Each block spans how long the incident stayed open'),
+    card('By hour (UTC)', columns(d.by_hour.map((v, hr) => ({ label: pad2(hr), value: v, tick: hr % 6 === 0 })), 'Occurrences by hour of day')),
+    card('By weekday', columns(MONDAY_FIRST.map((w) => ({ label: WEEKDAYS[w], value: d.by_weekday[w], tick: true })), 'Occurrences by weekday')),
+    card('Time to resolve', fixBars(d), 'Amber waited for an acknowledgement; blue was worked'),
+    card('All occurrences', table(d), `Newest first${d.count > d.occurrences.length ? `, latest ${d.occurrences.length} of ${d.count}` : ''}`)));
+}

internal/web/stats_test.go

@@ -20,7 +20,9 @@ func TestStatsPageIsEmbedded(t *testing.T) {
 			t.Errorf("index.html lacks %s", want)
 		}
 	}
-	if _, err := fs.Stat(sub, "js/stats.js"); err != nil {
-		t.Errorf("js/stats.js not embedded: %v", err)
+	for _, f := range []string{"js/stats.js", "js/statsalert.js", "js/charts.js"} {
+		if _, err := fs.Stat(sub, f); err != nil {
+			t.Errorf("%s not embedded: %v", f, err)
+		}
 	}
 }