Graph the metric behind each alert on the incident page

Each alert carries a generatorURL pointing at the Prometheus rule that fired
it. The server now parses that link, queries the same Prometheus for the
rule's expression and draws it under the alert, with the current value
beside it. The browser never talks to Prometheus, so the CSP stays at
connect-src 'self'.

The URL comes out of a webhook payload, so whoever holds an integration key
chooses it. Querying it is therefore opt-in and bounded by an allowlist,
TERDUT_PROMETHEUS_ALLOWED_HOSTS (names, "*" as one label, a leading "**" as
several, IPs, CIDRs). The check runs on the address the dialer is about to
use, a name that resolves to loopback or link-local is refused even when it
matches, and redirects and proxy variables are ignored.

A worker takes a snapshot when an incident opens and another when it
closes, so the graph outlives Prometheus' retention. It finds its work in
the database rather than being triggered from the webhook, so ingestion is
untouched and a restart loses nothing. While an incident is open the
metrics endpoint also queries live, cached for 15 seconds.

The rule expression keeps its threshold, so Prometheus returns data only
while the alert condition holds; "no data" for the current value means the
condition cleared. Stripping the threshold needs a PromQL parser and is left
out.

Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN

internal/web/static/app.css

@@ -600,6 +600,33 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
 .alert-item-summary { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
 .alert-item-foot { display: flex; flex-wrap: wrap; gap: 4px 12px; font-size: 13px; color: var(--faint); }
 .alert-item-foot a { color: var(--accent); font-weight: 600; }
+
+/* Metrics under an alert */
+.metrics { display: grid; gap: 6px; }
+.metrics-note { color: var(--faint); font-size: 13px; }
+.metrics-now { display: flex; align-items: baseline; gap: 8px; font-size: 14px; }
+.metrics-now strong { font-size: 18px; font-variant-numeric: tabular-nums; }
+.metrics-now.none { color: var(--muted); }
+.metrics-now-label { color: var(--faint); font-size: 12px; font-weight: 650; text-transform: uppercase; letter-spacing: .04em; }
+.metric-chart { width: 100%; height: auto; display: block; }
+.mc-grid { stroke: var(--border); stroke-width: 1; }
+.mc-label { fill: var(--faint); font-size: 10px; }
+.mc-marker { stroke-width: 1; stroke-dasharray: 3 3; }
+.mc-marker.start { stroke: var(--crit); }
+.mc-marker.end { stroke: var(--ok); }
+.mc-line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; }
+.mc-line.s0, .mc-swatch.s0 { stroke: var(--accent); background: var(--accent); }
+.mc-line.s1, .mc-swatch.s1 { stroke: var(--warn); background: var(--warn); }
+.mc-line.s2, .mc-swatch.s2 { stroke: var(--ok); background: var(--ok); }
+.mc-line.s3, .mc-swatch.s3 { stroke: var(--crit); background: var(--crit); }
+.mc-line.s4, .mc-swatch.s4 { stroke: var(--muted); background: var(--muted); }
+.mc-dot.s0 { fill: var(--accent); } .mc-dot.s1 { fill: var(--warn); } .mc-dot.s2 { fill: var(--ok); }
+.mc-dot.s3 { fill: var(--crit); } .mc-dot.s4 { fill: var(--muted); }
+.mc-swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; flex: none; }
+.metrics-legend { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 2px 14px; font-size: 12px; color: var(--muted); }
+.metrics-legend li { display: flex; align-items: center; gap: 6px; overflow-wrap: anywhere; }
+.metrics-expr { font-size: 12px; color: var(--faint); overflow-wrap: anywhere; }
+
 /* A disclosure is a button: a chevron that turns, and a 44px target on a
    touch screen (the bare triangle it replaces was a few pixels wide). */
 details > summary {

internal/web/static/js/api.js

@@ -108,6 +108,7 @@ export const unsnooze = (id) => call('DELETE', `/incidents/${id}/snooze`);
 export const archive = (id) => call('POST', `/incidents/${id}/archive`);
 export const unarchive = (id) => call('DELETE', `/incidents/${id}/archive`);
 export const addNote = (id, content, pinned = false) => call('POST', `/incidents/${id}/notes`, { body: { content, pinned } });
+export const metrics = (id) => call('GET', `/incidents/${id}/metrics`);
 export const similar = (id) => call('GET', `/incidents/${id}/similar`);
 export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/notes/${eventID}`);
 

internal/web/static/js/incident.js

@@ -11,6 +11,7 @@ import {
 } from './format.js';
 import { myID, users } from './state.js';
 import { back } from './app.js';
+import { alertMetrics } from './metricchart.js';
 
 const pane = () => document.getElementById('detail');
 
@@ -18,6 +19,7 @@ let currentID = null;
 let inc = null;
 let events = [];
 let similarList = [];
+let metricsByAlert = {};
 let error = null;
 let busy = false;
 
@@ -26,6 +28,7 @@ export function show(id) {
   currentID = id;
   inc = null;
   events = [];
+  metricsByAlert = {};
   error = null;
   if (id == null) {
     renderPlaceholder();
@@ -41,13 +44,17 @@ export async function refresh() {
   try {
     // Similar incidents are a courtesy: an older server answers 404 and a
     // failure here must not hide the incident itself.
-    const [i, t, sim] = await Promise.all([
-      api.incident(id), api.timeline(id), api.similar(id).catch(() => []),
+    // Metrics are a courtesy too: an older server has no such endpoint, and a
+    // Prometheus that is slow or down must not hold up the incident.
+    const [i, t, sim, met] = await Promise.all([
+      api.incident(id), api.timeline(id), api.similar(id).catch(() => []), api.metrics(id).catch(() => null),
     ]);
     if (id !== currentID) return;
     inc = i;
     events = t;
     similarList = sim;
+    metricsByAlert = {};
+    if (met && met.enabled) for (const m of met.alerts) metricsByAlert[m.alert_id] = m;
     error = null;
   } catch (err) {
     if (id !== currentID) return;
@@ -203,6 +210,7 @@ function alertItem(a) {
       h('span', { class: 'alert-item-name', text: a.name }),
       badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`)),
     summary && h('div', { class: 'alert-item-summary', text: summary }),
+    alertMetrics(metricsByAlert[a.id], a.starts_at, inc.resolved_at),
     h('div', { class: 'alert-item-foot' },
       h('span', { text: `Started ${ago(a.starts_at)}` }),
       h('span', { text: `Last seen ${ago(a.received_at)}` }),

internal/web/static/js/metricchart.js

@@ -0,0 +1,196 @@
+// The metrics under one alert: a line chart of the expression that fired it, and
+// what that expression reads right now. The data comes from this server, which
+// asked the alert's own Prometheus (see GET /api/incidents/{id}/metrics); the
+// browser never talks to Prometheus.
+
+import { h } from './ui.js';
+import { when } from './format.js';
+
+const SVG_NS = 'http://www.w3.org/2000/svg';
+const MAX_SERIES = 5;
+
+function svg(tag, attrs = {}, text) {
+  const el = document.createElementNS(SVG_NS, tag);
+  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
+  if (text != null) el.textContent = text;
+  return el;
+}
+
+const iso = (sec) => new Date(sec * 1000).toISOString();
+
+// Compact number for an axis or a badge: 1.2k, 0.034, 12.
+export function fmtValue(v) {
+  if (v == null || Number.isNaN(v)) return '–';
+  const a = Math.abs(v);
+  if (a >= 1e9) return `${+(v / 1e9).toPrecision(3)}G`;
+  if (a >= 1e6) return `${+(v / 1e6).toPrecision(3)}M`;
+  if (a >= 1e4) return `${+(v / 1e3).toPrecision(3)}k`;
+  if (a === 0) return '0';
+  if (a >= 100) return String(Math.round(v));
+  return String(+v.toPrecision(3));
+}
+
+const keyOf = (labels) => JSON.stringify(Object.entries(labels || {}).sort());
+
+// seriesName is the labels that tell the series apart, e.g. {instance="web-1"}.
+function seriesName(labels) {
+  const parts = Object.entries(labels || {}).filter(([k]) => k !== '__name__').map(([k, v]) => `${k}="${v}"`);
+  return parts.length ? `{${parts.join(', ')}}` : 'value';
+}
+
+// merge combines the snapshot taken when the incident opened with the live
+// window, so the chart keeps history Prometheus may since have dropped. On the
+// same timestamp the later source wins.
+function merge(...sources) {
+  const byKey = new Map();
+  for (const src of sources) {
+    for (const s of src || []) {
+      const k = keyOf(s.labels);
+      const cur = byKey.get(k) || { labels: s.labels, points: new Map() };
+      for (const [t, v] of s.points || []) cur.points.set(t, v);
+      byKey.set(k, cur);
+    }
+  }
+  return [...byKey.values()].map((s) => ({
+    labels: s.labels,
+    points: [...s.points.entries()].sort((a, b) => a[0] - b[0]),
+  }));
+}
+
+function parseSeries(raw) {
+  if (Array.isArray(raw)) return raw;
+  return [];
+}
+
+// pick chooses what to draw: while the incident is open, the opened snapshot
+// plus live data; once it is closed, the closed snapshot (the whole incident),
+// falling back to the opened one.
+function pick(entry) {
+  const snaps = entry.snapshots || {};
+  const live = entry.live;
+  if (live && !live.error) {
+    return {
+      series: merge(parseSeries(snaps.opened && snaps.opened.series), live.series),
+      from: Math.min(live.window_start, snaps.opened ? snaps.opened.window_start : Infinity),
+      to: live.window_end,
+      current: live.current || [],
+      isLive: true,
+    };
+  }
+  const snap = snaps.closed || snaps.opened;
+  if (snap && !snap.error) {
+    return {
+      series: merge(parseSeries(snap.series)), from: snap.window_start, to: snap.window_end, current: [], isLive: false,
+    };
+  }
+  return null;
+}
+
+// alertMetrics renders the metrics block for one alert, or null when there is
+// nothing worth saying (the feature off, or no Prometheus link on the alert).
+export function alertMetrics(entry, startsAtISO, resolvedISO) {
+  if (!entry) return null;
+  if (!entry.available) {
+    return entry.reason && entry.expr
+      ? h('div', { class: 'metrics-note', text: `Metrics unavailable: ${entry.reason}.` })
+      : null;
+  }
+  const data = pick(entry);
+  if (!data) {
+    const err = (entry.live && entry.live.error)
+      || (entry.snapshots && ((entry.snapshots.closed && entry.snapshots.closed.error) || (entry.snapshots.opened && entry.snapshots.opened.error)));
+    return h('div', { class: 'metrics-note', text: err ? `Metrics could not be fetched: ${err}` : 'Metrics are being collected…' });
+  }
+  const startSec = startsAtISO ? Date.parse(startsAtISO) / 1000 : null;
+  const endSec = resolvedISO ? Date.parse(resolvedISO) / 1000 : null;
+  return h('div', { class: 'metrics' },
+    currentLine(entry, data),
+    data.series.length ? chart(data, startSec, endSec) : h('div', { class: 'metrics-note', text: 'No samples in this window.' }),
+    legend(data.series),
+    entry.expr && h('code', { class: 'metrics-expr', text: entry.expr }),
+  );
+}
+
+function currentLine(entry, data) {
+  if (!data.isLive) return null;
+  const vals = data.current.map((s) => s.points && s.points[0] && s.points[0][1]).filter((v) => v != null);
+  if (!vals.length) {
+    return h('div', { class: 'metrics-now none' },
+      h('span', { class: 'metrics-now-label', text: 'Now' }),
+      h('span', { text: 'no data — the alert condition no longer holds' }));
+  }
+  return h('div', { class: 'metrics-now' },
+    h('span', { class: 'metrics-now-label', text: 'Now' }),
+    h('strong', { text: vals.length === 1 ? fmtValue(vals[0]) : vals.map(fmtValue).join(' · ') }));
+}
+
+function legend(series) {
+  if (series.length < 2 && !(series[0] && Object.keys(series[0].labels || {}).length)) return null;
+  return h('ul', { class: 'metrics-legend' }, series.slice(0, MAX_SERIES).map((s, i) =>
+    h('li', {}, h('span', { class: `mc-swatch s${i}` }), h('span', { text: seriesName(s.labels) }))));
+}
+
+function chart(data, startSec, endSec) {
+  const W = 480;
+  const H = 130;
+  const padL = 40;
+  const padR = 8;
+  const padT = 8;
+  const padB = 18;
+  const series = data.series.slice(0, MAX_SERIES);
+
+  let t0 = Infinity; let t1 = -Infinity; let lo = Infinity; let hi = -Infinity;
+  for (const s of series) {
+    for (const [t, v] of s.points) {
+      if (t < t0) t0 = t;
+      if (t > t1) t1 = t;
+      if (v < lo) lo = v;
+      if (v > hi) hi = v;
+    }
+  }
+  if (!Number.isFinite(t0)) { t0 = data.from; t1 = data.to; lo = 0; hi = 1; }
+  // Show the whole window, not just where there are samples, so the shape of
+  // "it only exists while firing" is visible.
+  t0 = Math.min(t0, data.from);
+  t1 = Math.max(t1, data.to);
+  if (t1 === t0) t1 = t0 + 1;
+  if (lo === hi) { lo -= 1; hi += 1; }
+  if (lo > 0 && lo < (hi - lo) * 0.5) lo = 0;
+
+  const x = (t) => padL + ((t - t0) / (t1 - t0)) * (W - padL - padR);
+  const y = (v) => padT + (1 - (v - lo) / (hi - lo)) * (H - padT - padB);
+
+  const root = svg('svg', {
+    class: 'metric-chart', viewBox: `0 0 ${W} ${H}`, role: 'img',
+    'aria-label': `Metric over time, ${fmtValue(lo)} to ${fmtValue(hi)}, from ${when(iso(t0))} to ${when(iso(t1))}`,
+  });
+  // Gridlines and y labels at the extremes and the middle.
+  for (const v of [lo, (lo + hi) / 2, hi]) {
+    root.appendChild(svg('line', { class: 'mc-grid', x1: padL, x2: W - padR, y1: y(v), y2: y(v) }));
+    root.appendChild(svg('text', { class: 'mc-label', x: padL - 4, y: y(v) + 3, 'text-anchor': 'end' }, fmtValue(v)));
+  }
+  root.appendChild(svg('text', { class: 'mc-label', x: padL, y: H - 4, 'text-anchor': 'start' }, when(iso(t0))));
+  root.appendChild(svg('text', { class: 'mc-label', x: W - padR, y: H - 4, 'text-anchor': 'end' }, when(iso(t1))));
+
+  const marker = (t, cls, text) => {
+    if (t == null || t < t0 || t > t1) return;
+    const m = svg('line', { class: `mc-marker ${cls}`, x1: x(t), x2: x(t), y1: padT, y2: H - padB });
+    m.appendChild(svg('title', {}, `${text} ${when(iso(t))}`));
+    root.appendChild(m);
+  };
+  marker(startSec, 'start', 'Alert started');
+  marker(endSec, 'end', 'Incident resolved');
+
+  series.forEach((s, i) => {
+    if (!s.points.length) return;
+    const d = s.points.map(([t, v], n) => `${n ? 'L' : 'M'}${x(t).toFixed(1)},${y(v).toFixed(1)}`).join(' ');
+    const line = svg('path', { class: `mc-line s${i}`, d });
+    line.appendChild(svg('title', {}, `${seriesName(s.labels)}: ${fmtValue(s.points[s.points.length - 1][1])} at ${when(iso(s.points[s.points.length - 1][0]))}`));
+    root.appendChild(line);
+    // A single sample has no line to draw; show it as a dot.
+    if (s.points.length === 1) {
+      root.appendChild(svg('circle', { class: `mc-dot s${i}`, cx: x(s.points[0][0]), cy: y(s.points[0][1]), r: 3 }));
+    }
+  });
+  return root;
+}