diff --git a/README.md b/README.md index af55a8e..5e1ddaa 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ The [documentation index](./docs/README.md) lists everything. The main pages: |---|---| | [Deployment](./docs/deployment.md) | Docker, Helm chart, database, backups, the operator | | [Configuration](./docs/configuration.md) | Environment variables and settings | +| [Metrics](./docs/metrics.md) | Graphs from the alert's Prometheus on the incident page | | [Alertmanager configuration](./docs/alertmanager.md) | Routes, keys, webhooks | | [Alerts and incidents](./docs/incidents.md) | Correlation, lifecycle, on-call assignment | | [Single sign-on](./docs/single-sign-on.md) | OIDC and the terminal device flow | diff --git a/charts/terdut-server/templates/deployment.yaml b/charts/terdut-server/templates/deployment.yaml index 49acee1..ddbabcd 100644 --- a/charts/terdut-server/templates/deployment.yaml +++ b/charts/terdut-server/templates/deployment.yaml @@ -120,6 +120,14 @@ spec: value: {{ .Values.trustedProxies | quote }} - name: TERDUT_OPERATOR_MODE value: {{ .Values.operatorMode | quote }} + {{- if .Values.prometheus.allowedHosts }} + - name: TERDUT_PROMETHEUS_ALLOWED_HOSTS + value: {{ join "," .Values.prometheus.allowedHosts | quote }} + - name: TERDUT_PROMETHEUS_PREROLL + value: {{ .Values.prometheus.preroll | quote }} + - name: TERDUT_PROMETHEUS_TIMEOUT + value: {{ .Values.prometheus.timeout | quote }} + {{- end }} {{- if .Values.oidc.enabled }} - name: TERDUT_OIDC_ISSUER value: {{ required "oidc.issuer is required when oidc.enabled" .Values.oidc.issuer | quote }} diff --git a/charts/terdut-server/values.yaml b/charts/terdut-server/values.yaml index eee7949..732b82d 100644 --- a/charts/terdut-server/values.yaml +++ b/charts/terdut-server/values.yaml @@ -114,6 +114,18 @@ passwordLogin: true # configuration from here on. operatorMode: false +# Metrics on incidents: the server queries the Prometheus an alert came from +# (the host in its generatorURL) and graphs the alert's expression. The URL is +# taken from a webhook payload, so only hosts listed here are ever contacted; +# empty turns the feature off. Entries are host names, with "*" as one DNS label +# and a leading "**" as one or more (e.g. "*.monitoring.svc.cluster.local", +# "prometheus-*.monitoring"), IPs or CIDR ranges, each with an optional ":port". +prometheus: + allowedHosts: [] + # How far before an alert started its graph begins. + preroll: 1h + timeout: 10s + # Single sign-on through an OpenID Connect provider such as Authentik. # # At the provider, create an OAuth2/OpenID application whose redirect URI is diff --git a/cmd/terdut/main.go b/cmd/terdut/main.go index 970f77c..ba5ad03 100644 --- a/cmd/terdut/main.go +++ b/cmd/terdut/main.go @@ -64,6 +64,7 @@ func main() { go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notifyCfg) go api.StartNotifier(ctx, database, notifyCfg) + go api.StartMetricsWorker(ctx, database, cfg) go func() { log.Printf("terdut-server %s listening on %s", version, cfg.Addr) diff --git a/docs/api.md b/docs/api.md index 3ba087d..b580dab 100644 --- a/docs/api.md +++ b/docs/api.md @@ -236,6 +236,7 @@ the team gets the same `404` as anybody else. | `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline | | `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident | | `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological | +| `GET` | `/api/incidents/{id}/metrics` | Metrics behind the incident's alerts, from their Prometheus — see [Metrics](./metrics.md) | | `POST` | `/api/incidents/{id}/acknowledge` | Acknowledge (stamps authed user + time) | | `DELETE` | `/api/incidents/{id}/acknowledge` | Clear acknowledgement, back to `triggered` | | `POST` | `/api/incidents/{id}/resolve` | Close by hand — **terminal**, see above | diff --git a/docs/configuration.md b/docs/configuration.md index 4632561..0a02bc6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -30,6 +30,9 @@ over an administrator's edit. | `TERDUT_TRUSTED_PROXIES` | `1` | How many reverse proxies in front of the server append to `X-Forwarded-For`; the per-address rate limits use the entry that many hops from the right. `0` ignores the header | | `TERDUT_OPERATOR_KEY` | — | At least 32 characters. When set, the instance-scoped service account `terdut-operator` is created if missing and its `seed` key replaced with this value at every start — how terdut-operator authenticates without a bootstrap handshake. An instance-scoped account acts as owner of every team (team configuration) but is not a member of any, so it reads no incidents | | `TERDUT_OPERATOR_MODE` | `false` | Declares this install gitops-managed: a session's or a user's own API key's writes to teams, escalation policies, dead man's switches and integrations are refused (`403 reason:"operator_managed"`); a [service account](./api.md#service-accounts)'s are not. Team membership and the schedule stay editable regardless | +| `TERDUT_PROMETHEUS_ALLOWED_HOSTS` | — | Comma-separated Prometheus servers the [metrics on an incident](./metrics.md) may be fetched from, matched against the host in each alert's `generatorURL`. Host names (`prometheus.monitoring.svc`), wildcards (`*` is one DNS label, so `*.example.com` or `prom-*.monitoring.svc`; a leading `**` is one or more labels), IPs or CIDR ranges, each with an optional `:port`. Empty turns the feature off. A bare `*` is refused at startup | +| `TERDUT_PROMETHEUS_PREROLL` | `1h` | How far before an alert started its metrics graph begins | +| `TERDUT_PROMETHEUS_TIMEOUT` | `10s` | Per-request timeout for a Prometheus query | | `TERDUT_OIDC_ISSUER` | — | Turns single sign-on on. The provider's issuer URL; discovery is read from `/.well-known/openid-configuration`. See [Single sign-on](./single-sign-on.md#single-sign-on-oidc) | | `TERDUT_OIDC_CLIENT_ID` / `TERDUT_OIDC_CLIENT_SECRET` | — | **Required with an issuer.** The confidential client registered at the provider. Keep the secret in a Secret, not in values | | `TERDUT_OIDC_NAME` | `SSO` | What the sign-in button calls the provider | diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..81fbccb --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,75 @@ +# Metrics on incidents + +Alertmanager sends each alert with a `generatorURL`: the link to the rule's graph in +the Prometheus that fired it. With the feature on, terdut-server parses that link, +queries the same Prometheus for the rule's expression, and shows a graph and the +current value under each alert on the incident page. The browser never talks to +Prometheus; the server does. + +## Turning it on + +Set `TERDUT_PROMETHEUS_ALLOWED_HOSTS` (Helm: `prometheus.allowedHosts`). The URL arrives in a +webhook payload, so whoever holds an integration key chooses it; without an allowlist the +server would fetch whatever it was told to. Empty, the default, means off. + +| Entry | Matches | +|---|---| +| `prometheus.monitoring.svc` | that host, any port | +| `prometheus.monitoring.svc:9090` | that host and port | +| `*.example.com` | `a.example.com`; not `example.com`, not `a.b.example.com` | +| `prom-*.monitoring.svc` | `prom-0.monitoring.svc` | +| `**.example.com` | `a.example.com` and `a.b.example.com` | +| `10.0.0.0/8`, `[fd00::]/8:9090` | addresses in the range | + +`*` stands for one DNS label (or part of one); `**` is allowed only as the first label and means +one or more. Matching ignores case. A bare `*` or `**`, or any entry whose last label is only a +wildcard, is refused at startup. + +The check is made on the address the connection is about to use, not just on the name, so a +name that resolves somewhere else (DNS rebinding) is caught. A host admitted by name is still +refused if it resolves to a loopback, link-local (which includes the cloud metadata address), +unspecified or multicast address; to allow one deliberately, list it as an IP or CIDR. Redirects +and proxy environment variables are ignored. + +## What is captured + +- **Snapshots.** A background worker stores each alert's series when the incident opens + (`opened`) and when it closes (`closed`), in Postgres, so the graph outlives Prometheus' + own retention. Incidents closed more than 24 hours before the feature was switched on are not + backfilled. A failed capture is retried a few times; a host that is not allowed, or a link that is + not a Prometheus graph link, is recorded as failed once. +- **Live data.** While an incident is open, `GET /api/incidents/{id}/metrics` also queries + Prometheus (answers are reused for 15 seconds) and returns the current value. +- **Window.** From `TERDUT_PROMETHEUS_PREROLL` (default 1h) before the alert started until now, + or until the incident was resolved. + +## Reading the graph + +The expression is the alert rule's own, threshold included (`latency > 1`), so Prometheus only +returns data while the condition is true. The graph therefore shows the firing period, and **"no +data" for the current value means the condition no longer holds**, not that Prometheus is down. +A rule expression matches every series, not just the one that fired; only series that agree with the +alert's labels are kept, at most five per alert. + +## Endpoint + +`GET /api/incidents/{id}/metrics` — same team scoping as the other incident routes. + +```json +{ + "enabled": true, + "alerts": [{ + "alert_id": 12, "alert_name": "HighLatency", "expr": "latency>1", + "available": true, + "snapshots": { "opened": { "window_start": 1, "window_end": 2, "step_seconds": 15, + "series": [{"labels": {"instance": "web-1"}, "points": [[100, 1.5]]}], + "captured_at": 3 } }, + "live": { "window_start": 1, "window_end": 2, "step_seconds": 15, + "series": [], "current": [] } + }] +} +``` + +`enabled` is `false` when no host is allowed. `available` is `false`, with a `reason`, when this +alert has no Prometheus link or its host is not allowed. A snapshot that failed carries `error`. +`live` is absent once the incident is resolved. Points are `[unix seconds, value]`. diff --git a/internal/api/export_test.go b/internal/api/export_test.go index 9928297..0aa74c6 100644 --- a/internal/api/export_test.go +++ b/internal/api/export_test.go @@ -1,8 +1,12 @@ package api import ( + "context" + "database/sql" "strings" "time" + + "git.ryuvia.com/niklas/terdut-server/internal/config" ) // DeadmanConfig is a test fixture only: a set of matchers with one timeout and @@ -29,3 +33,9 @@ func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string) } return cfg } + +// MetricsSweep runs one pass of the metrics worker, so a test need not wait on +// its ticker. +func MetricsSweep(ctx context.Context, db *sql.DB, cfg config.Config) { + newMetricsService(cfg).sweep(ctx, db) +} diff --git a/internal/api/metrics.go b/internal/api/metrics.go new file mode 100644 index 0000000..c52d22b --- /dev/null +++ b/internal/api/metrics.go @@ -0,0 +1,456 @@ +package api + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "log" + "net/http" + "sync" + "time" + + "git.ryuvia.com/niklas/terdut-server/internal/config" + "git.ryuvia.com/niklas/terdut-server/internal/prom" +) + +const ( + // metricsInterval is how often the worker looks for incidents that need a + // snapshot. Short, because the "opened" snapshot is most useful when it is + // taken close to the moment the incident opened. + metricsInterval = 15 * time.Second + + // metricsBatch caps one pass, and metricsConcurrency how many Prometheus + // requests run at once within it. + metricsBatch = 50 + metricsConcurrency = 4 + + // metricsMaxAttempts stops a failing capture being retried forever, and + // metricsRetryStep spaces the retries: attempt n waits n*metricsRetryStep. + metricsMaxAttempts = 4 + metricsRetryStep = time.Minute + + // metricsLookback is how far back the worker goes for incidents that are + // already closed. Without it, switching the feature on would try to + // reconstruct the whole history from a Prometheus that has probably aged it + // out. Incidents that are still open are always eligible. + metricsLookback = 24 * time.Hour + + // maxAlertsPerIncident bounds the Prometheus work one incident can cause. + maxAlertsPerIncident = 10 + + // metricsLiveTTL is how long a live answer is reused, so a room full of + // people watching one incident is one query per alert, not one each. + metricsLiveTTL = 15 * time.Second + + // metricsLockKey is the advisory lock a pass holds, distinct from the + // archiver's and the notifier's. + metricsLockKey int64 = 7265_0004 +) + +// metricsService is the Prometheus side of the incident page: a client that only +// reaches allowed hosts, and a short-lived cache of live answers. +type metricsService struct { + client *prom.Client + preroll time.Duration + + mu sync.Mutex + cache map[int64]liveEntry +} + +type liveEntry struct { + at time.Time + live *liveMetrics +} + +func newMetricsService(cfg config.Config) *metricsService { + // Validate has already refused a bad list at startup; a failure here can only + // be a test building a Config by hand, and no allowlist is the safe reading. + allow, err := prom.ParseAllowlist(cfg.PrometheusAllowedHosts) + if err != nil { + allow = nil + } + preroll := cfg.PrometheusPreroll + if preroll <= 0 { + preroll = time.Hour + } + return &metricsService{ + client: prom.NewClient(allow, cfg.PrometheusTimeout), + preroll: preroll, + cache: map[int64]liveEntry{}, + } +} + +func (m *metricsService) enabled() bool { return m.client.Enabled() } + +// StartMetricsWorker captures the opened and closed snapshots until ctx is +// cancelled. With no allowed hosts configured it does nothing. +func StartMetricsWorker(ctx context.Context, db *sql.DB, cfg config.Config) { + m := newMetricsService(cfg) + if !m.enabled() { + log.Print("metrics: disabled (TERDUT_PROMETHEUS_ALLOWED_HOSTS is empty)") + return + } + log.Printf("metrics: capturing from %d allowed Prometheus host entries", len(cfg.PrometheusAllowedHosts)) + + ticker := time.NewTicker(metricsInterval) + defer ticker.Stop() + sweep := func() { + withAdvisoryLock(ctx, db, metricsLockKey, "metrics", func() { + m.sweep(ctx, db) + }) + } + sweep() + for { + select { + case <-ticker.C: + sweep() + case <-ctx.Done(): + return + } + } +} + +// snapshotJob is one (incident, alert, kind) that still needs a snapshot. +type snapshotJob struct { + incidentID int64 + alertID int64 + kind string + resolvedAt sql.NullInt64 + startsAt int64 + labels map[string]string + generatorURL string +} + +// sweep finds the snapshots that are missing, or failed and are due a retry, and +// takes them. +func (m *metricsService) sweep(ctx context.Context, db *sql.DB) { + now := time.Now() + rows, err := db.QueryContext(ctx, ` + SELECT i.id, a.id, k.kind, i.resolved_at, a.starts_at, a.labels, a.generator_url + FROM incidents i + JOIN incident_alerts ia ON ia.incident_id = i.id + JOIN alerts a ON a.id = ia.alert_id + CROSS JOIN (VALUES ('opened'), ('closed')) AS k(kind) + LEFT JOIN incident_metric_snapshots s + ON s.incident_id = i.id AND s.alert_id = a.id AND s.kind = k.kind + WHERE a.generator_url <> '' + AND i.archived_at IS NULL + AND (i.resolved_at IS NULL OR i.resolved_at >= $1) + AND (k.kind = 'opened' OR i.resolved_at IS NOT NULL) + AND (s.id IS NULL + OR (s.error IS NOT NULL AND s.attempts < $2 + AND s.captured_at + s.attempts * $3 <= $4)) + ORDER BY i.id, a.id, k.kind + LIMIT $5`, + now.Add(-metricsLookback).Unix(), metricsMaxAttempts, int64(metricsRetryStep/time.Second), now.Unix(), metricsBatch) + if err != nil { + log.Printf("metrics: find work: %v", err) + return + } + var jobs []snapshotJob + for rows.Next() { + var j snapshotJob + var labels []byte + if err := rows.Scan(&j.incidentID, &j.alertID, &j.kind, &j.resolvedAt, &j.startsAt, &labels, &j.generatorURL); err != nil { + log.Printf("metrics: scan: %v", err) + rows.Close() + return + } + _ = json.Unmarshal(labels, &j.labels) + jobs = append(jobs, j) + } + rows.Close() + if err := rows.Err(); err != nil { + log.Printf("metrics: find work: %v", err) + return + } + + sem := make(chan struct{}, metricsConcurrency) + var wg sync.WaitGroup + for _, j := range jobs { + sem <- struct{}{} + wg.Add(1) + go func() { + defer func() { <-sem; wg.Done() }() + m.capture(ctx, db, j) + }() + } + wg.Wait() +} + +// window is the span a snapshot covers: from the pre-roll before the alert +// started to the end of the incident, or to now while it is open. +func (m *metricsService) window(startsAt int64, resolvedAt sql.NullInt64, now time.Time) (time.Time, time.Time) { + start := time.Unix(startsAt, 0).Add(-m.preroll) + end := now + if resolvedAt.Valid { + end = time.Unix(resolvedAt.Int64, 0) + } + if !end.After(start) { + end = start.Add(time.Minute) + } + return start, end +} + +func (m *metricsService) capture(ctx context.Context, db *sql.DB, j snapshotJob) { + now := time.Now() + start, end := m.window(j.startsAt, j.resolvedAt, now) + step := prom.StepFor(start, end) + + series, expr, err := m.fetchRange(ctx, j.generatorURL, j.labels, start, end, step) + + attempts := 1 + var errText any + if err != nil { + errText = err.Error() + if errors.Is(err, prom.ErrNotAllowed) || errors.Is(err, errNotGraphLink) { + attempts = metricsMaxAttempts // retrying cannot change the answer + } + } + if series == nil { + series = []prom.Series{} + } + body, _ := json.Marshal(series) + + _, dbErr := db.ExecContext(ctx, ` + INSERT INTO incident_metric_snapshots + (incident_id, alert_id, kind, expr, window_start, window_end, step_seconds, series, error, attempts, captured_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (incident_id, alert_id, kind) DO UPDATE SET + expr = excluded.expr, window_start = excluded.window_start, window_end = excluded.window_end, + step_seconds = excluded.step_seconds, series = excluded.series, error = excluded.error, + attempts = CASE WHEN excluded.error IS NULL THEN 1 + ELSE GREATEST(incident_metric_snapshots.attempts + 1, excluded.attempts) END, + captured_at = excluded.captured_at`, + j.incidentID, j.alertID, j.kind, expr, start.Unix(), end.Unix(), int(step/time.Second), + string(body), errText, attempts, now.Unix()) + if dbErr != nil { + log.Printf("metrics: store snapshot (incident %d, alert %d, %s): %v", j.incidentID, j.alertID, j.kind, dbErr) + } +} + +var errNotGraphLink = errors.New("the alert's generator URL is not a Prometheus graph link") + +// fetchRange parses the generator URL and returns the series of the alert's own +// expression over [start, end]. expr is returned even on error where known. +func (m *metricsService) fetchRange(ctx context.Context, generatorURL string, labels map[string]string, start, end time.Time, step time.Duration) ([]prom.Series, string, error) { + base, expr, ok := prom.ParseGeneratorURL(generatorURL) + if !ok { + return nil, "", errNotGraphLink + } + series, err := m.client.QueryRange(ctx, base, expr, start, end, step) + if err != nil { + return nil, expr, err + } + return prom.SelectSeries(series, labels), expr, nil +} + +// --- REST --------------------------------------------------------------- + +type metricsResponse struct { + // Enabled is false when no Prometheus host is allowed, so a client can hide + // the section rather than show every alert as unavailable. + Enabled bool `json:"enabled"` + Alerts []alertMetrics `json:"alerts"` +} + +type alertMetrics struct { + AlertID int64 `json:"alert_id"` + AlertName string `json:"alert_name"` + Expr string `json:"expr,omitempty"` + + // Available says whether this alert can be queried at all; Reason says why + // not when it cannot. + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + + Snapshots map[string]*snapshotView `json:"snapshots"` + Live *liveMetrics `json:"live,omitempty"` +} + +type snapshotView struct { + WindowStart int64 `json:"window_start"` + WindowEnd int64 `json:"window_end"` + Step int `json:"step_seconds"` + Series json.RawMessage `json:"series"` + Error string `json:"error,omitempty"` + CapturedAt int64 `json:"captured_at"` +} + +type liveMetrics struct { + WindowStart int64 `json:"window_start"` + WindowEnd int64 `json:"window_end"` + Step int `json:"step_seconds"` + Series []prom.Series `json:"series"` + // Current is the expression's value right now, one point per series. Empty + // means the condition that fires the alert no longer holds. + Current []prom.Series `json:"current"` + Error string `json:"error,omitempty"` +} + +func handleIncidentMetrics(db *sql.DB, m *metricsService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, ok := incidentIDParam(w, r, db) + if !ok { + return + } + resp := metricsResponse{Enabled: m.enabled(), Alerts: []alertMetrics{}} + if !resp.Enabled { + respond(w, http.StatusOK, resp) + return + } + + var resolvedAt sql.NullInt64 + if err := db.QueryRowContext(r.Context(), "SELECT resolved_at FROM incidents WHERE id = $1", id).Scan(&resolvedAt); err != nil { + serverError(w, r, err) + return + } + + rows, err := db.QueryContext(r.Context(), ` + SELECT a.id, a.name, a.labels, a.generator_url, a.starts_at + FROM alerts a + JOIN incident_alerts ia ON ia.alert_id = a.id + WHERE ia.incident_id = $1 + ORDER BY a.starts_at, a.id + LIMIT $2`, id, maxAlertsPerIncident) + if err != nil { + serverError(w, r, err) + return + } + type alertRow struct { + am alertMetrics + labels map[string]string + url string + startsAt int64 + } + var alerts []*alertRow + for rows.Next() { + var a alertRow + var labels []byte + if err := rows.Scan(&a.am.AlertID, &a.am.AlertName, &labels, &a.url, &a.startsAt); err != nil { + rows.Close() + serverError(w, r, err) + return + } + _ = json.Unmarshal(labels, &a.labels) + a.am.Snapshots = map[string]*snapshotView{} + alerts = append(alerts, &a) + } + rows.Close() + if err := rows.Err(); err != nil { + serverError(w, r, err) + return + } + + snaps, err := loadSnapshots(r.Context(), db, id) + if err != nil { + serverError(w, r, err) + return + } + + var wg sync.WaitGroup + for _, a := range alerts { + a := a + if base, expr, ok := prom.ParseGeneratorURL(a.url); !ok { + a.am.Reason = "no Prometheus link on this alert" + } else if !m.client.Allowed(base) { + a.am.Reason = "Prometheus host is not allowed" + a.am.Expr = expr + } else { + a.am.Available = true + a.am.Expr = expr + } + for kind, s := range snaps[a.am.AlertID] { + a.am.Snapshots[kind] = s + } + if a.am.Available && !resolvedAt.Valid { + wg.Add(1) + go func() { + defer wg.Done() + a.am.Live = m.live(r.Context(), a.am.AlertID, a.url, a.labels, a.startsAt) + }() + } + } + wg.Wait() + + for _, a := range alerts { + resp.Alerts = append(resp.Alerts, a.am) + } + respond(w, http.StatusOK, resp) + } +} + +func loadSnapshots(ctx context.Context, db *sql.DB, incidentID int64) (map[int64]map[string]*snapshotView, error) { + rows, err := db.QueryContext(ctx, ` + SELECT alert_id, kind, window_start, window_end, step_seconds, series, COALESCE(error, ''), captured_at + FROM incident_metric_snapshots + WHERE incident_id = $1`, incidentID) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[int64]map[string]*snapshotView{} + for rows.Next() { + var alertID int64 + var kind string + var s snapshotView + var series []byte + if err := rows.Scan(&alertID, &kind, &s.WindowStart, &s.WindowEnd, &s.Step, &series, &s.Error, &s.CapturedAt); err != nil { + return nil, err + } + s.Series = json.RawMessage(series) + if out[alertID] == nil { + out[alertID] = map[string]*snapshotView{} + } + out[alertID][kind] = &s + } + return out, rows.Err() +} + +// live queries Prometheus now, reusing an answer younger than metricsLiveTTL. +// The cache is per alert; the window start is fixed by the alert, only its end +// moves, so one entry serves every viewer. +func (m *metricsService) live(ctx context.Context, alertID int64, generatorURL string, labels map[string]string, startsAt int64) *liveMetrics { + m.mu.Lock() + if e, ok := m.cache[alertID]; ok && time.Since(e.at) < metricsLiveTTL { + m.mu.Unlock() + return e.live + } + m.mu.Unlock() + + now := time.Now() + start, end := m.window(startsAt, sql.NullInt64{}, now) + step := prom.StepFor(start, end) + lm := &liveMetrics{WindowStart: start.Unix(), WindowEnd: end.Unix(), Step: int(step / time.Second), + Series: []prom.Series{}, Current: []prom.Series{}} + + base, expr, ok := prom.ParseGeneratorURL(generatorURL) + if !ok { + lm.Error = errNotGraphLink.Error() + return lm + } + series, err := m.client.QueryRange(ctx, base, expr, start, end, step) + if err != nil { + lm.Error = err.Error() + } else { + lm.Series = prom.SelectSeries(series, labels) + if cur, err := m.client.Query(ctx, base, expr, now); err == nil { + lm.Current = prom.SelectSeries(cur, labels) + } else { + lm.Error = err.Error() + } + } + + // A failed answer is cached too, for the same TTL: a Prometheus that is down + // should not be hit once per viewer per poll. + m.mu.Lock() + for k, e := range m.cache { + if time.Since(e.at) > 10*metricsLiveTTL { + delete(m.cache, k) + } + } + m.cache[alertID] = liveEntry{at: now, live: lm} + m.mu.Unlock() + return lm +} diff --git a/internal/api/metrics_test.go b/internal/api/metrics_test.go new file mode 100644 index 0000000..d5cf24b --- /dev/null +++ b/internal/api/metrics_test.go @@ -0,0 +1,189 @@ +package api_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "git.ryuvia.com/niklas/terdut-server/internal/api" +) + +// fakePrometheus answers the two query endpoints and counts the requests. +func fakePrometheus(t *testing.T) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/query_range": + w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[ + {"metric":{"instance":"web-1"},"values":[[100,"1"],[160,"2"]]}, + {"metric":{"instance":"web-2"},"values":[[100,"9"]]}]}}`)) + case "/api/v1/query": + w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[ + {"metric":{"instance":"web-1"},"value":[170,"3"]}]}}`)) + default: + w.WriteHeader(404) + } + })) + t.Cleanup(srv.Close) + return srv, &hits +} + +type metricsBody struct { + Enabled bool `json:"enabled"` + Alerts []struct { + AlertID int64 `json:"alert_id"` + Expr string `json:"expr"` + Available bool `json:"available"` + Reason string `json:"reason"` + Snapshots map[string]struct { + Series json.RawMessage `json:"series"` + Error string `json:"error"` + } `json:"snapshots"` + Live *struct { + Series []struct{ Labels map[string]string } `json:"series"` + Current []struct{ Labels map[string]string } `json:"current"` + } `json:"live"` + } `json:"alerts"` +} + +func getMetrics(t *testing.T, s *ts, id int) metricsBody { + t.Helper() + var out metricsBody + decode(t, s.req(t, http.MethodGet, "/api/incidents/"+itoa(id)+"/metrics", nil), &out) + return out +} + +func itoa(n int) string { + b, _ := json.Marshal(n) + return string(b) +} + +func metricsAlert(promURL string) map[string]any { + a := amAlert("fp-metrics", "HighLatency", "firing", "2026-05-20T10:00:00Z", "0001-01-01T00:00:00Z", + map[string]string{"instance": "web-1"}) + a["generatorURL"] = promURL + "/graph?g0.expr=latency%3E1&g0.tab=1" + return a +} + +func TestMetrics_SnapshotsOpenedAndClosedAndLive(t *testing.T) { + prom, _ := fakePrometheus(t) + conf := testConfig() + conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"} // httptest listens on loopback + s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf) + + postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-metrics") + incs := listIncidents(t, s, "") + if len(incs) != 1 { + t.Fatalf("want one incident, got %d", len(incs)) + } + id := int(incs[0]["id"].(float64)) + + api.MetricsSweep(context.Background(), s.db, conf) + + m := getMetrics(t, s, id) + if !m.Enabled || len(m.Alerts) != 1 { + t.Fatalf("metrics = %+v", m) + } + a := m.Alerts[0] + if !a.Available || a.Expr != "latency>1" { + t.Fatalf("alert = %+v", a) + } + opened, ok := a.Snapshots["opened"] + if !ok || opened.Error != "" { + t.Fatalf("opened snapshot = %+v", a.Snapshots) + } + var series []struct{ Labels map[string]string } + json.Unmarshal(opened.Series, &series) + if len(series) != 1 || series[0].Labels["instance"] != "web-1" { + t.Errorf("snapshot should hold only this alert's series (web-1), got %+v", series) + } + if _, ok := a.Snapshots["closed"]; ok { + t.Error("closed snapshot exists while the incident is open") + } + if a.Live == nil || len(a.Live.Series) != 1 || len(a.Live.Current) != 1 { + t.Errorf("an open incident should have live data, got %+v", a.Live) + } + + // A second pass must not capture again. + var n int + s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n) + api.MetricsSweep(context.Background(), s.db, conf) + var n2 int + s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n2) + if n != 1 || n2 != 1 { + t.Errorf("snapshot rows = %d then %d, want 1 and 1", n, n2) + } + + // Resolving the incident gets the closed snapshot, and no more live data. + resp := s.req(t, http.MethodPost, "/api/incidents/"+itoa(id)+"/resolve", nil) + resp.Body.Close() + api.MetricsSweep(context.Background(), s.db, conf) + m = getMetrics(t, s, id) + a = m.Alerts[0] + if _, ok := a.Snapshots["closed"]; !ok { + t.Errorf("no closed snapshot after resolve: %+v", a.Snapshots) + } + if a.Live != nil { + t.Error("a resolved incident should not query live") + } +} + +func TestMetrics_DisabledWithoutAllowedHosts(t *testing.T) { + prom, hits := fakePrometheus(t) + s := newTS(t) + postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-off") + id := int(listIncidents(t, s, "")[0]["id"].(float64)) + + api.MetricsSweep(context.Background(), s.db, testConfig()) + m := getMetrics(t, s, id) + if m.Enabled || len(m.Alerts) != 0 { + t.Errorf("metrics = %+v, want disabled and empty", m) + } + if hits.Load() != 0 { + t.Errorf("Prometheus was contacted %d times", hits.Load()) + } +} + +func TestMetrics_UnlistedHostIsRecordedNotQueried(t *testing.T) { + prom, hits := fakePrometheus(t) + conf := testConfig() + conf.PrometheusAllowedHosts = []string{"*.example.com"} + s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf) + postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-unlisted") + id := int(listIncidents(t, s, "")[0]["id"].(float64)) + + api.MetricsSweep(context.Background(), s.db, conf) + api.MetricsSweep(context.Background(), s.db, conf) // must not retry a permanent failure + + m := getMetrics(t, s, id) + a := m.Alerts[0] + if a.Available || a.Reason == "" { + t.Errorf("alert should be unavailable with a reason: %+v", a) + } + if hits.Load() != 0 { + t.Errorf("an unlisted host was contacted %d times", hits.Load()) + } + var attempts int + var errText *string + s.db.QueryRow("SELECT attempts, error FROM incident_metric_snapshots").Scan(&attempts, &errText) + if errText == nil || attempts < 4 { + t.Errorf("failure should be recorded as final: attempts=%d error=%v", attempts, errText) + } +} + +func TestMetrics_IncidentScoping(t *testing.T) { + conf := testConfig() + conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"} + s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf) + resp := s.req(t, http.MethodGet, "/api/incidents/9999/metrics", nil) + resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("unknown incident: %d, want 404", resp.StatusCode) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index ed4f402..cb768ae 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -28,6 +28,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin signupLimiter := newLoginLimiter(db) oidcLimit := newLoginLimiter(db) + metrics := newMetricsService(cfg) + r := chi.NewRouter() r.Use(requestLogger) r.Use(middleware.Recoverer) @@ -149,6 +151,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin r.Get("/api/incidents/clusters", handleListClusters(db)) r.Get("/api/incidents/{id}", handleGetIncident(db)) r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db)) + r.Get("/api/incidents/{id}/metrics", handleIncidentMetrics(db, metrics)) r.Get("/api/incidents/{id}/similar", handleIncidentSimilar(db)) r.Post("/api/incidents/{id}/acknowledge", handleIncidentAcknowledge(db)) r.Delete("/api/incidents/{id}/acknowledge", handleIncidentUnacknowledge(db)) diff --git a/internal/config/config.go b/internal/config/config.go index 581bbd4..fad749b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" "time" + + "git.ryuvia.com/niklas/terdut-server/internal/prom" ) // MinOperatorKeyLength is the shortest TERDUT_OPERATOR_KEY accepted: it is a @@ -79,6 +81,20 @@ type Config struct { // "where this server is plugged in" — it is a statement about who owns // this install's configuration, not a per-request toggle. OperatorMode bool + + // PrometheusAllowedHosts lists the Prometheus servers this install may query + // for the metrics behind an alert, matched against the host in each alert's + // generatorURL. Entries are host names (with "*" and a leading "**" as + // wildcards), IPs or CIDR ranges, optionally with a port; see prom.Allowlist. + // Empty turns the feature off, which is the default: the URL comes from a + // webhook payload, so querying it is opt-in. + PrometheusAllowedHosts []string + + // PrometheusPreroll is how far before an alert started its graph begins. + PrometheusPreroll time.Duration + + // PrometheusTimeout bounds one request to a Prometheus. + PrometheusTimeout time.Duration } // OIDC is the single sign-on configuration. Groups from the provider decide @@ -151,6 +167,10 @@ func Load() Config { OIDC: loadOIDC(), OperatorMode: boolean("TERDUT_OPERATOR_MODE", false), + + PrometheusAllowedHosts: list("TERDUT_PROMETHEUS_ALLOWED_HOSTS", ""), + PrometheusPreroll: duration("TERDUT_PROMETHEUS_PREROLL", time.Hour), + PrometheusTimeout: duration("TERDUT_PROMETHEUS_TIMEOUT", 10*time.Second), } } @@ -180,6 +200,9 @@ func (c Config) Validate() error { if c.OperatorKey != "" && len(c.OperatorKey) < MinOperatorKeyLength { return fmt.Errorf("TERDUT_OPERATOR_KEY must be at least %d characters", MinOperatorKeyLength) } + if _, err := prom.ParseAllowlist(c.PrometheusAllowedHosts); err != nil { + return fmt.Errorf("TERDUT_PROMETHEUS_ALLOWED_HOSTS: %w", err) + } o := c.OIDC if !o.Enabled() { if c.DisablePasswordLogin { diff --git a/internal/db/migrations/002_incident_metrics.sql b/internal/db/migrations/002_incident_metrics.sql new file mode 100644 index 0000000..3829eb8 --- /dev/null +++ b/internal/db/migrations/002_incident_metrics.sql @@ -0,0 +1,25 @@ +-- Metrics captured from the Prometheus an alert came from (its generatorURL), +-- once when the incident opens and once when it closes, so the graph outlives +-- Prometheus' own retention. One row per incident, alert and kind. +-- +-- A row exists whether or not the capture worked: a failure is recorded in +-- error with the attempt count, which is what stops the worker from retrying a +-- host that is not on the allowlist (or is gone) forever. + +CREATE TABLE incident_metric_snapshots ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + incident_id bigint NOT NULL REFERENCES incidents(id) ON DELETE CASCADE, + alert_id bigint NOT NULL REFERENCES alerts(id) ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN ('opened', 'closed')), + expr text NOT NULL DEFAULT '', + window_start bigint NOT NULL, + window_end bigint NOT NULL, + step_seconds integer NOT NULL DEFAULT 0, + series jsonb NOT NULL DEFAULT '[]'::jsonb, + error text, + attempts integer NOT NULL DEFAULT 1, + captured_at bigint NOT NULL, + UNIQUE (incident_id, alert_id, kind) +); + +CREATE INDEX incident_metric_snapshots_alert_id_idx ON incident_metric_snapshots (alert_id); diff --git a/internal/prom/allow.go b/internal/prom/allow.go new file mode 100644 index 0000000..3849e70 --- /dev/null +++ b/internal/prom/allow.go @@ -0,0 +1,268 @@ +// Package prom fetches metrics from the Prometheus an alert came from. +// +// The address to query is the alert's generatorURL, which arrives in a webhook +// payload and is therefore chosen by whoever holds an integration key. Left +// unchecked, that would let the key holder make this server issue requests to +// any host it can reach. Everything here is built around the Allowlist so that +// cannot happen: a host is queried only if the operator listed it. +package prom + +import ( + "fmt" + "net" + "net/netip" + "net/url" + "path" + "strings" +) + +// Allowlist is the set of Prometheus servers this install may query. The zero +// value, and an empty list, permit nothing, which is what keeps the feature off +// until an operator opts in. +// +// An entry is one of: +// +// - a host name, optionally with a port: prometheus.monitoring.svc, prom:9090 +// - a wildcard host name: "*" stands for exactly one DNS label (or part of +// one), so *.example.com matches a.example.com but neither example.com nor +// a.b.example.com, and prometheus-*.monitoring.svc matches prometheus-0. +// A leading "**" label matches one or more labels, so **.example.com also +// matches a.b.example.com. +// - an IP address or a CIDR range, optionally with a port: 10.0.0.0/8, +// [fd00::]:9090 +// +// Without a port an entry matches any port. Matching is case-insensitive. +type Allowlist struct { + hosts []hostPattern + nets []netEntry +} + +type hostPattern struct { + labels []string // lower-cased; a leading "**" label is kept as is + port string // empty matches any port +} + +type netEntry struct { + prefix netip.Prefix + port string +} + +// ParseAllowlist validates and compiles the configured entries. It refuses an +// entry that would admit every host (a bare "*", "*.*"): a typo there would +// quietly turn the guard off. +func ParseAllowlist(entries []string) (*Allowlist, error) { + a := &Allowlist{} + for _, raw := range entries { + e := strings.ToLower(strings.TrimSpace(raw)) + if e == "" { + continue + } + if err := a.add(e); err != nil { + return nil, fmt.Errorf("allowed host %q: %w", raw, err) + } + } + return a, nil +} + +func (a *Allowlist) add(e string) error { + // CIDR first: its colons (IPv6) and slash would confuse the host:port split. + if strings.Contains(e, "/") { + host, port := e, "" + if strings.HasPrefix(e, "[") { + end := strings.Index(e, "]") + if end < 0 { + return fmt.Errorf("unterminated [") + } + host = e[1:end] + rest := e[end+1:] + if strings.HasPrefix(rest, ":") { + port = rest[1:] + } + } + p, err := netip.ParsePrefix(host) + if err != nil { + return fmt.Errorf("not a CIDR range: %w", err) + } + if err := checkPort(port); err != nil { + return err + } + a.nets = append(a.nets, netEntry{prefix: p.Masked(), port: port}) + return nil + } + + // A bare IP address, IPv6 included. + if ip, err := netip.ParseAddr(e); err == nil { + a.nets = append(a.nets, netEntry{prefix: netip.PrefixFrom(ip.Unmap(), ip.Unmap().BitLen())}) + return nil + } + + host, port := e, "" + if strings.HasPrefix(e, "[") || strings.Count(e, ":") == 1 { + h, p, err := net.SplitHostPort(e) + if err != nil { + return fmt.Errorf("bad host:port: %w", err) + } + host, port = h, p + if ip, err := netip.ParseAddr(host); err == nil { + if err := checkPort(port); err != nil { + return err + } + a.nets = append(a.nets, netEntry{prefix: netip.PrefixFrom(ip.Unmap(), ip.Unmap().BitLen()), port: port}) + return nil + } + } + if err := checkPort(port); err != nil { + return err + } + + host = strings.TrimSuffix(host, ".") + if host == "" { + return fmt.Errorf("empty host") + } + for _, r := range host { + ok := r == '.' || r == '-' || r == '_' || r == '*' || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if !ok { + return fmt.Errorf("unexpected character %q", r) + } + } + labels := strings.Split(host, ".") + for i, l := range labels { + if l == "" { + return fmt.Errorf("empty label") + } + if l == "**" && i != 0 { + return fmt.Errorf("** is only allowed as the first label") + } + if strings.Contains(l, "**") && l != "**" { + return fmt.Errorf("** must be a whole label") + } + } + // The last label has to say something, or the entry is allow-all. + if strings.Trim(labels[len(labels)-1], "*") == "" { + return fmt.Errorf("would match every host: the last label cannot be a wildcard alone") + } + a.hosts = append(a.hosts, hostPattern{labels: labels, port: port}) + return nil +} + +func checkPort(p string) error { + if p == "" { + return nil + } + for _, r := range p { + if r < '0' || r > '9' { + return fmt.Errorf("bad port %q", p) + } + } + return nil +} + +// Empty reports whether nothing is allowed, i.e. the feature is off. +func (a *Allowlist) Empty() bool { + return a == nil || (len(a.hosts) == 0 && len(a.nets) == 0) +} + +// Port returns the effective port of u: the explicit one, else the scheme's. +func Port(u *url.URL) string { + if p := u.Port(); p != "" { + return p + } + if u.Scheme == "https" { + return "443" + } + return "80" +} + +// permits decides whether host:port may be contacted at ip. host is what the +// URL said (a name or an IP literal), ip is where it actually resolved to; the +// check runs on the resolved address at dial time so that a name that resolves +// somewhere else than it did when it was listed (DNS rebinding) is caught. +// +// - An address inside a listed CIDR range or equal to a listed IP is allowed. +// - A host name that matches a listed pattern is allowed, unless it resolved +// to an address that is never a Prometheus (loopback, link-local, which +// includes the cloud metadata address, unspecified, multicast). A wildcard +// over a domain somebody else controls must not be a way to reach internal +// services. To allow such an address deliberately, list it as a CIDR. +func (a *Allowlist) permits(host, port string, ip netip.Addr) bool { + if a == nil { + return false + } + ip = ip.Unmap() + for _, n := range a.nets { + if n.prefix.Contains(ip) && (n.port == "" || n.port == port) { + return true + } + } + if forbiddenAddr(ip) { + return false + } + if _, isIP := parseHostIP(host); isIP { + return false // an IP literal is allowed only through a CIDR entry + } + return a.matchesName(host, port) +} + +func parseHostIP(host string) (netip.Addr, bool) { + ip, err := netip.ParseAddr(strings.Trim(host, "[]")) + return ip, err == nil +} + +// MatchesHost reports whether the URL's host is worth dialing at all: a listed +// name, or an IP that falls in a listed range. It is the cheap early answer; +// the dial-time check in permits is the one that is authoritative for names. +func (a *Allowlist) MatchesHost(u *url.URL) bool { + if a.Empty() { + return false + } + host := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) + port := Port(u) + if ip, ok := parseHostIP(host); ok { + return a.permits(host, port, ip) + } + if a.matchesName(host, port) { + return true + } + // A name outside the patterns can still be admitted by a CIDR entry once it + // resolves; leave that to the dialer. + return len(a.nets) > 0 +} + +func (a *Allowlist) matchesName(host, port string) bool { + labels := strings.Split(strings.TrimSuffix(strings.ToLower(host), "."), ".") + for _, p := range a.hosts { + if (p.port == "" || p.port == port) && matchLabels(p.labels, labels) { + return true + } + } + return false +} + +func matchLabels(pattern, host []string) bool { + if len(pattern) > 0 && pattern[0] == "**" { + rest := pattern[1:] + if len(host) <= len(rest) { + return false + } + return matchLabels(rest, host[len(host)-len(rest):]) + } + if len(pattern) != len(host) { + return false + } + for i := range pattern { + // The pattern was restricted to [a-z0-9_.-*] when parsed, so path.Match + // sees nothing but literals and "*" here. + if ok, _ := path.Match(pattern[i], host[i]); !ok { + return false + } + } + return true +} + +// forbiddenAddr is the set of addresses a Prometheus is never reached at and +// that are dangerous to reach by mistake. Private ranges are deliberately not +// in it: a Prometheus inside the cluster lives there. +func forbiddenAddr(ip netip.Addr) bool { + return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() +} diff --git a/internal/prom/client.go b/internal/prom/client.go new file mode 100644 index 0000000..add2e41 --- /dev/null +++ b/internal/prom/client.go @@ -0,0 +1,274 @@ +package prom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "net/netip" + "net/url" + "strconv" + "strings" + "time" +) + +const ( + // maxResponseBytes bounds what one query may make us read. + maxResponseBytes = 5 << 20 + + // maxSeries bounds how many series one query may return before we give up on + // it; a rule expression without aggregation can match thousands. + maxSeries = 2000 + + // TargetPoints is how many points a graph is asked for. + TargetPoints = 300 +) + +// ErrNotAllowed means the target host is not on the allowlist. +var ErrNotAllowed = errors.New("host is not in the allowed Prometheus hosts") + +// Series is one time series, ready to chart. Points are [unix seconds, value]. +type Series struct { + Labels map[string]string `json:"labels"` + Points [][2]float64 `json:"points"` +} + +// Client queries Prometheus servers that the Allowlist permits. +type Client struct { + http *http.Client + allow *Allowlist +} + +// NewClient returns a client that only ever connects to allowed addresses. The +// check is made on the address the dialer is about to use, after name +// resolution, and redirects and environment proxies are off: either would let a +// permitted host send the request somewhere that is not. +func NewClient(allow *Allowlist, timeout time.Duration) *Client { + if timeout <= 0 { + timeout = 10 * time.Second + } + dialer := &net.Dialer{Timeout: timeout} + tr := &http.Transport{ + Proxy: nil, + DialContext: guardedDial(allow, dialer), + TLSHandshakeTimeout: timeout, + MaxIdleConns: 4, + IdleConnTimeout: 30 * time.Second, + } + return &Client{ + allow: allow, + http: &http.Client{ + Timeout: timeout, + Transport: tr, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + } +} + +// Enabled reports whether any host is allowed at all. +func (c *Client) Enabled() bool { return c != nil && !c.allow.Empty() } + +// Allowed is the early, resolution-free answer: whether base could be queried. +func (c *Client) Allowed(base *url.URL) bool { + return c != nil && c.allow.MatchesHost(base) +} + +func guardedDial(allow *Allowlist, d *net.Dialer) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + var ips []netip.Addr + if ip, err := netip.ParseAddr(host); err == nil { + ips = []netip.Addr{ip} + } else { + ips, err = net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil { + return nil, err + } + } + var lastErr error = ErrNotAllowed + for _, ip := range ips { + if !allow.permits(host, port, ip) { + continue + } + // Dial the checked address itself, so a second lookup cannot return + // something different. + conn, err := d.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) + if err == nil { + return conn, nil + } + lastErr = err + } + return nil, lastErr + } +} + +// QueryRange runs expr over [start, end] at the given step. +func (c *Client) QueryRange(ctx context.Context, base *url.URL, expr string, start, end time.Time, step time.Duration) ([]Series, error) { + form := url.Values{ + "query": {expr}, + "start": {strconv.FormatInt(start.Unix(), 10)}, + "end": {strconv.FormatInt(end.Unix(), 10)}, + "step": {strconv.FormatInt(int64(step/time.Second), 10)}, + } + res, err := c.do(ctx, base, "/api/v1/query_range", form) + if err != nil { + return nil, err + } + if res.ResultType != "matrix" { + return nil, fmt.Errorf("unexpected result type %q", res.ResultType) + } + var raw []struct { + Metric map[string]string `json:"metric"` + Values [][2]json.RawMessage `json:"values"` + } + if err := json.Unmarshal(res.Result, &raw); err != nil { + return nil, fmt.Errorf("decode matrix: %w", err) + } + out := make([]Series, 0, len(raw)) + for _, r := range raw { + s := Series{Labels: r.Metric, Points: make([][2]float64, 0, len(r.Values))} + for _, v := range r.Values { + if p, ok := point(v); ok { + s.Points = append(s.Points, p) + } + } + out = append(out, s) + } + return out, nil +} + +// Query runs expr at a single instant and returns one point per series. +func (c *Client) Query(ctx context.Context, base *url.URL, expr string, at time.Time) ([]Series, error) { + form := url.Values{ + "query": {expr}, + "time": {strconv.FormatInt(at.Unix(), 10)}, + } + res, err := c.do(ctx, base, "/api/v1/query", form) + if err != nil { + return nil, err + } + if res.ResultType != "vector" { + return nil, fmt.Errorf("unexpected result type %q", res.ResultType) + } + var raw []struct { + Metric map[string]string `json:"metric"` + Value [2]json.RawMessage `json:"value"` + } + if err := json.Unmarshal(res.Result, &raw); err != nil { + return nil, fmt.Errorf("decode vector: %w", err) + } + out := make([]Series, 0, len(raw)) + for _, r := range raw { + s := Series{Labels: r.Metric} + if p, ok := point(r.Value); ok { + s.Points = [][2]float64{p} + } + out = append(out, s) + } + return out, nil +} + +type apiResult struct { + ResultType string `json:"resultType"` + Result json.RawMessage `json:"result"` +} + +func (c *Client) do(ctx context.Context, base *url.URL, apiPath string, form url.Values) (*apiResult, error) { + if !c.Allowed(base) { + return nil, ErrNotAllowed + } + u := *base + u.Path = strings.TrimSuffix(base.Path, "/") + apiPath + // POST, so a long expression is not limited by URL length. + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Accept", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil { + return nil, err + } + if len(body) > maxResponseBytes { + return nil, errors.New("response too large") + } + var env struct { + Status string `json:"status"` + Data *apiResult `json:"data"` + ErrorType string `json:"errorType"` + Error string `json:"error"` + } + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("prometheus answered %d with a body that is not JSON", resp.StatusCode) + } + if resp.StatusCode/100 != 2 || env.Status != "success" || env.Data == nil { + msg := env.Error + if msg == "" { + msg = "status " + strconv.Itoa(resp.StatusCode) + } + return nil, fmt.Errorf("prometheus: %s", truncate(msg, 200)) + } + if n := countResults(env.Data.Result); n > maxSeries { + return nil, fmt.Errorf("query matches %d series, over the limit of %d", n, maxSeries) + } + return env.Data, nil +} + +func countResults(raw json.RawMessage) int { + var items []json.RawMessage + if json.Unmarshal(raw, &items) != nil { + return 0 + } + return len(items) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// point decodes a [timestamp, "value"] pair. Prometheus writes values as +// strings so that NaN and ±Inf survive; JSON cannot carry them, so those points +// are dropped. +func point(v [2]json.RawMessage) ([2]float64, bool) { + var ts float64 + var s string + if json.Unmarshal(v[0], &ts) != nil || json.Unmarshal(v[1], &s) != nil { + return [2]float64{}, false + } + f, err := strconv.ParseFloat(s, 64) + if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { + return [2]float64{}, false + } + return [2]float64{ts, f}, true +} + +// StepFor picks a query step that yields about TargetPoints over the span, never +// finer than a second. +func StepFor(start, end time.Time) time.Duration { + span := end.Sub(start) + step := span / TargetPoints + if step < time.Second { + return time.Second + } + return step.Round(time.Second) +} diff --git a/internal/prom/filter.go b/internal/prom/filter.go new file mode 100644 index 0000000..ba5e81f --- /dev/null +++ b/internal/prom/filter.go @@ -0,0 +1,51 @@ +package prom + +import "sort" + +// MaxSeriesPerAlert caps how many series are kept for one alert. +const MaxSeriesPerAlert = 5 + +// SelectSeries keeps the series that belong to this alert. The rule expression +// returns every series it matches, not just the one that fired, so a series is +// kept only if it agrees with the alert's labels on every label name they both +// carry. Labels only one side has (the rule's severity, the series' __name__) +// say nothing either way, and alertname is the rule's, not the series'. +// +// The result is capped at MaxSeriesPerAlert, best-labelled first: the more +// labels a series shares with the alert, the likelier it is the one that fired. +func SelectSeries(series []Series, alertLabels map[string]string) []Series { + type scored struct { + s Series + score int + } + var kept []scored + for _, s := range series { + score, ok := 0, true + for k, v := range s.Labels { + if k == "__name__" || k == "alertname" { + continue + } + av, has := alertLabels[k] + if !has { + continue + } + if av != v { + ok = false + break + } + score++ + } + if ok && len(s.Points) > 0 { + kept = append(kept, scored{s, score}) + } + } + sort.SliceStable(kept, func(i, j int) bool { return kept[i].score > kept[j].score }) + if len(kept) > MaxSeriesPerAlert { + kept = kept[:MaxSeriesPerAlert] + } + out := make([]Series, len(kept)) + for i, k := range kept { + out[i] = k.s + } + return out +} diff --git a/internal/prom/parse.go b/internal/prom/parse.go new file mode 100644 index 0000000..f16fa0f --- /dev/null +++ b/internal/prom/parse.go @@ -0,0 +1,38 @@ +package prom + +import ( + "net/url" + "strings" +) + +// ParseGeneratorURL splits an Alertmanager generatorURL into the Prometheus it +// points at and the expression of the rule that fired. Prometheus builds it as +// /graph?g0.expr=&g0.tab=1 (/query on Prometheus 3), and the +// external URL may carry a path prefix, which is kept in base. +// +// ok is false for anything else: a non-HTTP scheme, credentials in the URL, a +// page that is not the graph page, or no expression. The caller then has +// nothing to query and falls back to showing the link. +func ParseGeneratorURL(raw string) (base *url.URL, expr string, ok bool) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || u.User != nil { + return nil, "", false + } + if u.Scheme != "http" && u.Scheme != "https" { + return nil, "", false + } + prefix := "" + switch { + case strings.HasSuffix(u.Path, "/graph"): + prefix = strings.TrimSuffix(u.Path, "/graph") + case strings.HasSuffix(u.Path, "/query"): + prefix = strings.TrimSuffix(u.Path, "/query") + default: + return nil, "", false + } + expr = strings.TrimSpace(u.Query().Get("g0.expr")) + if expr == "" { + return nil, "", false + } + return &url.URL{Scheme: u.Scheme, Host: u.Host, Path: prefix}, expr, true +} diff --git a/internal/prom/prom_test.go b/internal/prom/prom_test.go new file mode 100644 index 0000000..142010c --- /dev/null +++ b/internal/prom/prom_test.go @@ -0,0 +1,237 @@ +package prom + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "testing" + "time" +) + +func TestParseAllowlistRejectsAllowAll(t *testing.T) { + for _, e := range []string{"*", "**", "*.*", "**.*", "a.**", "pro**.example.com", "ex ample.com", "host:abc", "10.0.0.0/33"} { + if _, err := ParseAllowlist([]string{e}); err == nil { + t.Errorf("%q was accepted", e) + } + } +} + +func TestAllowlistNames(t *testing.T) { + a, err := ParseAllowlist([]string{ + "prometheus.monitoring.svc", + "prom-*.monitoring.svc", + "*.example.com", + "**.corp.example", + "fixed.example.org:9090", + }) + if err != nil { + t.Fatal(err) + } + pub := netip.MustParseAddr("10.1.2.3") + cases := []struct { + host, port string + want bool + }{ + {"prometheus.monitoring.svc", "9090", true}, + {"PROMETHEUS.monitoring.svc", "80", true}, + {"prom-0.monitoring.svc", "9090", true}, + {"prom-0.other.svc", "9090", false}, + {"a.example.com", "443", true}, + {"example.com", "443", false}, + {"a.b.example.com", "443", false}, + {"a.corp.example", "80", true}, + {"a.b.c.corp.example", "80", true}, + {"corp.example", "80", false}, + {"fixed.example.org", "9090", true}, + {"fixed.example.org", "80", false}, + {"evilexample.com", "80", false}, + {"a.example.com.evil.net", "80", false}, + } + for _, c := range cases { + if got := a.permits(c.host, c.port, pub); got != c.want { + t.Errorf("permits(%s:%s) = %v, want %v", c.host, c.port, got, c.want) + } + } +} + +func TestAllowlistForbiddenAddressesAndCIDR(t *testing.T) { + a, _ := ParseAllowlist([]string{"*.example.com", "192.168.0.0/16", "[fd00::]:9090", "127.0.0.9"}) + cases := []struct { + name, host, port, ip string + want bool + }{ + {"wildcard to private is fine", "a.example.com", "80", "10.0.0.5", true}, + {"wildcard rebinding to loopback", "a.example.com", "80", "127.0.0.1", false}, + {"wildcard to metadata", "a.example.com", "80", "169.254.169.254", false}, + {"wildcard to unspecified", "a.example.com", "80", "0.0.0.0", false}, + {"wildcard to v4-mapped loopback", "a.example.com", "80", "::ffff:127.0.0.1", false}, + {"cidr literal", "192.168.4.5", "80", "192.168.4.5", true}, + {"outside cidr", "10.9.9.9", "80", "10.9.9.9", false}, + {"cidr admits unlisted name", "anything.test", "80", "192.168.1.1", true}, + {"explicit loopback entry", "127.0.0.9", "80", "127.0.0.9", true}, + {"other loopback", "127.0.0.1", "80", "127.0.0.1", false}, + {"ipv6 with port ok", "fd00::1", "9090", "fd00::1", false}, // entry is a single address fd00::, not the range + {"ipv6 exact", "fd00::", "9090", "fd00::", true}, + {"ipv6 wrong port", "fd00::", "80", "fd00::", false}, + } + for _, c := range cases { + if got := a.permits(c.host, c.port, netip.MustParseAddr(c.ip)); got != c.want { + t.Errorf("%s: permits = %v, want %v", c.name, got, c.want) + } + } +} + +func TestEmptyAllowlistPermitsNothing(t *testing.T) { + var nilList *Allowlist + empty, _ := ParseAllowlist(nil) + for _, a := range []*Allowlist{nilList, empty} { + if !a.Empty() || a.permits("x.example.com", "80", netip.MustParseAddr("10.0.0.1")) { + t.Error("empty allowlist permitted something") + } + } +} + +func TestParseGeneratorURL(t *testing.T) { + cases := []struct { + raw string + wantBase string + wantExpr string + ok bool + }{ + {"http://prom:9090/graph?g0.expr=up%3D%3D0&g0.tab=1", "http://prom:9090", "up==0", true}, + {"https://p.example.com/prom/graph?g0.expr=rate%28x%5B5m%5D%29%3E1", "https://p.example.com/prom", "rate(x[5m])>1", true}, + {"https://p.example.com/query?g0.expr=up", "https://p.example.com", "up", true}, + {"http://prom:9090/graph?g0.tab=1", "", "", false}, + {"http://prom:9090/other?g0.expr=up", "", "", false}, + {"ftp://prom/graph?g0.expr=up", "", "", false}, + {"http://user:pw@prom/graph?g0.expr=up", "", "", false}, + {"", "", "", false}, + {"not a url", "", "", false}, + } + for _, c := range cases { + base, expr, ok := ParseGeneratorURL(c.raw) + if ok != c.ok || expr != c.wantExpr || (ok && base.String() != c.wantBase) { + t.Errorf("%q = (%v, %q, %v), want (%q, %q, %v)", c.raw, base, expr, ok, c.wantBase, c.wantExpr, c.ok) + } + } +} + +func TestSelectSeries(t *testing.T) { + mk := func(l map[string]string) Series { return Series{Labels: l, Points: [][2]float64{{1, 1}}} } + series := []Series{ + mk(map[string]string{"__name__": "up", "instance": "a", "job": "x"}), + mk(map[string]string{"__name__": "up", "instance": "b", "job": "x"}), + mk(map[string]string{"instance": "a", "job": "y"}), + {Labels: map[string]string{"instance": "a"}}, // no points + } + got := SelectSeries(series, map[string]string{"alertname": "Down", "instance": "a", "job": "x", "severity": "page"}) + if len(got) != 1 || got[0].Labels["job"] != "x" || got[0].Labels["instance"] != "a" { + t.Fatalf("got %+v", got) + } + if n := len(SelectSeries(series, nil)); n != 3 { + t.Errorf("no alert labels should keep every series with points, got %d", n) + } +} + +func TestStepFor(t *testing.T) { + now := time.Now() + if s := StepFor(now.Add(-time.Minute), now); s != time.Second { + t.Errorf("short span step = %v", s) + } + if s := StepFor(now.Add(-5*time.Hour), now); s != time.Minute { + t.Errorf("5h step = %v, want 1m", s) + } +} + +func fakeProm(t *testing.T) (*httptest.Server, *int) { + t.Helper() + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + r.ParseForm() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/query_range": + w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[ + {"metric":{"instance":"a"},"values":[[100,"1"],[110,"NaN"],[120,"2.5"]]}]}}`)) + case "/api/v1/query": + w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[ + {"metric":{"instance":"a"},"value":[130,"3"]}]}}`)) + case "/redirect/api/v1/query": + http.Redirect(w, r, "http://127.0.0.1:1/", http.StatusFound) + default: + w.WriteHeader(400) + w.Write([]byte(`{"status":"error","errorType":"bad_data","error":"nope"}`)) + } + })) + t.Cleanup(srv.Close) + return srv, &hits +} + +func TestClientQueries(t *testing.T) { + srv, _ := fakeProm(t) + u, _ := url.Parse(srv.URL) + // httptest listens on loopback, which is refused unless listed as a CIDR. + allow, _ := ParseAllowlist([]string{"127.0.0.0/8"}) + c := NewClient(allow, 5*time.Second) + + got, err := c.QueryRange(context.Background(), u, "up", time.Unix(100, 0), time.Unix(120, 0), 10*time.Second) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || len(got[0].Points) != 2 || got[0].Points[1] != [2]float64{120, 2.5} { + t.Fatalf("range = %+v (NaN point should be dropped)", got) + } + inst, err := c.Query(context.Background(), u, "up", time.Unix(130, 0)) + if err != nil || len(inst) != 1 || inst[0].Points[0][1] != 3 { + t.Fatalf("instant = %+v, %v", inst, err) + } +} + +func TestClientRefusesUnlistedAndLoopback(t *testing.T) { + srv, hits := fakeProm(t) + u, _ := url.Parse(srv.URL) + + // Listed by name, but the name resolves to loopback: refused at dial time. + local, _ := url.Parse("http://localhost:" + u.Port()) + wild, _ := ParseAllowlist([]string{"local*"}) + if _, err := NewClient(wild, time.Second).Query(context.Background(), local, "up", time.Now()); err == nil { + t.Error("wildcard-matched name resolving to loopback was queried") + } + + // Not listed at all. + other, _ := ParseAllowlist([]string{"*.example.com"}) + if _, err := NewClient(other, time.Second).Query(context.Background(), u, "up", time.Now()); !errors.Is(err, ErrNotAllowed) { + t.Errorf("unlisted host: err = %v", err) + } + + // Disabled. + if _, err := NewClient(nil, time.Second).Query(context.Background(), u, "up", time.Now()); !errors.Is(err, ErrNotAllowed) { + t.Errorf("disabled: err = %v", err) + } + if *hits != 0 { + t.Errorf("server was contacted %d times", *hits) + } +} + +func TestClientDoesNotFollowRedirects(t *testing.T) { + srv, _ := fakeProm(t) + u, _ := url.Parse(srv.URL + "/redirect") + allow, _ := ParseAllowlist([]string{"127.0.0.0/8"}) + if _, err := NewClient(allow, time.Second).Query(context.Background(), u, "up", time.Now()); err == nil { + t.Error("redirect response was treated as success") + } +} + +func TestClientSurfacesPrometheusError(t *testing.T) { + srv, _ := fakeProm(t) + u, _ := url.Parse(srv.URL + "/missing") + allow, _ := ParseAllowlist([]string{"127.0.0.0/8"}) + _, err := NewClient(allow, time.Second).Query(context.Background(), u, "up", time.Now()) + if err == nil { + t.Fatal("expected an error") + } +} diff --git a/internal/web/static/app.css b/internal/web/static/app.css index d53a38e..3aa0bbe 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -600,6 +600,33 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh .alert-item-summary { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; } .alert-item-foot { display: flex; flex-wrap: wrap; gap: 4px 12px; font-size: 13px; color: var(--faint); } .alert-item-foot a { color: var(--accent); font-weight: 600; } + +/* Metrics under an alert */ +.metrics { display: grid; gap: 6px; } +.metrics-note { color: var(--faint); font-size: 13px; } +.metrics-now { display: flex; align-items: baseline; gap: 8px; font-size: 14px; } +.metrics-now strong { font-size: 18px; font-variant-numeric: tabular-nums; } +.metrics-now.none { color: var(--muted); } +.metrics-now-label { color: var(--faint); font-size: 12px; font-weight: 650; text-transform: uppercase; letter-spacing: .04em; } +.metric-chart { width: 100%; height: auto; display: block; } +.mc-grid { stroke: var(--border); stroke-width: 1; } +.mc-label { fill: var(--faint); font-size: 10px; } +.mc-marker { stroke-width: 1; stroke-dasharray: 3 3; } +.mc-marker.start { stroke: var(--crit); } +.mc-marker.end { stroke: var(--ok); } +.mc-line { fill: none; stroke-width: 1.75; stroke-linejoin: round; stroke-linecap: round; } +.mc-line.s0, .mc-swatch.s0 { stroke: var(--accent); background: var(--accent); } +.mc-line.s1, .mc-swatch.s1 { stroke: var(--warn); background: var(--warn); } +.mc-line.s2, .mc-swatch.s2 { stroke: var(--ok); background: var(--ok); } +.mc-line.s3, .mc-swatch.s3 { stroke: var(--crit); background: var(--crit); } +.mc-line.s4, .mc-swatch.s4 { stroke: var(--muted); background: var(--muted); } +.mc-dot.s0 { fill: var(--accent); } .mc-dot.s1 { fill: var(--warn); } .mc-dot.s2 { fill: var(--ok); } +.mc-dot.s3 { fill: var(--crit); } .mc-dot.s4 { fill: var(--muted); } +.mc-swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; flex: none; } +.metrics-legend { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 2px 14px; font-size: 12px; color: var(--muted); } +.metrics-legend li { display: flex; align-items: center; gap: 6px; overflow-wrap: anywhere; } +.metrics-expr { font-size: 12px; color: var(--faint); overflow-wrap: anywhere; } + /* A disclosure is a button: a chevron that turns, and a 44px target on a touch screen (the bare triangle it replaces was a few pixels wide). */ details > summary { diff --git a/internal/web/static/js/api.js b/internal/web/static/js/api.js index 41dc235..94953e7 100644 --- a/internal/web/static/js/api.js +++ b/internal/web/static/js/api.js @@ -108,6 +108,7 @@ export const unsnooze = (id) => call('DELETE', `/incidents/${id}/snooze`); export const archive = (id) => call('POST', `/incidents/${id}/archive`); export const unarchive = (id) => call('DELETE', `/incidents/${id}/archive`); export const addNote = (id, content, pinned = false) => call('POST', `/incidents/${id}/notes`, { body: { content, pinned } }); +export const metrics = (id) => call('GET', `/incidents/${id}/metrics`); export const similar = (id) => call('GET', `/incidents/${id}/similar`); export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/notes/${eventID}`); diff --git a/internal/web/static/js/incident.js b/internal/web/static/js/incident.js index f96ce67..40caf6b 100644 --- a/internal/web/static/js/incident.js +++ b/internal/web/static/js/incident.js @@ -11,6 +11,7 @@ import { } from './format.js'; import { myID, users } from './state.js'; import { back } from './app.js'; +import { alertMetrics } from './metricchart.js'; const pane = () => document.getElementById('detail'); @@ -18,6 +19,7 @@ let currentID = null; let inc = null; let events = []; let similarList = []; +let metricsByAlert = {}; let error = null; let busy = false; @@ -26,6 +28,7 @@ export function show(id) { currentID = id; inc = null; events = []; + metricsByAlert = {}; error = null; if (id == null) { renderPlaceholder(); @@ -41,13 +44,17 @@ export async function refresh() { try { // Similar incidents are a courtesy: an older server answers 404 and a // failure here must not hide the incident itself. - const [i, t, sim] = await Promise.all([ - api.incident(id), api.timeline(id), api.similar(id).catch(() => []), + // Metrics are a courtesy too: an older server has no such endpoint, and a + // Prometheus that is slow or down must not hold up the incident. + const [i, t, sim, met] = await Promise.all([ + api.incident(id), api.timeline(id), api.similar(id).catch(() => []), api.metrics(id).catch(() => null), ]); if (id !== currentID) return; inc = i; events = t; similarList = sim; + metricsByAlert = {}; + if (met && met.enabled) for (const m of met.alerts) metricsByAlert[m.alert_id] = m; error = null; } catch (err) { if (id !== currentID) return; @@ -203,6 +210,7 @@ function alertItem(a) { h('span', { class: 'alert-item-name', text: a.name }), badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`)), summary && h('div', { class: 'alert-item-summary', text: summary }), + alertMetrics(metricsByAlert[a.id], a.starts_at, inc.resolved_at), h('div', { class: 'alert-item-foot' }, h('span', { text: `Started ${ago(a.starts_at)}` }), h('span', { text: `Last seen ${ago(a.received_at)}` }), diff --git a/internal/web/static/js/metricchart.js b/internal/web/static/js/metricchart.js new file mode 100644 index 0000000..e0f3ac1 --- /dev/null +++ b/internal/web/static/js/metricchart.js @@ -0,0 +1,196 @@ +// The metrics under one alert: a line chart of the expression that fired it, and +// what that expression reads right now. The data comes from this server, which +// asked the alert's own Prometheus (see GET /api/incidents/{id}/metrics); the +// browser never talks to Prometheus. + +import { h } from './ui.js'; +import { when } from './format.js'; + +const SVG_NS = 'http://www.w3.org/2000/svg'; +const MAX_SERIES = 5; + +function svg(tag, attrs = {}, text) { + const el = document.createElementNS(SVG_NS, tag); + for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v)); + if (text != null) el.textContent = text; + return el; +} + +const iso = (sec) => new Date(sec * 1000).toISOString(); + +// Compact number for an axis or a badge: 1.2k, 0.034, 12. +export function fmtValue(v) { + if (v == null || Number.isNaN(v)) return '–'; + const a = Math.abs(v); + if (a >= 1e9) return `${+(v / 1e9).toPrecision(3)}G`; + if (a >= 1e6) return `${+(v / 1e6).toPrecision(3)}M`; + if (a >= 1e4) return `${+(v / 1e3).toPrecision(3)}k`; + if (a === 0) return '0'; + if (a >= 100) return String(Math.round(v)); + return String(+v.toPrecision(3)); +} + +const keyOf = (labels) => JSON.stringify(Object.entries(labels || {}).sort()); + +// seriesName is the labels that tell the series apart, e.g. {instance="web-1"}. +function seriesName(labels) { + const parts = Object.entries(labels || {}).filter(([k]) => k !== '__name__').map(([k, v]) => `${k}="${v}"`); + return parts.length ? `{${parts.join(', ')}}` : 'value'; +} + +// merge combines the snapshot taken when the incident opened with the live +// window, so the chart keeps history Prometheus may since have dropped. On the +// same timestamp the later source wins. +function merge(...sources) { + const byKey = new Map(); + for (const src of sources) { + for (const s of src || []) { + const k = keyOf(s.labels); + const cur = byKey.get(k) || { labels: s.labels, points: new Map() }; + for (const [t, v] of s.points || []) cur.points.set(t, v); + byKey.set(k, cur); + } + } + return [...byKey.values()].map((s) => ({ + labels: s.labels, + points: [...s.points.entries()].sort((a, b) => a[0] - b[0]), + })); +} + +function parseSeries(raw) { + if (Array.isArray(raw)) return raw; + return []; +} + +// pick chooses what to draw: while the incident is open, the opened snapshot +// plus live data; once it is closed, the closed snapshot (the whole incident), +// falling back to the opened one. +function pick(entry) { + const snaps = entry.snapshots || {}; + const live = entry.live; + if (live && !live.error) { + return { + series: merge(parseSeries(snaps.opened && snaps.opened.series), live.series), + from: Math.min(live.window_start, snaps.opened ? snaps.opened.window_start : Infinity), + to: live.window_end, + current: live.current || [], + isLive: true, + }; + } + const snap = snaps.closed || snaps.opened; + if (snap && !snap.error) { + return { + series: merge(parseSeries(snap.series)), from: snap.window_start, to: snap.window_end, current: [], isLive: false, + }; + } + return null; +} + +// alertMetrics renders the metrics block for one alert, or null when there is +// nothing worth saying (the feature off, or no Prometheus link on the alert). +export function alertMetrics(entry, startsAtISO, resolvedISO) { + if (!entry) return null; + if (!entry.available) { + return entry.reason && entry.expr + ? h('div', { class: 'metrics-note', text: `Metrics unavailable: ${entry.reason}.` }) + : null; + } + const data = pick(entry); + if (!data) { + const err = (entry.live && entry.live.error) + || (entry.snapshots && ((entry.snapshots.closed && entry.snapshots.closed.error) || (entry.snapshots.opened && entry.snapshots.opened.error))); + return h('div', { class: 'metrics-note', text: err ? `Metrics could not be fetched: ${err}` : 'Metrics are being collected…' }); + } + const startSec = startsAtISO ? Date.parse(startsAtISO) / 1000 : null; + const endSec = resolvedISO ? Date.parse(resolvedISO) / 1000 : null; + return h('div', { class: 'metrics' }, + currentLine(entry, data), + data.series.length ? chart(data, startSec, endSec) : h('div', { class: 'metrics-note', text: 'No samples in this window.' }), + legend(data.series), + entry.expr && h('code', { class: 'metrics-expr', text: entry.expr }), + ); +} + +function currentLine(entry, data) { + if (!data.isLive) return null; + const vals = data.current.map((s) => s.points && s.points[0] && s.points[0][1]).filter((v) => v != null); + if (!vals.length) { + return h('div', { class: 'metrics-now none' }, + h('span', { class: 'metrics-now-label', text: 'Now' }), + h('span', { text: 'no data — the alert condition no longer holds' })); + } + return h('div', { class: 'metrics-now' }, + h('span', { class: 'metrics-now-label', text: 'Now' }), + h('strong', { text: vals.length === 1 ? fmtValue(vals[0]) : vals.map(fmtValue).join(' · ') })); +} + +function legend(series) { + if (series.length < 2 && !(series[0] && Object.keys(series[0].labels || {}).length)) return null; + return h('ul', { class: 'metrics-legend' }, series.slice(0, MAX_SERIES).map((s, i) => + h('li', {}, h('span', { class: `mc-swatch s${i}` }), h('span', { text: seriesName(s.labels) })))); +} + +function chart(data, startSec, endSec) { + const W = 480; + const H = 130; + const padL = 40; + const padR = 8; + const padT = 8; + const padB = 18; + const series = data.series.slice(0, MAX_SERIES); + + let t0 = Infinity; let t1 = -Infinity; let lo = Infinity; let hi = -Infinity; + for (const s of series) { + for (const [t, v] of s.points) { + if (t < t0) t0 = t; + if (t > t1) t1 = t; + if (v < lo) lo = v; + if (v > hi) hi = v; + } + } + if (!Number.isFinite(t0)) { t0 = data.from; t1 = data.to; lo = 0; hi = 1; } + // Show the whole window, not just where there are samples, so the shape of + // "it only exists while firing" is visible. + t0 = Math.min(t0, data.from); + t1 = Math.max(t1, data.to); + if (t1 === t0) t1 = t0 + 1; + if (lo === hi) { lo -= 1; hi += 1; } + if (lo > 0 && lo < (hi - lo) * 0.5) lo = 0; + + const x = (t) => padL + ((t - t0) / (t1 - t0)) * (W - padL - padR); + const y = (v) => padT + (1 - (v - lo) / (hi - lo)) * (H - padT - padB); + + const root = svg('svg', { + class: 'metric-chart', viewBox: `0 0 ${W} ${H}`, role: 'img', + 'aria-label': `Metric over time, ${fmtValue(lo)} to ${fmtValue(hi)}, from ${when(iso(t0))} to ${when(iso(t1))}`, + }); + // Gridlines and y labels at the extremes and the middle. + for (const v of [lo, (lo + hi) / 2, hi]) { + root.appendChild(svg('line', { class: 'mc-grid', x1: padL, x2: W - padR, y1: y(v), y2: y(v) })); + root.appendChild(svg('text', { class: 'mc-label', x: padL - 4, y: y(v) + 3, 'text-anchor': 'end' }, fmtValue(v))); + } + root.appendChild(svg('text', { class: 'mc-label', x: padL, y: H - 4, 'text-anchor': 'start' }, when(iso(t0)))); + root.appendChild(svg('text', { class: 'mc-label', x: W - padR, y: H - 4, 'text-anchor': 'end' }, when(iso(t1)))); + + const marker = (t, cls, text) => { + if (t == null || t < t0 || t > t1) return; + const m = svg('line', { class: `mc-marker ${cls}`, x1: x(t), x2: x(t), y1: padT, y2: H - padB }); + m.appendChild(svg('title', {}, `${text} ${when(iso(t))}`)); + root.appendChild(m); + }; + marker(startSec, 'start', 'Alert started'); + marker(endSec, 'end', 'Incident resolved'); + + series.forEach((s, i) => { + if (!s.points.length) return; + const d = s.points.map(([t, v], n) => `${n ? 'L' : 'M'}${x(t).toFixed(1)},${y(v).toFixed(1)}`).join(' '); + const line = svg('path', { class: `mc-line s${i}`, d }); + line.appendChild(svg('title', {}, `${seriesName(s.labels)}: ${fmtValue(s.points[s.points.length - 1][1])} at ${when(iso(s.points[s.points.length - 1][0]))}`)); + root.appendChild(line); + // A single sample has no line to draw; show it as a dot. + if (s.points.length === 1) { + root.appendChild(svg('circle', { class: `mc-dot s${i}`, cx: x(s.points[0][0]), cy: y(s.points[0][1]), r: 3 })); + } + }); + return root; +}