From a955356821e347a0e3af47f30ea5b785fce777a1 Mon Sep 17 00:00:00 2001 From: Niklas Ye Date: Thu, 8 Oct 2026 18:23:28 +0200 Subject: [PATCH] Filter the queue by cluster A team with a cluster per alert source can now narrow the queue to one cluster. A dropdown under the status chips lists the clusters, and appears once there are two or more to choose between, the rule the team selector follows. The choice is kept per browser, like the selected team, and dropped if the server no longer knows the cluster rather than leaving an empty list with no explanation. The status chip counts follow it, and a row leaves out its cluster chip once the queue is narrowed to one. The filter is on the server. The list is capped at 50 rows, so filtering what is on screen would quietly miss older incidents in the Resolved and Archived lists. GET /api/incidents takes ?cluster=, matched against the incident's `cluster` group label, and GET /api/incidents/clusters lists the distinct values from the last 90 days (optionally for one team), so the dropdown is not limited to what the current page happens to show. Both are additive: no existing parameter or JSON shape changed, so terdut-tui keeps working unchanged and has nothing it must mirror. An incident carries the label only when `cluster` is in Alertmanager's group_by, so the filter only sees those; the README says so. --- README.md | 7 +- internal/api/incidents.go | 52 +++++++++++++++ internal/api/incidents_test.go | 35 ++++++++++ internal/api/notify_test.go | 9 ++- internal/api/router.go | 1 + internal/web/static/app.css | 10 +++ internal/web/static/index.html | 3 + internal/web/static/js/api.js | 1 + internal/web/static/js/queue.js | 112 +++++++++++++++++++++++++++++--- 9 files changed, 218 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 137c609..da37499 100644 --- a/README.md +++ b/README.md @@ -463,6 +463,10 @@ queue, the incident and the alert list, instead of leaving it in the title. An alert that is not grouped by `cluster` still shows the chip on the alert list, which reads the label from the alert itself. +The queue has a cluster dropdown once there are two or more values to choose +between. It filters on the incident's `cluster` group label +(`GET /api/incidents?cluster=...`), so it only sees incidents grouped by it. + ### An incident opens only on a new occurrence An incident opens when an alert **transitions into firing**: a fingerprint that @@ -945,7 +949,8 @@ the team gets the same `404` as anybody else. | Method | Path | Description | |---|---|---| -| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?limit=` (default 50, max 500) | +| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?cluster=`, `?limit=` (default 50, max 500) | +| `GET` | `/api/incidents/clusters` | The distinct `cluster` values on the caller's incidents from the last 90 days, sorted (`?team_id=` narrows it). An empty array when nothing carries the label | | `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline | | `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident | | `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological | diff --git a/internal/api/incidents.go b/internal/api/incidents.go index f0aa6f1..b98a3ff 100644 --- a/internal/api/incidents.go +++ b/internal/api/incidents.go @@ -13,6 +13,52 @@ import ( "github.com/go-chi/chi/v5" ) +// handleListClusters answers GET /api/incidents/clusters: the distinct values of +// the origin label across the caller's incidents from the last 90 days, sorted, +// so the queue can offer them as a filter. Optional team_id narrows it to one +// team. Empty when nothing carries the label, which is how the UI knows to show +// no filter at all. +func handleListClusters(db *sql.DB) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + args := &sqlArgs{} + where := []string{ + "team_id = ANY(" + args.add(callerTeamIDs(r.Context())) + ")", + "triggered_at >= " + args.add(time.Now().AddDate(0, 0, -90).Unix()), + } + if team := r.URL.Query().Get("team_id"); team != "" { + if n, err := strconv.ParseInt(team, 10, 64); err == nil { + where = append(where, "team_id = "+args.add(n)) + } + } + label := args.add(originLabel) + + rows, err := db.QueryContext(r.Context(), + "SELECT DISTINCT group_labels->>"+label+" AS v FROM incidents WHERE "+ + strings.Join(where, " AND ")+" AND group_labels->>"+label+" <> '' ORDER BY v LIMIT 200", + args.all()...) + if err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + defer rows.Close() + + clusters := []string{} + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + clusters = append(clusters, v) + } + if err := rows.Err(); err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + respond(w, http.StatusOK, clusters) + } +} + func handleListIncidents(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() @@ -53,6 +99,12 @@ func handleListIncidents(db *sql.DB) http.HandlerFunc { if severity := q.Get("severity"); severity != "" { where = append(where, "i.severity = "+args.add(severity)) } + // Where it came from: the value of the origin label (originLabel, by + // convention "cluster") among the incident's group labels. An incident has + // it only when the label is in Alertmanager's group_by. + if cluster := q.Get("cluster"); cluster != "" { + where = append(where, "i.group_labels->>"+args.add(originLabel)+" = "+args.add(cluster)) + } if assignee := q.Get("assigned_to"); assignee != "" { if n, err := strconv.ParseInt(assignee, 10, 64); err == nil { where = append(where, "i.assigned_to = "+args.add(n)) diff --git a/internal/api/incidents_test.go b/internal/api/incidents_test.go index 2fdca38..6fbd466 100644 --- a/internal/api/incidents_test.go +++ b/internal/api/incidents_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "testing" "time" @@ -972,3 +973,37 @@ func TestServiceAccount_AssignArchiveUnarchiveRecordActor(t *testing.T) { } } } + +// The queue can be narrowed to one cluster, and the distinct clusters are +// offered so the filter has something to list. +func TestIncidents_FilterByCluster(t *testing.T) { + s := newTS(t) + + for _, c := range []string{"prod-eu", "prod-us"} { + fireGroupedAs(t, s, "fp-"+c, + map[string]string{"alertname": "PodRestarting", "cluster": c, "namespace": "n"}, + `{}:{alertname="PodRestarting",cluster="`+c+`",namespace="n"}`) + } + // A cluster-less incident exists too, and must never match a cluster filter. + fireGroupedAs(t, s, "fp-none", map[string]string{"alertname": "DiskFull"}, `{}:{alertname="DiskFull"}`) + + if got := len(listIncidents(t, s, "")); got != 3 { + t.Fatalf("expected 3 open incidents, got %d", got) + } + eu := listIncidents(t, s, "?cluster=prod-eu") + if len(eu) != 1 { + t.Fatalf("expected 1 incident for prod-eu, got %d", len(eu)) + } + if labels, _ := eu[0]["group_labels"].(map[string]any); labels["cluster"] != "prod-eu" { + t.Errorf("filtered to the wrong cluster: %v", eu[0]["group_labels"]) + } + if got := len(listIncidents(t, s, "?cluster=nowhere")); got != 0 { + t.Errorf("an unknown cluster should match nothing, got %d", got) + } + + var clusters []string + decode(t, s.req(t, http.MethodGet, "/api/incidents/clusters", nil), &clusters) + if want := "prod-eu,prod-us"; strings.Join(clusters, ",") != want { + t.Errorf("clusters = %v, want %s", clusters, want) + } +} diff --git a/internal/api/notify_test.go b/internal/api/notify_test.go index 703c077..a73a082 100644 --- a/internal/api/notify_test.go +++ b/internal/api/notify_test.go @@ -168,10 +168,17 @@ func fireCritical(t *testing.T, s *ts) { // fireGrouped posts one critical alert whose Alertmanager group carries the // given labels, the way group_by puts them on the webhook. func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) { + t.Helper() + fireGroupedAs(t, s, "fp-grouped", groupLabels, groupKey) +} + +// fireGroupedAs is fireGrouped with its own alert fingerprint, for a test that +// needs several alerts open at once. +func fireGroupedAs(t *testing.T, s *ts, fingerprint string, groupLabels map[string]string, groupKey string) { t.Helper() payload := map[string]any{ "version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels, - "alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing", + "alerts": []map[string]any{amAlert(fingerprint, "PodRestarting", "firing", "2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})}, } data, _ := json.Marshal(payload) diff --git a/internal/api/router.go b/internal/api/router.go index f0c5ae5..1b24f08 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -145,6 +145,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin r.Get("/api/alerts/{id}", handleGetAlert(db)) r.Get("/api/incidents", handleListIncidents(db)) + r.Get("/api/incidents/clusters", handleListClusters(db)) r.Get("/api/incidents/{id}", handleGetIncident(db)) r.Get("/api/incidents/{id}/alerts", handleIncidentAlerts(db)) r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db)) diff --git a/internal/web/static/app.css b/internal/web/static/app.css index 679df47..eca6fae 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -404,6 +404,16 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh mask-image: linear-gradient(90deg, #000 calc(100% - 28px), transparent); } +/* The queue's cluster filter: a pill-shaped select under the status chips. */ +.queue-origin { position: relative; display: flex; align-items: center; padding: 0 16px 8px; } +.origin-select-icon { position: absolute; left: 28px; width: 16px; height: 16px; color: var(--muted); pointer-events: none; } +.origin-select { + min-height: 34px; max-width: 100%; padding: 0 12px 0 34px; + border: 1px solid var(--border-strong); border-radius: 999px; + background: var(--surface); color: var(--text); + font: inherit; font-size: 13px; font-weight: 600; cursor: pointer; +} + /* ---------- lists ---------- */ .list { padding: 0 16px 16px; display: grid; gap: 8px; } diff --git a/internal/web/static/index.html b/internal/web/static/index.html index f515bb2..4e9b4b6 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -154,6 +154,9 @@
+ +
diff --git a/internal/web/static/js/api.js b/internal/web/static/js/api.js index 300a304..c76d563 100644 --- a/internal/web/static/js/api.js +++ b/internal/web/static/js/api.js @@ -83,6 +83,7 @@ export const setNotifyTarget = (id, ntfyTopic) => // incidents export const incidents = (query, opts) => call('GET', '/incidents', { query, ...opts }); +export const clusters = (query) => call('GET', '/incidents/clusters', { query }); export const incident = (id) => call('GET', `/incidents/${id}`); export const timeline = (id) => call('GET', `/incidents/${id}/timeline`); diff --git a/internal/web/static/js/queue.js b/internal/web/static/js/queue.js index b2b12c7..43fb091 100644 --- a/internal/web/static/js/queue.js +++ b/internal/web/static/js/queue.js @@ -1,7 +1,7 @@ // The incident queue: filter chips and a list of incident rows. import * as api from './api.js'; -import { h, clear, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js'; +import { h, clear, icon, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js'; import { ago, until, isFuture, severityClass, labelSummary, originOf, titleWithoutOrigin } from './format.js'; import { state, myID, onTeamChange } from './state.js'; import * as onboarding from './onboarding.js'; @@ -30,8 +30,10 @@ onboarding.onRerender(() => renderList()); // The queue used to keep its own team filter (a per-tab sessionStorage value, // out of step with team.js's own picker); both now defer to the global // selector's shared state, so re-render whenever it changes. -onTeamChange(() => { +onTeamChange(async () => { renderChips(); + await loadClusters(true); + renderOrigin(); refresh({ fresh: true }); }); @@ -52,6 +54,63 @@ function loadFilter() { return 'open'; } +// Which cluster the queue is narrowed to, or null for all of them. Kept in +// localStorage like the selected team: it is a standing choice, not a +// per-tab one. +const ORIGIN_KEY = 'terdut.queue.cluster'; +let origin = loadOrigin(); +let clusters = []; // the clusters the server has seen, for the dropdown +let clustersAt = 0; + +function loadOrigin() { + try { + return localStorage.getItem(ORIGIN_KEY) || null; + } catch { + return null; // storage unavailable + } +} + +function saveOrigin() { + try { + if (origin) localStorage.setItem(ORIGIN_KEY, origin); + else localStorage.removeItem(ORIGIN_KEY); + } catch { + /* storage unavailable */ + } +} + +// The clusters to offer, from the server because the list on screen is only +// the fifty most recent. Asked again at most every five minutes; a cluster +// first met in an incident on screen is added at once (see noteClusters). A +// remembered choice the server no longer knows would filter to nothing with no +// hint why, so it is dropped. +async function loadClusters(force = false) { + if (!force && Date.now() - clustersAt < 5 * 60 * 1000) return; + try { + const query = state.selectedTeamID != null ? { team_id: state.selectedTeamID } : undefined; + clusters = await api.clusters(query); + clustersAt = Date.now(); + if (origin && !clusters.includes(origin)) { + origin = null; + saveOrigin(); + } + } catch { + /* the filter is a convenience: without the list it stays hidden */ + } +} + +function noteClusters(list) { + let added = false; + for (const inc of list || []) { + const c = originOf(inc.group_labels); + if (c && !clusters.includes(c)) { + clusters = [...clusters, c].sort(); + added = true; + } + } + return added; +} + function saveFilter() { try { sessionStorage.setItem('terdut.queue.filter', filter); @@ -64,6 +123,7 @@ export function show(incidentID) { selected = incidentID; if (!built) { renderChips(); + renderOrigin(); built = true; } renderList(); @@ -73,15 +133,22 @@ export async function refresh({ fresh = false } = {}) { const f = FILTERS.find((x) => x.id === filter); const requested = filter; try { + // First, so a remembered cluster the server no longer knows is dropped + // before it is sent as a filter. + await loadClusters(); // The open list is already fetched for the badges; no need to ask twice. // The cached open queue covers every team, so it can only be reused when // no team filter is applied. - const query = state.selectedTeamID != null ? { ...f.query, team_id: state.selectedTeamID } : f.query; - const cached = filter === 'open' && !fresh && state.selectedTeamID == null; + const query = { ...f.query }; + if (state.selectedTeamID != null) query.team_id = state.selectedTeamID; + if (origin) query.cluster = origin; + const cached = filter === 'open' && !fresh && state.selectedTeamID == null && !origin; const result = cached ? state.open : await api.incidents(query); await onboarding.load(); if (requested !== filter) return; items = result; + noteClusters(state.open); + noteClusters(items); error = null; } catch (err) { if (requested !== filter) return; @@ -91,6 +158,7 @@ export async function refresh({ fresh = false } = {}) { // whichever caller updated it before calling here — app.js's poll, or the // `cached` branch above. renderChips(); + renderOrigin(); renderList(); } @@ -109,9 +177,9 @@ function setFilter(id) { // for the badges — Snoozed/Resolved/Archived would need a request of their // own, so those chips stay count-less for now. function chipCount(id) { - const open = state.selectedTeamID == null - ? state.open - : state.open.filter((i) => i.team_id === state.selectedTeamID); + const open = state.open.filter((i) => + (state.selectedTeamID == null || i.team_id === state.selectedTeamID) + && (!origin || originOf(i.group_labels) === origin)); if (id === 'open') return open.length; if (id === 'triggered') return open.filter((i) => i.status === 'triggered').length; if (id === 'acknowledged') return open.filter((i) => i.status === 'acknowledged').length; @@ -144,6 +212,30 @@ function renderChips() { chipsFade(); } +// The cluster dropdown, under the status chips. Not shown at all until there +// are two clusters to choose between, the same rule the team selector follows. +function renderOrigin() { + const el = document.getElementById('queue-origin'); + if (clusters.length < 2) { + el.hidden = true; + return; + } + const select = h('select', { class: 'origin-select', 'aria-label': 'Cluster' }, + h('option', { value: '', text: 'All clusters', selected: !origin }), + clusters.map((c) => h('option', { value: c, text: c, selected: c === origin }))); + select.addEventListener('change', () => { + origin = select.value || null; + saveOrigin(); + items = null; + cursor = -1; + renderChips(); + renderList(); + refresh({ fresh: true }); + }); + clear(el, icon('server', 'icon origin-select-icon'), select); + el.hidden = false; +} + function renderList() { const el = document.getElementById('queue-list'); const checklist = onboarding.card(); @@ -201,7 +293,7 @@ function row(inc, index) { ? h('span', { class: 'row-team', text: inc.team_name }) : null; - const origin = originOf(inc.group_labels); + const rowOrigin = originOf(inc.group_labels); return h('a', { class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`, @@ -209,10 +301,10 @@ function row(inc, index) { 'aria-current': inc.id === selected ? 'true' : null, dataset: { index: String(index) }, }, - h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, origin) }), + h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, rowOrigin) }), h('div', { class: 'row-age', title: inc.triggered_at, text: `Triggered ${ago(inc.triggered_at)}` }), h('div', { class: 'row-meta' }, - origin && originChip(origin), + rowOrigin && !origin && originChip(rowOrigin), !impliedStatus && status, // The left-border colour alone doesn't say what it means; spell it out // too, same badge the incident detail page uses for severity.