diff --git a/README.md b/README.md index 137c609..da37499 100644 --- a/README.md +++ b/README.md @@ -463,6 +463,10 @@ queue, the incident and the alert list, instead of leaving it in the title. An alert that is not grouped by `cluster` still shows the chip on the alert list, which reads the label from the alert itself. +The queue has a cluster dropdown once there are two or more values to choose +between. It filters on the incident's `cluster` group label +(`GET /api/incidents?cluster=...`), so it only sees incidents grouped by it. + ### An incident opens only on a new occurrence An incident opens when an alert **transitions into firing**: a fingerprint that @@ -945,7 +949,8 @@ the team gets the same `404` as anybody else. | Method | Path | Description | |---|---|---| -| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?limit=` (default 50, max 500) | +| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?cluster=`, `?limit=` (default 50, max 500) | +| `GET` | `/api/incidents/clusters` | The distinct `cluster` values on the caller's incidents from the last 90 days, sorted (`?team_id=` narrows it). An empty array when nothing carries the label | | `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline | | `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident | | `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological | diff --git a/internal/api/incidents.go b/internal/api/incidents.go index f0aa6f1..b98a3ff 100644 --- a/internal/api/incidents.go +++ b/internal/api/incidents.go @@ -13,6 +13,52 @@ import ( "github.com/go-chi/chi/v5" ) +// handleListClusters answers GET /api/incidents/clusters: the distinct values of +// the origin label across the caller's incidents from the last 90 days, sorted, +// so the queue can offer them as a filter. Optional team_id narrows it to one +// team. Empty when nothing carries the label, which is how the UI knows to show +// no filter at all. +func handleListClusters(db *sql.DB) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + args := &sqlArgs{} + where := []string{ + "team_id = ANY(" + args.add(callerTeamIDs(r.Context())) + ")", + "triggered_at >= " + args.add(time.Now().AddDate(0, 0, -90).Unix()), + } + if team := r.URL.Query().Get("team_id"); team != "" { + if n, err := strconv.ParseInt(team, 10, 64); err == nil { + where = append(where, "team_id = "+args.add(n)) + } + } + label := args.add(originLabel) + + rows, err := db.QueryContext(r.Context(), + "SELECT DISTINCT group_labels->>"+label+" AS v FROM incidents WHERE "+ + strings.Join(where, " AND ")+" AND group_labels->>"+label+" <> '' ORDER BY v LIMIT 200", + args.all()...) + if err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + defer rows.Close() + + clusters := []string{} + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + clusters = append(clusters, v) + } + if err := rows.Err(); err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + respond(w, http.StatusOK, clusters) + } +} + func handleListIncidents(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() @@ -53,6 +99,12 @@ func handleListIncidents(db *sql.DB) http.HandlerFunc { if severity := q.Get("severity"); severity != "" { where = append(where, "i.severity = "+args.add(severity)) } + // Where it came from: the value of the origin label (originLabel, by + // convention "cluster") among the incident's group labels. An incident has + // it only when the label is in Alertmanager's group_by. + if cluster := q.Get("cluster"); cluster != "" { + where = append(where, "i.group_labels->>"+args.add(originLabel)+" = "+args.add(cluster)) + } if assignee := q.Get("assigned_to"); assignee != "" { if n, err := strconv.ParseInt(assignee, 10, 64); err == nil { where = append(where, "i.assigned_to = "+args.add(n)) diff --git a/internal/api/incidents_test.go b/internal/api/incidents_test.go index 2fdca38..6fbd466 100644 --- a/internal/api/incidents_test.go +++ b/internal/api/incidents_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "testing" "time" @@ -972,3 +973,37 @@ func TestServiceAccount_AssignArchiveUnarchiveRecordActor(t *testing.T) { } } } + +// The queue can be narrowed to one cluster, and the distinct clusters are +// offered so the filter has something to list. +func TestIncidents_FilterByCluster(t *testing.T) { + s := newTS(t) + + for _, c := range []string{"prod-eu", "prod-us"} { + fireGroupedAs(t, s, "fp-"+c, + map[string]string{"alertname": "PodRestarting", "cluster": c, "namespace": "n"}, + `{}:{alertname="PodRestarting",cluster="`+c+`",namespace="n"}`) + } + // A cluster-less incident exists too, and must never match a cluster filter. + fireGroupedAs(t, s, "fp-none", map[string]string{"alertname": "DiskFull"}, `{}:{alertname="DiskFull"}`) + + if got := len(listIncidents(t, s, "")); got != 3 { + t.Fatalf("expected 3 open incidents, got %d", got) + } + eu := listIncidents(t, s, "?cluster=prod-eu") + if len(eu) != 1 { + t.Fatalf("expected 1 incident for prod-eu, got %d", len(eu)) + } + if labels, _ := eu[0]["group_labels"].(map[string]any); labels["cluster"] != "prod-eu" { + t.Errorf("filtered to the wrong cluster: %v", eu[0]["group_labels"]) + } + if got := len(listIncidents(t, s, "?cluster=nowhere")); got != 0 { + t.Errorf("an unknown cluster should match nothing, got %d", got) + } + + var clusters []string + decode(t, s.req(t, http.MethodGet, "/api/incidents/clusters", nil), &clusters) + if want := "prod-eu,prod-us"; strings.Join(clusters, ",") != want { + t.Errorf("clusters = %v, want %s", clusters, want) + } +} diff --git a/internal/api/notify_test.go b/internal/api/notify_test.go index 703c077..a73a082 100644 --- a/internal/api/notify_test.go +++ b/internal/api/notify_test.go @@ -168,10 +168,17 @@ func fireCritical(t *testing.T, s *ts) { // fireGrouped posts one critical alert whose Alertmanager group carries the // given labels, the way group_by puts them on the webhook. func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) { + t.Helper() + fireGroupedAs(t, s, "fp-grouped", groupLabels, groupKey) +} + +// fireGroupedAs is fireGrouped with its own alert fingerprint, for a test that +// needs several alerts open at once. +func fireGroupedAs(t *testing.T, s *ts, fingerprint string, groupLabels map[string]string, groupKey string) { t.Helper() payload := map[string]any{ "version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels, - "alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing", + "alerts": []map[string]any{amAlert(fingerprint, "PodRestarting", "firing", "2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})}, } data, _ := json.Marshal(payload) diff --git a/internal/api/router.go b/internal/api/router.go index f0c5ae5..1b24f08 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -145,6 +145,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin r.Get("/api/alerts/{id}", handleGetAlert(db)) r.Get("/api/incidents", handleListIncidents(db)) + r.Get("/api/incidents/clusters", handleListClusters(db)) r.Get("/api/incidents/{id}", handleGetIncident(db)) r.Get("/api/incidents/{id}/alerts", handleIncidentAlerts(db)) r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db)) diff --git a/internal/web/static/app.css b/internal/web/static/app.css index 679df47..eca6fae 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -404,6 +404,16 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh mask-image: linear-gradient(90deg, #000 calc(100% - 28px), transparent); } +/* The queue's cluster filter: a pill-shaped select under the status chips. */ +.queue-origin { position: relative; display: flex; align-items: center; padding: 0 16px 8px; } +.origin-select-icon { position: absolute; left: 28px; width: 16px; height: 16px; color: var(--muted); pointer-events: none; } +.origin-select { + min-height: 34px; max-width: 100%; padding: 0 12px 0 34px; + border: 1px solid var(--border-strong); border-radius: 999px; + background: var(--surface); color: var(--text); + font: inherit; font-size: 13px; font-weight: 600; cursor: pointer; +} + /* ---------- lists ---------- */ .list { padding: 0 16px 16px; display: grid; gap: 8px; } diff --git a/internal/web/static/index.html b/internal/web/static/index.html index f515bb2..4e9b4b6 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -154,6 +154,9 @@
+ +
diff --git a/internal/web/static/js/api.js b/internal/web/static/js/api.js index 300a304..c76d563 100644 --- a/internal/web/static/js/api.js +++ b/internal/web/static/js/api.js @@ -83,6 +83,7 @@ export const setNotifyTarget = (id, ntfyTopic) => // incidents export const incidents = (query, opts) => call('GET', '/incidents', { query, ...opts }); +export const clusters = (query) => call('GET', '/incidents/clusters', { query }); export const incident = (id) => call('GET', `/incidents/${id}`); export const timeline = (id) => call('GET', `/incidents/${id}/timeline`); diff --git a/internal/web/static/js/queue.js b/internal/web/static/js/queue.js index b2b12c7..43fb091 100644 --- a/internal/web/static/js/queue.js +++ b/internal/web/static/js/queue.js @@ -1,7 +1,7 @@ // The incident queue: filter chips and a list of incident rows. import * as api from './api.js'; -import { h, clear, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js'; +import { h, clear, icon, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js'; import { ago, until, isFuture, severityClass, labelSummary, originOf, titleWithoutOrigin } from './format.js'; import { state, myID, onTeamChange } from './state.js'; import * as onboarding from './onboarding.js'; @@ -30,8 +30,10 @@ onboarding.onRerender(() => renderList()); // The queue used to keep its own team filter (a per-tab sessionStorage value, // out of step with team.js's own picker); both now defer to the global // selector's shared state, so re-render whenever it changes. -onTeamChange(() => { +onTeamChange(async () => { renderChips(); + await loadClusters(true); + renderOrigin(); refresh({ fresh: true }); }); @@ -52,6 +54,63 @@ function loadFilter() { return 'open'; } +// Which cluster the queue is narrowed to, or null for all of them. Kept in +// localStorage like the selected team: it is a standing choice, not a +// per-tab one. +const ORIGIN_KEY = 'terdut.queue.cluster'; +let origin = loadOrigin(); +let clusters = []; // the clusters the server has seen, for the dropdown +let clustersAt = 0; + +function loadOrigin() { + try { + return localStorage.getItem(ORIGIN_KEY) || null; + } catch { + return null; // storage unavailable + } +} + +function saveOrigin() { + try { + if (origin) localStorage.setItem(ORIGIN_KEY, origin); + else localStorage.removeItem(ORIGIN_KEY); + } catch { + /* storage unavailable */ + } +} + +// The clusters to offer, from the server because the list on screen is only +// the fifty most recent. Asked again at most every five minutes; a cluster +// first met in an incident on screen is added at once (see noteClusters). A +// remembered choice the server no longer knows would filter to nothing with no +// hint why, so it is dropped. +async function loadClusters(force = false) { + if (!force && Date.now() - clustersAt < 5 * 60 * 1000) return; + try { + const query = state.selectedTeamID != null ? { team_id: state.selectedTeamID } : undefined; + clusters = await api.clusters(query); + clustersAt = Date.now(); + if (origin && !clusters.includes(origin)) { + origin = null; + saveOrigin(); + } + } catch { + /* the filter is a convenience: without the list it stays hidden */ + } +} + +function noteClusters(list) { + let added = false; + for (const inc of list || []) { + const c = originOf(inc.group_labels); + if (c && !clusters.includes(c)) { + clusters = [...clusters, c].sort(); + added = true; + } + } + return added; +} + function saveFilter() { try { sessionStorage.setItem('terdut.queue.filter', filter); @@ -64,6 +123,7 @@ export function show(incidentID) { selected = incidentID; if (!built) { renderChips(); + renderOrigin(); built = true; } renderList(); @@ -73,15 +133,22 @@ export async function refresh({ fresh = false } = {}) { const f = FILTERS.find((x) => x.id === filter); const requested = filter; try { + // First, so a remembered cluster the server no longer knows is dropped + // before it is sent as a filter. + await loadClusters(); // The open list is already fetched for the badges; no need to ask twice. // The cached open queue covers every team, so it can only be reused when // no team filter is applied. - const query = state.selectedTeamID != null ? { ...f.query, team_id: state.selectedTeamID } : f.query; - const cached = filter === 'open' && !fresh && state.selectedTeamID == null; + const query = { ...f.query }; + if (state.selectedTeamID != null) query.team_id = state.selectedTeamID; + if (origin) query.cluster = origin; + const cached = filter === 'open' && !fresh && state.selectedTeamID == null && !origin; const result = cached ? state.open : await api.incidents(query); await onboarding.load(); if (requested !== filter) return; items = result; + noteClusters(state.open); + noteClusters(items); error = null; } catch (err) { if (requested !== filter) return; @@ -91,6 +158,7 @@ export async function refresh({ fresh = false } = {}) { // whichever caller updated it before calling here — app.js's poll, or the // `cached` branch above. renderChips(); + renderOrigin(); renderList(); } @@ -109,9 +177,9 @@ function setFilter(id) { // for the badges — Snoozed/Resolved/Archived would need a request of their // own, so those chips stay count-less for now. function chipCount(id) { - const open = state.selectedTeamID == null - ? state.open - : state.open.filter((i) => i.team_id === state.selectedTeamID); + const open = state.open.filter((i) => + (state.selectedTeamID == null || i.team_id === state.selectedTeamID) + && (!origin || originOf(i.group_labels) === origin)); if (id === 'open') return open.length; if (id === 'triggered') return open.filter((i) => i.status === 'triggered').length; if (id === 'acknowledged') return open.filter((i) => i.status === 'acknowledged').length; @@ -144,6 +212,30 @@ function renderChips() { chipsFade(); } +// The cluster dropdown, under the status chips. Not shown at all until there +// are two clusters to choose between, the same rule the team selector follows. +function renderOrigin() { + const el = document.getElementById('queue-origin'); + if (clusters.length < 2) { + el.hidden = true; + return; + } + const select = h('select', { class: 'origin-select', 'aria-label': 'Cluster' }, + h('option', { value: '', text: 'All clusters', selected: !origin }), + clusters.map((c) => h('option', { value: c, text: c, selected: c === origin }))); + select.addEventListener('change', () => { + origin = select.value || null; + saveOrigin(); + items = null; + cursor = -1; + renderChips(); + renderList(); + refresh({ fresh: true }); + }); + clear(el, icon('server', 'icon origin-select-icon'), select); + el.hidden = false; +} + function renderList() { const el = document.getElementById('queue-list'); const checklist = onboarding.card(); @@ -201,7 +293,7 @@ function row(inc, index) { ? h('span', { class: 'row-team', text: inc.team_name }) : null; - const origin = originOf(inc.group_labels); + const rowOrigin = originOf(inc.group_labels); return h('a', { class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`, @@ -209,10 +301,10 @@ function row(inc, index) { 'aria-current': inc.id === selected ? 'true' : null, dataset: { index: String(index) }, }, - h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, origin) }), + h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, rowOrigin) }), h('div', { class: 'row-age', title: inc.triggered_at, text: `Triggered ${ago(inc.triggered_at)}` }), h('div', { class: 'row-meta' }, - origin && originChip(origin), + rowOrigin && !origin && originChip(rowOrigin), !impliedStatus && status, // The left-border colour alone doesn't say what it means; spell it out // too, same badge the incident detail page uses for severity.