diff --git a/README.md b/README.md index 16d03d6..137c609 100644 --- a/README.md +++ b/README.md @@ -444,6 +444,25 @@ high-water mark — the highest `severity` label any of its alerts has carried an incident that hit `critical` still reads as critical after the critical alert clears. +### Several clusters, one team + +A team with one Alertmanager per Kubernetes cluster, each posting to its own +source, needs two settings or the clusters run together. + +1. Give every alert a `cluster` label at the source. In Prometheus that is + `externalLabels: {cluster: prod-eu}` (kube-prometheus-stack: + `prometheus.prometheusSpec.externalLabels`). +2. Add `cluster` to `group_by` in `alertmanager.yml`. + +The second one is the one that matters. Incidents are matched on the team and +Alertmanager's `groupKey`, and the `groupKey` does not include external labels: +without `cluster` in `group_by`, the same alert in two clusters has the same +key and joins one incident. With it, each cluster gets its own, `cluster` is in +the incident's `group_labels`, and the web UI shows it as a coloured chip on the +queue, the incident and the alert list, instead of leaving it in the title. +An alert that is not grouped by `cluster` still shows the chip on the alert +list, which reads the label from the alert itself. + ### An incident opens only on a new occurrence An incident opens when an alert **transitions into firing**: a fingerprint that diff --git a/internal/web/static/app.css b/internal/web/static/app.css index 53da2f8..679df47 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -488,7 +488,15 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh } .badge::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: currentColor; } .badge.plain::before { display: none; } -.badge .badge-icon { width: 12px; height: 12px; stroke-width: 2.4; } +.badge .badge-icon, .origin-chip .badge-icon { width: 12px; height: 12px; stroke-width: 2.4; } +/* The cluster an incident or alert came from. Same shape as a badge, coloured + from the rcN palette (see originClass in format.js), never the severity one. */ +.origin-chip { + display: inline-flex; align-items: center; gap: 5px; max-width: 100%; + padding: 1px 8px; border-radius: 999px; + font-size: 12px; font-weight: 700; letter-spacing: 0.01em; white-space: nowrap; + overflow: hidden; text-overflow: ellipsis; +} .badge.st-triggered, .badge.st-firing { background: var(--crit-soft); color: var(--crit); } .badge.st-acknowledged { background: var(--warn-soft); color: var(--warn); } .badge.st-snoozed { background: var(--snooze-soft); color: var(--snooze); } diff --git a/internal/web/static/js/alerts.js b/internal/web/static/js/alerts.js index 28fd585..92fe385 100644 --- a/internal/web/static/js/alerts.js +++ b/internal/web/static/js/alerts.js @@ -2,8 +2,8 @@ // incident it belongs to, which is where anything can be done about it. import * as api from './api.js'; -import { h, clear, badge, severityBadge, emptyState, spinner } from './ui.js'; -import { age, labelSummary } from './format.js'; +import { h, clear, badge, severityBadge, originChip, emptyState, spinner } from './ui.js'; +import { age, labelSummary, originOf, ORIGIN_LABEL } from './format.js'; const FILTERS = [ { id: 'firing', label: 'Firing', query: { status: 'firing' } }, @@ -77,7 +77,8 @@ function row(a) { const summary = (a.annotations && a.annotations.summary) || ''; const sev = a.labels && a.labels.severity; const labels = labelSummary(Object.fromEntries( - Object.entries(a.labels || {}).filter(([k]) => k !== 'severity'))); + Object.entries(a.labels || {}).filter(([k]) => k !== 'severity' && k !== ORIGIN_LABEL))); + const origin = originOf(a.labels); const linked = a.incident_id != null; return h(linked ? 'a' : 'div', { class: `row st-${a.status} ${linked ? '' : 'no-link'}`, @@ -86,6 +87,7 @@ function row(a) { h('div', { class: 'row-title', text: a.name }), h('div', { class: 'row-age', title: a.starts_at, text: age(a.status === 'firing' ? a.starts_at : a.received_at) }), h('div', { class: 'row-meta' }, + origin && originChip(origin), badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`), sev && severityBadge(sev), summary && h('span', { text: summary }), diff --git a/internal/web/static/js/format.js b/internal/web/static/js/format.js index 49ed148..9bf994f 100644 --- a/internal/web/static/js/format.js +++ b/internal/web/static/js/format.js @@ -117,3 +117,36 @@ export function labelSummary(labels, skip = 'alertname') { export function initial(name) { return (name || '?').trim().charAt(0) || '?'; } + +// Where an incident or alert came from, for a team with several Kubernetes +// clusters (or other origins) behind it: the value of one label, `cluster` by +// convention. It comes from Prometheus's externalLabels, so it is on every +// alert; an incident carries it only when it is in Alertmanager's group_by, +// which is also what keeps two clusters' identical alerts from merging into one +// incident (see the README, "Several clusters, one team"). +export const ORIGIN_LABEL = 'cluster'; + +export function originOf(labels) { + const v = labels && labels[ORIGIN_LABEL]; + return v ? String(v) : null; +} + +// A stable colour for an origin, from the six-colour rcN palette app.css has, +// so "prod-eu" is the same colour on every row without anything being stored. +export function originClass(value) { + let n = 0; + for (const ch of value) n = (n * 31 + ch.charCodeAt(0)) >>> 0; + return `rc${(n % 6) + 1}`; +} + +// The title without its `cluster=...` part, for a row that shows the origin as +// a chip: the server puts every grouping label in the title, so the cluster +// would otherwise appear twice. Anything that does not look like the server's +// "name (k=v, k=v)" shape is returned unchanged. +export function titleWithoutOrigin(title, value) { + if (!value) return title; + const m = title.match(/^(.*?) \((.*)\)$/s); + if (!m) return title; + const rest = m[2].split(', ').filter((p) => p !== `${ORIGIN_LABEL}=${value}`); + return rest.length ? `${m[1]} (${rest.join(', ')})` : m[1]; +} diff --git a/internal/web/static/js/incident.js b/internal/web/static/js/incident.js index 0b63648..143d6b8 100644 --- a/internal/web/static/js/incident.js +++ b/internal/web/static/js/incident.js @@ -4,10 +4,10 @@ import * as api from './api.js'; import * as poll from './poll.js'; import { - h, clear, icon, badge, severityBadge, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState, + h, clear, icon, badge, severityBadge, originChip, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState, } from './ui.js'; import { - ago, when, until, duration, isFuture, STATUS_LABEL, + ago, when, until, duration, isFuture, originOf, STATUS_LABEL, } from './format.js'; import { myID, users } from './state.js'; import { back } from './app.js'; @@ -99,6 +99,8 @@ function render() { function statusBadges() { const out = []; + const origin = originOf(inc.group_labels); + if (origin) out.push(originChip(origin)); if (inc.severity) out.push(severityBadge(inc.severity)); out.push(badge(STATUS_LABEL[inc.status] || inc.status, `st-${inc.status}`)); if (inc.status !== 'resolved' && isFuture(inc.snoozed_until)) { diff --git a/internal/web/static/js/queue.js b/internal/web/static/js/queue.js index 111f718..b2b12c7 100644 --- a/internal/web/static/js/queue.js +++ b/internal/web/static/js/queue.js @@ -1,8 +1,8 @@ // The incident queue: filter chips and a list of incident rows. import * as api from './api.js'; -import { h, clear, badge, severityBadge, emptyState, spinner, fadeOnOverflow } from './ui.js'; -import { ago, until, isFuture, severityClass, labelSummary } from './format.js'; +import { h, clear, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js'; +import { ago, until, isFuture, severityClass, labelSummary, originOf, titleWithoutOrigin } from './format.js'; import { state, myID, onTeamChange } from './state.js'; import * as onboarding from './onboarding.js'; import { navigate } from './app.js'; @@ -201,15 +201,18 @@ function row(inc, index) { ? h('span', { class: 'row-team', text: inc.team_name }) : null; + const origin = originOf(inc.group_labels); + return h('a', { class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`, href: `/incidents/${inc.id}`, 'aria-current': inc.id === selected ? 'true' : null, dataset: { index: String(index) }, }, - h('div', { class: 'row-title', text: inc.title }), + h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, origin) }), h('div', { class: 'row-age', title: inc.triggered_at, text: `Triggered ${ago(inc.triggered_at)}` }), h('div', { class: 'row-meta' }, + origin && originChip(origin), !impliedStatus && status, // The left-border colour alone doesn't say what it means; spell it out // too, same badge the incident detail page uses for severity. diff --git a/internal/web/static/js/ui.js b/internal/web/static/js/ui.js index 6b771b5..04fe645 100644 --- a/internal/web/static/js/ui.js +++ b/internal/web/static/js/ui.js @@ -1,6 +1,6 @@ // DOM helpers, the bottom sheet, confirmation and toasts. -import { severityClass } from './format.js'; +import { severityClass, originClass } from './format.js'; // h builds an element. attrs: class, text, on, dataset, aria/other // attributes; boolean true sets an empty attribute, false/null skips it. @@ -49,6 +49,7 @@ export function fadeOnOverflow(el) { // Stroke icons, 24×24. Built as SVG nodes so the CSP needs no inline anything. const ICONS = { + server: ['rect:3,4,18,6,2', 'rect:3,14,18,6,2', 'M7 7h.01', 'M7 17h.01'], alertTriangle: ['M12 4l9.5 16.5h-19z', 'M12 10v4.5', 'M12 17.5h.01'], alertCircle: ['circle:12,12,9', 'M12 7.5v5', 'M12 16h.01'], infoCircle: ['circle:12,12,9', 'M12 11v5.5', 'M12 7.5h.01'], @@ -193,6 +194,13 @@ export function badge(text, cls = '') { return h('span', { class: `badge ${cls}`, text }); } +// Where it came from (the cluster): a chip in that origin's colour, with a +// server icon so it reads as a place and not as a status. +export function originChip(value) { + return h('span', { class: `origin-chip ${originClass(value)}`, title: `Origin: ${value}` }, + icon('server', 'icon badge-icon'), value); +} + // The severity as a badge with a shape as well as a colour: a triangle for // critical, a circle with ! for warning, a circle with i for anything else, so // the coloured bar and badge are not the only thing that says it.