Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| def0f68d00 | |||
| fb86a18988 |
@@ -444,6 +444,25 @@ high-water mark — the highest `severity` label any of its alerts has carried
|
||||
an incident that hit `critical` still reads as critical after the critical alert
|
||||
clears.
|
||||
|
||||
### Several clusters, one team
|
||||
|
||||
A team with one Alertmanager per Kubernetes cluster, each posting to its own
|
||||
source, needs two settings or the clusters run together.
|
||||
|
||||
1. Give every alert a `cluster` label at the source. In Prometheus that is
|
||||
`externalLabels: {cluster: prod-eu}` (kube-prometheus-stack:
|
||||
`prometheus.prometheusSpec.externalLabels`).
|
||||
2. Add `cluster` to `group_by` in `alertmanager.yml`.
|
||||
|
||||
The second one is the one that matters. Incidents are matched on the team and
|
||||
Alertmanager's `groupKey`, and the `groupKey` does not include external labels:
|
||||
without `cluster` in `group_by`, the same alert in two clusters has the same
|
||||
key and joins one incident. With it, each cluster gets its own, `cluster` is in
|
||||
the incident's `group_labels`, and the web UI shows it as a coloured chip on the
|
||||
queue, the incident and the alert list, instead of leaving it in the title.
|
||||
An alert that is not grouped by `cluster` still shows the chip on the alert
|
||||
list, which reads the label from the alert itself.
|
||||
|
||||
### An incident opens only on a new occurrence
|
||||
|
||||
An incident opens when an alert **transitions into firing**: a fingerprint that
|
||||
|
||||
@@ -15,5 +15,5 @@ type: application
|
||||
# appVersion and image.tag in values.yaml no longer agree, and that is not an oversight:
|
||||
# image.tag stays "latest", which is what a local install actually pulls. appVersion is
|
||||
# metadata and drives nothing.
|
||||
version: 0.41.3
|
||||
appVersion: "v0.41.3"
|
||||
version: 0.42.0
|
||||
appVersion: "v0.42.0"
|
||||
|
||||
@@ -488,7 +488,15 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
|
||||
}
|
||||
.badge::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: currentColor; }
|
||||
.badge.plain::before { display: none; }
|
||||
.badge .badge-icon { width: 12px; height: 12px; stroke-width: 2.4; }
|
||||
.badge .badge-icon, .origin-chip .badge-icon { width: 12px; height: 12px; stroke-width: 2.4; }
|
||||
/* The cluster an incident or alert came from. Same shape as a badge, coloured
|
||||
from the rcN palette (see originClass in format.js), never the severity one. */
|
||||
.origin-chip {
|
||||
display: inline-flex; align-items: center; gap: 5px; max-width: 100%;
|
||||
padding: 1px 8px; border-radius: 999px;
|
||||
font-size: 12px; font-weight: 700; letter-spacing: 0.01em; white-space: nowrap;
|
||||
overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.badge.st-triggered, .badge.st-firing { background: var(--crit-soft); color: var(--crit); }
|
||||
.badge.st-acknowledged { background: var(--warn-soft); color: var(--warn); }
|
||||
.badge.st-snoozed { background: var(--snooze-soft); color: var(--snooze); }
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// incident it belongs to, which is where anything can be done about it.
|
||||
|
||||
import * as api from './api.js';
|
||||
import { h, clear, badge, severityBadge, emptyState, spinner } from './ui.js';
|
||||
import { age, labelSummary } from './format.js';
|
||||
import { h, clear, badge, severityBadge, originChip, emptyState, spinner } from './ui.js';
|
||||
import { age, labelSummary, originOf, ORIGIN_LABEL } from './format.js';
|
||||
|
||||
const FILTERS = [
|
||||
{ id: 'firing', label: 'Firing', query: { status: 'firing' } },
|
||||
@@ -77,7 +77,8 @@ function row(a) {
|
||||
const summary = (a.annotations && a.annotations.summary) || '';
|
||||
const sev = a.labels && a.labels.severity;
|
||||
const labels = labelSummary(Object.fromEntries(
|
||||
Object.entries(a.labels || {}).filter(([k]) => k !== 'severity')));
|
||||
Object.entries(a.labels || {}).filter(([k]) => k !== 'severity' && k !== ORIGIN_LABEL)));
|
||||
const origin = originOf(a.labels);
|
||||
const linked = a.incident_id != null;
|
||||
return h(linked ? 'a' : 'div', {
|
||||
class: `row st-${a.status} ${linked ? '' : 'no-link'}`,
|
||||
@@ -86,6 +87,7 @@ function row(a) {
|
||||
h('div', { class: 'row-title', text: a.name }),
|
||||
h('div', { class: 'row-age', title: a.starts_at, text: age(a.status === 'firing' ? a.starts_at : a.received_at) }),
|
||||
h('div', { class: 'row-meta' },
|
||||
origin && originChip(origin),
|
||||
badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`),
|
||||
sev && severityBadge(sev),
|
||||
summary && h('span', { text: summary }),
|
||||
|
||||
@@ -117,3 +117,36 @@ export function labelSummary(labels, skip = 'alertname') {
|
||||
export function initial(name) {
|
||||
return (name || '?').trim().charAt(0) || '?';
|
||||
}
|
||||
|
||||
// Where an incident or alert came from, for a team with several Kubernetes
|
||||
// clusters (or other origins) behind it: the value of one label, `cluster` by
|
||||
// convention. It comes from Prometheus's externalLabels, so it is on every
|
||||
// alert; an incident carries it only when it is in Alertmanager's group_by,
|
||||
// which is also what keeps two clusters' identical alerts from merging into one
|
||||
// incident (see the README, "Several clusters, one team").
|
||||
export const ORIGIN_LABEL = 'cluster';
|
||||
|
||||
export function originOf(labels) {
|
||||
const v = labels && labels[ORIGIN_LABEL];
|
||||
return v ? String(v) : null;
|
||||
}
|
||||
|
||||
// A stable colour for an origin, from the six-colour rcN palette app.css has,
|
||||
// so "prod-eu" is the same colour on every row without anything being stored.
|
||||
export function originClass(value) {
|
||||
let n = 0;
|
||||
for (const ch of value) n = (n * 31 + ch.charCodeAt(0)) >>> 0;
|
||||
return `rc${(n % 6) + 1}`;
|
||||
}
|
||||
|
||||
// The title without its `cluster=...` part, for a row that shows the origin as
|
||||
// a chip: the server puts every grouping label in the title, so the cluster
|
||||
// would otherwise appear twice. Anything that does not look like the server's
|
||||
// "name (k=v, k=v)" shape is returned unchanged.
|
||||
export function titleWithoutOrigin(title, value) {
|
||||
if (!value) return title;
|
||||
const m = title.match(/^(.*?) \((.*)\)$/s);
|
||||
if (!m) return title;
|
||||
const rest = m[2].split(', ').filter((p) => p !== `${ORIGIN_LABEL}=${value}`);
|
||||
return rest.length ? `${m[1]} (${rest.join(', ')})` : m[1];
|
||||
}
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
import * as api from './api.js';
|
||||
import * as poll from './poll.js';
|
||||
import {
|
||||
h, clear, icon, badge, severityBadge, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState,
|
||||
h, clear, icon, badge, severityBadge, originChip, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState,
|
||||
} from './ui.js';
|
||||
import {
|
||||
ago, when, until, duration, isFuture, STATUS_LABEL,
|
||||
ago, when, until, duration, isFuture, originOf, STATUS_LABEL,
|
||||
} from './format.js';
|
||||
import { myID, users } from './state.js';
|
||||
import { back } from './app.js';
|
||||
@@ -99,6 +99,8 @@ function render() {
|
||||
|
||||
function statusBadges() {
|
||||
const out = [];
|
||||
const origin = originOf(inc.group_labels);
|
||||
if (origin) out.push(originChip(origin));
|
||||
if (inc.severity) out.push(severityBadge(inc.severity));
|
||||
out.push(badge(STATUS_LABEL[inc.status] || inc.status, `st-${inc.status}`));
|
||||
if (inc.status !== 'resolved' && isFuture(inc.snoozed_until)) {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
// The incident queue: filter chips and a list of incident rows.
|
||||
|
||||
import * as api from './api.js';
|
||||
import { h, clear, badge, severityBadge, emptyState, spinner, fadeOnOverflow } from './ui.js';
|
||||
import { ago, until, isFuture, severityClass, labelSummary } from './format.js';
|
||||
import { h, clear, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js';
|
||||
import { ago, until, isFuture, severityClass, labelSummary, originOf, titleWithoutOrigin } from './format.js';
|
||||
import { state, myID, onTeamChange } from './state.js';
|
||||
import * as onboarding from './onboarding.js';
|
||||
import { navigate } from './app.js';
|
||||
@@ -201,15 +201,18 @@ function row(inc, index) {
|
||||
? h('span', { class: 'row-team', text: inc.team_name })
|
||||
: null;
|
||||
|
||||
const origin = originOf(inc.group_labels);
|
||||
|
||||
return h('a', {
|
||||
class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`,
|
||||
href: `/incidents/${inc.id}`,
|
||||
'aria-current': inc.id === selected ? 'true' : null,
|
||||
dataset: { index: String(index) },
|
||||
},
|
||||
h('div', { class: 'row-title', text: inc.title }),
|
||||
h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, origin) }),
|
||||
h('div', { class: 'row-age', title: inc.triggered_at, text: `Triggered ${ago(inc.triggered_at)}` }),
|
||||
h('div', { class: 'row-meta' },
|
||||
origin && originChip(origin),
|
||||
!impliedStatus && status,
|
||||
// The left-border colour alone doesn't say what it means; spell it out
|
||||
// too, same badge the incident detail page uses for severity.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// DOM helpers, the bottom sheet, confirmation and toasts.
|
||||
|
||||
import { severityClass } from './format.js';
|
||||
import { severityClass, originClass } from './format.js';
|
||||
|
||||
// h builds an element. attrs: class, text, on<event>, dataset, aria/other
|
||||
// attributes; boolean true sets an empty attribute, false/null skips it.
|
||||
@@ -49,6 +49,7 @@ export function fadeOnOverflow(el) {
|
||||
|
||||
// Stroke icons, 24×24. Built as SVG nodes so the CSP needs no inline anything.
|
||||
const ICONS = {
|
||||
server: ['rect:3,4,18,6,2', 'rect:3,14,18,6,2', 'M7 7h.01', 'M7 17h.01'],
|
||||
alertTriangle: ['M12 4l9.5 16.5h-19z', 'M12 10v4.5', 'M12 17.5h.01'],
|
||||
alertCircle: ['circle:12,12,9', 'M12 7.5v5', 'M12 16h.01'],
|
||||
infoCircle: ['circle:12,12,9', 'M12 11v5.5', 'M12 7.5h.01'],
|
||||
@@ -193,6 +194,13 @@ export function badge(text, cls = '') {
|
||||
return h('span', { class: `badge ${cls}`, text });
|
||||
}
|
||||
|
||||
// Where it came from (the cluster): a chip in that origin's colour, with a
|
||||
// server icon so it reads as a place and not as a status.
|
||||
export function originChip(value) {
|
||||
return h('span', { class: `origin-chip ${originClass(value)}`, title: `Origin: ${value}` },
|
||||
icon('server', 'icon badge-icon'), value);
|
||||
}
|
||||
|
||||
// The severity as a badge with a shape as well as a colour: a triangle for
|
||||
// critical, a circle with ! for warning, a circle with i for anything else, so
|
||||
// the coloured bar and badge are not the only thing that says it.
|
||||
|
||||
Reference in New Issue
Block a user