Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| db474ca909 | |||
| a23e88c16d | |||
| def0f68d00 | |||
| fb86a18988 |
@@ -444,6 +444,25 @@ high-water mark — the highest `severity` label any of its alerts has carried
|
||||
an incident that hit `critical` still reads as critical after the critical alert
|
||||
clears.
|
||||
|
||||
### Several clusters, one team
|
||||
|
||||
A team with one Alertmanager per Kubernetes cluster, each posting to its own
|
||||
source, needs two settings or the clusters run together.
|
||||
|
||||
1. Give every alert a `cluster` label at the source. In Prometheus that is
|
||||
`externalLabels: {cluster: prod-eu}` (kube-prometheus-stack:
|
||||
`prometheus.prometheusSpec.externalLabels`).
|
||||
2. Add `cluster` to `group_by` in `alertmanager.yml`.
|
||||
|
||||
The second one is the one that matters. Incidents are matched on the team and
|
||||
Alertmanager's `groupKey`, and the `groupKey` does not include external labels:
|
||||
without `cluster` in `group_by`, the same alert in two clusters has the same
|
||||
key and joins one incident. With it, each cluster gets its own, `cluster` is in
|
||||
the incident's `group_labels`, and the web UI shows it as a coloured chip on the
|
||||
queue, the incident and the alert list, instead of leaving it in the title.
|
||||
An alert that is not grouped by `cluster` still shows the chip on the alert
|
||||
list, which reads the label from the alert itself.
|
||||
|
||||
### An incident opens only on a new occurrence
|
||||
|
||||
An incident opens when an alert **transitions into firing**: a fingerprint that
|
||||
|
||||
@@ -15,5 +15,5 @@ type: application
|
||||
# appVersion and image.tag in values.yaml no longer agree, and that is not an oversight:
|
||||
# image.tag stays "latest", which is what a local install actually pulls. appVersion is
|
||||
# metadata and drives nothing.
|
||||
version: 0.41.3
|
||||
appVersion: "v0.41.3"
|
||||
version: 0.42.1
|
||||
appVersion: "v0.42.1"
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -409,9 +410,11 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
|
||||
strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID)
|
||||
}
|
||||
|
||||
title := pageTitle(inc)
|
||||
|
||||
switch n.kind {
|
||||
case notifyResolved:
|
||||
msg.Title = "Resolved: " + inc.Title
|
||||
msg.Title = "Resolved: " + title
|
||||
msg.Message = "All alerts stopped firing after " +
|
||||
humanDuration(time.Since(inc.TriggeredAt))
|
||||
msg.Priority = ntfyPriorityLow
|
||||
@@ -419,9 +422,9 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
|
||||
return msg
|
||||
|
||||
case notifyReminder:
|
||||
msg.Title = "Still unacknowledged: " + inc.Title
|
||||
msg.Title = "Still unacknowledged: " + title
|
||||
default:
|
||||
msg.Title = inc.Title
|
||||
msg.Title = title
|
||||
}
|
||||
|
||||
severity := derefString(inc.Severity)
|
||||
@@ -443,6 +446,48 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
|
||||
return msg
|
||||
}
|
||||
|
||||
// originLabel is the label that says where an alert came from, for a team with
|
||||
// several Kubernetes clusters behind it. It comes from Prometheus's
|
||||
// externalLabels and reaches an incident through Alertmanager's group_by; the
|
||||
// web UI reads the same label, and the README ("Several clusters, one team")
|
||||
// explains how to set it up.
|
||||
const originLabel = "cluster"
|
||||
|
||||
// pageTitle is the incident's title for a notification. A phone's lock screen
|
||||
// cuts a long title off at the end, and the incident title puts the grouping
|
||||
// labels there, so the cluster would be the first thing lost. When the incident
|
||||
// has an origin it leads instead, "[prod-eu] PodRestarting (namespace=foo)", and
|
||||
// is dropped from the parenthesis so it is not said twice. A title that is not
|
||||
// in incidentTitle's "name (k=v, k=v)" shape keeps its text and gains the prefix.
|
||||
func pageTitle(inc models.Incident) string {
|
||||
origin := inc.GroupLabels[originLabel]
|
||||
if origin == "" {
|
||||
return inc.Title
|
||||
}
|
||||
return "[" + origin + "] " + titleWithoutLabel(inc.Title, originLabel, origin)
|
||||
}
|
||||
|
||||
var titleShape = regexp.MustCompile(`(?s)^(.*?) \((.*)\)$`)
|
||||
|
||||
// titleWithoutLabel removes "key=value" from the parenthesised tail of a title
|
||||
// built by incidentTitle, and the parentheses with it if nothing else is left.
|
||||
func titleWithoutLabel(title, key, value string) string {
|
||||
m := titleShape.FindStringSubmatch(title)
|
||||
if m == nil {
|
||||
return title
|
||||
}
|
||||
var rest []string
|
||||
for _, part := range strings.Split(m[2], ", ") {
|
||||
if part != key+"="+value {
|
||||
rest = append(rest, part)
|
||||
}
|
||||
}
|
||||
if len(rest) == 0 {
|
||||
return m[1]
|
||||
}
|
||||
return m[1] + " (" + strings.Join(rest, ", ") + ")"
|
||||
}
|
||||
|
||||
// ntfy's priority scale. Max is the one that overrides the phone's quiet
|
||||
// settings, which is the whole point of paging on critical.
|
||||
const (
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -164,10 +165,84 @@ func fireCritical(t *testing.T, s *ts) {
|
||||
}, "{}:{alertname=\"DiskFull\"}")
|
||||
}
|
||||
|
||||
// fireGrouped posts one critical alert whose Alertmanager group carries the
|
||||
// given labels, the way group_by puts them on the webhook.
|
||||
func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) {
|
||||
t.Helper()
|
||||
payload := map[string]any{
|
||||
"version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels,
|
||||
"alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing",
|
||||
"2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})},
|
||||
}
|
||||
data, _ := json.Marshal(payload)
|
||||
resp, err := http.Post(s.URL+"/api/integrations/"+s.ingestKey+"/alertmanager",
|
||||
"application/json", bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("post webhook: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("webhook returned %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Delivery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// A phone cuts a long title off at the end, and the incident title keeps the
|
||||
// grouping labels there, so the cluster leads the page instead.
|
||||
func TestNotify_ClusterLeadsTheTitle(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||
|
||||
fireGrouped(t, s, map[string]string{
|
||||
"alertname": "PodRestarting", "cluster": "prod-eu", "namespace": "shop",
|
||||
}, `{}:{alertname="PodRestarting",cluster="prod-eu",namespace="shop"}`)
|
||||
s.sweepNotify(t)
|
||||
|
||||
msgs := f.messages()
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||
}
|
||||
if want := "[prod-eu] PodRestarting (namespace=shop)"; msgs[0].Title != want {
|
||||
t.Errorf("title = %q, want %q", msgs[0].Title, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the cluster is the whole grouping: no parenthesis is left behind.
|
||||
func TestNotify_ClusterAloneLeavesNoParenthesis(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{})
|
||||
|
||||
fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "cluster": "prod-eu"},
|
||||
`{}:{alertname="PodRestarting",cluster="prod-eu"}`)
|
||||
s.sweepNotify(t)
|
||||
|
||||
msgs := f.messages()
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||
}
|
||||
if want := "[prod-eu] PodRestarting"; msgs[0].Title != want {
|
||||
t.Errorf("title = %q, want %q", msgs[0].Title, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing changes for a team whose alerts have no cluster label.
|
||||
func TestNotify_NoClusterKeepsTheTitle(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{})
|
||||
|
||||
fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "namespace": "shop"},
|
||||
`{}:{alertname="PodRestarting",namespace="shop"}`)
|
||||
s.sweepNotify(t)
|
||||
|
||||
msgs := f.messages()
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||
}
|
||||
if want := "PodRestarting (namespace=shop)"; msgs[0].Title != want {
|
||||
t.Errorf("title = %q, want %q", msgs[0].Title, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||
|
||||
|
||||
@@ -488,7 +488,15 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
|
||||
}
|
||||
.badge::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: currentColor; }
|
||||
.badge.plain::before { display: none; }
|
||||
.badge .badge-icon { width: 12px; height: 12px; stroke-width: 2.4; }
|
||||
.badge .badge-icon, .origin-chip .badge-icon { width: 12px; height: 12px; stroke-width: 2.4; }
|
||||
/* The cluster an incident or alert came from. Same shape as a badge, coloured
|
||||
from the rcN palette (see originClass in format.js), never the severity one. */
|
||||
.origin-chip {
|
||||
display: inline-flex; align-items: center; gap: 5px; max-width: 100%;
|
||||
padding: 1px 8px; border-radius: 999px;
|
||||
font-size: 12px; font-weight: 700; letter-spacing: 0.01em; white-space: nowrap;
|
||||
overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.badge.st-triggered, .badge.st-firing { background: var(--crit-soft); color: var(--crit); }
|
||||
.badge.st-acknowledged { background: var(--warn-soft); color: var(--warn); }
|
||||
.badge.st-snoozed { background: var(--snooze-soft); color: var(--snooze); }
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// incident it belongs to, which is where anything can be done about it.
|
||||
|
||||
import * as api from './api.js';
|
||||
import { h, clear, badge, severityBadge, emptyState, spinner } from './ui.js';
|
||||
import { age, labelSummary } from './format.js';
|
||||
import { h, clear, badge, severityBadge, originChip, emptyState, spinner } from './ui.js';
|
||||
import { age, labelSummary, originOf, ORIGIN_LABEL } from './format.js';
|
||||
|
||||
const FILTERS = [
|
||||
{ id: 'firing', label: 'Firing', query: { status: 'firing' } },
|
||||
@@ -77,7 +77,8 @@ function row(a) {
|
||||
const summary = (a.annotations && a.annotations.summary) || '';
|
||||
const sev = a.labels && a.labels.severity;
|
||||
const labels = labelSummary(Object.fromEntries(
|
||||
Object.entries(a.labels || {}).filter(([k]) => k !== 'severity')));
|
||||
Object.entries(a.labels || {}).filter(([k]) => k !== 'severity' && k !== ORIGIN_LABEL)));
|
||||
const origin = originOf(a.labels);
|
||||
const linked = a.incident_id != null;
|
||||
return h(linked ? 'a' : 'div', {
|
||||
class: `row st-${a.status} ${linked ? '' : 'no-link'}`,
|
||||
@@ -86,6 +87,7 @@ function row(a) {
|
||||
h('div', { class: 'row-title', text: a.name }),
|
||||
h('div', { class: 'row-age', title: a.starts_at, text: age(a.status === 'firing' ? a.starts_at : a.received_at) }),
|
||||
h('div', { class: 'row-meta' },
|
||||
origin && originChip(origin),
|
||||
badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`),
|
||||
sev && severityBadge(sev),
|
||||
summary && h('span', { text: summary }),
|
||||
|
||||
@@ -117,3 +117,36 @@ export function labelSummary(labels, skip = 'alertname') {
|
||||
export function initial(name) {
|
||||
return (name || '?').trim().charAt(0) || '?';
|
||||
}
|
||||
|
||||
// Where an incident or alert came from, for a team with several Kubernetes
|
||||
// clusters (or other origins) behind it: the value of one label, `cluster` by
|
||||
// convention. It comes from Prometheus's externalLabels, so it is on every
|
||||
// alert; an incident carries it only when it is in Alertmanager's group_by,
|
||||
// which is also what keeps two clusters' identical alerts from merging into one
|
||||
// incident (see the README, "Several clusters, one team").
|
||||
export const ORIGIN_LABEL = 'cluster';
|
||||
|
||||
export function originOf(labels) {
|
||||
const v = labels && labels[ORIGIN_LABEL];
|
||||
return v ? String(v) : null;
|
||||
}
|
||||
|
||||
// A stable colour for an origin, from the six-colour rcN palette app.css has,
|
||||
// so "prod-eu" is the same colour on every row without anything being stored.
|
||||
export function originClass(value) {
|
||||
let n = 0;
|
||||
for (const ch of value) n = (n * 31 + ch.charCodeAt(0)) >>> 0;
|
||||
return `rc${(n % 6) + 1}`;
|
||||
}
|
||||
|
||||
// The title without its `cluster=...` part, for a row that shows the origin as
|
||||
// a chip: the server puts every grouping label in the title, so the cluster
|
||||
// would otherwise appear twice. Anything that does not look like the server's
|
||||
// "name (k=v, k=v)" shape is returned unchanged.
|
||||
export function titleWithoutOrigin(title, value) {
|
||||
if (!value) return title;
|
||||
const m = title.match(/^(.*?) \((.*)\)$/s);
|
||||
if (!m) return title;
|
||||
const rest = m[2].split(', ').filter((p) => p !== `${ORIGIN_LABEL}=${value}`);
|
||||
return rest.length ? `${m[1]} (${rest.join(', ')})` : m[1];
|
||||
}
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
import * as api from './api.js';
|
||||
import * as poll from './poll.js';
|
||||
import {
|
||||
h, clear, icon, badge, severityBadge, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState,
|
||||
h, clear, icon, badge, severityBadge, originChip, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState,
|
||||
} from './ui.js';
|
||||
import {
|
||||
ago, when, until, duration, isFuture, STATUS_LABEL,
|
||||
ago, when, until, duration, isFuture, originOf, STATUS_LABEL,
|
||||
} from './format.js';
|
||||
import { myID, users } from './state.js';
|
||||
import { back } from './app.js';
|
||||
@@ -99,6 +99,8 @@ function render() {
|
||||
|
||||
function statusBadges() {
|
||||
const out = [];
|
||||
const origin = originOf(inc.group_labels);
|
||||
if (origin) out.push(originChip(origin));
|
||||
if (inc.severity) out.push(severityBadge(inc.severity));
|
||||
out.push(badge(STATUS_LABEL[inc.status] || inc.status, `st-${inc.status}`));
|
||||
if (inc.status !== 'resolved' && isFuture(inc.snoozed_until)) {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
// The incident queue: filter chips and a list of incident rows.
|
||||
|
||||
import * as api from './api.js';
|
||||
import { h, clear, badge, severityBadge, emptyState, spinner, fadeOnOverflow } from './ui.js';
|
||||
import { ago, until, isFuture, severityClass, labelSummary } from './format.js';
|
||||
import { h, clear, badge, severityBadge, originChip, emptyState, spinner, fadeOnOverflow } from './ui.js';
|
||||
import { ago, until, isFuture, severityClass, labelSummary, originOf, titleWithoutOrigin } from './format.js';
|
||||
import { state, myID, onTeamChange } from './state.js';
|
||||
import * as onboarding from './onboarding.js';
|
||||
import { navigate } from './app.js';
|
||||
@@ -201,15 +201,18 @@ function row(inc, index) {
|
||||
? h('span', { class: 'row-team', text: inc.team_name })
|
||||
: null;
|
||||
|
||||
const origin = originOf(inc.group_labels);
|
||||
|
||||
return h('a', {
|
||||
class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`,
|
||||
href: `/incidents/${inc.id}`,
|
||||
'aria-current': inc.id === selected ? 'true' : null,
|
||||
dataset: { index: String(index) },
|
||||
},
|
||||
h('div', { class: 'row-title', text: inc.title }),
|
||||
h('div', { class: 'row-title', text: titleWithoutOrigin(inc.title, origin) }),
|
||||
h('div', { class: 'row-age', title: inc.triggered_at, text: `Triggered ${ago(inc.triggered_at)}` }),
|
||||
h('div', { class: 'row-meta' },
|
||||
origin && originChip(origin),
|
||||
!impliedStatus && status,
|
||||
// The left-border colour alone doesn't say what it means; spell it out
|
||||
// too, same badge the incident detail page uses for severity.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// DOM helpers, the bottom sheet, confirmation and toasts.
|
||||
|
||||
import { severityClass } from './format.js';
|
||||
import { severityClass, originClass } from './format.js';
|
||||
|
||||
// h builds an element. attrs: class, text, on<event>, dataset, aria/other
|
||||
// attributes; boolean true sets an empty attribute, false/null skips it.
|
||||
@@ -49,6 +49,7 @@ export function fadeOnOverflow(el) {
|
||||
|
||||
// Stroke icons, 24×24. Built as SVG nodes so the CSP needs no inline anything.
|
||||
const ICONS = {
|
||||
server: ['rect:3,4,18,6,2', 'rect:3,14,18,6,2', 'M7 7h.01', 'M7 17h.01'],
|
||||
alertTriangle: ['M12 4l9.5 16.5h-19z', 'M12 10v4.5', 'M12 17.5h.01'],
|
||||
alertCircle: ['circle:12,12,9', 'M12 7.5v5', 'M12 16h.01'],
|
||||
infoCircle: ['circle:12,12,9', 'M12 11v5.5', 'M12 7.5h.01'],
|
||||
@@ -193,6 +194,13 @@ export function badge(text, cls = '') {
|
||||
return h('span', { class: `badge ${cls}`, text });
|
||||
}
|
||||
|
||||
// Where it came from (the cluster): a chip in that origin's colour, with a
|
||||
// server icon so it reads as a place and not as a status.
|
||||
export function originChip(value) {
|
||||
return h('span', { class: `origin-chip ${originClass(value)}`, title: `Origin: ${value}` },
|
||||
icon('server', 'icon badge-icon'), value);
|
||||
}
|
||||
|
||||
// The severity as a badge with a shape as well as a colour: a triangle for
|
||||
// critical, a circle with ! for warning, a circle with i for anything else, so
|
||||
// the coloured bar and badge are not the only thing that says it.
|
||||
|
||||
Reference in New Issue
Block a user