niklas/terdut-server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 35 Wiki Activity

List dead man's switches with their status on Team -> Switches

Browse Source 
The page was a bare form: it did not say which switches existed or
whether they were alive. It now lists them, each with a Healthy, Dead or
Dormant badge, when its heartbeat was last heard and when it last opened
an incident (linked while that incident is open). A matcher that several
clusters satisfy is broken down per cluster, since a live cluster must
not hide a dead one. The form moved into a "New switch" sheet, and each
row has a Remove with a confirm.

That needed a switch to be a thing, so switches are rows now
(migration 009) with their own name, matcher, timeout and severity,
instead of one string with one team-wide timeout in deadman_configs.
Existing configuration is split into one row per matcher; a team whose
timeout was zero simply has none. The sweeper and the status endpoint
share one death rule (deadmanAlert.dead), so the page cannot disagree
with the pager. Incident group keys are unchanged, so incidents that
are open across the upgrade keep working.

The environment defaults (TERDUT_DEADMAN_*) are seeded into teams once
per install, recorded in settings, so a team that deletes its last
switch does not get it back on the next restart. Installs that already
had per-team rows are marked as seeded by the migration.

Removing a switch stops the watching but leaves an incident it already
opened open until someone resolves it.

API: GET/PUT /api/teams/{id}/deadman are replaced by
GET/POST /deadman/switches and DELETE /deadman/switches/{switchID}.
terdut-tui does not call them, so nothing to mirror there.
This commit is contained in:
Niklas Ye
2026-09-25 23:40:55 +02:00
parent 3ee8583f6f
commit f3918b863c
11 changed files with 868 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/web/static/js/api.js
+5 -2
View File
@@ -136,8 +136,11 @@ export const createIntegration = (id, name) =>
export const deleteIntegration = (id, integrationID) =>
call('DELETE', `/teams/${id}/integrations/${integrationID}`);
export const deadman = (id) => call('GET', `/teams/${id}/deadman`);
export const setDeadman = (id, body) => call('PUT', `/teams/${id}/deadman`, { body });
export const deadmanSwitches = (id) => call('GET', `/teams/${id}/deadman/switches`);
export const createDeadmanSwitch = (id, body) =>
call('POST', `/teams/${id}/deadman/switches`, { body });
export const deleteDeadmanSwitch = (id, switchID) =>
call('DELETE', `/teams/${id}/deadman/switches/${switchID}`);
export const escalation = (id) => call('GET', `/teams/${id}/escalation`);
export const setEscalation = (id, body) => call('PUT', `/teams/${id}/escalation`, { body });
internal/web/static/js/team.js
+141 -40
View File
@@ -18,9 +18,9 @@
// than no form, but it is not the thing enforcing anything.
import * as api from './api.js';
import { h, clear, spinner, confirm, icon, openSheet, closeSheet, menuCard } from './ui.js';
import { h, clear, spinner, confirm, icon, openSheet, closeSheet, menuCard, badge, labelChip } from './ui.js';
import { state, currentTeam, users as allUsers, myID } from './state.js';
import { isoDate, addDays, mondayOf, initial } from './format.js';
import { isoDate, addDays, mondayOf, initial, ago, when, duration } from './format.js';
const view = () => document.getElementById('view-team');
@@ -111,14 +111,14 @@ async function load(id) {
return { members, escalation };
}
if (tab === 'sources') return { integrations: await api.integrations(id) };
if (tab === 'deadman') return { deadman: await api.deadman(id) };
if (tab === 'deadman') return { deadman: await api.deadmanSwitches(id) };
const grid = gridDays();
const [members, integrations, escalation, deadman, schedule] = await Promise.all([
api.teamMembers(id),
api.integrations(id),
api.escalation(id),
api.deadman(id),
api.deadmanSwitches(id),
api.schedule(id, isoDate(grid.start), isoDate(addDays(grid.start, grid.count - 1))),
]);
return { members, integrations, escalation, deadman, schedule };
@@ -203,8 +203,8 @@ function overview() {
const levels = (data.escalation?.levels || []).length;
const keys = (data.integrations || []).length;
const unused = (data.integrations || []).filter((i) => !i.last_used_at).length;
const switches = (data.deadman?.matchers || '')
.split(';').map((x) => x.trim()).filter(Boolean).length;
const switches = (data.deadman || []).length;
const dead = (data.deadman || []).filter((s) => s.status === 'dead').length;
return h('div', { class: 'overview-menu' },
menuCard('/team/rota', 'Rota', null,
@@ -220,7 +220,9 @@ function overview() {
? (unused ? `${unused} of them never used.` : 'All in use.')
: 'No key yet, so nothing can reach this team.'),
menuCard('/team/deadman', 'Dead man’s switches', switches || null,
switches ? 'Alerts whose absence opens an incident.' : 'Nothing watched.'),
switches
? (dead ? `${dead} of them silent.` : 'All quiet, as they should be.')
: 'Nothing watched.'),
);
}
@@ -650,47 +652,146 @@ function newIntegrationForm() {
// --- dead man's switches ---------------------------------------------------
const SWITCH_STATUS = {
healthy: { label: 'Healthy', hint: 'Heard from within its timeout.' },
dead: { label: 'Dead', hint: 'Silent for longer than its timeout.' },
dormant: { label: 'Dormant', hint: 'Nothing has matched yet, so there is nothing to lose.' },
};
function switchBadge(status) {
const s = SWITCH_STATUS[status] || SWITCH_STATUS.dormant;
const el = badge(s.label, `st-${status}`);
el.title = s.hint;
return el;
}
const timeCell = (iso) => iso
? h('span', { title: when(iso), text: ago(iso) })
: h('span', { class: 'muted', text: 'never' });
// When it last opened an incident. An incident that is still open is a link,
// because that is the thing somebody looking at a red row wants next.
const triggeredCell = (iso, incidentID) => {
if (!iso) return h('span', { class: 'muted', text: 'never' });
return incidentID
? h('a', { href: `/incidents/${incidentID}`, title: when(iso) }, `#${incidentID} · ${ago(iso)}`)
: h('span', { title: when(iso), text: ago(iso) });
};
function switchRows(sw) {
const main = h('tr', {},
h('td', {}, switchBadge(sw.status)),
h('td', {},
h('strong', { text: sw.name }),
sw.name !== sw.matcher && h('div', { class: 'muted small' }, h('code', { text: sw.matcher }))),
h('td', { class: 'muted small' }, timeCell(sw.last_heartbeat_at)),
h('td', { class: 'muted small' }, triggeredCell(sw.last_triggered_at, sw.open_incident_id)),
h('td', { class: 'muted small', text: duration(sw.timeout_seconds * 1000) }),
h('td', {}, isOwner() && h('button', {
class: 'btn-sm danger', type: 'button', text: 'Remove',
onclick: async () => {
if (!(await confirm({
title: `Remove ${sw.name}?`,
text: 'It stops being watched. An incident it already opened stays open until it is resolved.',
confirmLabel: 'Remove',
danger: true,
}))) return;
act(() => api.deleteDeadmanSwitch(teamID, sw.id));
},
})),
);
// One heartbeat is the switch's own times; several are worth telling apart,
// since a live cluster must not hide a dead one.
const sources = sw.sources.length > 1
? sw.sources.map((src) => h('tr', { class: 'source-row' },
h('td', {}, switchBadge(src.status)),
h('td', { class: 'source-labels' },
...Object.entries(src.labels || {})
.filter(([k]) => k !== 'alertname')
.map(([k, v]) => labelChip(k, v)),
!Object.keys(src.labels || {}).some((k) => k !== 'alertname')
&& h('code', { class: 'small', text: src.fingerprint })),
h('td', { class: 'muted small' }, timeCell(src.last_heartbeat_at)),
h('td', { class: 'muted small' }, triggeredCell(src.last_triggered_at, src.incident_id)),
h('td'), h('td')))
: [];
return [main, ...sources];
}
function deadmanCard() {
const d = data.deadman || {};
const matchers = h('input', {
type: 'text', value: d.matchers || '', placeholder: 'alertname=Watchdog',
class: 'wide',
});
const timeout = h('input', {
type: 'number', min: '0', class: 'setting-value',
value: String(Math.round((d.timeout_seconds || 0) / 60)),
});
const severity = h('select', {},
...['critical', 'error', 'warning', 'info'].map((s) =>
h('option', { value: s, text: s, selected: (d.severity || 'critical') === s })));
const form = h('form', { class: 'stacked-form' },
h('label', {}, 'Heartbeat alerts ', matchers),
h('label', {}, 'Declare dead after ', timeout, ' minutes of silence'),
h('label', {}, 'Open the incident at severity ', severity),
h('button', { class: 'btn', type: 'submit', text: 'Save switches' }));
form.addEventListener('submit', (e) => {
e.preventDefault();
act(() => api.setDeadman(teamID, {
matchers: matchers.value.trim(),
timeout_seconds: Number(timeout.value) * 60,
severity: severity.value,
}));
});
const switches = data.deadman || [];
return h('div', { class: 'card' },
h('h2', { text: 'Dead man’s switches' }),
h('div', { class: 'card-head' },
h('h2', { text: 'Dead man’s switches' }),
isOwner() && h('button', {
class: 'btn', type: 'button', text: 'New switch', onclick: openNewSwitch,
})),
h('p', { class: 'muted small' },
'Alerts whose ABSENCE is the signal. Receiving one opens nothing; going ',
'quiet for longer than the timeout opens an incident. ',
h('code', { text: 'alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat' }),
' — semicolons separate switches, commas separate conditions, and every ',
'switch must name an alertname. Leave empty to watch nothing.'),
isOwner() ? form : h('p', { class: 'muted', text: d.matchers || 'Nothing watched.' }),
'quiet for longer than the switch’s timeout opens an incident.'),
switches.length
? h('div', { class: 'table-scroll' },
h('table', { class: 'admin-table switch-table' },
h('thead', {}, h('tr', {},
h('th', { text: 'Status' }), h('th', { text: 'Switch' }),
h('th', { text: 'Last heartbeat' }), h('th', { text: 'Last triggered' }),
h('th', { text: 'Silent after' }), h('th'))),
h('tbody', {}, switches.flatMap(switchRows))))
: h('p', { class: 'muted', text: 'Nothing watched.' }),
);
}
// The form lives in the sheet, not on the page: most visits are to look at the
// list, and a form that is always open is the page this replaced.
function openNewSwitch() {
const name = h('input', { type: 'text', placeholder: 'Prod Watchdog', autofocus: true });
const matcher = h('input', {
type: 'text', placeholder: 'alertname=Watchdog,cluster=prod', class: 'wide', required: true,
});
const timeout = h('input', {
type: 'number', min: '1', value: '15', class: 'setting-value', required: true,
});
const severity = h('select', {},
...['critical', 'error', 'warning', 'info'].map((s) => h('option', { value: s, text: s })));
const problem = h('p', { class: 'load-error', hidden: true });
const form = h('form', { class: 'stacked-form' },
h('label', {}, 'Name (optional) ', name),
h('label', {}, 'Heartbeat alert ', matcher),
h('p', { class: 'muted small' },
'Conditions are ', h('code', { text: 'label=value' }), ' separated by commas, and one ',
'must be ', h('code', { text: 'alertname' }), '. Every distinct label set that ',
'matches is watched on its own.'),
h('label', {}, 'Declare dead after ', timeout, ' minutes of silence'),
h('label', {}, 'Open the incident at severity ', severity),
problem,
h('div', { class: 'sheet-actions' },
h('button', { class: 'btn', type: 'button', text: 'Cancel', onclick: () => closeSheet(false) }),
h('button', { class: 'btn btn-primary', type: 'submit', text: 'Add switch' })));
form.addEventListener('submit', async (e) => {
e.preventDefault();
try {
await api.createDeadmanSwitch(teamID, {
name: name.value.trim(),
matcher: matcher.value.trim(),
timeout_seconds: Math.round(Number(timeout.value) * 60),
severity: severity.value,
});
} catch (err) {
problem.textContent = err.message;
problem.hidden = false;
return;
}
closeSheet(true);
refresh();
});
openSheet(() => [h('h2', { class: 'sheet-title', text: 'New switch' }), form]);
}
// --- members ---------------------------------------------------------------
function membersCard() {