-- Splits the single alerts row into two objects, the way an incident management -- tool needs them: alerts stay the machine-owned signal record that Alertmanager -- writes, and incidents become the human work item people acknowledge, assign, -- snooze, discuss and resolve. -- -- Correlation uses Alertmanager's own groupKey, so incidents follow the group_by -- routing tree the operator already tuned rather than a second grouping scheme -- invented here. CREATE TABLE incidents ( id INTEGER PRIMARY KEY AUTOINCREMENT, group_key TEXT NOT NULL, -- Alertmanager groupKey, opaque title TEXT NOT NULL, -- rendered from group_labels group_labels TEXT NOT NULL DEFAULT '{}', -- JSON status TEXT NOT NULL CHECK(status IN ('triggered', 'acknowledged', 'resolved')), severity TEXT, -- highest `severity` label across firing members triggered_at INTEGER NOT NULL, acknowledged_by INTEGER REFERENCES users(id) ON DELETE SET NULL, acknowledged_at INTEGER, assigned_to INTEGER REFERENCES users(id) ON DELETE SET NULL, snoozed_until INTEGER, resolved_at INTEGER, resolution_source TEXT, -- 'alerts' | 'manual' archived_at INTEGER ); -- Load-bearing: at most one OPEN incident per group_key. This is what makes -- "resolved incident + a new alert occurrence = a new incident" work, and it is -- the constraint the webhook's find-or-open lookup relies on. CREATE UNIQUE INDEX incidents_open_group_key_idx ON incidents(group_key) WHERE resolved_at IS NULL; CREATE INDEX incidents_status_idx ON incidents(status); CREATE INDEX incidents_triggered_at_idx ON incidents(triggered_at DESC); CREATE INDEX incidents_archived_at_idx ON incidents(archived_at); -- Membership is historical, not a pointer on alerts: one alert row (one -- fingerprint) resolves and re-fires over time and belongs to a different -- incident each occurrence. CREATE TABLE incident_alerts ( incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE, alert_id INTEGER NOT NULL REFERENCES alerts(id) ON DELETE CASCADE, added_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), PRIMARY KEY (incident_id, alert_id) ); CREATE INDEX incident_alerts_alert_id_idx ON incident_alerts(alert_id); -- The timeline. Append-only, and the only history this server keeps: alert rows -- are mutated in place, so without this there is no record that anything -- happened. Notes are events too, so one query renders the whole story. CREATE TABLE incident_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE, -- triggered | alert_added | alert_resolved | acknowledged | unacknowledged -- | assigned | snoozed | unsnoozed | resolved | note type TEXT NOT NULL, user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, -- NULL = the server acted alert_id INTEGER REFERENCES alerts(id) ON DELETE SET NULL, detail TEXT, created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) ); CREATE INDEX incident_events_incident_idx ON incident_events(incident_id, created_at); -- --------------------------------------------------------------------------- -- Backfill -- -- Every pre-existing alert gets its own incident, archived ones included, so no -- acknowledgement and no comment is orphaned. There is no historical groupKey to -- correlate on, hence one incident per fingerprint under a 'backfill:' prefix -- that can never collide with a real Alertmanager groupKey. -- --------------------------------------------------------------------------- INSERT INTO incidents (group_key, title, group_labels, status, severity, triggered_at, acknowledged_by, acknowledged_at, assigned_to, resolved_at, resolution_source, archived_at) SELECT 'backfill:' || a.fingerprint, a.name, json_object('alertname', a.name), CASE WHEN a.status = 'resolved' THEN 'resolved' WHEN a.acknowledged_by IS NOT NULL THEN 'acknowledged' ELSE 'triggered' END, json_extract(a.labels, '$.severity'), a.starts_at, a.acknowledged_by, a.acknowledged_at, a.acknowledged_by, CASE WHEN a.status = 'resolved' THEN COALESCE(a.ends_at, a.received_at) END, CASE WHEN a.status = 'resolved' THEN 'alerts' END, a.archived_at FROM alerts a; INSERT INTO incident_alerts (incident_id, alert_id, added_at) SELECT i.id, a.id, a.starts_at FROM alerts a JOIN incidents i ON i.group_key = 'backfill:' || a.fingerprint; INSERT INTO incident_events (incident_id, type, alert_id, created_at) SELECT i.id, 'triggered', ia.alert_id, i.triggered_at FROM incidents i JOIN incident_alerts ia ON ia.incident_id = i.id; INSERT INTO incident_events (incident_id, type, user_id, created_at) SELECT i.id, 'acknowledged', i.acknowledged_by, i.acknowledged_at FROM incidents i WHERE i.acknowledged_at IS NOT NULL; INSERT INTO incident_events (incident_id, type, created_at) SELECT i.id, 'resolved', i.resolved_at FROM incidents i WHERE i.resolved_at IS NOT NULL; INSERT INTO incident_events (incident_id, type, user_id, alert_id, detail, created_at) SELECT ia.incident_id, 'note', c.user_id, c.alert_id, c.content, c.created_at FROM alert_comments c JOIN incident_alerts ia ON ia.alert_id = c.alert_id; -- --------------------------------------------------------------------------- -- Workflow state now lives on incidents only. Leaving these behind would keep -- the bug they caused: the webhook upsert owns the alerts row and never cleared -- the acknowledgement, so a re-fire days later still read as acknowledged. -- --------------------------------------------------------------------------- DROP TABLE alert_comments; ALTER TABLE alerts DROP COLUMN acknowledged_by; ALTER TABLE alerts DROP COLUMN acknowledged_at;