-- Escalation: page somebody else when the first person does not answer. -- -- This is the gap the whole multi-tenancy line of work was opened to close. -- Until now an unacknowledged incident re-paged the same topic every -- notify_repeat forever, which is a louder version of the same silence: if the -- person on call is asleep, has no signal, or has left, nothing else happens. -- -- Shape: one policy per team, an ordered list of levels, each level with a -- timeout and a set of targets. When a level's timeout passes and the incident -- is still triggered, the next level is paged. When the last level passes, the -- chain repeats repeat_count times, and then the team's fallback topic is paged -- once as the end of the line. -- -- A team WITHOUT a policy keeps exactly today's behaviour: page the assignee, -- then remind on the same topic. Escalation is opt-in per team, and the two -- never both run for one incident -- see enqueueReminders. CREATE TABLE escalation_policies ( -- One per team for now, hence the team as the key rather than an id with a -- unique index: routing different alerts to different chains needs the -- alert to carry something to route ON, which is a separate question. team_id BIGINT PRIMARY KEY REFERENCES teams(id) ON DELETE CASCADE, -- How many extra times to run the whole chain after it has been walked -- once. 0 means walk it once and stop at the fallback. repeat_count BIGINT NOT NULL DEFAULT 0 CHECK (repeat_count >= 0 AND repeat_count <= 10), -- Where the last page goes when every level has been tried. Per team now: -- TERDUT_NTFY_FALLBACK_TOPIC was one topic for the whole install, which in -- a multi-team server pages the wrong people. Empty means the chain simply -- ends. fallback_topic TEXT NOT NULL DEFAULT '', updated_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM now()))::bigint ); CREATE TABLE escalation_levels ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, team_id BIGINT NOT NULL REFERENCES escalation_policies(team_id) ON DELETE CASCADE, -- 1-based, dense. The API rewrites the whole ladder on every edit rather -- than patching one rung, so there is no way to leave a gap. position BIGINT NOT NULL, -- How long this level has to produce an acknowledgement before the next one -- is paged. Seconds, like every other duration in this schema. timeout_seconds BIGINT NOT NULL CHECK (timeout_seconds > 0), UNIQUE (team_id, position) ); -- Who a level pages. Either a named person, or whoever the team's rota says is -- on call today -- which is the target that keeps working when the rota -- changes and nobody remembers to edit the policy. CREATE TABLE escalation_targets ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, level_id BIGINT NOT NULL REFERENCES escalation_levels(id) ON DELETE CASCADE, kind TEXT NOT NULL CHECK (kind IN ('user', 'oncall')), -- Set for kind='user', NULL for kind='oncall'. user_id BIGINT REFERENCES users(id) ON DELETE CASCADE, CHECK ((kind = 'user' AND user_id IS NOT NULL) OR (kind = 'oncall' AND user_id IS NULL)) ); CREATE INDEX escalation_targets_level_idx ON escalation_targets(level_id); -- --------------------------------------------------------------------------- -- Where an incident is in its chain. -- -- On the incident rather than in a side table: it is read on every notifier -- tick alongside the incident's status, and one row per incident is exactly -- what the state is. -- --------------------------------------------------------------------------- -- 0 means no level has been paged yet, which is the state of every incident -- that existed before escalation and of every incident in a team with no -- policy. 1 is the first level. ALTER TABLE incidents ADD COLUMN escalation_level BIGINT NOT NULL DEFAULT 0; -- When the current level was entered, and therefore what its timeout is -- measured from. NULL while escalation_level is 0. ALTER TABLE incidents ADD COLUMN escalation_level_at BIGINT; -- How many times the chain has been walked in full. Compared against the -- policy's repeat_count. ALTER TABLE incidents ADD COLUMN escalation_round BIGINT NOT NULL DEFAULT 0; -- The notifier's escalation query: incidents still waiting, oldest level first. CREATE INDEX incidents_escalation_idx ON incidents(escalation_level_at) WHERE resolved_at IS NULL AND status = 'triggered'; -- 'escalated' joins the outbox kinds: a page that went out because nobody -- answered the last one, which is worth telling apart from the first page and -- from a reminder when reading the timeline or debugging a delivery. ALTER TABLE notifications DROP CONSTRAINT notifications_kind_check; ALTER TABLE notifications ADD CONSTRAINT notifications_kind_check CHECK (kind IN ('triggered', 'reminder', 'resolved', 'escalated'));