Niklas Ye
60ebb75cd2
Each incident gets a signature: the alert name plus the group labels that
say what is broken, minus the ones that only say where it ran (instance,
pod, container, ...). GET /api/incidents/{id}/similar returns resolved
incidents in the same team with the same signature that have notes.
Notes can be marked as the resolution note, "what fixed it", either with a
resolution field on resolve or pinned on a note. Those lead the similar
list, show on the incident page as "Seen before", and the triggered
notification carries the latest one.
Claude-Session: https://claude.ai/code/session_01MMados3BD1oSjevHxbmVqU
355 lines
12 KiB
Go
355 lines
12 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
|
)
|
|
|
|
// Values for incidents.resolution_source, recording who closed the incident:
|
|
// every member alert stopped firing, or a person decided it was done.
|
|
const (
|
|
incidentResolutionAlerts = "alerts"
|
|
incidentResolutionManual = "manual"
|
|
|
|
// incidentResolutionRecovered closes a dead man's switch incident whose
|
|
// heartbeat started arriving again. It cannot be "alerts": these incidents
|
|
// have no member alerts for the cascade to work from.
|
|
incidentResolutionRecovered = "recovered"
|
|
)
|
|
|
|
// Incident timeline event types. Stored as free text so adding one later is not
|
|
// a migration, but these are the ones the server writes.
|
|
const (
|
|
evTriggered = "triggered"
|
|
evAlertAdded = "alert_added"
|
|
evAlertResolved = "alert_resolved"
|
|
evAcknowledged = "acknowledged"
|
|
evUnacknowledged = "unacknowledged"
|
|
evAssigned = "assigned"
|
|
evSnoozed = "snoozed"
|
|
evUnsnoozed = "unsnoozed"
|
|
evResolved = "resolved"
|
|
evNote = "note"
|
|
// evResolutionNote is the note worth finding again: what fixed it. The
|
|
// similar-incidents lookup and the page lead with these; plain notes are
|
|
// the working chatter and stay one click away.
|
|
evResolutionNote = "resolution_note"
|
|
evDeadmanSilent = "deadman_silent"
|
|
)
|
|
|
|
// severityLabel is the Alertmanager label an incident's severity is derived from.
|
|
const severityLabel = "severity"
|
|
|
|
// querier is satisfied by both *sql.DB and *sql.Tx, so the helpers below work
|
|
// inside the webhook's transaction and standalone from handlers and the sweeper.
|
|
type querier interface {
|
|
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
|
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
|
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
|
}
|
|
|
|
const incidentSelectFrom = `
|
|
SELECT i.id, i.team_id, t.name, i.group_key, i.title, i.group_labels, i.status, i.severity,
|
|
i.escalation_level,
|
|
-- When this level runs out. Computed here rather than in Go because
|
|
-- the timeout lives beside the level in the policy, and one join is
|
|
-- cheaper than a second query per incident in a list.
|
|
(SELECT i.escalation_level_at + el.timeout_seconds
|
|
FROM escalation_levels el
|
|
WHERE el.team_id = i.team_id AND el.position = i.escalation_level),
|
|
i.triggered_at,
|
|
i.acknowledged_by, i.acknowledged_at, ack.username,
|
|
i.assigned_to, asg.username, i.snoozed_until,
|
|
i.resolved_at, i.resolution_source, i.archived_at
|
|
FROM incidents i
|
|
JOIN teams t ON t.id = i.team_id
|
|
LEFT JOIN users ack ON ack.id = i.acknowledged_by
|
|
LEFT JOIN users asg ON asg.id = i.assigned_to`
|
|
|
|
func scanIncident(s scanner) (models.Incident, error) {
|
|
var i models.Incident
|
|
var groupLabelsJSON string
|
|
var triggeredAt int64
|
|
var ackAt, snoozedUntil, resolvedAt, archivedAt, escalationDue *int64
|
|
|
|
if err := s.Scan(
|
|
&i.ID, &i.TeamID, &i.TeamName, &i.GroupKey, &i.Title, &groupLabelsJSON, &i.Status, &i.Severity,
|
|
&i.EscalationLevel, &escalationDue,
|
|
&triggeredAt,
|
|
&i.AcknowledgedByID, &ackAt, &i.AcknowledgedByUser,
|
|
&i.AssignedToID, &i.AssignedToUser, &snoozedUntil,
|
|
&resolvedAt, &i.ResolutionSource, &archivedAt,
|
|
); err != nil {
|
|
return i, err
|
|
}
|
|
|
|
json.Unmarshal([]byte(groupLabelsJSON), &i.GroupLabels) //nolint:errcheck
|
|
i.TriggeredAt = time.Unix(triggeredAt, 0).UTC()
|
|
i.AcknowledgedAt = unixPtr(ackAt)
|
|
i.SnoozedUntil = unixPtr(snoozedUntil)
|
|
i.ResolvedAt = unixPtr(resolvedAt)
|
|
i.ArchivedAt = unixPtr(archivedAt)
|
|
i.EscalationDueAt = unixPtr(escalationDue)
|
|
return i, nil
|
|
}
|
|
|
|
// unixPtr converts a nullable Unix-second column to a nullable UTC time.
|
|
func unixPtr(sec *int64) *time.Time {
|
|
if sec == nil {
|
|
return nil
|
|
}
|
|
t := time.Unix(*sec, 0).UTC()
|
|
return &t
|
|
}
|
|
|
|
func fetchIncident(ctx context.Context, q querier, id int64) (models.Incident, error) {
|
|
return scanIncident(q.QueryRowContext(ctx, incidentSelectFrom+" WHERE i.id = $1", id))
|
|
}
|
|
|
|
// logEvent appends one entry to an incident's timeline. A nil userID means the
|
|
// server acted rather than a person.
|
|
func logEvent(ctx context.Context, q querier, incidentID int64, evType string, userID, alertID *int64, detail *string) error {
|
|
_, err := q.ExecContext(ctx, `
|
|
INSERT INTO incident_events (incident_id, type, user_id, alert_id, detail, created_at)
|
|
VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
incidentID, evType, userID, alertID, detail, time.Now().Unix())
|
|
return err
|
|
}
|
|
|
|
// todayUTC is the schedule's day key. The schedule's smallest unit is one UTC day.
|
|
func todayUTC() string {
|
|
return time.Now().UTC().Format("2006-01-02")
|
|
}
|
|
|
|
// currentOnCall returns a team's on-call user for today, or nil when nobody is
|
|
// scheduled. A missing schedule entry is not an error — incidents just open
|
|
// unassigned.
|
|
//
|
|
// Per team: each team keeps its own rota, so two teams can have two different
|
|
// people on call on the same day, which was the point of scoping the schedule.
|
|
func currentOnCall(ctx context.Context, q querier, teamID int64) (*int64, error) {
|
|
var userID int64
|
|
err := q.QueryRowContext(ctx,
|
|
"SELECT user_id FROM schedule_entries WHERE team_id = $1 AND date = $2",
|
|
teamID, todayUTC()).Scan(&userID)
|
|
if err == sql.ErrNoRows {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &userID, nil
|
|
}
|
|
|
|
// severityRank orders the conventional Alertmanager severity label values.
|
|
// Anything unrecognised sorts below all of them rather than being dropped.
|
|
func severityRank(s string) int {
|
|
switch strings.ToLower(s) {
|
|
case "critical":
|
|
return 4
|
|
case "error":
|
|
return 3
|
|
case "warning":
|
|
return 2
|
|
case "info":
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
// refreshSeverity raises an incident's severity to the highest `severity` label
|
|
// seen across its alerts.
|
|
//
|
|
// It is a high-water mark, never lowered: an incident that hit critical was a
|
|
// critical incident, even after the critical alert clears and a warning is all
|
|
// that is left firing. Downgrading a live incident would also quietly demote it
|
|
// in the queue while the work is still open.
|
|
func refreshSeverity(ctx context.Context, q querier, incidentID int64) error {
|
|
rows, err := q.QueryContext(ctx, `
|
|
SELECT a.labels ->> $1
|
|
FROM incident_alerts ia
|
|
JOIN alerts a ON a.id = ia.alert_id
|
|
WHERE ia.incident_id = $2`, severityLabel, incidentID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
best := ""
|
|
for rows.Next() {
|
|
var sev *string
|
|
if err := rows.Scan(&sev); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
if sev != nil && severityRank(*sev) > severityRank(best) {
|
|
best = *sev
|
|
}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
rows.Close()
|
|
|
|
if best == "" {
|
|
return nil
|
|
}
|
|
// The comparison lives in SQL so an unrelated concurrent update cannot be
|
|
// clobbered by a stale read.
|
|
_, err = q.ExecContext(ctx, `
|
|
UPDATE incidents SET severity = $1
|
|
WHERE id = $2
|
|
AND (severity IS NULL OR `+severityRankSQL("severity")+` < $3)`,
|
|
best, incidentID, severityRank(best))
|
|
return err
|
|
}
|
|
|
|
// severityRankSQL mirrors severityRank for use inside a statement. SQL cannot
|
|
// order these strings meaningfully on its own.
|
|
func severityRankSQL(col string) string {
|
|
return `CASE lower(COALESCE(` + col + `, ''))
|
|
WHEN 'critical' THEN 4
|
|
WHEN 'error' THEN 3
|
|
WHEN 'warning' THEN 2
|
|
WHEN 'info' THEN 1
|
|
ELSE 0 END`
|
|
}
|
|
|
|
// resolveIfSettled closes an incident once every alert under it has stopped
|
|
// firing — PagerDuty's cascade, and the only automatic route out of the open
|
|
// state. Reports whether it actually resolved anything.
|
|
func resolveIfSettled(ctx context.Context, q querier, incidentID int64) (bool, error) {
|
|
res, err := q.ExecContext(ctx, `
|
|
UPDATE incidents
|
|
SET status = 'resolved',
|
|
resolved_at = $1,
|
|
resolution_source = $2
|
|
WHERE id = $3
|
|
AND resolved_at IS NULL
|
|
-- An incident with no members yet is mid-creation, not settled.
|
|
AND EXISTS (SELECT 1 FROM incident_alerts ia WHERE ia.incident_id = incidents.id)
|
|
AND NOT EXISTS (SELECT 1
|
|
FROM incident_alerts ia
|
|
JOIN alerts a ON a.id = ia.alert_id
|
|
WHERE ia.incident_id = incidents.id
|
|
AND a.status = 'firing')`,
|
|
time.Now().Unix(), incidentResolutionAlerts, incidentID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
if n == 0 {
|
|
return false, nil
|
|
}
|
|
if err := stopEscalation(ctx, q, incidentID); err != nil {
|
|
return false, err
|
|
}
|
|
if err := logEvent(ctx, q, incidentID, evResolved, nil, nil, nil); err != nil {
|
|
return false, err
|
|
}
|
|
// The all-clear goes only to whoever was paged in the first place, which
|
|
// enqueueResolved works out from the incident's own notification history.
|
|
// Manual resolution sends nothing: the person who closed it already knows.
|
|
return true, enqueueResolved(ctx, q, incidentID)
|
|
}
|
|
|
|
// acknowledgeIncident records that userID has picked an incident up, and reports
|
|
// whether it changed anything — an already-resolved incident is left alone.
|
|
// Shared by the authenticated handler and the Acknowledge button in a push
|
|
// notification, so both write the same state and the same timeline entry.
|
|
func acknowledgeIncident(ctx context.Context, q querier, incidentID, userID int64) (bool, error) {
|
|
res, err := q.ExecContext(ctx, `
|
|
UPDATE incidents
|
|
SET status = 'acknowledged', acknowledged_by = $1, acknowledged_at = $2
|
|
WHERE id = $3 AND resolved_at IS NULL`,
|
|
userID, time.Now().Unix(), incidentID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return false, nil
|
|
}
|
|
// Somebody has it: stop waking anybody else.
|
|
if err := stopEscalation(ctx, q, incidentID); err != nil {
|
|
return false, err
|
|
}
|
|
return true, logEvent(ctx, q, incidentID, evAcknowledged, &userID, nil, nil)
|
|
}
|
|
|
|
// openIncidentForAlert returns the open incident an alert currently belongs to,
|
|
// or 0 when it has none. Used when an alert resolves or expires so the event
|
|
// lands on the right timeline.
|
|
func openIncidentForAlert(ctx context.Context, q querier, alertID int64) (int64, error) {
|
|
var id int64
|
|
err := q.QueryRowContext(ctx, `
|
|
SELECT i.id
|
|
FROM incident_alerts ia
|
|
JOIN incidents i ON i.id = ia.incident_id
|
|
WHERE ia.alert_id = $1 AND i.resolved_at IS NULL`, alertID).Scan(&id)
|
|
if err == sql.ErrNoRows {
|
|
return 0, nil
|
|
}
|
|
return id, err
|
|
}
|
|
|
|
// volatileLabels say where a problem ran this time, not what the problem is, so
|
|
// they stay out of the signature. Migration 008's backfill lists the same set.
|
|
var volatileLabels = map[string]bool{
|
|
"instance": true, "pod": true, "pod_name": true, "pod_ip": true,
|
|
"container": true, "container_name": true, "endpoint": true,
|
|
}
|
|
|
|
// incidentSignature identifies "the same problem" across incidents: the alert
|
|
// name plus the stable group labels, sorted. Incidents in one team with equal
|
|
// signatures are what the similar-incidents lookup returns. title stands in for
|
|
// the name when the payload carried no alertname (groupless and dead man's
|
|
// switch incidents).
|
|
func incidentSignature(groupLabels map[string]string, title string) string {
|
|
name := groupLabels["alertname"]
|
|
if name == "" {
|
|
name = title
|
|
}
|
|
rest := make([]string, 0, len(groupLabels))
|
|
for k, v := range groupLabels {
|
|
if k == "alertname" || volatileLabels[k] {
|
|
continue
|
|
}
|
|
rest = append(rest, k+"="+v)
|
|
}
|
|
sort.Strings(rest)
|
|
return name + "|" + strings.Join(rest, ",")
|
|
}
|
|
|
|
// incidentTitle renders a human-readable title from Alertmanager's groupLabels,
|
|
// leading with the alert name and appending whatever else the operator grouped
|
|
// by. Falls back to the alert's own name when the payload carried no groupLabels.
|
|
func incidentTitle(groupLabels map[string]string, fallback string) string {
|
|
name := groupLabels["alertname"]
|
|
if name == "" {
|
|
name = fallback
|
|
}
|
|
if name == "" {
|
|
name = "Incident"
|
|
}
|
|
|
|
rest := make([]string, 0, len(groupLabels))
|
|
for k, v := range groupLabels {
|
|
if k == "alertname" {
|
|
continue
|
|
}
|
|
rest = append(rest, k+"="+v)
|
|
}
|
|
if len(rest) == 0 {
|
|
return name
|
|
}
|
|
sort.Strings(rest)
|
|
return name + " (" + strings.Join(rest, ", ") + ")"
|
|
}
|