Files
terdut-server/internal/api/incidents.go
T
Niklas Ye 60ebb75cd2 Show notes from similar earlier incidents
Each incident gets a signature: the alert name plus the group labels that
say what is broken, minus the ones that only say where it ran (instance,
pod, container, ...). GET /api/incidents/{id}/similar returns resolved
incidents in the same team with the same signature that have notes.

Notes can be marked as the resolution note, "what fixed it", either with a
resolution field on resolve or pinned on a note. Those lead the similar
list, show on the incident page as "Seen before", and the triggered
notification carries the latest one.

Claude-Session: https://claude.ai/code/session_01MMados3BD1oSjevHxbmVqU
2026-09-25 15:42:25 +02:00

600 lines
18 KiB
Go

package api
import (
"database/sql"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"git.ryuvia.com/niklas/terdut-server/internal/models"
"github.com/go-chi/chi/v5"
)
func handleListIncidents(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
where := []string{}
args := &sqlArgs{}
// The combined queue: every team the caller belongs to, in one list. A
// caller in no team sees an empty queue rather than everybody's.
where = append(where, "i.team_id = ANY("+args.add(callerTeamIDs(r.Context()))+")")
if team := q.Get("team_id"); team != "" {
if n, err := strconv.ParseInt(team, 10, 64); err == nil {
where = append(where, "i.team_id = "+args.add(n))
}
}
// Without an explicit status the queue shows open work, which is what an
// on-call person opens the tool to see.
if status := q.Get("status"); status != "" {
where = append(where, "i.status = "+args.add(status))
} else {
where = append(where, "i.resolved_at IS NULL")
}
if q.Get("archived") == "true" {
where = append(where, "i.archived_at IS NOT NULL")
} else {
where = append(where, "i.archived_at IS NULL")
}
// A snooze expires by simply falling into the past; nothing sweeps it.
if q.Get("snoozed") == "true" {
where = append(where, "i.snoozed_until > "+args.add(time.Now().Unix()))
} else {
where = append(where, "(i.snoozed_until IS NULL OR i.snoozed_until <= "+args.add(time.Now().Unix())+")")
}
if severity := q.Get("severity"); severity != "" {
where = append(where, "i.severity = "+args.add(severity))
}
if assignee := q.Get("assigned_to"); assignee != "" {
if n, err := strconv.ParseInt(assignee, 10, 64); err == nil {
where = append(where, "i.assigned_to = "+args.add(n))
}
}
if from := q.Get("from"); from != "" {
if t, err := time.Parse("2006-01-02", from); err == nil {
where = append(where, "i.triggered_at >= "+args.add(t.UTC().Unix()))
}
}
if to := q.Get("to"); to != "" {
if t, err := time.Parse("2006-01-02", to); err == nil {
where = append(where, "i.triggered_at < "+args.add(t.UTC().AddDate(0, 0, 1).Unix()))
}
}
limit := 50
if l := q.Get("limit"); l != "" {
if n, err := strconv.Atoi(l); err == nil && n > 0 && n <= 500 {
limit = n
}
}
order := "i.triggered_at DESC"
if q.Get("sort") == "severity" {
order = severityRankSQL("i.severity") + " DESC, i.triggered_at DESC"
}
rows, err := db.QueryContext(r.Context(),
fmt.Sprintf("%s WHERE %s ORDER BY %s LIMIT %s",
incidentSelectFrom, strings.Join(where, " AND "), order, args.add(limit)),
args.all()...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
incidents := []models.Incident{}
for rows.Next() {
i, err := scanIncident(rows)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
incidents = append(incidents, i)
}
respond(w, http.StatusOK, incidents)
}
}
func handleGetIncident(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
inc, err := fetchIncident(r.Context(), db, id)
if err == sql.ErrNoRows {
respond(w, http.StatusNotFound, errResp("incident not found"))
return
}
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if inc.Alerts, err = incidentAlerts(r, db, id); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, inc)
}
}
func handleIncidentAlerts(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
if !incidentExists(w, r, db, id) {
return
}
alerts, err := incidentAlerts(r, db, id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, alerts)
}
}
func handleIncidentTimeline(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
if !incidentExists(w, r, db, id) {
return
}
rows, err := db.QueryContext(r.Context(), `
SELECT e.id, e.incident_id, e.type, e.user_id, u.username,
e.alert_id, e.detail, e.created_at
FROM incident_events e
LEFT JOIN users u ON u.id = e.user_id
WHERE e.incident_id = $1
ORDER BY e.created_at ASC, e.id ASC`, id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
events := []models.IncidentEvent{}
for rows.Next() {
var e models.IncidentEvent
var ts int64
if err := rows.Scan(&e.ID, &e.IncidentID, &e.Type, &e.UserID, &e.Username,
&e.AlertID, &e.Detail, &ts); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
e.CreatedAt = time.Unix(ts, 0).UTC()
events = append(events, e)
}
respond(w, http.StatusOK, events)
}
}
func handleIncidentAcknowledge(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
acked, err := acknowledgeIncident(r.Context(), db, id, user.ID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if !acked {
if !incidentExists(w, r, db, id) {
return
}
respond(w, http.StatusConflict, errResp("incident is resolved"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentUnacknowledge(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
`UPDATE incidents SET status = 'triggered', acknowledged_by = NULL, acknowledged_at = NULL
WHERE id = $1 AND resolved_at IS NULL`, id) {
return
}
if err := logEvent(r.Context(), db, id, evUnacknowledged, &user.ID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
// handleIncidentResolve closes an incident by hand. This is terminal: a later
// occurrence opens a new incident rather than reopening this one, which is what
// stops a resolved incident from reappearing on the next repeat_interval
// re-send of an alert that never stopped firing. Use snooze for "not now".
func handleIncidentResolve(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
// The body is optional: clients that predate resolution notes send none.
var req struct {
Resolution string `json:"resolution"`
}
if err := decodeJSON(r, &req); err != nil && err != io.EOF {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
req.Resolution = strings.TrimSpace(req.Resolution)
if !updateOpenIncident(w, r, db, id,
`UPDATE incidents SET status = 'resolved', resolved_at = $1, resolution_source = $2
WHERE id = $3 AND resolved_at IS NULL`,
time.Now().Unix(), incidentResolutionManual, id) {
return
}
// A person closing an incident is the clearest possible "I have this".
if err := stopEscalation(r.Context(), db, id); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if err := logEvent(r.Context(), db, id, evResolved, &user.ID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if req.Resolution != "" {
if err := logEvent(r.Context(), db, id, evResolutionNote, &user.ID, nil, &req.Resolution); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
}
respondIncident(w, r, db, id)
}
}
func handleIncidentAssign(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
var req struct {
UserID int64 `json:"user_id"`
}
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
if req.UserID == 0 {
respond(w, http.StatusBadRequest, errResp("user_id is required"))
return
}
var exists int
if err := db.QueryRowContext(r.Context(),
"SELECT 1 FROM users WHERE id = $1", req.UserID).Scan(&exists); err != nil {
respond(w, http.StatusNotFound, errResp("user not found"))
return
}
if !updateOpenIncident(w, r, db, id,
"UPDATE incidents SET assigned_to = $1 WHERE id = $2 AND resolved_at IS NULL",
req.UserID, id) {
return
}
// On an "assigned" event user_id is the assignee, not the actor.
if err := logEvent(r.Context(), db, id, evAssigned, &req.UserID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respondIncident(w, r, db, id)
}
}
// handleIncidentSnooze hides an incident from the default queue without closing
// it. Accepts either an absolute {"until": RFC3339} or a relative
// {"duration": "2h"}.
func handleIncidentSnooze(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
var req struct {
Until string `json:"until"`
Duration string `json:"duration"`
}
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
var until time.Time
switch {
case req.Until != "":
t, err := time.Parse(time.RFC3339, req.Until)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid until (expected RFC3339)"))
return
}
until = t
case req.Duration != "":
d, err := time.ParseDuration(req.Duration)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid duration"))
return
}
until = time.Now().Add(d)
default:
respond(w, http.StatusBadRequest, errResp("until or duration is required"))
return
}
if !until.After(time.Now()) {
respond(w, http.StatusBadRequest, errResp("snooze must end in the future"))
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
"UPDATE incidents SET snoozed_until = $1 WHERE id = $2 AND resolved_at IS NULL",
until.Unix(), id) {
return
}
detail := until.UTC().Format(time.RFC3339)
if err := logEvent(r.Context(), db, id, evSnoozed, &user.ID, nil, &detail); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentUnsnooze(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
"UPDATE incidents SET snoozed_until = NULL WHERE id = $1 AND resolved_at IS NULL", id) {
return
}
if err := logEvent(r.Context(), db, id, evUnsnoozed, &user.ID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
func handleIncidentArchive(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
res, err := db.ExecContext(r.Context(),
"UPDATE incidents SET archived_at = "+nowEpoch+" WHERE id = $1", id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("incident not found"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentUnarchive(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
res, err := db.ExecContext(r.Context(),
"UPDATE incidents SET archived_at = NULL WHERE id = $1", id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("incident not found"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
// handleCreateNote adds a note to the timeline. Notes are ordinary events, so a
// single query renders the whole story of an incident in order.
func handleCreateNote(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
var req struct {
Content string `json:"content"`
// Pinned files the note as the resolution note: what fixed it.
Pinned bool `json:"pinned"`
}
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
noteType := evNote
if req.Pinned {
noteType = evResolutionNote
}
if req.Content == "" {
respond(w, http.StatusBadRequest, errResp("content is required"))
return
}
if !incidentExists(w, r, db, id) {
return
}
user, _ := userFromContext(r.Context())
now := time.Now()
var eventID int64
err := db.QueryRowContext(r.Context(), `
INSERT INTO incident_events (incident_id, type, user_id, detail, created_at)
VALUES ($1, $2, $3, $4, $5)
RETURNING id`, id, noteType, user.ID, req.Content, now.Unix()).Scan(&eventID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusCreated, models.IncidentEvent{
ID: eventID,
IncidentID: id,
Type: noteType,
UserID: &user.ID,
Username: &user.Username,
Detail: &req.Content,
CreatedAt: now.UTC().Truncate(time.Second),
})
}
}
// handleDeleteNote removes one of your own notes. Only notes are deletable — the
// rest of the timeline is what actually happened, and is not editable.
func handleDeleteNote(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
eventID, err := strconv.ParseInt(chi.URLParam(r, "eventID"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid note id"))
return
}
user, _ := userFromContext(r.Context())
res, err := db.ExecContext(r.Context(), `
DELETE FROM incident_events
WHERE id = $1 AND incident_id = $2 AND type IN ($3, $4) AND user_id = $5`,
eventID, id, evNote, evResolutionNote, user.ID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("note not found"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
// ---------------------------------------------------------------------------
// Shared handler plumbing
// ---------------------------------------------------------------------------
// incidentIDParam reads {id} from the path AND confirms the incident belongs to
// a team the caller is in. Both in one place, deliberately: every incident route
// goes through here, so scoping cannot be forgotten by writing a new handler
// that only remembers the first half.
//
// An incident in somebody else's team is reported as not found rather than
// forbidden, because "there is an incident 41 you may not see" is itself
// something only that team should know.
func incidentIDParam(w http.ResponseWriter, r *http.Request, db *sql.DB) (int64, bool) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid incident id"))
return 0, false
}
if !incidentExists(w, r, db, id) {
return 0, false
}
return id, true
}
// incidentExists reports whether the incident is one the caller may see at all.
func incidentExists(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) bool {
var exists int
if err := db.QueryRowContext(r.Context(),
"SELECT 1 FROM incidents WHERE id = $1 AND team_id = ANY($2)",
id, callerTeamIDs(r.Context())).Scan(&exists); err != nil {
respond(w, http.StatusNotFound, errResp("incident not found"))
return false
}
return true
}
// updateOpenIncident runs a mutation that is only valid while an incident is
// open. The query must be constrained to `resolved_at IS NULL`, so no rows means
// either the incident does not exist or it is already closed — two different
// answers the caller should not have to distinguish itself.
func updateOpenIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64, query string, args ...any) bool {
res, err := db.ExecContext(r.Context(), query, args...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return false
}
if n, _ := res.RowsAffected(); n > 0 {
return true
}
if !incidentExists(w, r, db, id) {
return false
}
respond(w, http.StatusConflict, errResp("incident is resolved"))
return false
}
func respondIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) {
inc, err := fetchIncident(r.Context(), db, id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, inc)
}
// incidentAlerts loads the alerts under an incident, newest signal first.
func incidentAlerts(r *http.Request, db *sql.DB, id int64) ([]models.Alert, error) {
rows, err := db.QueryContext(r.Context(), alertSelectFrom+`
JOIN incident_alerts m ON m.alert_id = a.id
WHERE m.incident_id = $1
ORDER BY a.received_at DESC`, id)
if err != nil {
return nil, err
}
defer rows.Close()
alerts := []models.Alert{}
for rows.Next() {
a, err := scanAlert(rows)
if err != nil {
return nil, err
}
alerts = append(alerts, a)
}
return alerts, rows.Err()
}