Files
terdut-server/internal/api/incidents.go
T
Niklas Ye a4fbd60441
CI / chart (pull_request) Successful in 1s
CI / security (pull_request) Successful in 13s
CI / test (pull_request) Successful in 1m49s
Scope everything to a team, and route alerts by integration key
The core of #4, and what #1 is for: terdut stops being one shared space.
A team owns its incidents, alerts, schedule and integrations; a user sees
exactly the teams they are in. Everything that existed moves into one
Default team and every existing user becomes an owner of it, so the
upgrade is a no-op for the people using it.

Ingestion is the load-bearing half. An alert arrives on a team's
integration key, and the key is both the credential and the routing: it
says that the sender may post, and which team the alerts belong to. That
also closes the unauthenticated webhook -- the old path stays for one
release, deprecated and routed to the oldest team, so an upgrade does not
stop delivering while somebody edits the Alertmanager config.

Scoping is enforced in as few places as possible, because the failure
mode is silent. serveAs loads the caller's memberships once; list queries
carry `team_id = ANY(...)`; and every incident route goes through
incidentIDParam, which now parses the id AND checks the team in the same
call, so a new handler cannot remember the first half and forget the
second. Anything in another team is 404, never 403: whether an incident
exists is that team's business.

Two bugs this found, both of which would have been silent:

  * upsertAlerts decided "is this a new occurrence" by looking up the
    fingerprint alone. Across teams that made team B's first alert look
    like a re-send of team A's, so it opened no incident at all. The
    lookups are keyed on (team_id, fingerprint) now, as the index is.

  * Every uniqueness rule was written for one tenant. Two teams watching
    two clusters legitimately see the same fingerprint, the same
    groupKey, and want somebody on call on the same day; all three
    constraints move to include team_id.

Roles inside a team are separate from the system administrator flag: an
owner configures the team, a member works its incidents, and an admin is
NOT implicitly in every team -- administration is about accounts, not
about reading other people's incidents. An admin can still repair a team
whose owner has left, which is why requireTeamOwner lets them through.

A shift can only be given to somebody in the team. Paging a person who
cannot open the incident is worse than paging nobody.

The UI is updated only as far as keeping it working: it loads the
viewer's teams with the session and uses the first one, since nobody has
a second yet. "On call now" shows every team the viewer is in, named only
when there is more than one, so the common case reads exactly as before.
The team switcher, badges and per-team settings pages are the next step.

Breaking for API clients: the schedule endpoints moved under the team,
and /api/schedule/current returns an array rather than an object or a
404. terdut-tui will need a version for that.

Per-team dead-man configuration is deliberately not here. A heartbeat's
incident already opens in the team whose key received it, which is the
part that matters for isolation; moving the matchers out of env into
per-team rows is a change to how deadman.go is configured rather than to
who sees what.

Claude-Session: https://claude.ai/code/session_01RHPj4ggeFdEjKKfm4SHbD7
2026-09-20 13:36:24 +02:00

573 lines
17 KiB
Go

package api
import (
"database/sql"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"git.ryuvia.com/niklas/terdut-server/internal/models"
"github.com/go-chi/chi/v5"
)
func handleListIncidents(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
where := []string{}
args := &sqlArgs{}
// The combined queue: every team the caller belongs to, in one list. A
// caller in no team sees an empty queue rather than everybody's.
where = append(where, "i.team_id = ANY("+args.add(callerTeamIDs(r.Context()))+")")
if team := q.Get("team_id"); team != "" {
if n, err := strconv.ParseInt(team, 10, 64); err == nil {
where = append(where, "i.team_id = "+args.add(n))
}
}
// Without an explicit status the queue shows open work, which is what an
// on-call person opens the tool to see.
if status := q.Get("status"); status != "" {
where = append(where, "i.status = "+args.add(status))
} else {
where = append(where, "i.resolved_at IS NULL")
}
if q.Get("archived") == "true" {
where = append(where, "i.archived_at IS NOT NULL")
} else {
where = append(where, "i.archived_at IS NULL")
}
// A snooze expires by simply falling into the past; nothing sweeps it.
if q.Get("snoozed") == "true" {
where = append(where, "i.snoozed_until > "+args.add(time.Now().Unix()))
} else {
where = append(where, "(i.snoozed_until IS NULL OR i.snoozed_until <= "+args.add(time.Now().Unix())+")")
}
if severity := q.Get("severity"); severity != "" {
where = append(where, "i.severity = "+args.add(severity))
}
if assignee := q.Get("assigned_to"); assignee != "" {
if n, err := strconv.ParseInt(assignee, 10, 64); err == nil {
where = append(where, "i.assigned_to = "+args.add(n))
}
}
if from := q.Get("from"); from != "" {
if t, err := time.Parse("2006-01-02", from); err == nil {
where = append(where, "i.triggered_at >= "+args.add(t.UTC().Unix()))
}
}
if to := q.Get("to"); to != "" {
if t, err := time.Parse("2006-01-02", to); err == nil {
where = append(where, "i.triggered_at < "+args.add(t.UTC().AddDate(0, 0, 1).Unix()))
}
}
limit := 50
if l := q.Get("limit"); l != "" {
if n, err := strconv.Atoi(l); err == nil && n > 0 && n <= 500 {
limit = n
}
}
order := "i.triggered_at DESC"
if q.Get("sort") == "severity" {
order = severityRankSQL("i.severity") + " DESC, i.triggered_at DESC"
}
rows, err := db.QueryContext(r.Context(),
fmt.Sprintf("%s WHERE %s ORDER BY %s LIMIT %s",
incidentSelectFrom, strings.Join(where, " AND "), order, args.add(limit)),
args.all()...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
incidents := []models.Incident{}
for rows.Next() {
i, err := scanIncident(rows)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
incidents = append(incidents, i)
}
respond(w, http.StatusOK, incidents)
}
}
func handleGetIncident(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
inc, err := fetchIncident(r.Context(), db, id)
if err == sql.ErrNoRows {
respond(w, http.StatusNotFound, errResp("incident not found"))
return
}
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if inc.Alerts, err = incidentAlerts(r, db, id); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, inc)
}
}
func handleIncidentAlerts(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
if !incidentExists(w, r, db, id) {
return
}
alerts, err := incidentAlerts(r, db, id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, alerts)
}
}
func handleIncidentTimeline(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
if !incidentExists(w, r, db, id) {
return
}
rows, err := db.QueryContext(r.Context(), `
SELECT e.id, e.incident_id, e.type, e.user_id, u.username,
e.alert_id, e.detail, e.created_at
FROM incident_events e
LEFT JOIN users u ON u.id = e.user_id
WHERE e.incident_id = $1
ORDER BY e.created_at ASC, e.id ASC`, id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
events := []models.IncidentEvent{}
for rows.Next() {
var e models.IncidentEvent
var ts int64
if err := rows.Scan(&e.ID, &e.IncidentID, &e.Type, &e.UserID, &e.Username,
&e.AlertID, &e.Detail, &ts); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
e.CreatedAt = time.Unix(ts, 0).UTC()
events = append(events, e)
}
respond(w, http.StatusOK, events)
}
}
func handleIncidentAcknowledge(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
acked, err := acknowledgeIncident(r.Context(), db, id, user.ID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if !acked {
if !incidentExists(w, r, db, id) {
return
}
respond(w, http.StatusConflict, errResp("incident is resolved"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentUnacknowledge(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
`UPDATE incidents SET status = 'triggered', acknowledged_by = NULL, acknowledged_at = NULL
WHERE id = $1 AND resolved_at IS NULL`, id) {
return
}
if err := logEvent(r.Context(), db, id, evUnacknowledged, &user.ID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
// handleIncidentResolve closes an incident by hand. This is terminal: a later
// occurrence opens a new incident rather than reopening this one, which is what
// stops a resolved incident from reappearing on the next repeat_interval
// re-send of an alert that never stopped firing. Use snooze for "not now".
func handleIncidentResolve(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
`UPDATE incidents SET status = 'resolved', resolved_at = $1, resolution_source = $2
WHERE id = $3 AND resolved_at IS NULL`,
time.Now().Unix(), incidentResolutionManual, id) {
return
}
if err := logEvent(r.Context(), db, id, evResolved, &user.ID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentAssign(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
var req struct {
UserID int64 `json:"user_id"`
}
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
if req.UserID == 0 {
respond(w, http.StatusBadRequest, errResp("user_id is required"))
return
}
var exists int
if err := db.QueryRowContext(r.Context(),
"SELECT 1 FROM users WHERE id = $1", req.UserID).Scan(&exists); err != nil {
respond(w, http.StatusNotFound, errResp("user not found"))
return
}
if !updateOpenIncident(w, r, db, id,
"UPDATE incidents SET assigned_to = $1 WHERE id = $2 AND resolved_at IS NULL",
req.UserID, id) {
return
}
// On an "assigned" event user_id is the assignee, not the actor.
if err := logEvent(r.Context(), db, id, evAssigned, &req.UserID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respondIncident(w, r, db, id)
}
}
// handleIncidentSnooze hides an incident from the default queue without closing
// it. Accepts either an absolute {"until": RFC3339} or a relative
// {"duration": "2h"}.
func handleIncidentSnooze(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
var req struct {
Until string `json:"until"`
Duration string `json:"duration"`
}
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
var until time.Time
switch {
case req.Until != "":
t, err := time.Parse(time.RFC3339, req.Until)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid until (expected RFC3339)"))
return
}
until = t
case req.Duration != "":
d, err := time.ParseDuration(req.Duration)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid duration"))
return
}
until = time.Now().Add(d)
default:
respond(w, http.StatusBadRequest, errResp("until or duration is required"))
return
}
if !until.After(time.Now()) {
respond(w, http.StatusBadRequest, errResp("snooze must end in the future"))
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
"UPDATE incidents SET snoozed_until = $1 WHERE id = $2 AND resolved_at IS NULL",
until.Unix(), id) {
return
}
detail := until.UTC().Format(time.RFC3339)
if err := logEvent(r.Context(), db, id, evSnoozed, &user.ID, nil, &detail); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentUnsnooze(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
user, _ := userFromContext(r.Context())
if !updateOpenIncident(w, r, db, id,
"UPDATE incidents SET snoozed_until = NULL WHERE id = $1 AND resolved_at IS NULL", id) {
return
}
if err := logEvent(r.Context(), db, id, evUnsnoozed, &user.ID, nil, nil); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
func handleIncidentArchive(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
res, err := db.ExecContext(r.Context(),
"UPDATE incidents SET archived_at = "+nowEpoch+" WHERE id = $1", id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("incident not found"))
return
}
respondIncident(w, r, db, id)
}
}
func handleIncidentUnarchive(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
res, err := db.ExecContext(r.Context(),
"UPDATE incidents SET archived_at = NULL WHERE id = $1", id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("incident not found"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
// handleCreateNote adds a note to the timeline. Notes are ordinary events, so a
// single query renders the whole story of an incident in order.
func handleCreateNote(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
var req struct {
Content string `json:"content"`
}
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
if req.Content == "" {
respond(w, http.StatusBadRequest, errResp("content is required"))
return
}
if !incidentExists(w, r, db, id) {
return
}
user, _ := userFromContext(r.Context())
now := time.Now()
var eventID int64
err := db.QueryRowContext(r.Context(), `
INSERT INTO incident_events (incident_id, type, user_id, detail, created_at)
VALUES ($1, $2, $3, $4, $5)
RETURNING id`, id, evNote, user.ID, req.Content, now.Unix()).Scan(&eventID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusCreated, models.IncidentEvent{
ID: eventID,
IncidentID: id,
Type: evNote,
UserID: &user.ID,
Username: &user.Username,
Detail: &req.Content,
CreatedAt: now.UTC().Truncate(time.Second),
})
}
}
// handleDeleteNote removes one of your own notes. Only notes are deletable — the
// rest of the timeline is what actually happened, and is not editable.
func handleDeleteNote(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, ok := incidentIDParam(w, r, db)
if !ok {
return
}
eventID, err := strconv.ParseInt(chi.URLParam(r, "eventID"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid note id"))
return
}
user, _ := userFromContext(r.Context())
res, err := db.ExecContext(r.Context(), `
DELETE FROM incident_events
WHERE id = $1 AND incident_id = $2 AND type = $3 AND user_id = $4`,
eventID, id, evNote, user.ID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("note not found"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
// ---------------------------------------------------------------------------
// Shared handler plumbing
// ---------------------------------------------------------------------------
// incidentIDParam reads {id} from the path AND confirms the incident belongs to
// a team the caller is in. Both in one place, deliberately: every incident route
// goes through here, so scoping cannot be forgotten by writing a new handler
// that only remembers the first half.
//
// An incident in somebody else's team is reported as not found rather than
// forbidden, because "there is an incident 41 you may not see" is itself
// something only that team should know.
func incidentIDParam(w http.ResponseWriter, r *http.Request, db *sql.DB) (int64, bool) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid incident id"))
return 0, false
}
if !incidentExists(w, r, db, id) {
return 0, false
}
return id, true
}
// incidentExists reports whether the incident is one the caller may see at all.
func incidentExists(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) bool {
var exists int
if err := db.QueryRowContext(r.Context(),
"SELECT 1 FROM incidents WHERE id = $1 AND team_id = ANY($2)",
id, callerTeamIDs(r.Context())).Scan(&exists); err != nil {
respond(w, http.StatusNotFound, errResp("incident not found"))
return false
}
return true
}
// updateOpenIncident runs a mutation that is only valid while an incident is
// open. The query must be constrained to `resolved_at IS NULL`, so no rows means
// either the incident does not exist or it is already closed — two different
// answers the caller should not have to distinguish itself.
func updateOpenIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64, query string, args ...any) bool {
res, err := db.ExecContext(r.Context(), query, args...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return false
}
if n, _ := res.RowsAffected(); n > 0 {
return true
}
if !incidentExists(w, r, db, id) {
return false
}
respond(w, http.StatusConflict, errResp("incident is resolved"))
return false
}
func respondIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) {
inc, err := fetchIncident(r.Context(), db, id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, inc)
}
// incidentAlerts loads the alerts under an incident, newest signal first.
func incidentAlerts(r *http.Request, db *sql.DB, id int64) ([]models.Alert, error) {
rows, err := db.QueryContext(r.Context(), alertSelectFrom+`
JOIN incident_alerts m ON m.alert_id = a.id
WHERE m.incident_id = $1
ORDER BY a.received_at DESC`, id)
if err != nil {
return nil, err
}
defer rows.Close()
alerts := []models.Alert{}
for rows.Next() {
a, err := scanAlert(rows)
if err != nil {
return nil, err
}
alerts = append(alerts, a)
}
return alerts, rows.Err()
}