bc285799d1
An incident opened, got assigned to whoever held today's schedule entry,
and then sat there silently until somebody thought to look. The schedule
and the incident model were both built; nothing reached the person
holding the pager.
Notifications go out through ntfy, over plain HTTP with no new
dependencies. Delivery is an outbox rather than an inline call: the pool
is limited to a single connection, so a POST made while holding the
webhook's transaction would stall every other request behind it. The
webhook inserts a row and a notifier goroutine sends it within a tick,
retrying with exponential backoff.
Only opening an incident has to resolve a topic from scratch. Reminders
and all-clears reuse whatever that first notification chose, which keeps
configuration out of resolveIfSettled and gives the right rule for free:
you only hear that something resolved if you were told it started.
Each push carries an Acknowledge button, because the useful thing to do
at 3am is stop the pager without unlocking anything. It POSTs to an
unauthenticated /api/notify/ack/{token} — a notification body lives on
the ntfy server and in the device cache, so a real API key must never
appear in one. The token is minted per delivery, scoped to one incident
and one action, and expires in a day.
Reminders repeat until the incident stops being untouched. The stop
conditions are the states that already mean somebody has it: acknowledged,
snoozed, resolved, archived. Snooze is the mute button, so there is no
separate reminder cap.
Notifications sent to the fallback topic carry no Acknowledge button. The
topic is shared, and a button on it would let any subscriber acknowledge
as somebody else.
558 lines
16 KiB
Go
558 lines
16 KiB
Go
package api
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/yeniklas/terdut-server/internal/models"
|
|
)
|
|
|
|
func handleListIncidents(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
q := r.URL.Query()
|
|
|
|
where := []string{}
|
|
args := []any{}
|
|
|
|
// Without an explicit status the queue shows open work, which is what an
|
|
// on-call person opens the tool to see.
|
|
if status := q.Get("status"); status != "" {
|
|
where = append(where, "i.status = ?")
|
|
args = append(args, status)
|
|
} else {
|
|
where = append(where, "i.resolved_at IS NULL")
|
|
}
|
|
|
|
if q.Get("archived") == "true" {
|
|
where = append(where, "i.archived_at IS NOT NULL")
|
|
} else {
|
|
where = append(where, "i.archived_at IS NULL")
|
|
}
|
|
|
|
// A snooze expires by simply falling into the past; nothing sweeps it.
|
|
if q.Get("snoozed") == "true" {
|
|
where = append(where, "i.snoozed_until > ?")
|
|
args = append(args, time.Now().Unix())
|
|
} else {
|
|
where = append(where, "(i.snoozed_until IS NULL OR i.snoozed_until <= ?)")
|
|
args = append(args, time.Now().Unix())
|
|
}
|
|
|
|
if severity := q.Get("severity"); severity != "" {
|
|
where = append(where, "i.severity = ?")
|
|
args = append(args, severity)
|
|
}
|
|
if assignee := q.Get("assigned_to"); assignee != "" {
|
|
if n, err := strconv.ParseInt(assignee, 10, 64); err == nil {
|
|
where = append(where, "i.assigned_to = ?")
|
|
args = append(args, n)
|
|
}
|
|
}
|
|
if from := q.Get("from"); from != "" {
|
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
|
where = append(where, "i.triggered_at >= ?")
|
|
args = append(args, t.UTC().Unix())
|
|
}
|
|
}
|
|
if to := q.Get("to"); to != "" {
|
|
if t, err := time.Parse("2006-01-02", to); err == nil {
|
|
where = append(where, "i.triggered_at < ?")
|
|
args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
|
|
}
|
|
}
|
|
|
|
limit := 50
|
|
if l := q.Get("limit"); l != "" {
|
|
if n, err := strconv.Atoi(l); err == nil && n > 0 && n <= 500 {
|
|
limit = n
|
|
}
|
|
}
|
|
|
|
order := "i.triggered_at DESC"
|
|
if q.Get("sort") == "severity" {
|
|
order = severityRankSQL("i.severity") + " DESC, i.triggered_at DESC"
|
|
}
|
|
args = append(args, limit)
|
|
|
|
rows, err := db.QueryContext(r.Context(),
|
|
fmt.Sprintf("%s WHERE %s ORDER BY %s LIMIT ?",
|
|
incidentSelectFrom, strings.Join(where, " AND "), order),
|
|
args...)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
incidents := []models.Incident{}
|
|
for rows.Next() {
|
|
i, err := scanIncident(rows)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
incidents = append(incidents, i)
|
|
}
|
|
respond(w, http.StatusOK, incidents)
|
|
}
|
|
}
|
|
|
|
func handleGetIncident(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
inc, err := fetchIncident(r.Context(), db, id)
|
|
if err == sql.ErrNoRows {
|
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
|
return
|
|
}
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
if inc.Alerts, err = incidentAlerts(r, db, id); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respond(w, http.StatusOK, inc)
|
|
}
|
|
}
|
|
|
|
func handleIncidentAlerts(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !incidentExists(w, r, db, id) {
|
|
return
|
|
}
|
|
alerts, err := incidentAlerts(r, db, id)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respond(w, http.StatusOK, alerts)
|
|
}
|
|
}
|
|
|
|
func handleIncidentTimeline(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !incidentExists(w, r, db, id) {
|
|
return
|
|
}
|
|
|
|
rows, err := db.QueryContext(r.Context(), `
|
|
SELECT e.id, e.incident_id, e.type, e.user_id, u.username,
|
|
e.alert_id, e.detail, e.created_at
|
|
FROM incident_events e
|
|
LEFT JOIN users u ON u.id = e.user_id
|
|
WHERE e.incident_id = ?
|
|
ORDER BY e.created_at ASC, e.id ASC`, id)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
events := []models.IncidentEvent{}
|
|
for rows.Next() {
|
|
var e models.IncidentEvent
|
|
var ts int64
|
|
if err := rows.Scan(&e.ID, &e.IncidentID, &e.Type, &e.UserID, &e.Username,
|
|
&e.AlertID, &e.Detail, &ts); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
e.CreatedAt = time.Unix(ts, 0).UTC()
|
|
events = append(events, e)
|
|
}
|
|
respond(w, http.StatusOK, events)
|
|
}
|
|
}
|
|
|
|
func handleIncidentAcknowledge(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
user, _ := userFromContext(r.Context())
|
|
acked, err := acknowledgeIncident(r.Context(), db, id, user.ID)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
if !acked {
|
|
if !incidentExists(w, r, db, id) {
|
|
return
|
|
}
|
|
respond(w, http.StatusConflict, errResp("incident is resolved"))
|
|
return
|
|
}
|
|
respondIncident(w, r, db, id)
|
|
}
|
|
}
|
|
|
|
func handleIncidentUnacknowledge(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
user, _ := userFromContext(r.Context())
|
|
if !updateOpenIncident(w, r, db, id,
|
|
`UPDATE incidents SET status = 'triggered', acknowledged_by = NULL, acknowledged_at = NULL
|
|
WHERE id = ? AND resolved_at IS NULL`, id) {
|
|
return
|
|
}
|
|
if err := logEvent(r.Context(), db, id, evUnacknowledged, &user.ID, nil, nil); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}
|
|
|
|
// handleIncidentResolve closes an incident by hand. This is terminal: a later
|
|
// occurrence opens a new incident rather than reopening this one, which is what
|
|
// stops a resolved incident from reappearing on the next repeat_interval
|
|
// re-send of an alert that never stopped firing. Use snooze for "not now".
|
|
func handleIncidentResolve(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
user, _ := userFromContext(r.Context())
|
|
if !updateOpenIncident(w, r, db, id,
|
|
`UPDATE incidents SET status = 'resolved', resolved_at = ?, resolution_source = ?
|
|
WHERE id = ? AND resolved_at IS NULL`,
|
|
time.Now().Unix(), incidentResolutionManual, id) {
|
|
return
|
|
}
|
|
if err := logEvent(r.Context(), db, id, evResolved, &user.ID, nil, nil); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respondIncident(w, r, db, id)
|
|
}
|
|
}
|
|
|
|
func handleIncidentAssign(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
var req struct {
|
|
UserID int64 `json:"user_id"`
|
|
}
|
|
if err := decodeJSON(r, &req); err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
|
return
|
|
}
|
|
if req.UserID == 0 {
|
|
respond(w, http.StatusBadRequest, errResp("user_id is required"))
|
|
return
|
|
}
|
|
var exists int
|
|
if err := db.QueryRowContext(r.Context(),
|
|
"SELECT 1 FROM users WHERE id = ?", req.UserID).Scan(&exists); err != nil {
|
|
respond(w, http.StatusNotFound, errResp("user not found"))
|
|
return
|
|
}
|
|
|
|
if !updateOpenIncident(w, r, db, id,
|
|
"UPDATE incidents SET assigned_to = ? WHERE id = ? AND resolved_at IS NULL",
|
|
req.UserID, id) {
|
|
return
|
|
}
|
|
// On an "assigned" event user_id is the assignee, not the actor.
|
|
if err := logEvent(r.Context(), db, id, evAssigned, &req.UserID, nil, nil); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respondIncident(w, r, db, id)
|
|
}
|
|
}
|
|
|
|
// handleIncidentSnooze hides an incident from the default queue without closing
|
|
// it. Accepts either an absolute {"until": RFC3339} or a relative
|
|
// {"duration": "2h"}.
|
|
func handleIncidentSnooze(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
var req struct {
|
|
Until string `json:"until"`
|
|
Duration string `json:"duration"`
|
|
}
|
|
if err := decodeJSON(r, &req); err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
|
return
|
|
}
|
|
|
|
var until time.Time
|
|
switch {
|
|
case req.Until != "":
|
|
t, err := time.Parse(time.RFC3339, req.Until)
|
|
if err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid until (expected RFC3339)"))
|
|
return
|
|
}
|
|
until = t
|
|
case req.Duration != "":
|
|
d, err := time.ParseDuration(req.Duration)
|
|
if err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid duration"))
|
|
return
|
|
}
|
|
until = time.Now().Add(d)
|
|
default:
|
|
respond(w, http.StatusBadRequest, errResp("until or duration is required"))
|
|
return
|
|
}
|
|
if !until.After(time.Now()) {
|
|
respond(w, http.StatusBadRequest, errResp("snooze must end in the future"))
|
|
return
|
|
}
|
|
|
|
user, _ := userFromContext(r.Context())
|
|
if !updateOpenIncident(w, r, db, id,
|
|
"UPDATE incidents SET snoozed_until = ? WHERE id = ? AND resolved_at IS NULL",
|
|
until.Unix(), id) {
|
|
return
|
|
}
|
|
detail := until.UTC().Format(time.RFC3339)
|
|
if err := logEvent(r.Context(), db, id, evSnoozed, &user.ID, nil, &detail); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respondIncident(w, r, db, id)
|
|
}
|
|
}
|
|
|
|
func handleIncidentUnsnooze(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
user, _ := userFromContext(r.Context())
|
|
if !updateOpenIncident(w, r, db, id,
|
|
"UPDATE incidents SET snoozed_until = NULL WHERE id = ? AND resolved_at IS NULL", id) {
|
|
return
|
|
}
|
|
if err := logEvent(r.Context(), db, id, evUnsnoozed, &user.ID, nil, nil); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}
|
|
|
|
func handleIncidentArchive(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
res, err := db.ExecContext(r.Context(),
|
|
"UPDATE incidents SET archived_at = unixepoch() WHERE id = ?", id)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
|
return
|
|
}
|
|
respondIncident(w, r, db, id)
|
|
}
|
|
}
|
|
|
|
func handleIncidentUnarchive(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
res, err := db.ExecContext(r.Context(),
|
|
"UPDATE incidents SET archived_at = NULL WHERE id = ?", id)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}
|
|
|
|
// handleCreateNote adds a note to the timeline. Notes are ordinary events, so a
|
|
// single query renders the whole story of an incident in order.
|
|
func handleCreateNote(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
var req struct {
|
|
Content string `json:"content"`
|
|
}
|
|
if err := decodeJSON(r, &req); err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
|
return
|
|
}
|
|
if req.Content == "" {
|
|
respond(w, http.StatusBadRequest, errResp("content is required"))
|
|
return
|
|
}
|
|
if !incidentExists(w, r, db, id) {
|
|
return
|
|
}
|
|
|
|
user, _ := userFromContext(r.Context())
|
|
now := time.Now()
|
|
res, err := db.ExecContext(r.Context(), `
|
|
INSERT INTO incident_events (incident_id, type, user_id, detail, created_at)
|
|
VALUES (?, ?, ?, ?, ?)`, id, evNote, user.ID, req.Content, now.Unix())
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
eventID, _ := res.LastInsertId()
|
|
|
|
respond(w, http.StatusCreated, models.IncidentEvent{
|
|
ID: eventID,
|
|
IncidentID: id,
|
|
Type: evNote,
|
|
UserID: &user.ID,
|
|
Username: &user.Username,
|
|
Detail: &req.Content,
|
|
CreatedAt: now.UTC().Truncate(time.Second),
|
|
})
|
|
}
|
|
}
|
|
|
|
// handleDeleteNote removes one of your own notes. Only notes are deletable — the
|
|
// rest of the timeline is what actually happened, and is not editable.
|
|
func handleDeleteNote(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
id, ok := incidentIDParam(w, r)
|
|
if !ok {
|
|
return
|
|
}
|
|
eventID, err := strconv.ParseInt(chi.URLParam(r, "eventID"), 10, 64)
|
|
if err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid note id"))
|
|
return
|
|
}
|
|
|
|
user, _ := userFromContext(r.Context())
|
|
res, err := db.ExecContext(r.Context(), `
|
|
DELETE FROM incident_events
|
|
WHERE id = ? AND incident_id = ? AND type = ? AND user_id = ?`,
|
|
eventID, id, evNote, user.ID)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
respond(w, http.StatusNotFound, errResp("note not found"))
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Shared handler plumbing
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func incidentIDParam(w http.ResponseWriter, r *http.Request) (int64, bool) {
|
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
if err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid incident id"))
|
|
return 0, false
|
|
}
|
|
return id, true
|
|
}
|
|
|
|
func incidentExists(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) bool {
|
|
var exists int
|
|
if err := db.QueryRowContext(r.Context(),
|
|
"SELECT 1 FROM incidents WHERE id = ?", id).Scan(&exists); err != nil {
|
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// updateOpenIncident runs a mutation that is only valid while an incident is
|
|
// open. The query must be constrained to `resolved_at IS NULL`, so no rows means
|
|
// either the incident does not exist or it is already closed — two different
|
|
// answers the caller should not have to distinguish itself.
|
|
func updateOpenIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64, query string, args ...any) bool {
|
|
res, err := db.ExecContext(r.Context(), query, args...)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return false
|
|
}
|
|
if n, _ := res.RowsAffected(); n > 0 {
|
|
return true
|
|
}
|
|
if !incidentExists(w, r, db, id) {
|
|
return false
|
|
}
|
|
respond(w, http.StatusConflict, errResp("incident is resolved"))
|
|
return false
|
|
}
|
|
|
|
func respondIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) {
|
|
inc, err := fetchIncident(r.Context(), db, id)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respond(w, http.StatusOK, inc)
|
|
}
|
|
|
|
// incidentAlerts loads the alerts under an incident, newest signal first.
|
|
func incidentAlerts(r *http.Request, db *sql.DB, id int64) ([]models.Alert, error) {
|
|
rows, err := db.QueryContext(r.Context(), alertSelectFrom+`
|
|
JOIN incident_alerts m ON m.alert_id = a.id
|
|
WHERE m.incident_id = ?
|
|
ORDER BY a.received_at DESC`, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
alerts := []models.Alert{}
|
|
for rows.Next() {
|
|
a, err := scanAlert(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
alerts = append(alerts, a)
|
|
}
|
|
return alerts, rows.Err()
|
|
}
|