chart: back up the database through a python sidecar

The image is FROM scratch, so there is no interpreter to run a k8up
backupcommand in, and the database runs in WAL mode, where a file-level
copy of the volume is not crash-consistent.

Also switches to strategy: Recreate. The data PVC is ReadWriteOnce, so a
RollingUpdate deadlocks the new pod against the old one holding it.

.github/workflows/chart-release.yml

@@ -29,5 +29,10 @@ jobs:
 
       - name: Run chart-releaser
         uses: helm/chart-releaser-action@v1.6.0
+        with:
+          # A charts/** push without a Chart.yaml version bump would otherwise
+          # fail trying to re-release the current version. Tagged releases also
+          # publish the chart from release.yml, so the two can race.
+          skip_existing: true
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/ci.yml

@@ -0,0 +1,34 @@
+name: CI
+
+# The release workflow gates a tag, which is late: a broken commit sits green
+# until somebody decides to publish. This runs the same checks on the way in.
+#
+# push is scoped to main rather than all branches for two reasons: a branch
+# pushed as part of a pull request would otherwise be checked twice, and
+# gh-pages holds the published Helm chart index with no Go code in it, so
+# `go vet ./...` there would fail on a missing go.mod.
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# A rapid series of pushes only needs the last one checked.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Test
+        run: go test ./...

.github/workflows/release.yml

@@ -4,9 +4,28 @@ on:
   push:
     tags:
       - 'v*'
+  workflow_dispatch:
 
 jobs:
+  # Gates every publishing job below. A tag that fails here publishes nothing:
+  # the binaries, the image and the chart are all downstream of it.
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Test
+        run: go test ./...
+
   build:
+    needs: test
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -43,6 +62,7 @@ jobs:
           path: terdut-${{ github.ref_name }}-${{ matrix.goos }}-${{ matrix.goarch }}
 
   docker:
+    needs: test
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -75,6 +95,7 @@ jobs:
             ghcr.io/yeniklas/terdut-server:${{ github.ref_name }}
 
   chart:
+    needs: test
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -94,12 +115,16 @@ jobs:
       - name: Update chart versions
         run: |
           VERSION="${{ github.ref_name }}"
-          CHART_VERSION="${VERSION#v}"
+          if [[ "$VERSION" =~ ^v[0-9] ]]; then
-          sed -i "s/^version:.*/version: ${CHART_VERSION}/" charts/terdut-server/Chart.yaml
+            CHART_VERSION="${VERSION#v}"
-          sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" charts/terdut-server/Chart.yaml
+            sed -i "s/^version:.*/version: ${CHART_VERSION}/" charts/terdut-server/Chart.yaml
+            sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" charts/terdut-server/Chart.yaml
+          fi
 
       - name: Run chart-releaser
         uses: helm/chart-releaser-action@v1.6.0
+        with:
+          skip_existing: true
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 

README.md

@@ -1,11 +1,12 @@
 # Terminal Duty (terdut-server)
 
-On-call alert management server for teams using Prometheus Alertmanager.
+Incident management server for teams using Prometheus Alertmanager.
 
 - Receives Alertmanager webhooks directly — no adapter needed
-- Stores and queries alerts (acknowledge, comment)
+- Turns alerts into **incidents**, correlated by Alertmanager's own `groupKey`
-- On-call schedule management (user-to-day assignments)
+- Incident workflow: acknowledge, assign, snooze, note, resolve, with a full timeline
-- Alert statistics (by status, by hour, by day)
+- On-call schedule management, with new incidents auto-assigned to whoever is on call
+- Alert and incident statistics, including MTTA and MTTR
 - REST API with per-user API key authentication
 - Single binary, SQLite storage — trivial to self-host
 
@@ -49,6 +50,46 @@ docker run -p 8080:8080 -v $(pwd)/data:/data \
   terdut-server
 ```
 
+### Kubernetes
+
+A Helm chart is published from this repository:
+
+```bash
+helm repo add terdut-server https://yeniklas.github.io/terdut-server
+helm upgrade --install terdut-server terdut-server/terdut-server \
+  --namespace terdut-server --create-namespace \
+  --set networking.hostname=terdut.example.com
+```
+
+The chart expects a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway named `envoy-main` in
+the `envoy-gateway-system` namespace to already exist — it renders an `HTTPRoute` against it rather
+than an `Ingress`. TLS is terminated at the gateway, so the server itself never sees a certificate.
+
+| Value | Default | Description |
+|---|---|---|
+| `networking.hostname` | `terdut.example.com` | Hostname the `HTTPRoute` serves |
+| `networking.listener` | `""` | Gateway listener (`sectionName`) to bind to. Empty attaches to every matching listener, **including plaintext HTTP** — set it to the HTTPS listener's name to serve TLS only |
+| `networking.servicePort` | `8080` | Port the route forwards to; keep in sync with `service.port` |
+| `bootstrap.enabled` | `true` | Runs a post-install hook that creates the first user and stores its API key in the `<release>-admin-key` Secret. Already-bootstrapped servers are left alone |
+| `backupSidecar.enabled` | `true` | Adds an idle `python` sidecar and the [k8up](https://k8up.io/) annotations that dump the database through it |
+
+The API key travels in an `Authorization: Bearer` header, so set `networking.listener` whenever the
+hostname is reachable outside a trusted network.
+
+#### Backups
+
+The server image is `FROM scratch` — the binary and nothing else — so there is no interpreter to
+run a database dump in, and the database runs in WAL mode, where a file-level copy of the volume is
+not crash-consistent. The chart therefore ships an idle `python:*-alpine` sidecar that shares the
+data volume, and points k8up's `backupcommand` at it with `k8up.io/backupcommand-container`. Without
+that annotation k8up execs into `.spec.containers[0]` and the dump fails.
+
+The dump is buffered and sanity-checked before its first byte reaches stdout, because k8up streams
+stdout straight into Restic: a dump that dies partway is otherwise stored as a silently truncated
+snapshot that k8up still reports as successful.
+
+Set `backupSidecar.enabled=false` if you back the volume up some other way.
+
 ---
 
 ## Configuration
@@ -57,6 +98,17 @@ docker run -p 8080:8080 -v $(pwd)/data:/data \
 |---|---|---|
 | `TERDUT_ADDR` | `:8080` | TCP address to listen on |
 | `TERDUT_DB_PATH` | `terdut.db` | Path to the SQLite database file |
+| `TERDUT_ARCHIVE_AFTER` | `168h` (7d) | How long a resolved alert or incident stays in the default list before being auto-archived |
+| `TERDUT_STALE_AFTER` | `6h` | How long a firing alert may go without a refreshing webhook before it is treated as resolved — **must exceed your Alertmanager `repeat_interval`** |
+| `TERDUT_NTFY_URL` | — | ntfy server to publish push notifications to. Empty disables notifications entirely |
+| `TERDUT_NTFY_TOKEN` | — | Bearer token for an access-controlled ntfy |
+| `TERDUT_NTFY_FALLBACK_TOPIC` | — | Topic used when nobody is on call |
+| `TERDUT_PUBLIC_URL` | — | Base URL a phone uses to reach this server, for the link and Acknowledge button inside a notification |
+| `TERDUT_NOTIFY_REPEAT` | `15m` | How long an incident may sit unacknowledged before it is paged again. `0` notifies once and never repeats |
+
+Durations use Go syntax (`30m`, `12h`, `168h`). An unparseable value falls back to the default.
+
+In the Helm chart the two sweeper durations are set via `sweeper.staleAfter` and `sweeper.archiveAfter`, and notifications via the `notify.*` values.
 
 ---
 
@@ -79,6 +131,124 @@ The webhook endpoint requires no authentication.
 
 ---
 
+## Alerts and incidents
+
+There are two objects, and the difference between them is the whole design.
+
+**An alert is Alertmanager's record.** It has two states, `firing` and
+`resolved`, one row per fingerprint, and no human ever writes to it. The API
+exposes alerts read-only.
+
+**An incident is the work item.** It goes `triggered → acknowledged → resolved`,
+carries an assignee, a snooze, notes and a timeline, and is the only thing people
+act on. Many alerts belong to one incident.
+
+### Correlation uses Alertmanager's `groupKey`
+
+Alertmanager has already grouped alerts according to the `group_by` routing tree
+you configured, and it sends the resulting `groupKey` and `groupLabels` on every
+webhook. Incidents adopt that answer rather than re-grouping alerts a second
+time — if you want different correlation, change `group_by` in
+`alertmanager.yml` and terdut follows.
+
+At most one incident is open per `groupKey` at a time. Alerts firing in a group
+that already has an open incident join it. The incident's `severity` is a
+high-water mark — the highest `severity` label any of its alerts has carried — so
+an incident that hit `critical` still reads as critical after the critical alert
+clears.
+
+### An incident opens only on a new occurrence
+
+An incident opens when an alert **transitions into firing**: a fingerprint that
+was never seen, an alert with a newer `startsAt`, or a resolved alert that
+started again. The unchanged firing notifications Alertmanager re-sends every
+`repeat_interval` are none of those, and open nothing.
+
+This is what makes closing an incident by hand mean something. Without the rule,
+`POST /api/incidents/{id}/resolve` would be undone by the next re-send of an
+alert that never stopped firing.
+
+### Leaving the open state
+
+- **Automatically**, once every alert under the incident has stopped firing —
+  whether by a resolved webhook or by the sweeper's
+  [stale-alert expiry](#stale-alert-expiry). The incident gets
+  `"resolution_source": "alerts"`.
+- **By hand**, via `POST /api/incidents/{id}/resolve`
+  (`"resolution_source": "manual"`). This is **terminal**: a later occurrence in
+  that group opens a *new* incident rather than reopening this one. If the alert
+  underneath never stops firing, the incident stays closed — that is what
+  resolving by hand asserts.
+
+To quieten an incident you expect to come back, snooze it instead
+(`POST /api/incidents/{id}/snooze`). A snooze hides the incident from the default
+list without closing it, and expires by simply falling into the past.
+
+### On-call assignment
+
+A new incident is assigned to whoever holds today's schedule entry at the moment
+it opens (`GET /api/schedule/current`). If nobody is scheduled it opens
+unassigned. Reassign with `POST /api/incidents/{id}/assign`.
+
+### Push notifications
+
+With `TERDUT_NTFY_URL` set, an incident that opens is pushed to the on-call
+person's phone through [ntfy](https://ntfy.sh). Set each user's topic with
+`PUT /api/users/{id}/notify`; a user with no topic falls back to
+`TERDUT_NTFY_FALLBACK_TOPIC`, as does an incident that opens with nobody on call.
+If neither yields a topic, nothing is queued.
+
+Three things get pushed:
+
+- **triggered** — an incident opened. Priority follows severity (`critical` maps
+  to ntfy's max priority, the one that overrides the phone's quiet settings).
+- **reminder** — the incident is still `triggered` after `TERDUT_NOTIFY_REPEAT`.
+  Repeats until somebody acts. Acknowledging, snoozing, resolving or archiving
+  all stop it — snooze is the mute button.
+- **resolved** — every alert under the incident stopped firing. Only sent to
+  whoever was paged in the first place, and only for the automatic cascade:
+  resolving by hand pushes nothing, since the person who did it already knows.
+
+Notifications carry an **Acknowledge** button that acknowledges the incident
+without opening anything. It POSTs to `/api/notify/ack/{token}`, an
+unauthenticated route authorised by the 256-bit single-use token in its path —
+minted fresh per notification, scoped to one incident and one action, and valid
+for 24 hours. A real API key is never put in a notification, because the message
+is stored on the ntfy server and cached on the device.
+
+Two consequences worth planning for:
+
+- `/api/notify/ack/{token}` **must stay publicly reachable**, or the button will
+  not work when the responder is off your network.
+- Notifications sent to the fallback topic carry **no** Acknowledge button. The
+  topic is shared, and a button on it would let any subscriber acknowledge as
+  somebody else.
+
+Delivery is a queue, not an inline call: the webhook writes a row and a
+background notifier sends it within 30 seconds, retrying with exponential
+backoff up to 8 attempts. Nothing about ingestion blocks on ntfy being reachable.
+
+### Stale alert expiry
+
+A resolved webhook is the only signal that an alert has stopped firing, so a
+notification that is dropped, silenced, or lost to a restart would otherwise pin
+that alert as firing forever. A background sweeper resolves firing alerts that
+Alertmanager has stopped refreshing, using either signal:
+
+- the `endsAt` watermark on the last notification has passed, or
+- no webhook has refreshed the alert within `TERDUT_STALE_AFTER`.
+
+Alertmanager re-sends firing notifications every `repeat_interval`, which is what
+keeps a live alert fresh — so `TERDUT_STALE_AFTER` must be comfortably larger
+than your `repeat_interval` (default 4h), or live alerts will be resolved
+prematurely. Alerts resolved this way are marked `"resolution_source": "expiry"`
+to distinguish them from a real Alertmanager resolve (`"alertmanager"`).
+
+An expiry cascades: once it leaves an incident with nothing firing under it, the
+incident resolves too, in the same sweep.
+
+---
+
 ## API reference
 
 ### Authentication
@@ -97,6 +267,7 @@ Authorization: Bearer <api-key>
 | `GET` | `/api/users` | List users |
 | `POST` | `/api/users` | Create user `{"username","email"}` |
 | `DELETE` | `/api/users/{id}` | Delete user (cascades to keys) |
+| `PUT` | `/api/users/{id}/notify` | Set push notification target `{"ntfy_topic"}` — empty string clears it |
 | `POST` | `/api/users/{id}/api-keys` | Issue API key `{"name"}` — key shown once |
 | `DELETE` | `/api/users/{id}/api-keys/{keyID}` | Revoke API key |
 
@@ -106,17 +277,170 @@ Authorization: Bearer <api-key>
 |---|---|---|
 | `POST` | `/api/alertmanager/webhook` | Alertmanager v4 webhook receiver (no auth) |
 
-### Alerts
+### Notifications
 
 | Method | Path | Description |
 |---|---|---|
-| `GET` | `/api/alerts` | List alerts. Filters: `?status=firing\|resolved`, `?name=`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?limit=` (default 50, max 500) |
+| `POST` | `/api/notify/ack/{token}` | Acknowledge an incident from a push notification's Acknowledge button. No auth: the single-use token in the path is the credential. Must stay publicly reachable |
+
+### Incidents
+
+| Method | Path | Description |
+|---|---|---|
+| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=<user id>`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?limit=` (default 50, max 500) |
+| `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline |
+| `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident |
+| `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological |
+| `POST` | `/api/incidents/{id}/acknowledge` | Acknowledge (stamps authed user + time) |
+| `DELETE` | `/api/incidents/{id}/acknowledge` | Clear acknowledgement, back to `triggered` |
+| `POST` | `/api/incidents/{id}/resolve` | Close by hand — **terminal**, see above |
+| `POST` | `/api/incidents/{id}/assign` | Reassign `{"user_id"}` |
+| `POST` | `/api/incidents/{id}/snooze` | Hide until `{"until": RFC3339}` or `{"duration": "2h"}` |
+| `DELETE` | `/api/incidents/{id}/snooze` | Un-snooze |
+| `POST` | `/api/incidents/{id}/archive` | Archive (hides from the default list) |
+| `DELETE` | `/api/incidents/{id}/archive` | Un-archive |
+| `POST` | `/api/incidents/{id}/notes` | Add a note `{"content"}` |
+| `DELETE` | `/api/incidents/{id}/notes/{eventID}` | Delete own note |
+
+With no `?status=` filter, `GET /api/incidents` returns **open** incidents only —
+the queue an on-call person wants. Currently snoozed and archived incidents are
+excluded unless asked for. Actions that only make sense on an open incident
+return `409` once it is resolved.
+
+Notes are ordinary timeline events of type `note`; only they are deletable, and
+only by their author. The rest of the timeline is a record of what happened.
+
+#### The incident object
+
+| Field | Type | Notes |
+|---|---|---|
+| `id` | integer | Server-assigned |
+| `group_key` | string | Alertmanager's `groupKey` — opaque, treat as an identifier |
+| `title` | string | Rendered from `groupLabels` |
+| `group_labels` | object | String→string, as sent by Alertmanager |
+| `status` | string | `"triggered"`, `"acknowledged"` or `"resolved"` |
+| `severity` | string | *optional* — high-water mark across the incident's alerts; never lowered |
+| `triggered_at` | timestamp | When the incident opened |
+| `acknowledged_by_id` / `acknowledged_by` / `acknowledged_at` | | *optional* — user id, username, time |
+| `assigned_to_id` / `assigned_to` | | *optional* — user id, username |
+| `snoozed_until` | timestamp | *optional* — a value in the past reads as not snoozed |
+| `resolved_at` | timestamp | *optional* |
+| `resolution_source` | string | *optional* — `"alerts"` or `"manual"` |
+| `archived_at` | timestamp | *optional* |
+| `alerts` | array | Only on `GET /api/incidents/{id}` |
+
+Treat `resolution_source` as an open set, as with the alert field of the same
+name: degrade unknown values to "resolved, reason unknown".
+
+#### The timeline event object
+
+| Field | Type | Notes |
+|---|---|---|
+| `id` | integer | |
+| `incident_id` | integer | |
+| `type` | string | See below — treat as an open set |
+| `user_id` / `username` | | *optional* — absent when the server acted rather than a person |
+| `alert_id` | integer | *optional* — the alert an `alert_added` / `alert_resolved` event refers to |
+| `detail` | string | *optional* — the note body, the snooze deadline, etc. |
+| `created_at` | timestamp | |
+
+Types written today: `triggered`, `alert_added`, `alert_resolved`,
+`acknowledged`, `unacknowledged`, `assigned`, `snoozed`, `unsnoozed`, `resolved`,
+`note`. On an `assigned` event `user_id` is the **assignee**, not the actor. New
+types may be added; render unknown ones generically rather than dropping them.
+
+### Alerts
+
+Alerts are read-only. Everything a person does happens on the incident.
+
+| Method | Path | Description |
+|---|---|---|
+| `GET` | `/api/alerts` | List alerts. Filters: `?status=firing\|resolved`, `?name=`, `?incident_id=`, `?archived=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?limit=` (default 50, max 500) |
 | `GET` | `/api/alerts/{id}` | Get single alert |
-| `POST` | `/api/alerts/{id}/acknowledge` | Acknowledge alert (stamps authed user + time) |
+
-| `DELETE` | `/api/alerts/{id}/acknowledge` | Clear acknowledgement |
+Archived alerts are hidden from `GET /api/alerts` unless `?archived=true` is
-| `GET` | `/api/alerts/{id}/comments` | List comments (chronological) |
+passed; alert archiving is automatic housekeeping by the sweeper, not a user
-| `POST` | `/api/alerts/{id}/comments` | Add comment `{"content"}` |
+action. Resolved alerts carry `resolution_source`: `"alertmanager"` for a real
-| `DELETE` | `/api/alerts/{id}/comments/{commentID}` | Delete own comment |
+resolved webhook, `"expiry"` when the sweeper inferred it (see
+[Stale alert expiry](#stale-alert-expiry)).
+
+#### The alert object
+
+Returned by `GET /api/alerts` (as an array) and `GET /api/alerts/{id}`.
+Timestamps are RFC 3339 in UTC. Fields marked *optional* are omitted entirely
+when unset, so clients must treat them as nullable.
+
+| Field | Type | Notes |
+|---|---|---|
+| `id` | integer | Server-assigned; stable for the life of the row |
+| `fingerprint` | string | Alertmanager's fingerprint — the upsert key |
+| `name` | string | From the `alertname` label |
+| `status` | string | `"firing"` or `"resolved"` |
+| `labels` | object | String→string, as sent by Alertmanager |
+| `annotations` | object | String→string, as sent by Alertmanager |
+| `starts_at` | timestamp | When the alert instance began, **per Prometheus** |
+| `ends_at` | timestamp | *optional* — absent while no end is known |
+| `generator_url` | string | Link back to the originating Prometheus |
+| `received_at` | timestamp | When the server last accepted a webhook for this alert — see below |
+| `incident_id` | integer | *optional* — the most recent incident this alert belongs to |
+| `resolution_source` | string | *optional* — `"alertmanager"` or `"expiry"` |
+| `archived_at` | timestamp | *optional* — set while archived |
+
+##### `received_at` is a liveness heartbeat
+
+`starts_at` comes from Prometheus and **never changes** for the lifetime of an
+alert instance. It says when the problem began, not whether it is still
+happening — an alert that started twelve days ago looks identical whether
+Alertmanager refreshed it a minute ago or went silent a week ago.
+
+`received_at` is the field that answers "is this still live". It is set to the
+server's clock on **every accepted webhook** for that fingerprint, including the
+unchanged firing notifications Alertmanager re-sends every `repeat_interval`.
+Clients may rely on this:
+
+- **A firing alert whose `received_at` is advancing is still being refreshed.**
+  Stale-dating it against `repeat_interval` is a valid liveness check, and it is
+  what the built-in sweeper does (see
+  [Stale alert expiry](#stale-alert-expiry)).
+- **`received_at` tracks accepted payloads, not delivery attempts.** A retry
+  that describes an older instance than the stored one is discarded, and a
+  discarded payload does not move `received_at`.
+- **It stops advancing once the alert resolves,** because Alertmanager stops
+  re-sending. On an alert resolved by the sweeper
+  (`"resolution_source": "expiry"`) it therefore marks the last time
+  Alertmanager was actually heard from, which is earlier than `ends_at`.
+
+`GET /api/alerts` is ordered by `received_at` descending — most recently
+refreshed first — and the `?from=` / `?to=` filters on both the alert and stats
+endpoints select on `received_at`, not `starts_at`.
+
+##### `resolution_source` says how much to trust `ends_at`
+
+An alert can leave the firing state two ways, and `resolution_source` records
+which happened. Clients may rely on this:
+
+- **Absent while firing.** It is set only on resolve, and a re-fire under the
+  same fingerprint clears it again, so its presence always agrees with
+  `"status": "resolved"`.
+- **`"alertmanager"` — a real resolved webhook arrived.** `ends_at` is the end
+  time Alertmanager reported. It is an observed value and can be displayed as
+  fact.
+- **`"expiry"` — the sweeper inferred the resolve** because Alertmanager stopped
+  refreshing the alert (see [Stale alert expiry](#stale-alert-expiry)). Nothing
+  ever reported an end, so **`ends_at` is approximate**: it is either the stale
+  `endsAt` watermark from the last notification, or — when that notification
+  carried none — the time the sweep ran, which lags the last real contact by up
+  to `TERDUT_STALE_AFTER` plus a sweep interval. Treat it as "no later than",
+  not as when the problem stopped.
+
+  On these alerts `received_at` is the more truthful signal: it marks the last
+  time Alertmanager was actually heard from. Surfacing the distinction is
+  worthwhile, since `"expiry"` can also mean the alert is still firing and the
+  notification path broke.
+
+Treat the value as an open set and tolerate ones you do not recognise — new
+sources may be added, and unknown values should degrade to "resolved, reason
+unknown" rather than being rejected.
 
 ### On-call schedule
 
@@ -129,15 +453,47 @@ Authorization: Bearer <api-key>
 
 ### Statistics
 
-All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD` to filter by `received_at`.
+All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats filter on `triggered_at`.
 
 | Method | Path | Description |
 |---|---|---|
+| `GET` | `/api/stats/incidents` | `{total, triggered, acknowledged, resolved, mtta_seconds, mttr_seconds}` |
 | `GET` | `/api/stats/alerts` | `{total, firing, resolved}` counts |
 | `GET` | `/api/stats/alerts/top` | Most frequent alert names. `?limit=` (default 10, max 100) |
 | `GET` | `/api/stats/alerts/by-hour` | Count per hour-of-day (UTC), all 24 slots returned |
 | `GET` | `/api/stats/alerts/by-day` | Count per day-of-week, all 7 slots with names returned |
 
+`mtta_seconds` (time to acknowledge) and `mttr_seconds` (time to resolve) are
+averages over incidents that have actually been acknowledged or resolved, and are
+**null** until there are any — null means "no data", not zero.
+
+---
+
+## Upgrading to incidents
+
+The incidents release moves the workflow off alerts, which is a **breaking API
+change**. These endpoints are gone:
+
+| Removed | Replacement |
+|---|---|
+| `POST`/`DELETE` `/api/alerts/{id}/acknowledge` | `POST`/`DELETE` `/api/incidents/{id}/acknowledge` |
+| `POST`/`DELETE` `/api/alerts/{id}/archive` | `POST`/`DELETE` `/api/incidents/{id}/archive` (alert archiving is now sweeper-only) |
+| `GET`/`POST` `/api/alerts/{id}/comments` | `GET /api/incidents/{id}/timeline`, `POST /api/incidents/{id}/notes` |
+| `DELETE /api/alerts/{id}/comments/{commentID}` | `DELETE /api/incidents/{id}/notes/{eventID}` |
+
+The alert object also drops `acknowledged_by_id`, `acknowledged_by` and
+`acknowledged_at`, and gains `incident_id`.
+
+Migration `008_incidents.sql` runs automatically on start and preserves existing
+data: every alert gets a backfilled incident carrying its acknowledgement, and
+comments become timeline notes. Backfilled incidents have a `group_key` of
+`backfill:<fingerprint>` — there is no historical `groupKey` to correlate on, so
+they are one-per-alert rather than grouped.
+
+Nothing about the two documented alert contracts changes: `received_at` is still
+advanced on every accepted webhook, and `resolution_source` still means what it
+did.
+
 ---
 
 ## Development

charts/terdut-server/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: terdut-server
 description: A Helm chart for Terminal Duty — on-call alert management server
 type: application
-version: 0.1.0
+version: 0.6.0
 appVersion: "latest"

charts/terdut-server/templates/_helpers.tpl

@@ -32,3 +32,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 app.kubernetes.io/name: {{ include "terdut-server.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+
+{{- define "terdut-server.bootstrapSecretName" -}}
+{{- .Values.bootstrap.secretName | default (printf "%s-admin-key" (include "terdut-server.fullname" .)) }}
+{{- end }}

charts/terdut-server/templates/bootstrap-job.yaml

@@ -0,0 +1,91 @@
+{{- if .Values.bootstrap.enabled }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "terdut-server.fullname" . }}-bootstrap
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "terdut-server.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "0"
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      labels:
+        {{- include "terdut-server.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: bootstrap
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: {{ include "terdut-server.fullname" . }}-bootstrap
+      containers:
+        - name: bootstrap
+          image: alpine:3
+          command:
+            - /bin/sh
+            - -c
+            - |
+              apk add --no-cache curl > /dev/null 2>&1
+
+              SERVICE_URL="http://{{ include "terdut-server.fullname" . }}:{{ .Values.service.port }}"
+              SECRET_NAME="{{ include "terdut-server.bootstrapSecretName" . }}"
+              K8S_API="https://kubernetes.default.svc"
+              SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+              NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)"
+
+              echo "Waiting for terdut-server to be ready..."
+              RETRIES=60
+              while [ "$RETRIES" -gt 0 ]; do
+                curl -sf "$SERVICE_URL/healthz" > /dev/null 2>&1 && break
+                RETRIES=$((RETRIES - 1))
+                sleep 2
+              done
+              if [ "$RETRIES" -eq 0 ]; then
+                echo "Timed out waiting for server to be ready."
+                exit 1
+              fi
+              echo "Server is ready."
+
+              RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$SERVICE_URL/api/bootstrap" \
+                -H "Content-Type: application/json" \
+                -d '{"username":"{{ .Values.bootstrap.username }}","email":"{{ .Values.bootstrap.email }}"}')
+
+              HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+              BODY=$(echo "$RESPONSE" | head -1)
+
+              if [ "$HTTP_CODE" = "403" ]; then
+                echo "Server already bootstrapped, nothing to do."
+                exit 0
+              fi
+
+              if [ "$HTTP_CODE" != "201" ]; then
+                echo "Bootstrap failed (HTTP $HTTP_CODE): $BODY"
+                exit 1
+              fi
+
+              API_KEY=$(echo "$BODY" | grep -o '"key":"[^"]*"' | cut -d'"' -f4)
+              if [ -z "$API_KEY" ]; then
+                echo "Failed to extract API key from response."
+                exit 1
+              fi
+
+              echo "Bootstrap succeeded. Storing API key in secret '$SECRET_NAME'."
+
+              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
+                -X POST "$K8S_API/api/v1/namespaces/$NAMESPACE/secrets" \
+                --cacert "$CA_CERT" \
+                -H "Authorization: Bearer $SA_TOKEN" \
+                -H "Content-Type: application/json" \
+                -d "$(printf '{"apiVersion":"v1","kind":"Secret","metadata":{"name":"%s"},"stringData":{"api-key":"%s"}}' "$SECRET_NAME" "$API_KEY")")
+
+              if [ "$HTTP_CODE" != "201" ]; then
+                echo "Failed to create secret (HTTP $HTTP_CODE)."
+                exit 1
+              fi
+
+              echo "Secret '$SECRET_NAME' created successfully."
+{{- end }}

charts/terdut-server/templates/bootstrap-rbac.yaml

@@ -0,0 +1,50 @@
+{{- if .Values.bootstrap.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "terdut-server.fullname" . }}-bootstrap
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "terdut-server.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "-1"
+    helm.sh/hook-delete-policy: before-hook-creation
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "terdut-server.fullname" . }}-bootstrap
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "terdut-server.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "-1"
+    helm.sh/hook-delete-policy: before-hook-creation
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "terdut-server.fullname" . }}-bootstrap
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "terdut-server.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "-1"
+    helm.sh/hook-delete-policy: before-hook-creation
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "terdut-server.fullname" . }}-bootstrap
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "terdut-server.fullname" . }}-bootstrap
+    namespace: {{ .Release.Namespace }}
+{{- end }}

charts/terdut-server/templates/deployment.yaml

@@ -10,10 +10,50 @@ spec:
   selector:
     matchLabels:
       {{- include "terdut-server.selectorLabels" . | nindent 6 }}
+  # The data PVC is ReadWriteOnce, so a RollingUpdate deadlocks: the new pod
+  # cannot attach the volume until the old one releases it, and the old one is
+  # not torn down until the new one is ready.
+  strategy:
+    type: Recreate
   template:
     metadata:
       labels:
         {{- include "terdut-server.selectorLabels" . | nindent 8 }}
+      {{- if .Values.backupSidecar.enabled }}
+      annotations:
+        # Dumps the whole database: incidents, alerts, users, API key hashes,
+        # the schedule and the notification outbox.
+        #
+        # Runs in the `backup` sidecar, NOT in the app container: the server
+        # image is FROM scratch and has no interpreter at all. k8up execs into
+        # .spec.containers[0] unless told otherwise, hence the explicit
+        # k8up.io/backupcommand-container.
+        #
+        # Buffered and sanity-checked before the first byte reaches stdout: k8up
+        # streams stdout straight into restic, so a dump that dies partway is
+        # stored as a silently-truncated snapshot that k8up still reports as
+        # Succeeded. The check counts users rather than incidents -- incidents
+        # are swept and archived, so an empty incidents table is a legitimate
+        # state, whereas a database with no users never is.
+        #
+        # The connection is read-only but the mount is not: the database runs in
+        # WAL mode, and opening it mode=ro still needs write access to the -shm
+        # wal-index.
+        #
+        # chr(10), not '\n': k8up parses this annotation with go-shellquote.
+        k8up.io/backupcommand-container: backup
+        k8up.io/backupcommand: >-
+          python3 -c "import sqlite3, sys;
+          con = sqlite3.connect('file:/data/terdut.db?mode=ro', uri=True);
+          con.execute('BEGIN');
+          users = con.execute('SELECT count(*) FROM users').fetchone()[0];
+          out = chr(10).join(con.iterdump()) + chr(10);
+          (users > 0 and out.rstrip().endswith('COMMIT;'))
+          or sys.exit('terdut: db dump failed sanity checks');
+          sys.stdout.write(out)"
+        k8up.io/file-extension: ".sql"
+        k8up.io/backup: "true"
+      {{- end }}
     spec:
       enableServiceLinks: false
       containers:
@@ -29,6 +69,27 @@ spec:
               value: ":{{ .Values.service.port }}"
             - name: TERDUT_DB_PATH
               value: "/data/terdut.db"
+            - name: TERDUT_STALE_AFTER
+              value: "{{ .Values.sweeper.staleAfter }}"
+            - name: TERDUT_ARCHIVE_AFTER
+              value: "{{ .Values.sweeper.archiveAfter }}"
+            {{- if .Values.notify.ntfyUrl }}
+            - name: TERDUT_NTFY_URL
+              value: "{{ .Values.notify.ntfyUrl }}"
+            - name: TERDUT_NTFY_FALLBACK_TOPIC
+              value: "{{ .Values.notify.fallbackTopic }}"
+            - name: TERDUT_NOTIFY_REPEAT
+              value: "{{ .Values.notify.repeatEvery }}"
+            - name: TERDUT_PUBLIC_URL
+              value: "{{ .Values.notify.publicUrl | default (printf "https://%s" .Values.networking.hostname) }}"
+            {{- if .Values.notify.tokenSecret.name }}
+            - name: TERDUT_NTFY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.notify.tokenSecret.name }}
+                  key: {{ .Values.notify.tokenSecret.key }}
+            {{- end }}
+            {{- end }}
           volumeMounts:
             - name: data
               mountPath: /data
@@ -42,6 +103,25 @@ spec:
               path: /healthz
               port: http
             initialDelaySeconds: 5
+
+        {{- if .Values.backupSidecar.enabled }}
+        # Idle sidecar. It exists only so k8up has a container with a sqlite3
+        # module to exec the backupcommand in. Mounted read-write on purpose:
+        # see the note on the backupcommand annotation above.
+        - name: backup
+          image: "{{ .Values.backupSidecar.image.repository }}:{{ .Values.backupSidecar.image.tag }}"
+          imagePullPolicy: {{ .Values.backupSidecar.image.pullPolicy }}
+          command: ["sleep", "infinity"]
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: "16Mi"
+              cpu: "10m"
+            limits:
+              memory: "64Mi"
+        {{- end }}
       volumes:
         - name: data
           persistentVolumeClaim:

charts/terdut-server/templates/httpproxy.yaml

@@ -9,6 +9,9 @@ spec:
   parentRefs:
   - name: envoy-main
     namespace: envoy-gateway-system
+    {{- with .Values.networking.listener }}
+    sectionName: {{ . | quote }}
+    {{- end }}
   rules:
   - backendRefs:
     - name: {{ include "terdut-server.fullname" . }}

charts/terdut-server/values.yaml

@@ -1,6 +1,10 @@
 networking:
   hostname: "terdut.example.com"
   servicePort: 8080
+  # Gateway listener to bind the HTTPRoute to. Empty attaches to every matching
+  # listener, including plaintext HTTP. Set this to the name of the HTTPS
+  # listener to serve the API over TLS only.
+  listener: ""
 
 image:
   repository: ghcr.io/yeniklas/terdut-server
@@ -14,3 +18,53 @@ storage:
 service:
   type: ClusterIP
   port: 8080
+
+sweeper:
+  # How long a firing alert may go without a refreshing webhook before it is
+  # treated as resolved. Must exceed your Alertmanager repeat_interval.
+  staleAfter: 6h
+  # How long a resolved alert stays in the default list before auto-archiving.
+  archiveAfter: 168h
+
+notify:
+  # ntfy server that push notifications are published to, e.g.
+  # http://ntfy.ntfy.svc.cluster.local. Empty disables notifications entirely.
+  ntfyUrl: ""
+  # Topic used when nobody is on call today. Notifications sent here carry no
+  # Acknowledge button: the topic is shared, so there is no user to attribute an
+  # acknowledgement to. Leave empty to send nothing when the schedule is unset.
+  fallbackTopic: ""
+  # How long an incident may sit unacknowledged before it is paged again.
+  # Set to 0 to notify once and never repeat.
+  repeatEvery: 15m
+  # Base URL a phone uses to reach this server, for the link and the Acknowledge
+  # button inside a notification. Defaults to https://<networking.hostname>.
+  #
+  # The Acknowledge button is a POST to /api/notify/ack/{token} from the
+  # responder's phone, so that path has to stay publicly reachable — it is
+  # authorised by the single-use token in the URL, not by network placement.
+  publicUrl: ""
+  # Optional bearer token for an access-controlled ntfy, read from an existing
+  # Secret. Leave name empty for an open ntfy.
+  tokenSecret:
+    name: ""
+    key: token
+
+# The server image is FROM scratch — just the binary, with no shell, no sqlite3
+# and no python — so a k8up backupcommand cannot run in the app container. This
+# idle sidecar shares the data volume and is selected with
+# k8up.io/backupcommand-container. Only the stdlib sqlite3 module is used, so any
+# python image works.
+backupSidecar:
+  enabled: true
+  image:
+    repository: python
+    tag: "3.13-alpine"
+    pullPolicy: IfNotPresent
+
+bootstrap:
+  enabled: true
+  username: admin
+  email: admin@example.com
+  # secretName overrides the default of <fullname>-admin-key
+  secretName: ""

cmd/terdut/main.go

@@ -28,7 +28,15 @@ func main() {
 		log.Fatalf("migrate: %v", err)
 	}
 
-	router := api.NewRouter(database)
+	notify := api.NotifyConfig{
+		BaseURL:       cfg.NtfyURL,
+		Token:         cfg.NtfyToken,
+		FallbackTopic: cfg.NtfyFallbackTopic,
+		PublicURL:     cfg.PublicURL,
+		RepeatEvery:   cfg.NotifyRepeat,
+	}
+
+	router := api.NewRouter(database, notify)
 
 	srv := &http.Server{
 		Addr:         cfg.Addr,
@@ -41,6 +49,9 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
+	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter)
+	go api.StartNotifier(ctx, database, notify)
+
 	go func() {
 		log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
 		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {

internal/api/alertmanager.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
 	"log"
@@ -8,11 +9,26 @@ import (
 	"time"
 )
 
+// Values for alerts.resolution_source, recording why an alert left the firing
+// state: a real Alertmanager notification, or inference by the sweeper.
+const (
+	resolutionAlertmanager = "alertmanager"
+	resolutionExpiry       = "expiry"
+)
+
 // amPayload mirrors the Alertmanager webhook v4 payload.
 type amPayload struct {
-	Version string    `json:"version"`
+	Version string `json:"version"`
-	Status  string    `json:"status"`
+	Status  string `json:"status"`
-	Alerts  []amAlert `json:"alerts"`
+
+	// GroupKey and GroupLabels are how alerts get correlated into incidents.
+	// Alertmanager has already done the grouping work according to the group_by
+	// routing tree the operator configured, so we adopt its answer instead of
+	// inventing a second grouping scheme here.
+	GroupKey    string            `json:"groupKey"`
+	GroupLabels map[string]string `json:"groupLabels"`
+
+	Alerts []amAlert `json:"alerts"`
 }
 
 type amAlert struct {
@@ -25,7 +41,24 @@ type amAlert struct {
 	Fingerprint  string            `json:"fingerprint"`
 }
 
-func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
+// ingested records what actually happened to one alert of a payload, which is
+// what decides whether an incident opens.
+type ingested struct {
+	id     int64
+	name   string
+	firing bool
+
+	// newOccurrence marks an alert that transitioned *into* firing: a
+	// fingerprint we had never seen, a newer startsAt, or a resolved alert that
+	// started again. A repeat_interval re-send of an already-firing alert is
+	// none of these, which is what keeps a manually resolved incident closed.
+	newOccurrence bool
+
+	// justResolved marks the firing → resolved edge, worth a timeline entry.
+	justResolved bool
+}
+
+func handleAlertmanagerWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var payload amPayload
 		if err := decodeJSON(r, &payload); err != nil {
@@ -33,40 +66,296 @@ func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
 			return
 		}
 
-		now := time.Now().Unix()
+		// Alertmanager retries anything that is not 2xx, and a retry of a payload
-		for _, a := range payload.Alerts {
+		// we failed to store is more useful than an error it cannot act on — so
-			name := a.Labels["alertname"]
+		// failures are logged, not surfaced.
-			labelsJSON, _ := json.Marshal(a.Labels)
+		if err := ingest(r.Context(), db, notify, payload); err != nil {
-			annotationsJSON, _ := json.Marshal(a.Annotations)
+			log.Printf("webhook ingest (group %q): %v", payload.GroupKey, err)
-
-			// Alertmanager uses zero time ("0001-01-01T00:00:00Z") to mean "still firing".
-			var endsAtUnix *int64
-			if a.EndsAt.Year() > 1 {
-				t := a.EndsAt.Unix()
-				endsAtUnix = &t
-			}
-
-			_, err := db.ExecContext(r.Context(), `
-				INSERT INTO alerts
-					(fingerprint, name, status, labels, annotations, starts_at, ends_at, generator_url, received_at)
-				VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-				ON CONFLICT(fingerprint) DO UPDATE SET
-					status        = excluded.status,
-					labels        = excluded.labels,
-					annotations   = excluded.annotations,
-					ends_at       = excluded.ends_at,
-					generator_url = excluded.generator_url,
-					received_at   = excluded.received_at`,
-				a.Fingerprint, name, a.Status,
-				string(labelsJSON), string(annotationsJSON),
-				a.StartsAt.Unix(), endsAtUnix,
-				a.GeneratorURL, now,
-			)
-			if err != nil {
-				log.Printf("upsert alert %s: %v", a.Fingerprint, err)
-			}
 		}
 
 		w.WriteHeader(http.StatusOK)
 	}
 }
+
+// ingest stores a payload's alerts and reconciles the incident for its group.
+// The whole payload is one transaction: an incident that opened but whose alerts
+// failed to link would be a work item nobody could act on.
+func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, payload amPayload) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback() //nolint:errcheck
+
+	accepted, err := upsertAlerts(ctx, tx, payload.Alerts)
+	if err != nil {
+		return err
+	}
+
+	// touched collects every incident this payload affected, so severity and the
+	// resolution cascade are recomputed once per incident at the end.
+	touched := map[int64]bool{}
+
+	incidentID, err := incidentForGroup(ctx, tx, notify, payload, accepted)
+	if err != nil {
+		return err
+	}
+	if incidentID != 0 {
+		touched[incidentID] = true
+		for _, a := range accepted {
+			if !a.firing {
+				continue
+			}
+			if err := linkAlert(ctx, tx, incidentID, a.id); err != nil {
+				return err
+			}
+		}
+	}
+
+	for _, a := range accepted {
+		if !a.justResolved {
+			continue
+		}
+		id, err := openIncidentForAlert(ctx, tx, a.id)
+		if err != nil {
+			return err
+		}
+		if id == 0 {
+			continue
+		}
+		touched[id] = true
+		alertID := a.id
+		if err := logEvent(ctx, tx, id, evAlertResolved, nil, &alertID, nil); err != nil {
+			return err
+		}
+	}
+
+	for id := range touched {
+		if err := refreshSeverity(ctx, tx, id); err != nil {
+			return err
+		}
+		if _, err := resolveIfSettled(ctx, tx, id); err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+// upsertAlerts stores each alert of a payload and reports what changed. Payloads
+// the ordering guard rejected are left out entirely.
+func upsertAlerts(ctx context.Context, tx *sql.Tx, alerts []amAlert) ([]ingested, error) {
+	now := time.Now().Unix()
+	accepted := make([]ingested, 0, len(alerts))
+
+	for _, a := range alerts {
+		name := a.Labels["alertname"]
+		labelsJSON, _ := json.Marshal(a.Labels)
+		annotationsJSON, _ := json.Marshal(a.Annotations)
+
+		// The stored state has to be read before the upsert overwrites it: it is
+		// the only way to tell a genuine new occurrence from a re-send.
+		var prevStatus string
+		var prevStartsAt int64
+		existed := true
+		switch err := tx.QueryRowContext(ctx,
+			"SELECT status, starts_at FROM alerts WHERE fingerprint = ?", a.Fingerprint,
+		).Scan(&prevStatus, &prevStartsAt); {
+		case err == sql.ErrNoRows:
+			existed = false
+		case err != nil:
+			return nil, err
+		}
+
+		// Zero time ("0001-01-01T00:00:00Z") means "no end known" — that is the
+		// convention of Alertmanager's ingest API. Outgoing notifications
+		// normally carry a real future endsAt instead, which is the watermark
+		// the sweeper uses to expire alerts that stop being refreshed.
+		var endsAtUnix *int64
+		if a.EndsAt.Year() > 1 {
+			t := a.EndsAt.Unix()
+			endsAtUnix = &t
+		}
+
+		var resolutionSource *string
+		if a.Status == "resolved" {
+			s := resolutionAlertmanager
+			resolutionSource = &s
+		}
+
+		// The WHERE clause discards payloads that describe an alert instance
+		// older than the stored one. Alertmanager retries failed notifications,
+		// so a stale firing retry can arrive after the resolved one; it carries
+		// the same startsAt, whereas a genuine re-fire carries a newer one.
+		// Within a single instance, resolution is terminal.
+		if _, err := tx.ExecContext(ctx, `
+			INSERT INTO alerts
+				(fingerprint, name, status, labels, annotations, starts_at, ends_at,
+				 generator_url, received_at, resolution_source)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			ON CONFLICT(fingerprint) DO UPDATE SET
+				status            = excluded.status,
+				labels            = excluded.labels,
+				annotations       = excluded.annotations,
+				starts_at         = excluded.starts_at,
+				ends_at           = excluded.ends_at,
+				generator_url     = excluded.generator_url,
+				-- Load-bearing: advancing received_at on every accepted
+				-- payload, re-sends included, is the documented liveness
+				-- heartbeat clients and the sweeper both read. Removing it
+				-- is a breaking API change — see models.Alert.ReceivedAt.
+				received_at       = excluded.received_at,
+				resolution_source = excluded.resolution_source,
+				-- A re-fire makes the alert current again, so it leaves the archive.
+				archived_at       = CASE WHEN excluded.status = 'firing'
+				                        THEN NULL ELSE alerts.archived_at END
+			WHERE excluded.starts_at > alerts.starts_at
+			   OR (excluded.starts_at = alerts.starts_at
+			       AND NOT (alerts.status = 'resolved' AND excluded.status = 'firing'))`,
+			a.Fingerprint, name, a.Status,
+			string(labelsJSON), string(annotationsJSON),
+			a.StartsAt.Unix(), endsAtUnix,
+			a.GeneratorURL, now, resolutionSource,
+		); err != nil {
+			return nil, err
+		}
+
+		var id int64
+		var curStatus string
+		var curStartsAt int64
+		if err := tx.QueryRowContext(ctx,
+			"SELECT id, status, starts_at FROM alerts WHERE fingerprint = ?", a.Fingerprint,
+		).Scan(&id, &curStatus, &curStartsAt); err != nil {
+			return nil, err
+		}
+
+		// The upsert copies status and starts_at straight from the payload, so a
+		// row that does not match it is one the ordering guard rejected. A
+		// discarded payload describes a past instance and must not touch the
+		// incident state either.
+		if existed && (curStatus != a.Status || curStartsAt != a.StartsAt.Unix()) {
+			continue
+		}
+
+		firing := a.Status == "firing"
+		accepted = append(accepted, ingested{
+			id:            id,
+			name:          name,
+			firing:        firing,
+			newOccurrence: firing && (!existed || a.StartsAt.Unix() > prevStartsAt || prevStatus == "resolved"),
+			justResolved:  !firing && existed && prevStatus == "firing",
+		})
+	}
+
+	return accepted, nil
+}
+
+// incidentForGroup returns the open incident that this payload's firing alerts
+// belong to, opening one if the group has none. It returns 0 when the payload
+// warrants no incident at all.
+//
+// The rule that matters: a group with no open incident gets a new one only if
+// something actually started firing. Without that, a manually resolved incident
+// would reappear on the next repeat_interval re-send of an alert that never
+// stopped, and manual resolution would be meaningless.
+func incidentForGroup(ctx context.Context, tx *sql.Tx, notify NotifyConfig, payload amPayload, accepted []ingested) (int64, error) {
+	var firstName string
+	anyFiring, anyNew := false, false
+	for _, a := range accepted {
+		if a.firing {
+			if !anyFiring {
+				firstName = a.name
+			}
+			anyFiring = true
+		}
+		if a.newOccurrence {
+			anyNew = true
+		}
+	}
+	if !anyFiring {
+		// A payload of nothing but resolutions never opens an incident.
+		return 0, nil
+	}
+
+	groupKey := payload.GroupKey
+	if groupKey == "" {
+		// Alertmanager always sends groupKey; a sender that does not still gets
+		// one incident per alert name rather than one giant shared incident.
+		groupKey = "groupless:" + firstName
+	}
+
+	var id int64
+	switch err := tx.QueryRowContext(ctx,
+		"SELECT id FROM incidents WHERE group_key = ? AND resolved_at IS NULL", groupKey,
+	).Scan(&id); {
+	case err == nil:
+		return id, nil
+	case err != sql.ErrNoRows:
+		return 0, err
+	}
+
+	if !anyNew {
+		return 0, nil
+	}
+	return openIncident(ctx, tx, notify, groupKey, payload.GroupLabels, firstName)
+}
+
+// openIncident creates an incident for a group and assigns it to whoever is on
+// call today, which is the point at which the schedule stops being decorative.
+func openIncident(ctx context.Context, tx *sql.Tx, notify NotifyConfig, groupKey string, groupLabels map[string]string, fallbackName string) (int64, error) {
+	onCall, err := currentOnCall(ctx, tx)
+	if err != nil {
+		return 0, err
+	}
+
+	labelsJSON, _ := json.Marshal(groupLabels)
+	if groupLabels == nil {
+		labelsJSON = []byte("{}")
+	}
+
+	res, err := tx.ExecContext(ctx, `
+		INSERT INTO incidents (group_key, title, group_labels, status, triggered_at, assigned_to)
+		VALUES (?, ?, ?, 'triggered', ?, ?)`,
+		groupKey, incidentTitle(groupLabels, fallbackName), string(labelsJSON),
+		time.Now().Unix(), onCall)
+	if err != nil {
+		return 0, err
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		return 0, err
+	}
+
+	if err := logEvent(ctx, tx, id, evTriggered, nil, nil, nil); err != nil {
+		return 0, err
+	}
+	if onCall != nil {
+		// On an "assigned" event user_id is the assignee, not the actor.
+		if err := logEvent(ctx, tx, id, evAssigned, onCall, nil, nil); err != nil {
+			return 0, err
+		}
+	}
+
+	// Queue the page, but do not send it here: this runs inside the webhook's
+	// transaction on a single-connection pool, so an HTTP call would hold up
+	// every other request. The notifier picks the row up within a tick.
+	if err := enqueueOpened(ctx, tx, notify, id, onCall); err != nil {
+		return 0, err
+	}
+	return id, nil
+}
+
+// linkAlert adds an alert to an incident, emitting a timeline entry only the
+// first time. Re-sends of an already-linked alert are silent.
+func linkAlert(ctx context.Context, tx *sql.Tx, incidentID, alertID int64) error {
+	res, err := tx.ExecContext(ctx, `
+		INSERT OR IGNORE INTO incident_alerts (incident_id, alert_id, added_at)
+		VALUES (?, ?, ?)`, incidentID, alertID, time.Now().Unix())
+	if err != nil {
+		return err
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return nil
+	}
+	return logEvent(ctx, tx, incidentID, evAlertAdded, nil, &alertID, nil)
+}

internal/api/alerts.go

@@ -15,14 +15,21 @@ import (
 )
 
 // alertSelectFrom is the shared SELECT … FROM … clause used by all alert queries.
-// It LEFT JOINs users so acknowledged_by username is always available.
+// The subquery resolves the alert's most recent incident: membership is kept in
+// incident_alerts rather than as a column here, because one alert row is reused
+// across occurrences and belongs to a different incident each time.
 const alertSelectFrom = `
 	SELECT a.id, a.fingerprint, a.name, a.status,
 	       a.labels, a.annotations,
 	       a.starts_at, a.ends_at, a.generator_url, a.received_at,
-	       a.acknowledged_by, a.acknowledged_at, u.username
+	       (SELECT ia.incident_id
-	FROM   alerts a
+	          FROM incident_alerts ia
-	LEFT   JOIN users u ON u.id = a.acknowledged_by`
+	          JOIN incidents i ON i.id = ia.incident_id
+	         WHERE ia.alert_id = a.id
+	         ORDER BY i.triggered_at DESC, i.id DESC
+	         LIMIT 1),
+	       a.resolution_source, a.archived_at
+	FROM   alerts a`
 
 func handleListAlerts(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -39,6 +46,18 @@ func handleListAlerts(db *sql.DB) http.HandlerFunc {
 			where = append(where, "a.name = ?")
 			args = append(args, name)
 		}
+		if archived := q.Get("archived"); archived == "true" {
+			where = append(where, "a.archived_at IS NOT NULL")
+		} else {
+			where = append(where, "a.archived_at IS NULL")
+		}
+		if incidentID := q.Get("incident_id"); incidentID != "" {
+			if n, err := strconv.ParseInt(incidentID, 10, 64); err == nil {
+				where = append(where, "a.id IN (SELECT alert_id FROM incident_alerts WHERE incident_id = ?)")
+				args = append(args, n)
+			}
+		}
+
 		if from := q.Get("from"); from != "" {
 			if t, err := time.Parse("2006-01-02", from); err == nil {
 				where = append(where, "a.received_at >= ?")
@@ -107,55 +126,7 @@ func handleGetAlert(db *sql.DB) http.HandlerFunc {
 	}
 }
 
-func handleAcknowledge(db *sql.DB) http.HandlerFunc {
+// fetchAlert loads a single alert by ID using the shared query.
-	return func(w http.ResponseWriter, r *http.Request) {
-		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid alert id"))
-			return
-		}
-		user, _ := userFromContext(r.Context())
-
-		res, err := db.ExecContext(r.Context(),
-			"UPDATE alerts SET acknowledged_by = ?, acknowledged_at = ? WHERE id = ?",
-			user.ID, time.Now().Unix(), id)
-		if err != nil {
-			respond(w, http.StatusInternalServerError, errResp("internal error"))
-			return
-		}
-		if n, _ := res.RowsAffected(); n == 0 {
-			respond(w, http.StatusNotFound, errResp("alert not found"))
-			return
-		}
-
-		a, _ := fetchAlert(r.Context(), db, id)
-		respond(w, http.StatusOK, a)
-	}
-}
-
-func handleUnacknowledge(db *sql.DB) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid alert id"))
-			return
-		}
-
-		res, err := db.ExecContext(r.Context(),
-			"UPDATE alerts SET acknowledged_by = NULL, acknowledged_at = NULL WHERE id = ?", id)
-		if err != nil {
-			respond(w, http.StatusInternalServerError, errResp("internal error"))
-			return
-		}
-		if n, _ := res.RowsAffected(); n == 0 {
-			respond(w, http.StatusNotFound, errResp("alert not found"))
-			return
-		}
-		w.WriteHeader(http.StatusNoContent)
-	}
-}
-
-// fetchAlert loads a single alert by ID using the shared JOIN query.
 func fetchAlert(ctx context.Context, db *sql.DB, id int64) (models.Alert, error) {
 	return scanAlert(db.QueryRowContext(ctx, alertSelectFrom+" WHERE a.id = ?", id))
 }
@@ -169,33 +140,24 @@ func scanAlert(s scanner) (models.Alert, error) {
 	var a models.Alert
 	var labelsJSON, annotationsJSON string
 	var startsAtUnix, receivedAtUnix int64
-	var endsAtUnix, ackAtUnix *int64
+	var endsAtUnix, archivedAtUnix *int64
-	var ackByID *int64
-	var ackByUser *string
 
 	if err := s.Scan(
 		&a.ID, &a.Fingerprint, &a.Name, &a.Status,
 		&labelsJSON, &annotationsJSON,
 		&startsAtUnix, &endsAtUnix,
 		&a.GeneratorURL, &receivedAtUnix,
-		&ackByID, &ackAtUnix, &ackByUser,
+		&a.IncidentID,
+		&a.ResolutionSource, &archivedAtUnix,
 	); err != nil {
 		return a, err
 	}
 
-	json.Unmarshal([]byte(labelsJSON), &a.Labels)          //nolint:errcheck
+	json.Unmarshal([]byte(labelsJSON), &a.Labels)           //nolint:errcheck
 	json.Unmarshal([]byte(annotationsJSON), &a.Annotations) //nolint:errcheck
 	a.StartsAt = time.Unix(startsAtUnix, 0).UTC()
 	a.ReceivedAt = time.Unix(receivedAtUnix, 0).UTC()
-	if endsAtUnix != nil {
+	a.EndsAt = unixPtr(endsAtUnix)
-		t := time.Unix(*endsAtUnix, 0).UTC()
+	a.ArchivedAt = unixPtr(archivedAtUnix)
-		a.EndsAt = &t
-	}
-	if ackByID != nil {
-		t := time.Unix(*ackAtUnix, 0).UTC()
-		a.AcknowledgedByID = ackByID
-		a.AcknowledgedByUser = ackByUser
-		a.AcknowledgedAt = &t
-	}
 	return a, nil
 }

internal/api/api_test.go

@@ -2,25 +2,38 @@ package api_test
 
 import (
 	"bytes"
+	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
+	"time"
 
 	"github.com/yeniklas/terdut-server/internal/api"
 	"github.com/yeniklas/terdut-server/internal/db"
 )
 
-// ts wraps httptest.Server with a pre-bootstrapped API key.
+// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
+// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
 type ts struct {
 	*httptest.Server
-	key string
+	key    string
+	db     *sql.DB
+	notify api.NotifyConfig
 }
 
-func newTS(t *testing.T) *ts {
+// newTS builds a server over a fresh in-memory database. Notifications are off
+// unless a NotifyConfig is passed, so tests that predate them are unaffected.
+func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
 	t.Helper()
+	var cfg api.NotifyConfig
+	if len(notify) > 0 {
+		cfg = notify[0]
+	}
+
 	database, err := db.Open(":memory:")
 	if err != nil {
 		t.Fatalf("open db: %v", err)
@@ -28,7 +41,7 @@ func newTS(t *testing.T) *ts {
 	if err := db.Migrate(database); err != nil {
 		t.Fatalf("migrate: %v", err)
 	}
-	srv := httptest.NewServer(api.NewRouter(database))
+	srv := httptest.NewServer(api.NewRouter(database, cfg))
 	t.Cleanup(func() { srv.Close(); database.Close() })
 
 	body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
@@ -44,7 +57,50 @@ func newTS(t *testing.T) *ts {
 	json.NewDecoder(resp.Body).Decode(&result)
 	key := result["api_key"].(map[string]any)["key"].(string)
 
-	return &ts{Server: srv, key: key}
+	return &ts{Server: srv, key: key, db: database, notify: cfg}
+}
+
+// exec runs a statement against the test database.
+func (s *ts) exec(t *testing.T, query string, args ...any) {
+	t.Helper()
+	if _, err := s.db.Exec(query, args...); err != nil {
+		t.Fatalf("exec %q: %v", query, err)
+	}
+}
+
+// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
+func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
+	t.Helper()
+	err := s.db.QueryRow(
+		"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = ?",
+		fingerprint).Scan(&status, &source, &archivedAt)
+	if err != nil {
+		t.Fatalf("read alert %s: %v", fingerprint, err)
+	}
+	return status, source, archivedAt
+}
+
+// alertTimes reads the timestamp columns that make up the received_at contract.
+func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
+	t.Helper()
+	err := s.db.QueryRow(
+		"SELECT starts_at, received_at FROM alerts WHERE fingerprint = ?",
+		fingerprint).Scan(&startsAt, &receivedAt)
+	if err != nil {
+		t.Fatalf("read alert times %s: %v", fingerprint, err)
+	}
+	return startsAt, receivedAt
+}
+
+// alertEndsAt reads the nullable ends_at column of one alert.
+func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
+	t.Helper()
+	var endsAt *int64
+	if err := s.db.QueryRow(
+		"SELECT ends_at FROM alerts WHERE fingerprint = ?", fingerprint).Scan(&endsAt); err != nil {
+		t.Fatalf("read ends_at %s: %v", fingerprint, err)
+	}
+	return endsAt
 }
 
 // req sends an authenticated request, optionally with a JSON body.
@@ -123,9 +179,22 @@ func TestBootstrap_SecondCallForbidden(t *testing.T) {
 // Alert upsert by fingerprint
 // ---------------------------------------------------------------------------
 
-func postWebhook(t *testing.T, s *ts, alerts []map[string]any) {
+// postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting
+// it exercises the fallback for senders that do not group, which is what most of
+// these tests want.
+func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) {
 	t.Helper()
 	payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
+	if len(groupKey) > 0 {
+		payload["groupKey"] = groupKey[0]
+		// Alertmanager groups by alertname by default, so the group labels echo
+		// the first alert's name.
+		if len(alerts) > 0 {
+			if labels, ok := alerts[0]["labels"].(map[string]string); ok {
+				payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]}
+			}
+		}
+	}
 	data, _ := json.Marshal(payload)
 	resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
 	if err != nil {
@@ -195,84 +264,6 @@ func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
 	}
 }
 
-// ---------------------------------------------------------------------------
-// Alert acknowledge
-// ---------------------------------------------------------------------------
-
-func TestAcknowledge(t *testing.T) {
-	s := newTS(t)
-	postWebhook(t, s, []map[string]any{{
-		"status": "firing", "labels": map[string]string{"alertname": "X"},
-		"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
-		"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-ack",
-	}})
-
-	resp := s.req(t, http.MethodPost, "/api/alerts/1/acknowledge", nil)
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("acknowledge returned %d", resp.StatusCode)
-	}
-	var alert map[string]any
-	decode(t, resp, &alert)
-	if alert["acknowledged_by"] == nil {
-		t.Error("expected acknowledged_by to be set")
-	}
-
-	// Clear it.
-	resp = s.req(t, http.MethodDelete, "/api/alerts/1/acknowledge", nil)
-	if resp.StatusCode != http.StatusNoContent {
-		t.Errorf("unacknowledge returned %d", resp.StatusCode)
-	}
-
-	resp = s.req(t, http.MethodGet, "/api/alerts/1", nil)
-	var alert2 map[string]any
-	decode(t, resp, &alert2)
-	if alert2["acknowledged_by"] != nil {
-		t.Error("expected acknowledged_by to be cleared")
-	}
-}
-
-// ---------------------------------------------------------------------------
-// Comments — own-only deletion
-// ---------------------------------------------------------------------------
-
-func TestComment_DeleteOwnOnly(t *testing.T) {
-	s := newTS(t)
-	postWebhook(t, s, []map[string]any{{
-		"status": "firing", "labels": map[string]string{"alertname": "Y"},
-		"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
-		"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-comment",
-	}})
-
-	// Create a second user and their own key.
-	s.req(t, http.MethodPost, "/api/users",
-		map[string]string{"username": "alice", "email": "alice@test.com"})
-	keyResp := s.req(t, http.MethodPost, "/api/users/2/api-keys",
-		map[string]string{"name": "alice-key"})
-	var keyData map[string]any
-	decode(t, keyResp, &keyData)
-	aliceKey := keyData["key"].(string)
-
-	// Admin posts a comment.
-	s.req(t, http.MethodPost, "/api/alerts/1/comments",
-		map[string]string{"content": "admin note"})
-
-	// Alice tries to delete admin's comment (should 404).
-	req, _ := http.NewRequest(http.MethodDelete, s.URL+"/api/alerts/1/comments/1", nil)
-	req.Header.Set("Authorization", "Bearer "+aliceKey)
-	resp, _ := http.DefaultClient.Do(req)
-	resp.Body.Close()
-	if resp.StatusCode != http.StatusNotFound {
-		t.Errorf("expected 404 when deleting another user's comment, got %d", resp.StatusCode)
-	}
-
-	// Admin deletes own comment (should 204).
-	resp = s.req(t, http.MethodDelete, "/api/alerts/1/comments/1", nil)
-	resp.Body.Close()
-	if resp.StatusCode != http.StatusNoContent {
-		t.Errorf("expected 204 when deleting own comment, got %d", resp.StatusCode)
-	}
-}
-
 // ---------------------------------------------------------------------------
 // Schedule conflict
 // ---------------------------------------------------------------------------
@@ -357,6 +348,332 @@ func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
 	}
 }
 
+// ---------------------------------------------------------------------------
+// Archive
+// ---------------------------------------------------------------------------
+
+// Alert archiving is sweeper-only housekeeping now — nobody archives an alert by
+// hand — but the list filter it drives is still part of the API.
+func TestArchive_AlertListFilter(t *testing.T) {
+	s := newTS(t)
+
+	postWebhook(t, s, []map[string]any{{
+		"status": "resolved", "fingerprint": "arch1",
+		"labels":       map[string]string{"alertname": "Archivable"},
+		"annotations":  map[string]string{},
+		"startsAt":     "2026-05-20T10:00:00Z",
+		"endsAt":       "2026-05-20T11:00:00Z",
+		"generatorURL": "",
+	}})
+
+	// 1. Alert appears in the default list.
+	var alerts []map[string]any
+	decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
+	if len(alerts) != 1 {
+		t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
+	}
+
+	// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
+	api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour)
+
+	// 3. Default list excludes it.
+	decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
+	if len(alerts) != 0 {
+		t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
+	}
+
+	// 4. archived=true shows it.
+	decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
+	if len(alerts) != 1 {
+		t.Fatalf("expected 1 archived alert, got %d", len(alerts))
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Stale-alert expiry
+// ---------------------------------------------------------------------------
+
+// noArchive is long enough that archiving never interferes with expiry tests.
+const noArchive = 365 * 24 * time.Hour
+
+// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
+const zeroTime = "0001-01-01T00:00:00Z"
+
+// postAlert sends a single-alert webhook.
+func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
+	t.Helper()
+	postWebhook(t, s, []map[string]any{{
+		"status":       status,
+		"labels":       map[string]string{"alertname": "Stale"},
+		"annotations":  map[string]string{},
+		"startsAt":     startsAt,
+		"endsAt":       endsAt,
+		"generatorURL": "",
+		"fingerprint":  fingerprint,
+	}})
+}
+
+func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
+	t.Helper()
+	api.Sweep(context.Background(), s.db, noArchive, staleAfter)
+}
+
+// A firing alert Alertmanager stopped refreshing is resolved via the
+// received_at heartbeat, even with no ends_at watermark to go on.
+func TestExpiry_StaleFiringAlert(t *testing.T) {
+	s := newTS(t)
+	postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
+
+	// Age the last-seen timestamp past the staleness window.
+	s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'stale1'",
+		time.Now().Add(-10*time.Hour).Unix())
+
+	sweep(t, s, 6*time.Hour)
+
+	status, source, _ := s.alertRow(t, "stale1")
+	if status != "resolved" {
+		t.Errorf("expected status resolved, got %q", status)
+	}
+	if source == nil || *source != "expiry" {
+		t.Errorf("expected resolution_source=expiry, got %v", source)
+	}
+}
+
+// A fresh webhook whose ends_at watermark has already passed is expired without
+// waiting out the full staleness window.
+func TestExpiry_PastEndsAt(t *testing.T) {
+	s := newTS(t)
+	postAlert(t, s, "stale2", "firing",
+		time.Now().Add(-2*time.Hour).Format(time.RFC3339),
+		time.Now().Add(-30*time.Minute).Format(time.RFC3339))
+
+	sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
+
+	status, source, _ := s.alertRow(t, "stale2")
+	if status != "resolved" {
+		t.Errorf("expected status resolved, got %q", status)
+	}
+	if source == nil || *source != "expiry" {
+		t.Errorf("expected resolution_source=expiry, got %v", source)
+	}
+}
+
+// The regression that matters most: a genuinely firing alert must survive a
+// sweep untouched.
+func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
+	s := newTS(t)
+	postAlert(t, s, "fresh1", "firing",
+		time.Now().Add(-10*time.Minute).Format(time.RFC3339),
+		time.Now().Add(1*time.Hour).Format(time.RFC3339))
+
+	sweep(t, s, 6*time.Hour)
+
+	status, source, _ := s.alertRow(t, "fresh1")
+	if status != "firing" {
+		t.Errorf("expected fresh alert to stay firing, got %q", status)
+	}
+	if source != nil {
+		t.Errorf("expected no resolution_source, got %q", *source)
+	}
+}
+
+// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
+func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
+	s := newTS(t)
+	postAlert(t, s, "grace1", "firing",
+		time.Now().Add(-time.Hour).Format(time.RFC3339),
+		time.Now().Add(-1*time.Minute).Format(time.RFC3339))
+
+	sweep(t, s, 6*time.Hour)
+
+	if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
+		t.Errorf("expected alert within grace period to stay firing, got %q", status)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Webhook resolution bookkeeping
+// ---------------------------------------------------------------------------
+
+func TestWebhook_ResolvedSetsSource(t *testing.T) {
+	s := newTS(t)
+	start := time.Now().Add(-time.Hour).Format(time.RFC3339)
+	postAlert(t, s, "src1", "firing", start, zeroTime)
+
+	if _, source, _ := s.alertRow(t, "src1"); source != nil {
+		t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
+	}
+
+	postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
+
+	status, source, _ := s.alertRow(t, "src1")
+	if status != "resolved" {
+		t.Errorf("expected status resolved, got %q", status)
+	}
+	if source == nil || *source != "alertmanager" {
+		t.Errorf("expected resolution_source=alertmanager, got %v", source)
+	}
+}
+
+// A re-fire under the same fingerprint must leave the archive and clear the
+// stale expiry marker, otherwise the alert stays invisible in the default list.
+func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
+	s := newTS(t)
+	postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
+
+	// Expire it, then archive it.
+	s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'refire1'",
+		time.Now().Add(-10*time.Hour).Unix())
+	sweep(t, s, 6*time.Hour)
+	s.exec(t, "UPDATE alerts SET archived_at = unixepoch() WHERE fingerprint = 'refire1'")
+
+	var alerts []map[string]any
+	decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
+	if len(alerts) != 0 {
+		t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
+	}
+
+	// Fires again: a new alert instance, so a newer startsAt.
+	postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
+
+	status, source, archivedAt := s.alertRow(t, "refire1")
+	if status != "firing" {
+		t.Errorf("expected status firing after re-fire, got %q", status)
+	}
+	if source != nil {
+		t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
+	}
+	if archivedAt != nil {
+		t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
+	}
+
+	decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
+	if len(alerts) != 1 {
+		t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
+	}
+}
+
+// Alertmanager retries failed notifications, so a firing payload for an
+// already-resolved instance can arrive late. It must not resurrect the alert.
+func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
+	s := newTS(t)
+	start := time.Now().Add(-time.Hour).Format(time.RFC3339)
+	end := time.Now().Format(time.RFC3339)
+
+	postAlert(t, s, "ooo1", "firing", start, zeroTime)
+	postAlert(t, s, "ooo1", "resolved", start, end)
+	postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
+
+	status, source, _ := s.alertRow(t, "ooo1")
+	if status != "resolved" {
+		t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
+	}
+	if source == nil || *source != "alertmanager" {
+		t.Errorf("expected resolution_source=alertmanager, got %v", source)
+	}
+}
+
+// An expiry resolve writes ends_at as an upper bound, not an observed end: an
+// Alertmanager watermark already on the row is preserved, and a row that never
+// carried one is stamped at sweep time. Clients are told to read it that way —
+// see "resolution_source says how much to trust ends_at" in the README.
+func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
+	s := newTS(t)
+
+	// No watermark: expires on the received_at heartbeat, so the sweeper has
+	// nothing to go on but its own clock.
+	postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
+	s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'ub-none'",
+		time.Now().Add(-10*time.Hour).Unix())
+
+	// Stale watermark: expires on the ends_at branch, and that reported time
+	// must survive the resolve rather than be overwritten with sweep time.
+	watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
+	postAlert(t, s, "ub-mark", "firing",
+		time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
+
+	sweep(t, s, 6*time.Hour)
+
+	if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
+		t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
+	}
+	stamped := s.alertEndsAt(t, "ub-none")
+	if stamped == nil {
+		t.Fatal("expected expiry to stamp ends_at when no watermark was known")
+	}
+	if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
+		t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
+	}
+
+	if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
+		t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
+	}
+	switch kept := s.alertEndsAt(t, "ub-mark"); {
+	case kept == nil:
+		t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
+	case *kept != watermark.Unix():
+		t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// received_at heartbeat contract
+//
+// received_at is documented as a public liveness signal, so these lock the
+// behaviour clients are told they may rely on. See "received_at is a liveness
+// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
+// ---------------------------------------------------------------------------
+
+// The heartbeat itself: an unchanged firing notification — what Alertmanager
+// re-sends every repeat_interval — must advance received_at, while leaving
+// starts_at, which identifies the alert instance, untouched.
+func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
+	s := newTS(t)
+	start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
+	postAlert(t, s, "beat1", "firing", start, zeroTime)
+
+	startsBefore, _ := s.alertTimes(t, "beat1")
+
+	// received_at has one-second granularity, so back-date it to make the bump
+	// observable instead of sleeping out a second.
+	aged := time.Now().Add(-2 * time.Hour).Unix()
+	s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat1'", aged)
+
+	// Identical re-send: same fingerprint, same startsAt, still firing.
+	postAlert(t, s, "beat1", "firing", start, zeroTime)
+
+	startsAfter, receivedAfter := s.alertTimes(t, "beat1")
+	if receivedAfter <= aged {
+		t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
+	}
+	if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
+		t.Errorf("expected received_at to track the server clock, off by %ds", skew)
+	}
+	if startsAfter != startsBefore {
+		t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
+	}
+}
+
+// received_at tracks accepted payloads, not delivery attempts: a retry
+// describing an already-resolved instance is discarded, so it must not register
+// as a heartbeat and revive the alert's apparent liveness.
+func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
+	s := newTS(t)
+	start := time.Now().Add(-time.Hour).Format(time.RFC3339)
+
+	postAlert(t, s, "beat2", "firing", start, zeroTime)
+	postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
+
+	aged := time.Now().Add(-2 * time.Hour).Unix()
+	s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat2'", aged)
+
+	postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
+
+	if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
+		t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
+	}
+}
+
 func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
 	s := newTS(t)
 	resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)

internal/api/archiver.go

@@ -0,0 +1,225 @@
+package api
+
+import (
+	"context"
+	"database/sql"
+	"log"
+	"strings"
+	"time"
+)
+
+const (
+	// sweepInterval is how often the background sweeper runs.
+	sweepInterval = 15 * time.Minute
+
+	// expiryGrace absorbs clock skew and notification latency before an alert
+	// whose ends_at watermark has passed is treated as stale.
+	expiryGrace = 5 * time.Minute
+)
+
+// StartArchiver runs the alert sweeper until ctx is cancelled, starting with an
+// immediate pass so a restart reconciles state right away.
+func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) {
+	ticker := time.NewTicker(sweepInterval)
+	defer ticker.Stop()
+
+	Sweep(ctx, db, archiveAfter, staleAfter)
+	for {
+		select {
+		case <-ticker.C:
+			Sweep(ctx, db, archiveAfter, staleAfter)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// Sweep runs a single pass, in dependency order: expire stale firing alerts,
+// close the incidents that leaves with nothing firing, then archive whatever has
+// been settled long enough. Running them in one pass means an alert can go stale
+// and its incident can close and archive without waiting three ticks.
+// Exported so tests can drive a pass without waiting on the ticker.
+func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration) {
+	expireStale(ctx, db, staleAfter)
+	resolveSettledIncidents(ctx, db)
+	archiveResolved(ctx, db, archiveAfter)
+	archiveResolvedIncidents(ctx, db, archiveAfter)
+	purgeAckTokens(ctx, db)
+}
+
+// expireStale resolves firing alerts that Alertmanager has stopped refreshing.
+//
+// A resolved webhook is otherwise the only way out of the firing state, so a
+// notification that is dropped, silenced, or lost to a restart would pin the
+// alert as firing forever. Two independent signals mark an alert stale:
+//
+//   - ends_at, the "valid until" watermark Alertmanager sets on outgoing firing
+//     notifications, has passed (plus expiryGrace for clock skew). Absent on
+//     rows whose payload carried no ends_at, hence the second signal.
+//   - received_at is older than staleAfter. Alertmanager re-sends firing
+//     notifications every repeat_interval, making received_at a liveness
+//     heartbeat — provided staleAfter exceeds that interval.
+//
+// The matching rows are collected before the update rather than updated in bulk,
+// because each one owes its incident a timeline entry.
+func expireStale(ctx context.Context, db *sql.DB, staleAfter time.Duration) {
+	now := time.Now()
+
+	ids, err := staleAlertIDs(ctx, db, now, staleAfter)
+	if err != nil {
+		log.Printf("sweeper: find stale: %v", err)
+		return
+	}
+	if len(ids) == 0 {
+		return
+	}
+
+	args := make([]any, 0, len(ids)+1)
+	args = append(args, resolutionExpiry)
+	for _, id := range ids {
+		args = append(args, id)
+	}
+	if _, err := db.ExecContext(ctx, `
+		UPDATE alerts
+		   SET status            = 'resolved',
+		       resolution_source = ?,
+		       ends_at           = COALESCE(ends_at, unixepoch())
+		 WHERE id IN (`+placeholders(len(ids))+`)`, args...); err != nil {
+		log.Printf("sweeper: expire stale: %v", err)
+		return
+	}
+	log.Printf("sweeper: expired %d stale firing alert(s)", len(ids))
+
+	for _, id := range ids {
+		incidentID, err := openIncidentForAlert(ctx, db, id)
+		if err != nil {
+			log.Printf("sweeper: incident for alert %d: %v", id, err)
+			continue
+		}
+		if incidentID == 0 {
+			continue
+		}
+		alertID := id
+		if err := logEvent(ctx, db, incidentID, evAlertResolved, nil, &alertID, nil); err != nil {
+			log.Printf("sweeper: log expiry event: %v", err)
+		}
+	}
+}
+
+// staleAlertIDs reads the ids in one go and closes the cursor before the caller
+// writes: the pool is limited to a single connection, so an open read would
+// block the update behind it.
+func staleAlertIDs(ctx context.Context, db *sql.DB, now time.Time, staleAfter time.Duration) ([]int64, error) {
+	rows, err := db.QueryContext(ctx, `
+		SELECT id FROM alerts
+		 WHERE status = 'firing'
+		   AND archived_at IS NULL
+		   AND ((ends_at IS NOT NULL AND ends_at < ?) OR received_at < ?)`,
+		now.Add(-expiryGrace).Unix(), now.Add(-staleAfter).Unix())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var ids []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		ids = append(ids, id)
+	}
+	return ids, rows.Err()
+}
+
+// resolveSettledIncidents closes incidents whose alerts have all stopped firing.
+// This is the cascade from alerts up to the work item, and it is what turns an
+// expiry into a closed incident rather than one that sits open forever.
+func resolveSettledIncidents(ctx context.Context, db *sql.DB) {
+	ids, err := settledIncidentIDs(ctx, db)
+	if err != nil {
+		log.Printf("sweeper: find settled incidents: %v", err)
+		return
+	}
+
+	resolved := 0
+	for _, id := range ids {
+		ok, err := resolveIfSettled(ctx, db, id)
+		if err != nil {
+			log.Printf("sweeper: resolve incident %d: %v", id, err)
+			continue
+		}
+		if ok {
+			resolved++
+		}
+	}
+	if resolved > 0 {
+		log.Printf("sweeper: resolved %d settled incident(s)", resolved)
+	}
+}
+
+func settledIncidentIDs(ctx context.Context, db *sql.DB) ([]int64, error) {
+	rows, err := db.QueryContext(ctx, `
+		SELECT i.id
+		FROM   incidents i
+		WHERE  i.resolved_at IS NULL
+		  AND  EXISTS (SELECT 1 FROM incident_alerts ia WHERE ia.incident_id = i.id)
+		  AND  NOT EXISTS (SELECT 1
+		                     FROM incident_alerts ia
+		                     JOIN alerts a ON a.id = ia.alert_id
+		                    WHERE ia.incident_id = i.id
+		                      AND a.status = 'firing')`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var ids []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		ids = append(ids, id)
+	}
+	return ids, rows.Err()
+}
+
+// archiveResolved hides resolved alerts that have been settled for archiveAfter.
+func archiveResolved(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
+	cutoff := time.Now().Add(-archiveAfter).Unix()
+	res, err := db.ExecContext(ctx,
+		`UPDATE alerts SET archived_at = unixepoch()
+		 WHERE status = 'resolved'
+		   AND archived_at IS NULL
+		   AND COALESCE(ends_at, received_at) < ?`, cutoff)
+	if err != nil {
+		log.Printf("archiver: %v", err)
+		return
+	}
+	if n, _ := res.RowsAffected(); n > 0 {
+		log.Printf("archiver: archived %d resolved alert(s)", n)
+	}
+}
+
+// archiveResolvedIncidents does the same for the work items, on the same clock.
+func archiveResolvedIncidents(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
+	cutoff := time.Now().Add(-archiveAfter).Unix()
+	res, err := db.ExecContext(ctx,
+		`UPDATE incidents SET archived_at = unixepoch()
+		 WHERE resolved_at IS NOT NULL
+		   AND archived_at IS NULL
+		   AND resolved_at < ?`, cutoff)
+	if err != nil {
+		log.Printf("archiver: incidents: %v", err)
+		return
+	}
+	if n, _ := res.RowsAffected(); n > 0 {
+		log.Printf("archiver: archived %d resolved incident(s)", n)
+	}
+}
+
+// placeholders builds "?, ?, …" for an IN clause of n values.
+func placeholders(n int) string {
+	return strings.TrimSuffix(strings.Repeat("?, ", n), ", ")
+}

internal/api/comments.go

@@ -1,131 +0,0 @@
-package api
-
-import (
-	"database/sql"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/yeniklas/terdut-server/internal/models"
-)
-
-func handleListComments(db *sql.DB) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		alertID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid alert id"))
-			return
-		}
-
-		// Verify the alert exists.
-		var exists int
-		if err := db.QueryRowContext(r.Context(), "SELECT 1 FROM alerts WHERE id = ?", alertID).Scan(&exists); err != nil {
-			respond(w, http.StatusNotFound, errResp("alert not found"))
-			return
-		}
-
-		rows, err := db.QueryContext(r.Context(), `
-			SELECT c.id, c.alert_id, c.user_id, u.username, c.content, c.created_at
-			FROM   alert_comments c
-			JOIN   users u ON u.id = c.user_id
-			WHERE  c.alert_id = ?
-			ORDER  BY c.created_at ASC`, alertID)
-		if err != nil {
-			respond(w, http.StatusInternalServerError, errResp("internal error"))
-			return
-		}
-		defer rows.Close()
-
-		comments := []models.Comment{}
-		for rows.Next() {
-			var c models.Comment
-			var ts int64
-			if err := rows.Scan(&c.ID, &c.AlertID, &c.UserID, &c.Username, &c.Content, &ts); err != nil {
-				respond(w, http.StatusInternalServerError, errResp("internal error"))
-				return
-			}
-			c.CreatedAt = time.Unix(ts, 0).UTC()
-			comments = append(comments, c)
-		}
-		respond(w, http.StatusOK, comments)
-	}
-}
-
-func handleCreateComment(db *sql.DB) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		alertID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid alert id"))
-			return
-		}
-
-		var req struct {
-			Content string `json:"content"`
-		}
-		if err := decodeJSON(r, &req); err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid request body"))
-			return
-		}
-		if req.Content == "" {
-			respond(w, http.StatusBadRequest, errResp("content is required"))
-			return
-		}
-
-		// Verify the alert exists.
-		var exists int
-		if err := db.QueryRowContext(r.Context(), "SELECT 1 FROM alerts WHERE id = ?", alertID).Scan(&exists); err != nil {
-			respond(w, http.StatusNotFound, errResp("alert not found"))
-			return
-		}
-
-		user, _ := userFromContext(r.Context())
-		res, err := db.ExecContext(r.Context(),
-			"INSERT INTO alert_comments (alert_id, user_id, content) VALUES (?, ?, ?)",
-			alertID, user.ID, req.Content)
-		if err != nil {
-			respond(w, http.StatusInternalServerError, errResp("internal error"))
-			return
-		}
-		commentID, _ := res.LastInsertId()
-
-		comment := models.Comment{
-			ID:        commentID,
-			AlertID:   alertID,
-			UserID:    user.ID,
-			Username:  user.Username,
-			Content:   req.Content,
-			CreatedAt: time.Now().UTC(),
-		}
-		respond(w, http.StatusCreated, comment)
-	}
-}
-
-func handleDeleteComment(db *sql.DB) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		alertID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid alert id"))
-			return
-		}
-		commentID, err := strconv.ParseInt(chi.URLParam(r, "commentID"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid comment id"))
-			return
-		}
-
-		user, _ := userFromContext(r.Context())
-		res, err := db.ExecContext(r.Context(),
-			"DELETE FROM alert_comments WHERE id = ? AND alert_id = ? AND user_id = ?",
-			commentID, alertID, user.ID)
-		if err != nil {
-			respond(w, http.StatusInternalServerError, errResp("internal error"))
-			return
-		}
-		if n, _ := res.RowsAffected(); n == 0 {
-			respond(w, http.StatusNotFound, errResp("comment not found"))
-			return
-		}
-		w.WriteHeader(http.StatusNoContent)
-	}
-}

internal/api/incident_store.go

@@ -0,0 +1,294 @@
+package api
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/yeniklas/terdut-server/internal/models"
+)
+
+// Values for incidents.resolution_source, recording who closed the incident:
+// every member alert stopped firing, or a person decided it was done.
+const (
+	incidentResolutionAlerts = "alerts"
+	incidentResolutionManual = "manual"
+)
+
+// Incident timeline event types. Stored as free text so adding one later is not
+// a migration, but these are the ones the server writes.
+const (
+	evTriggered      = "triggered"
+	evAlertAdded     = "alert_added"
+	evAlertResolved  = "alert_resolved"
+	evAcknowledged   = "acknowledged"
+	evUnacknowledged = "unacknowledged"
+	evAssigned       = "assigned"
+	evSnoozed        = "snoozed"
+	evUnsnoozed      = "unsnoozed"
+	evResolved       = "resolved"
+	evNote           = "note"
+)
+
+// severityLabel is the Alertmanager label an incident's severity is derived from.
+const severityLabel = "severity"
+
+// querier is satisfied by both *sql.DB and *sql.Tx, so the helpers below work
+// inside the webhook's transaction and standalone from handlers and the sweeper.
+type querier interface {
+	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
+	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
+}
+
+const incidentSelectFrom = `
+	SELECT i.id, i.group_key, i.title, i.group_labels, i.status, i.severity,
+	       i.triggered_at,
+	       i.acknowledged_by, i.acknowledged_at, ack.username,
+	       i.assigned_to, asg.username, i.snoozed_until,
+	       i.resolved_at, i.resolution_source, i.archived_at
+	FROM   incidents i
+	LEFT   JOIN users ack ON ack.id = i.acknowledged_by
+	LEFT   JOIN users asg ON asg.id = i.assigned_to`
+
+func scanIncident(s scanner) (models.Incident, error) {
+	var i models.Incident
+	var groupLabelsJSON string
+	var triggeredAt int64
+	var ackAt, snoozedUntil, resolvedAt, archivedAt *int64
+
+	if err := s.Scan(
+		&i.ID, &i.GroupKey, &i.Title, &groupLabelsJSON, &i.Status, &i.Severity,
+		&triggeredAt,
+		&i.AcknowledgedByID, &ackAt, &i.AcknowledgedByUser,
+		&i.AssignedToID, &i.AssignedToUser, &snoozedUntil,
+		&resolvedAt, &i.ResolutionSource, &archivedAt,
+	); err != nil {
+		return i, err
+	}
+
+	json.Unmarshal([]byte(groupLabelsJSON), &i.GroupLabels) //nolint:errcheck
+	i.TriggeredAt = time.Unix(triggeredAt, 0).UTC()
+	i.AcknowledgedAt = unixPtr(ackAt)
+	i.SnoozedUntil = unixPtr(snoozedUntil)
+	i.ResolvedAt = unixPtr(resolvedAt)
+	i.ArchivedAt = unixPtr(archivedAt)
+	return i, nil
+}
+
+// unixPtr converts a nullable Unix-second column to a nullable UTC time.
+func unixPtr(sec *int64) *time.Time {
+	if sec == nil {
+		return nil
+	}
+	t := time.Unix(*sec, 0).UTC()
+	return &t
+}
+
+func fetchIncident(ctx context.Context, q querier, id int64) (models.Incident, error) {
+	return scanIncident(q.QueryRowContext(ctx, incidentSelectFrom+" WHERE i.id = ?", id))
+}
+
+// logEvent appends one entry to an incident's timeline. A nil userID means the
+// server acted rather than a person.
+func logEvent(ctx context.Context, q querier, incidentID int64, evType string, userID, alertID *int64, detail *string) error {
+	_, err := q.ExecContext(ctx, `
+		INSERT INTO incident_events (incident_id, type, user_id, alert_id, detail, created_at)
+		VALUES (?, ?, ?, ?, ?, ?)`,
+		incidentID, evType, userID, alertID, detail, time.Now().Unix())
+	return err
+}
+
+// todayUTC is the schedule's day key. The schedule's smallest unit is one UTC day.
+func todayUTC() string {
+	return time.Now().UTC().Format("2006-01-02")
+}
+
+// currentOnCall returns today's on-call user, or nil when nobody is scheduled.
+// A missing schedule entry is not an error — incidents just open unassigned.
+func currentOnCall(ctx context.Context, q querier) (*int64, error) {
+	var userID int64
+	err := q.QueryRowContext(ctx,
+		"SELECT user_id FROM schedule_entries WHERE date = ?", todayUTC()).Scan(&userID)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &userID, nil
+}
+
+// severityRank orders the conventional Alertmanager severity label values.
+// Anything unrecognised sorts below all of them rather than being dropped.
+func severityRank(s string) int {
+	switch strings.ToLower(s) {
+	case "critical":
+		return 4
+	case "error":
+		return 3
+	case "warning":
+		return 2
+	case "info":
+		return 1
+	default:
+		return 0
+	}
+}
+
+// refreshSeverity raises an incident's severity to the highest `severity` label
+// seen across its alerts.
+//
+// It is a high-water mark, never lowered: an incident that hit critical was a
+// critical incident, even after the critical alert clears and a warning is all
+// that is left firing. Downgrading a live incident would also quietly demote it
+// in the queue while the work is still open.
+func refreshSeverity(ctx context.Context, q querier, incidentID int64) error {
+	rows, err := q.QueryContext(ctx, `
+		SELECT json_extract(a.labels, '$.'||?)
+		FROM   incident_alerts ia
+		JOIN   alerts a ON a.id = ia.alert_id
+		WHERE  ia.incident_id = ?`, severityLabel, incidentID)
+	if err != nil {
+		return err
+	}
+
+	best := ""
+	for rows.Next() {
+		var sev *string
+		if err := rows.Scan(&sev); err != nil {
+			rows.Close()
+			return err
+		}
+		if sev != nil && severityRank(*sev) > severityRank(best) {
+			best = *sev
+		}
+	}
+	if err := rows.Err(); err != nil {
+		rows.Close()
+		return err
+	}
+	rows.Close()
+
+	if best == "" {
+		return nil
+	}
+	// The comparison lives in SQL so an unrelated concurrent update cannot be
+	// clobbered by a stale read.
+	_, err = q.ExecContext(ctx, `
+		UPDATE incidents SET severity = ?
+		 WHERE id = ?
+		   AND (severity IS NULL OR `+severityRankSQL("severity")+` < ?)`,
+		best, incidentID, severityRank(best))
+	return err
+}
+
+// severityRankSQL mirrors severityRank for use inside a statement. SQL cannot
+// order these strings meaningfully on its own.
+func severityRankSQL(col string) string {
+	return `CASE lower(COALESCE(` + col + `, ''))
+		WHEN 'critical' THEN 4
+		WHEN 'error'    THEN 3
+		WHEN 'warning'  THEN 2
+		WHEN 'info'     THEN 1
+		ELSE 0 END`
+}
+
+// resolveIfSettled closes an incident once every alert under it has stopped
+// firing — PagerDuty's cascade, and the only automatic route out of the open
+// state. Reports whether it actually resolved anything.
+func resolveIfSettled(ctx context.Context, q querier, incidentID int64) (bool, error) {
+	res, err := q.ExecContext(ctx, `
+		UPDATE incidents
+		   SET status            = 'resolved',
+		       resolved_at       = ?,
+		       resolution_source = ?
+		 WHERE id = ?
+		   AND resolved_at IS NULL
+		   -- An incident with no members yet is mid-creation, not settled.
+		   AND EXISTS (SELECT 1 FROM incident_alerts ia WHERE ia.incident_id = incidents.id)
+		   AND NOT EXISTS (SELECT 1
+		                     FROM incident_alerts ia
+		                     JOIN alerts a ON a.id = ia.alert_id
+		                    WHERE ia.incident_id = incidents.id
+		                      AND a.status = 'firing')`,
+		time.Now().Unix(), incidentResolutionAlerts, incidentID)
+	if err != nil {
+		return false, err
+	}
+	n, _ := res.RowsAffected()
+	if n == 0 {
+		return false, nil
+	}
+	if err := logEvent(ctx, q, incidentID, evResolved, nil, nil, nil); err != nil {
+		return false, err
+	}
+	// The all-clear goes only to whoever was paged in the first place, which
+	// enqueueResolved works out from the incident's own notification history.
+	// Manual resolution sends nothing: the person who closed it already knows.
+	return true, enqueueResolved(ctx, q, incidentID)
+}
+
+// acknowledgeIncident records that userID has picked an incident up, and reports
+// whether it changed anything — an already-resolved incident is left alone.
+// Shared by the authenticated handler and the Acknowledge button in a push
+// notification, so both write the same state and the same timeline entry.
+func acknowledgeIncident(ctx context.Context, q querier, incidentID, userID int64) (bool, error) {
+	res, err := q.ExecContext(ctx, `
+		UPDATE incidents
+		   SET status = 'acknowledged', acknowledged_by = ?, acknowledged_at = ?
+		 WHERE id = ? AND resolved_at IS NULL`,
+		userID, time.Now().Unix(), incidentID)
+	if err != nil {
+		return false, err
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return false, nil
+	}
+	return true, logEvent(ctx, q, incidentID, evAcknowledged, &userID, nil, nil)
+}
+
+// openIncidentForAlert returns the open incident an alert currently belongs to,
+// or 0 when it has none. Used when an alert resolves or expires so the event
+// lands on the right timeline.
+func openIncidentForAlert(ctx context.Context, q querier, alertID int64) (int64, error) {
+	var id int64
+	err := q.QueryRowContext(ctx, `
+		SELECT i.id
+		FROM   incident_alerts ia
+		JOIN   incidents i ON i.id = ia.incident_id
+		WHERE  ia.alert_id = ? AND i.resolved_at IS NULL`, alertID).Scan(&id)
+	if err == sql.ErrNoRows {
+		return 0, nil
+	}
+	return id, err
+}
+
+// incidentTitle renders a human-readable title from Alertmanager's groupLabels,
+// leading with the alert name and appending whatever else the operator grouped
+// by. Falls back to the alert's own name when the payload carried no groupLabels.
+func incidentTitle(groupLabels map[string]string, fallback string) string {
+	name := groupLabels["alertname"]
+	if name == "" {
+		name = fallback
+	}
+	if name == "" {
+		name = "Incident"
+	}
+
+	rest := make([]string, 0, len(groupLabels))
+	for k, v := range groupLabels {
+		if k == "alertname" {
+			continue
+		}
+		rest = append(rest, k+"="+v)
+	}
+	if len(rest) == 0 {
+		return name
+	}
+	sort.Strings(rest)
+	return name + " (" + strings.Join(rest, ", ") + ")"
+}

internal/api/incidents.go

@@ -0,0 +1,557 @@
+package api
+
+import (
+	"database/sql"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/yeniklas/terdut-server/internal/models"
+)
+
+func handleListIncidents(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+
+		where := []string{}
+		args := []any{}
+
+		// Without an explicit status the queue shows open work, which is what an
+		// on-call person opens the tool to see.
+		if status := q.Get("status"); status != "" {
+			where = append(where, "i.status = ?")
+			args = append(args, status)
+		} else {
+			where = append(where, "i.resolved_at IS NULL")
+		}
+
+		if q.Get("archived") == "true" {
+			where = append(where, "i.archived_at IS NOT NULL")
+		} else {
+			where = append(where, "i.archived_at IS NULL")
+		}
+
+		// A snooze expires by simply falling into the past; nothing sweeps it.
+		if q.Get("snoozed") == "true" {
+			where = append(where, "i.snoozed_until > ?")
+			args = append(args, time.Now().Unix())
+		} else {
+			where = append(where, "(i.snoozed_until IS NULL OR i.snoozed_until <= ?)")
+			args = append(args, time.Now().Unix())
+		}
+
+		if severity := q.Get("severity"); severity != "" {
+			where = append(where, "i.severity = ?")
+			args = append(args, severity)
+		}
+		if assignee := q.Get("assigned_to"); assignee != "" {
+			if n, err := strconv.ParseInt(assignee, 10, 64); err == nil {
+				where = append(where, "i.assigned_to = ?")
+				args = append(args, n)
+			}
+		}
+		if from := q.Get("from"); from != "" {
+			if t, err := time.Parse("2006-01-02", from); err == nil {
+				where = append(where, "i.triggered_at >= ?")
+				args = append(args, t.UTC().Unix())
+			}
+		}
+		if to := q.Get("to"); to != "" {
+			if t, err := time.Parse("2006-01-02", to); err == nil {
+				where = append(where, "i.triggered_at < ?")
+				args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
+			}
+		}
+
+		limit := 50
+		if l := q.Get("limit"); l != "" {
+			if n, err := strconv.Atoi(l); err == nil && n > 0 && n <= 500 {
+				limit = n
+			}
+		}
+
+		order := "i.triggered_at DESC"
+		if q.Get("sort") == "severity" {
+			order = severityRankSQL("i.severity") + " DESC, i.triggered_at DESC"
+		}
+		args = append(args, limit)
+
+		rows, err := db.QueryContext(r.Context(),
+			fmt.Sprintf("%s WHERE %s ORDER BY %s LIMIT ?",
+				incidentSelectFrom, strings.Join(where, " AND "), order),
+			args...)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		defer rows.Close()
+
+		incidents := []models.Incident{}
+		for rows.Next() {
+			i, err := scanIncident(rows)
+			if err != nil {
+				respond(w, http.StatusInternalServerError, errResp("internal error"))
+				return
+			}
+			incidents = append(incidents, i)
+		}
+		respond(w, http.StatusOK, incidents)
+	}
+}
+
+func handleGetIncident(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		inc, err := fetchIncident(r.Context(), db, id)
+		if err == sql.ErrNoRows {
+			respond(w, http.StatusNotFound, errResp("incident not found"))
+			return
+		}
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if inc.Alerts, err = incidentAlerts(r, db, id); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		respond(w, http.StatusOK, inc)
+	}
+}
+
+func handleIncidentAlerts(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		if !incidentExists(w, r, db, id) {
+			return
+		}
+		alerts, err := incidentAlerts(r, db, id)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		respond(w, http.StatusOK, alerts)
+	}
+}
+
+func handleIncidentTimeline(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		if !incidentExists(w, r, db, id) {
+			return
+		}
+
+		rows, err := db.QueryContext(r.Context(), `
+			SELECT e.id, e.incident_id, e.type, e.user_id, u.username,
+			       e.alert_id, e.detail, e.created_at
+			FROM   incident_events e
+			LEFT   JOIN users u ON u.id = e.user_id
+			WHERE  e.incident_id = ?
+			ORDER  BY e.created_at ASC, e.id ASC`, id)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		defer rows.Close()
+
+		events := []models.IncidentEvent{}
+		for rows.Next() {
+			var e models.IncidentEvent
+			var ts int64
+			if err := rows.Scan(&e.ID, &e.IncidentID, &e.Type, &e.UserID, &e.Username,
+				&e.AlertID, &e.Detail, &ts); err != nil {
+				respond(w, http.StatusInternalServerError, errResp("internal error"))
+				return
+			}
+			e.CreatedAt = time.Unix(ts, 0).UTC()
+			events = append(events, e)
+		}
+		respond(w, http.StatusOK, events)
+	}
+}
+
+func handleIncidentAcknowledge(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		user, _ := userFromContext(r.Context())
+		acked, err := acknowledgeIncident(r.Context(), db, id, user.ID)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if !acked {
+			if !incidentExists(w, r, db, id) {
+				return
+			}
+			respond(w, http.StatusConflict, errResp("incident is resolved"))
+			return
+		}
+		respondIncident(w, r, db, id)
+	}
+}
+
+func handleIncidentUnacknowledge(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		user, _ := userFromContext(r.Context())
+		if !updateOpenIncident(w, r, db, id,
+			`UPDATE incidents SET status = 'triggered', acknowledged_by = NULL, acknowledged_at = NULL
+			  WHERE id = ? AND resolved_at IS NULL`, id) {
+			return
+		}
+		if err := logEvent(r.Context(), db, id, evUnacknowledged, &user.ID, nil, nil); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+// handleIncidentResolve closes an incident by hand. This is terminal: a later
+// occurrence opens a new incident rather than reopening this one, which is what
+// stops a resolved incident from reappearing on the next repeat_interval
+// re-send of an alert that never stopped firing. Use snooze for "not now".
+func handleIncidentResolve(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		user, _ := userFromContext(r.Context())
+		if !updateOpenIncident(w, r, db, id,
+			`UPDATE incidents SET status = 'resolved', resolved_at = ?, resolution_source = ?
+			  WHERE id = ? AND resolved_at IS NULL`,
+			time.Now().Unix(), incidentResolutionManual, id) {
+			return
+		}
+		if err := logEvent(r.Context(), db, id, evResolved, &user.ID, nil, nil); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		respondIncident(w, r, db, id)
+	}
+}
+
+func handleIncidentAssign(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		var req struct {
+			UserID int64 `json:"user_id"`
+		}
+		if err := decodeJSON(r, &req); err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid request body"))
+			return
+		}
+		if req.UserID == 0 {
+			respond(w, http.StatusBadRequest, errResp("user_id is required"))
+			return
+		}
+		var exists int
+		if err := db.QueryRowContext(r.Context(),
+			"SELECT 1 FROM users WHERE id = ?", req.UserID).Scan(&exists); err != nil {
+			respond(w, http.StatusNotFound, errResp("user not found"))
+			return
+		}
+
+		if !updateOpenIncident(w, r, db, id,
+			"UPDATE incidents SET assigned_to = ? WHERE id = ? AND resolved_at IS NULL",
+			req.UserID, id) {
+			return
+		}
+		// On an "assigned" event user_id is the assignee, not the actor.
+		if err := logEvent(r.Context(), db, id, evAssigned, &req.UserID, nil, nil); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		respondIncident(w, r, db, id)
+	}
+}
+
+// handleIncidentSnooze hides an incident from the default queue without closing
+// it. Accepts either an absolute {"until": RFC3339} or a relative
+// {"duration": "2h"}.
+func handleIncidentSnooze(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		var req struct {
+			Until    string `json:"until"`
+			Duration string `json:"duration"`
+		}
+		if err := decodeJSON(r, &req); err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid request body"))
+			return
+		}
+
+		var until time.Time
+		switch {
+		case req.Until != "":
+			t, err := time.Parse(time.RFC3339, req.Until)
+			if err != nil {
+				respond(w, http.StatusBadRequest, errResp("invalid until (expected RFC3339)"))
+				return
+			}
+			until = t
+		case req.Duration != "":
+			d, err := time.ParseDuration(req.Duration)
+			if err != nil {
+				respond(w, http.StatusBadRequest, errResp("invalid duration"))
+				return
+			}
+			until = time.Now().Add(d)
+		default:
+			respond(w, http.StatusBadRequest, errResp("until or duration is required"))
+			return
+		}
+		if !until.After(time.Now()) {
+			respond(w, http.StatusBadRequest, errResp("snooze must end in the future"))
+			return
+		}
+
+		user, _ := userFromContext(r.Context())
+		if !updateOpenIncident(w, r, db, id,
+			"UPDATE incidents SET snoozed_until = ? WHERE id = ? AND resolved_at IS NULL",
+			until.Unix(), id) {
+			return
+		}
+		detail := until.UTC().Format(time.RFC3339)
+		if err := logEvent(r.Context(), db, id, evSnoozed, &user.ID, nil, &detail); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		respondIncident(w, r, db, id)
+	}
+}
+
+func handleIncidentUnsnooze(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		user, _ := userFromContext(r.Context())
+		if !updateOpenIncident(w, r, db, id,
+			"UPDATE incidents SET snoozed_until = NULL WHERE id = ? AND resolved_at IS NULL", id) {
+			return
+		}
+		if err := logEvent(r.Context(), db, id, evUnsnoozed, &user.ID, nil, nil); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+func handleIncidentArchive(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		res, err := db.ExecContext(r.Context(),
+			"UPDATE incidents SET archived_at = unixepoch() WHERE id = ?", id)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			respond(w, http.StatusNotFound, errResp("incident not found"))
+			return
+		}
+		respondIncident(w, r, db, id)
+	}
+}
+
+func handleIncidentUnarchive(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		res, err := db.ExecContext(r.Context(),
+			"UPDATE incidents SET archived_at = NULL WHERE id = ?", id)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			respond(w, http.StatusNotFound, errResp("incident not found"))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+// handleCreateNote adds a note to the timeline. Notes are ordinary events, so a
+// single query renders the whole story of an incident in order.
+func handleCreateNote(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		var req struct {
+			Content string `json:"content"`
+		}
+		if err := decodeJSON(r, &req); err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid request body"))
+			return
+		}
+		if req.Content == "" {
+			respond(w, http.StatusBadRequest, errResp("content is required"))
+			return
+		}
+		if !incidentExists(w, r, db, id) {
+			return
+		}
+
+		user, _ := userFromContext(r.Context())
+		now := time.Now()
+		res, err := db.ExecContext(r.Context(), `
+			INSERT INTO incident_events (incident_id, type, user_id, detail, created_at)
+			VALUES (?, ?, ?, ?, ?)`, id, evNote, user.ID, req.Content, now.Unix())
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		eventID, _ := res.LastInsertId()
+
+		respond(w, http.StatusCreated, models.IncidentEvent{
+			ID:         eventID,
+			IncidentID: id,
+			Type:       evNote,
+			UserID:     &user.ID,
+			Username:   &user.Username,
+			Detail:     &req.Content,
+			CreatedAt:  now.UTC().Truncate(time.Second),
+		})
+	}
+}
+
+// handleDeleteNote removes one of your own notes. Only notes are deletable — the
+// rest of the timeline is what actually happened, and is not editable.
+func handleDeleteNote(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := incidentIDParam(w, r)
+		if !ok {
+			return
+		}
+		eventID, err := strconv.ParseInt(chi.URLParam(r, "eventID"), 10, 64)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid note id"))
+			return
+		}
+
+		user, _ := userFromContext(r.Context())
+		res, err := db.ExecContext(r.Context(), `
+			DELETE FROM incident_events
+			 WHERE id = ? AND incident_id = ? AND type = ? AND user_id = ?`,
+			eventID, id, evNote, user.ID)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			respond(w, http.StatusNotFound, errResp("note not found"))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Shared handler plumbing
+// ---------------------------------------------------------------------------
+
+func incidentIDParam(w http.ResponseWriter, r *http.Request) (int64, bool) {
+	id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
+	if err != nil {
+		respond(w, http.StatusBadRequest, errResp("invalid incident id"))
+		return 0, false
+	}
+	return id, true
+}
+
+func incidentExists(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) bool {
+	var exists int
+	if err := db.QueryRowContext(r.Context(),
+		"SELECT 1 FROM incidents WHERE id = ?", id).Scan(&exists); err != nil {
+		respond(w, http.StatusNotFound, errResp("incident not found"))
+		return false
+	}
+	return true
+}
+
+// updateOpenIncident runs a mutation that is only valid while an incident is
+// open. The query must be constrained to `resolved_at IS NULL`, so no rows means
+// either the incident does not exist or it is already closed — two different
+// answers the caller should not have to distinguish itself.
+func updateOpenIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64, query string, args ...any) bool {
+	res, err := db.ExecContext(r.Context(), query, args...)
+	if err != nil {
+		respond(w, http.StatusInternalServerError, errResp("internal error"))
+		return false
+	}
+	if n, _ := res.RowsAffected(); n > 0 {
+		return true
+	}
+	if !incidentExists(w, r, db, id) {
+		return false
+	}
+	respond(w, http.StatusConflict, errResp("incident is resolved"))
+	return false
+}
+
+func respondIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) {
+	inc, err := fetchIncident(r.Context(), db, id)
+	if err != nil {
+		respond(w, http.StatusInternalServerError, errResp("internal error"))
+		return
+	}
+	respond(w, http.StatusOK, inc)
+}
+
+// incidentAlerts loads the alerts under an incident, newest signal first.
+func incidentAlerts(r *http.Request, db *sql.DB, id int64) ([]models.Alert, error) {
+	rows, err := db.QueryContext(r.Context(), alertSelectFrom+`
+		JOIN incident_alerts m ON m.alert_id = a.id
+		WHERE m.incident_id = ?
+		ORDER BY a.received_at DESC`, id)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	alerts := []models.Alert{}
+	for rows.Next() {
+		a, err := scanAlert(rows)
+		if err != nil {
+			return nil, err
+		}
+		alerts = append(alerts, a)
+	}
+	return alerts, rows.Err()
+}

internal/api/incidents_test.go

@@ -0,0 +1,772 @@
+package api_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/yeniklas/terdut-server/internal/api"
+	"github.com/yeniklas/terdut-server/internal/db"
+)
+
+// amAlert builds one alert of a webhook payload.
+func amAlert(fingerprint, name, status, startsAt, endsAt string, labels map[string]string) map[string]any {
+	l := map[string]string{"alertname": name}
+	for k, v := range labels {
+		l[k] = v
+	}
+	return map[string]any{
+		"status":       status,
+		"labels":       l,
+		"annotations":  map[string]string{},
+		"startsAt":     startsAt,
+		"endsAt":       endsAt,
+		"generatorURL": "",
+		"fingerprint":  fingerprint,
+	}
+}
+
+func listIncidents(t *testing.T, s *ts, query string) []map[string]any {
+	t.Helper()
+	var out []map[string]any
+	decode(t, s.req(t, http.MethodGet, "/api/incidents"+query, nil), &out)
+	return out
+}
+
+func getIncident(t *testing.T, s *ts, id int) map[string]any {
+	t.Helper()
+	var out map[string]any
+	decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d", id), nil), &out)
+	return out
+}
+
+func timeline(t *testing.T, s *ts, id int) []map[string]any {
+	t.Helper()
+	var out []map[string]any
+	decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id), nil), &out)
+	return out
+}
+
+// eventTypes flattens a timeline to its event types, which is what the ordering
+// assertions actually care about.
+func eventTypes(events []map[string]any) []string {
+	types := make([]string, len(events))
+	for i, e := range events {
+		types[i] = e["type"].(string)
+	}
+	return types
+}
+
+// countIncidents counts rows directly, including resolved and archived ones that
+// no list view returns.
+func (s *ts) countIncidents(t *testing.T) int {
+	t.Helper()
+	var n int
+	if err := s.db.QueryRow("SELECT COUNT(*) FROM incidents").Scan(&n); err != nil {
+		t.Fatalf("count incidents: %v", err)
+	}
+	return n
+}
+
+// ---------------------------------------------------------------------------
+// Ingest: alerts becoming incidents
+// ---------------------------------------------------------------------------
+
+func TestWebhook_FiringOpensIncident(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-1", "HighCPU", "firing", "2026-05-20T10:00:00Z", zeroTime,
+			map[string]string{"severity": "critical"}),
+	}, "{}:{alertname=\"HighCPU\"}")
+
+	incidents := listIncidents(t, s, "")
+	if len(incidents) != 1 {
+		t.Fatalf("expected 1 incident, got %d", len(incidents))
+	}
+	inc := incidents[0]
+	if inc["status"] != "triggered" {
+		t.Errorf("expected status triggered, got %v", inc["status"])
+	}
+	if inc["severity"] != "critical" {
+		t.Errorf("expected severity critical, got %v", inc["severity"])
+	}
+	if inc["title"] != "HighCPU" {
+		t.Errorf("expected title from groupLabels, got %v", inc["title"])
+	}
+
+	// The alert points back at the incident it opened.
+	var alerts []map[string]any
+	decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
+	if len(alerts) != 1 || alerts[0]["incident_id"] == nil {
+		t.Fatalf("expected the alert to carry an incident_id, got %v", alerts)
+	}
+}
+
+// Alertmanager already grouped these; we adopt its answer rather than
+// correlating again.
+func TestWebhook_SameGroupKeyJoinsOneIncident(t *testing.T) {
+	s := newTS(t)
+	const groupKey = "{}:{alertname=\"DiskFull\"}"
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-a", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+		amAlert("fp-b", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	}, groupKey)
+
+	incidents := listIncidents(t, s, "")
+	if len(incidents) != 1 {
+		t.Fatalf("expected 1 incident for one groupKey, got %d", len(incidents))
+	}
+	id := int(incidents[0]["id"].(float64))
+
+	inc := getIncident(t, s, id)
+	members, _ := inc["alerts"].([]any)
+	if len(members) != 2 {
+		t.Fatalf("expected 2 alerts under the incident, got %d", len(members))
+	}
+
+	added := 0
+	for _, ty := range eventTypes(timeline(t, s, id)) {
+		if ty == "alert_added" {
+			added++
+		}
+	}
+	if added != 2 {
+		t.Errorf("expected 2 alert_added events, got %d", added)
+	}
+}
+
+// The load-bearing rule. Alertmanager re-sends firing notifications every
+// repeat_interval; if those re-sends reopened incidents, resolving one by hand
+// would mean nothing.
+func TestWebhook_HeartbeatDoesNotReopenResolvedIncident(t *testing.T) {
+	s := newTS(t)
+	const groupKey = "{}:{alertname=\"Flapper\"}"
+	alert := amAlert("fp-hb", "Flapper", "firing", "2026-05-20T10:00:00Z", zeroTime, nil)
+
+	postWebhook(t, s, []map[string]any{alert}, groupKey)
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("resolve returned %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	// Same startsAt, same fingerprint: a re-send, not a new occurrence.
+	postWebhook(t, s, []map[string]any{alert}, groupKey)
+
+	if n := s.countIncidents(t); n != 1 {
+		t.Fatalf("expected the heartbeat to open no incident, got %d total", n)
+	}
+	if inc := getIncident(t, s, 1); inc["resolved_at"] == nil {
+		t.Error("expected incident 1 to stay resolved")
+	}
+
+	// The alert itself is still firing and still being tracked — only the work
+	// item is closed.
+	status, _, _ := s.alertRow(t, "fp-hb")
+	if status != "firing" {
+		t.Errorf("expected the alert to still be firing, got %q", status)
+	}
+}
+
+func TestWebhook_NewOccurrenceOpensNewIncident(t *testing.T) {
+	s := newTS(t)
+	const groupKey = "{}:{alertname=\"Recurring\"}"
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-new", "Recurring", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	}, groupKey)
+	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
+
+	// A newer startsAt is a genuinely new occurrence, not a re-send.
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-new", "Recurring", "firing", "2026-05-21T09:00:00Z", zeroTime, nil),
+	}, groupKey)
+
+	if n := s.countIncidents(t); n != 2 {
+		t.Fatalf("expected a second incident for the new occurrence, got %d total", n)
+	}
+	open := listIncidents(t, s, "")
+	if len(open) != 1 || int(open[0]["id"].(float64)) != 2 {
+		t.Fatalf("expected incident 2 to be the open one, got %v", open)
+	}
+
+	// The new incident starts unacknowledged: that is the point of the split.
+	if open[0]["acknowledged_by"] != nil {
+		t.Error("expected a fresh occurrence to start unacknowledged")
+	}
+}
+
+func TestWebhook_ResolvedOnlyPayloadOpensNothing(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-res", "AlreadyOver", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
+	}, "{}:{alertname=\"AlreadyOver\"}")
+
+	if n := s.countIncidents(t); n != 0 {
+		t.Errorf("expected no incident from a resolved-only payload, got %d", n)
+	}
+	if status, _, _ := s.alertRow(t, "fp-res"); status != "resolved" {
+		t.Errorf("expected the alert itself to be stored, got %q", status)
+	}
+}
+
+// An incident that hit critical was a critical incident, even once the critical
+// alert clears and only a warning is left.
+func TestIncident_SeverityIsHighWaterMark(t *testing.T) {
+	s := newTS(t)
+	const groupKey = "{}:{alertname=\"Mixed\"}"
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-warn", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
+			map[string]string{"severity": "warning"}),
+		amAlert("fp-crit", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
+			map[string]string{"severity": "critical"}),
+	}, groupKey)
+
+	if inc := getIncident(t, s, 1); inc["severity"] != "critical" {
+		t.Fatalf("expected severity critical, got %v", inc["severity"])
+	}
+
+	// The critical alert clears; the warning keeps the incident open.
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-crit", "Mixed", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z",
+			map[string]string{"severity": "critical"}),
+	}, groupKey)
+
+	inc := getIncident(t, s, 1)
+	if inc["resolved_at"] != nil {
+		t.Fatal("expected the incident to stay open")
+	}
+	if inc["severity"] != "critical" {
+		t.Errorf("expected severity to stay critical, got %v", inc["severity"])
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Resolution cascade
+// ---------------------------------------------------------------------------
+
+func TestIncident_AllAlertsResolvedAutoResolves(t *testing.T) {
+	s := newTS(t)
+	const groupKey = "{}:{alertname=\"Pair\"}"
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-p1", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+		amAlert("fp-p2", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	}, groupKey)
+
+	// One down, one still firing: the work is not done.
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-p1", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
+	}, groupKey)
+	if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
+		t.Fatal("expected the incident to stay open while an alert is firing")
+	}
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-p2", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:30:00Z", nil),
+	}, groupKey)
+
+	inc := getIncident(t, s, 1)
+	if inc["status"] != "resolved" {
+		t.Errorf("expected status resolved, got %v", inc["status"])
+	}
+	if inc["resolution_source"] != "alerts" {
+		t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
+	}
+}
+
+// Expiry is inference, not observation, but it still has to close the work item
+// — otherwise a lost resolved notification leaves an incident open forever.
+func TestExpiry_CascadesToIncidentResolution(t *testing.T) {
+	s := newTS(t)
+	postAlert(t, s, "fp-exp", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
+
+	s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'fp-exp'",
+		time.Now().Add(-10*time.Hour).Unix())
+	sweep(t, s, 6*time.Hour)
+
+	inc := getIncident(t, s, 1)
+	if inc["status"] != "resolved" {
+		t.Errorf("expected the incident to resolve after expiry, got %v", inc["status"])
+	}
+	if inc["resolution_source"] != "alerts" {
+		t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
+	}
+
+	// The expiry is recorded against the alert, not the incident.
+	if _, source, _ := s.alertRow(t, "fp-exp"); source == nil || *source != "expiry" {
+		t.Errorf("expected the alert's resolution_source to stay expiry, got %v", source)
+	}
+	if types := eventTypes(timeline(t, s, 1)); !contains(types, "alert_resolved") {
+		t.Errorf("expected an alert_resolved event on the timeline, got %v", types)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Workflow actions
+// ---------------------------------------------------------------------------
+
+func TestIncident_Acknowledge(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-ack", "X", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("acknowledge returned %d", resp.StatusCode)
+	}
+	var inc map[string]any
+	decode(t, resp, &inc)
+	if inc["acknowledged_by"] == nil {
+		t.Error("expected acknowledged_by to be set")
+	}
+	if inc["status"] != "acknowledged" {
+		t.Errorf("expected status acknowledged, got %v", inc["status"])
+	}
+
+	resp = s.req(t, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Fatalf("unacknowledge returned %d", resp.StatusCode)
+	}
+	if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
+		t.Errorf("expected status back to triggered, got %v", inc["status"])
+	}
+}
+
+func TestIncident_ManualResolveIsTerminal(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-term", "Terminal", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("first resolve returned %d", resp.StatusCode)
+	}
+	var inc map[string]any
+	decode(t, resp, &inc)
+	if inc["resolution_source"] != "manual" {
+		t.Errorf("expected resolution_source manual, got %v", inc["resolution_source"])
+	}
+
+	resp = s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusConflict {
+		t.Errorf("expected 409 on re-resolve, got %d", resp.StatusCode)
+	}
+
+	// Acknowledging a closed incident is equally meaningless.
+	resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusConflict {
+		t.Errorf("expected 409 acknowledging a resolved incident, got %d", resp.StatusCode)
+	}
+}
+
+func TestIncident_SnoozeHiddenFromDefaultList(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-snz", "Noisy", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]string{"duration": "2h"})
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("snooze returned %d", resp.StatusCode)
+	}
+	var inc map[string]any
+	decode(t, resp, &inc)
+	if inc["snoozed_until"] == nil {
+		t.Error("expected snoozed_until to be set")
+	}
+
+	if got := listIncidents(t, s, ""); len(got) != 0 {
+		t.Errorf("expected the snoozed incident to be hidden, got %d", len(got))
+	}
+	if got := listIncidents(t, s, "?snoozed=true"); len(got) != 1 {
+		t.Errorf("expected snoozed=true to show it, got %d", len(got))
+	}
+
+	// A snooze is not a resolution: the incident is still open work.
+	if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
+		t.Error("expected a snoozed incident to stay open")
+	}
+
+	resp = s.req(t, http.MethodDelete, "/api/incidents/1/snooze", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Fatalf("unsnooze returned %d", resp.StatusCode)
+	}
+	if got := listIncidents(t, s, ""); len(got) != 1 {
+		t.Errorf("expected the incident back in the default list, got %d", len(got))
+	}
+}
+
+func TestIncident_SnoozeRejectsPastDeadline(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-past", "Past", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze",
+		map[string]string{"until": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Errorf("expected 400 for a snooze in the past, got %d", resp.StatusCode)
+	}
+}
+
+// The schedule stops being decorative here: it is read at trigger time.
+func TestIncident_AutoAssignedToCurrentOnCall(t *testing.T) {
+	s := newTS(t)
+
+	today := time.Now().UTC().Format("2006-01-02")
+	resp := s.req(t, http.MethodPost, "/api/schedule",
+		map[string]any{"user_id": 1, "dates": []string{today}})
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("schedule assignment returned %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-oncall", "PageMe", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	inc := getIncident(t, s, 1)
+	if inc["assigned_to"] != "admin" {
+		t.Errorf("expected the incident assigned to today's on-call, got %v", inc["assigned_to"])
+	}
+	if types := eventTypes(timeline(t, s, 1)); !contains(types, "assigned") {
+		t.Errorf("expected an assigned event, got %v", types)
+	}
+}
+
+func TestIncident_AssignToUser(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-asg", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	s.req(t, http.MethodPost, "/api/users",
+		map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2})
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("assign returned %d", resp.StatusCode)
+	}
+	var inc map[string]any
+	decode(t, resp, &inc)
+	if inc["assigned_to"] != "alice" {
+		t.Errorf("expected assigned_to alice, got %v", inc["assigned_to"])
+	}
+
+	resp = s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 99})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 assigning an unknown user, got %d", resp.StatusCode)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Timeline and notes
+// ---------------------------------------------------------------------------
+
+func TestIncident_TimelineOrdering(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-tl", "Storyline", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
+	s.req(t, http.MethodPost, "/api/incidents/1/notes",
+		map[string]string{"content": "looking into it"}).Body.Close()
+	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
+
+	events := timeline(t, s, 1)
+	want := []string{"triggered", "alert_added", "acknowledged", "note", "resolved"}
+	got := eventTypes(events)
+	if len(got) != len(want) {
+		t.Fatalf("expected timeline %v, got %v", want, got)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("expected timeline %v, got %v", want, got)
+		}
+	}
+
+	// The note carries its author; system events do not.
+	for _, e := range events {
+		if e["type"] == "note" {
+			if e["username"] != "admin" || e["detail"] != "looking into it" {
+				t.Errorf("unexpected note event: %v", e)
+			}
+		}
+		if e["type"] == "triggered" && e["username"] != nil {
+			t.Errorf("expected the triggered event to have no author, got %v", e["username"])
+		}
+	}
+}
+
+func TestIncident_NoteDeleteOwnOnly(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-note", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	s.req(t, http.MethodPost, "/api/users",
+		map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
+	var keyData map[string]any
+	decode(t, s.req(t, http.MethodPost, "/api/users/2/api-keys",
+		map[string]string{"name": "alice-key"}), &keyData)
+	aliceKey := keyData["key"].(string)
+
+	var note map[string]any
+	decode(t, s.req(t, http.MethodPost, "/api/incidents/1/notes",
+		map[string]string{"content": "admin note"}), &note)
+	noteID := int(note["id"].(float64))
+
+	// Alice cannot delete admin's note.
+	req, _ := http.NewRequest(http.MethodDelete,
+		fmt.Sprintf("%s/api/incidents/1/notes/%d", s.URL, noteID), nil)
+	req.Header.Set("Authorization", "Bearer "+aliceKey)
+	resp, _ := http.DefaultClient.Do(req)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 deleting another user's note, got %d", resp.StatusCode)
+	}
+
+	resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Errorf("expected 204 deleting own note, got %d", resp.StatusCode)
+	}
+}
+
+// Only notes are deletable — the rest of the timeline is what happened.
+func TestIncident_CannotDeleteSystemEvent(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-sys", "System", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	events := timeline(t, s, 1)
+	id := int(events[0]["id"].(float64))
+	resp := s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", id), nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 deleting a system event, got %d", resp.StatusCode)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Archive
+// ---------------------------------------------------------------------------
+
+func TestIncident_ArchiveRoundTrip(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-arc", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/archive", nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("archive returned %d", resp.StatusCode)
+	}
+	var inc map[string]any
+	decode(t, resp, &inc)
+	if inc["archived_at"] == nil {
+		t.Error("expected archived_at to be set")
+	}
+
+	if got := listIncidents(t, s, "?status=resolved"); len(got) != 0 {
+		t.Errorf("expected the archived incident to be hidden, got %d", len(got))
+	}
+	if got := listIncidents(t, s, "?status=resolved&archived=true"); len(got) != 1 {
+		t.Errorf("expected archived=true to show it, got %d", len(got))
+	}
+
+	resp = s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Fatalf("unarchive returned %d", resp.StatusCode)
+	}
+	if got := listIncidents(t, s, "?status=resolved"); len(got) != 1 {
+		t.Errorf("expected the incident back, got %d", len(got))
+	}
+}
+
+func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-swp", "Old", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
+
+	s.exec(t, "UPDATE incidents SET resolved_at = ? WHERE id = 1",
+		time.Now().Add(-30*24*time.Hour).Unix())
+	api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour)
+
+	if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
+		t.Error("expected the sweeper to archive a long-resolved incident")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Stats
+// ---------------------------------------------------------------------------
+
+func TestStats_Incidents(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-s1", "One", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	}, "g1")
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-s2", "Two", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	}, "g2")
+	s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
+	s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
+
+	var stats map[string]any
+	decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
+
+	if stats["total"].(float64) != 2 {
+		t.Errorf("expected total 2, got %v", stats["total"])
+	}
+	if stats["resolved"].(float64) != 1 {
+		t.Errorf("expected resolved 1, got %v", stats["resolved"])
+	}
+	if stats["triggered"].(float64) != 1 {
+		t.Errorf("expected triggered 1, got %v", stats["triggered"])
+	}
+	// One incident has been acknowledged and resolved, so both averages exist.
+	if stats["mtta_seconds"] == nil || stats["mttr_seconds"] == nil {
+		t.Errorf("expected mtta and mttr to be computable, got %v", stats)
+	}
+}
+
+// Nothing acknowledged yet means "no data", which is not the same claim as zero.
+func TestStats_IncidentsNullMTTAWhenNothingAcknowledged(t *testing.T) {
+	s := newTS(t)
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-s3", "Untouched", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
+	})
+
+	var stats map[string]any
+	decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
+	if stats["mtta_seconds"] != nil {
+		t.Errorf("expected mtta_seconds null, got %v", stats["mtta_seconds"])
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Migration backfill
+// ---------------------------------------------------------------------------
+
+// An upgrade must not drop the acknowledgements and comments people already
+// have, so 008 is replayed here over a database left at 007.
+func TestMigration_BackfillCarriesAckAndComments(t *testing.T) {
+	database, err := db.Open(":memory:")
+	if err != nil {
+		t.Fatalf("open db: %v", err)
+	}
+	t.Cleanup(func() { database.Close() })
+
+	files, err := filepath.Glob("../db/migrations/*.sql")
+	if err != nil || len(files) == 0 {
+		t.Fatalf("find migrations: %v", err)
+	}
+	sort.Strings(files)
+
+	var incidentsMigration string
+	for _, f := range files {
+		if filepath.Base(f) >= "008" {
+			incidentsMigration = f
+			break
+		}
+		data, err := os.ReadFile(f)
+		if err != nil {
+			t.Fatalf("read %s: %v", f, err)
+		}
+		if _, err := database.Exec(string(data)); err != nil {
+			t.Fatalf("apply %s: %v", f, err)
+		}
+	}
+	if incidentsMigration == "" {
+		t.Fatal("008 migration not found")
+	}
+
+	// A database as it would look on the old schema: an acknowledged firing
+	// alert with a comment on it.
+	if _, err := database.Exec(`
+		INSERT INTO users (id, username, email) VALUES (1, 'admin', 'admin@test.com');
+		INSERT INTO alerts (id, fingerprint, name, status, labels, annotations,
+		                    starts_at, received_at, acknowledged_by, acknowledged_at)
+		VALUES (1, 'legacy-fp', 'LegacyAlert', 'firing',
+		        '{"severity":"warning"}', '{}', 1000, 1000, 1, 1500);
+		INSERT INTO alert_comments (alert_id, user_id, content, created_at)
+		VALUES (1, 1, 'legacy comment', 1600);`); err != nil {
+		t.Fatalf("seed pre-008 data: %v", err)
+	}
+
+	data, err := os.ReadFile(incidentsMigration)
+	if err != nil {
+		t.Fatalf("read 008: %v", err)
+	}
+	if _, err := database.Exec(string(data)); err != nil {
+		t.Fatalf("apply 008: %v", err)
+	}
+
+	var status, groupKey string
+	var ackBy int64
+	var severity string
+	if err := database.QueryRow(
+		"SELECT status, group_key, acknowledged_by, severity FROM incidents WHERE id = 1",
+	).Scan(&status, &groupKey, &ackBy, &severity); err != nil {
+		t.Fatalf("read backfilled incident: %v", err)
+	}
+	if status != "acknowledged" {
+		t.Errorf("expected the ack to carry over as status, got %q", status)
+	}
+	if groupKey != "backfill:legacy-fp" {
+		t.Errorf("unexpected group_key %q", groupKey)
+	}
+	if ackBy != 1 {
+		t.Errorf("expected acknowledged_by 1, got %d", ackBy)
+	}
+	if severity != "warning" {
+		t.Errorf("expected severity carried from labels, got %q", severity)
+	}
+
+	var notes int
+	if err := database.QueryRow(
+		"SELECT COUNT(*) FROM incident_events WHERE type = 'note' AND detail = 'legacy comment'",
+	).Scan(&notes); err != nil {
+		t.Fatalf("count notes: %v", err)
+	}
+	if notes != 1 {
+		t.Errorf("expected the comment to become a note, got %d", notes)
+	}
+
+	// And the columns that caused the ack-survives-a-re-fire bug are gone.
+	if _, err := database.Exec("SELECT acknowledged_by FROM alerts"); err == nil {
+		t.Error("expected alerts.acknowledged_by to be dropped")
+	}
+}
+
+func contains(haystack []string, needle string) bool {
+	for _, s := range haystack {
+		if s == needle {
+			return true
+		}
+	}
+	return false
+}

internal/api/notifier.go

@@ -0,0 +1,532 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/yeniklas/terdut-server/internal/models"
+)
+
+const (
+	// notifyInterval is how often the notifier looks for work. The archiver's
+	// 15 minute tick is far too coarse for something that has to wake a person.
+	notifyInterval = 30 * time.Second
+
+	// notifyRetryBase and notifyRetryMax bound the delivery backoff. ntfy being
+	// briefly unreachable should not lose the page.
+	notifyRetryBase = 30 * time.Second
+	notifyRetryMax  = 15 * time.Minute
+
+	// notifyMaxAttempts stops a permanently undeliverable row from being retried
+	// forever. It keeps last_error so the reason survives.
+	notifyMaxAttempts = 8
+
+	// notifyBatch caps one delivery pass, so a large backlog cannot hold the
+	// single database connection for an unbounded stretch.
+	notifyBatch = 100
+
+	// ackTokenTTL is how long the Acknowledge button in a notification keeps
+	// working. Past this the notification is stale enough that the responder
+	// should look at the incident rather than blind-acknowledge it.
+	ackTokenTTL = 24 * time.Hour
+)
+
+// Notification kinds, recording why a push was sent.
+const (
+	notifyTriggered = "triggered"
+	notifyReminder  = "reminder"
+	notifyResolved  = "resolved"
+)
+
+// NotifyConfig is everything the notifier needs to reach ntfy and to build URLs
+// a phone can follow back to this server.
+type NotifyConfig struct {
+	// BaseURL is the ntfy server. Empty disables notifications entirely: no
+	// goroutine, and nothing is ever enqueued.
+	BaseURL string
+
+	// Token is an optional bearer token for an access-controlled ntfy.
+	Token string
+
+	// FallbackTopic receives incidents that open with nobody on call. Those
+	// notifications carry no Acknowledge button — there is no user to attribute
+	// the acknowledgement to, and putting one on a shared topic would let any
+	// subscriber acknowledge as somebody else.
+	FallbackTopic string
+
+	// PublicURL is the base URL a phone uses to reach this server, for the
+	// notification's click target and its Acknowledge action. Without it a
+	// notification is informational only.
+	PublicURL string
+
+	// RepeatEvery is how long an incident may sit unacknowledged before it is
+	// notified again. Zero disables reminders.
+	RepeatEvery time.Duration
+}
+
+// enabled reports whether notifications are configured at all.
+func (c NotifyConfig) enabled() bool { return c.BaseURL != "" }
+
+// notifyClient is shared: a page is small and infrequent, and the timeout is
+// what keeps a hung ntfy from stalling the delivery pass.
+var notifyClient = &http.Client{Timeout: 10 * time.Second}
+
+// StartNotifier delivers queued notifications until ctx is cancelled, starting
+// with an immediate pass so a restart flushes whatever the last one left behind.
+func StartNotifier(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
+	if !cfg.enabled() {
+		log.Print("notifier: disabled (no ntfy URL configured)")
+		return
+	}
+	log.Printf("notifier: publishing to %s", cfg.BaseURL)
+
+	ticker := time.NewTicker(notifyInterval)
+	defer ticker.Stop()
+
+	NotifySweep(ctx, db, cfg)
+	for {
+		select {
+		case <-ticker.C:
+			NotifySweep(ctx, db, cfg)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// NotifySweep runs a single pass: queue reminders for incidents nobody has
+// picked up, then deliver everything that is due. Reminders are queued first so
+// a freshly due one goes out in the same pass rather than a tick later.
+// Exported so tests can drive a pass without waiting on the ticker.
+func NotifySweep(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
+	enqueueReminders(ctx, db, cfg)
+	deliverPending(ctx, db, cfg)
+}
+
+// enqueueReminders re-notifies incidents that are still sitting untouched.
+//
+// The stop conditions are the incident states that already mean "somebody has
+// this": acknowledged, snoozed, resolved, archived. Snooze in particular is the
+// mute button — a deliberate "not now" that should not keep buzzing — which is
+// why there is no separate reminder cap.
+//
+// The previous notification must have actually been sent before another is
+// queued, so an ntfy outage produces a retry backlog rather than a reminder
+// backlog that all lands at once when it comes back.
+func enqueueReminders(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
+	if cfg.RepeatEvery <= 0 {
+		return
+	}
+	now := time.Now()
+
+	type due struct {
+		incidentID int64
+		userID     *int64
+		topic      string
+	}
+
+	rows, err := db.QueryContext(ctx, `
+		SELECT n.incident_id, n.user_id, n.topic
+		FROM   notifications n
+		JOIN   incidents i ON i.id = n.incident_id
+		WHERE  n.id = (SELECT MAX(id) FROM notifications WHERE incident_id = n.incident_id)
+		  AND  n.sent_at IS NOT NULL
+		  AND  n.created_at <= ?
+		  AND  i.resolved_at IS NULL
+		  AND  i.archived_at IS NULL
+		  AND  i.status = 'triggered'
+		  AND  (i.snoozed_until IS NULL OR i.snoozed_until <= ?)`,
+		now.Add(-cfg.RepeatEvery).Unix(), now.Unix())
+	if err != nil {
+		log.Printf("notifier: find reminders: %v", err)
+		return
+	}
+
+	// Collected before inserting: the pool is limited to a single connection, so
+	// an open cursor would block the writes behind it.
+	var pending []due
+	for rows.Next() {
+		var d due
+		if err := rows.Scan(&d.incidentID, &d.userID, &d.topic); err != nil {
+			rows.Close()
+			log.Printf("notifier: scan reminder: %v", err)
+			return
+		}
+		pending = append(pending, d)
+	}
+	if err := rows.Err(); err != nil {
+		rows.Close()
+		log.Printf("notifier: find reminders: %v", err)
+		return
+	}
+	rows.Close()
+
+	for _, d := range pending {
+		if err := enqueueNotification(ctx, db, d.incidentID, d.userID, d.topic, notifyReminder); err != nil {
+			log.Printf("notifier: queue reminder for incident %d: %v", d.incidentID, err)
+		}
+	}
+	if len(pending) > 0 {
+		log.Printf("notifier: queued %d reminder(s)", len(pending))
+	}
+}
+
+// outboxRow is one queued notification, read before any HTTP happens.
+type outboxRow struct {
+	id         int64
+	incidentID int64
+	userID     *int64
+	topic      string
+	kind       string
+	attempts   int
+}
+
+// deliverPending sends everything that is due and records the outcome.
+func deliverPending(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
+	batch, err := pendingNotifications(ctx, db)
+	if err != nil {
+		log.Printf("notifier: find pending: %v", err)
+		return
+	}
+
+	sent := 0
+	for _, n := range batch {
+		if err := deliver(ctx, db, cfg, n); err != nil {
+			log.Printf("notifier: deliver %d (incident %d): %v", n.id, n.incidentID, err)
+			markFailed(ctx, db, n, err)
+			continue
+		}
+		if _, err := db.ExecContext(ctx,
+			"UPDATE notifications SET sent_at = ?, attempts = attempts + 1, last_error = NULL WHERE id = ?",
+			time.Now().Unix(), n.id); err != nil {
+			log.Printf("notifier: mark sent %d: %v", n.id, err)
+		}
+		sent++
+	}
+	if sent > 0 {
+		log.Printf("notifier: delivered %d notification(s)", sent)
+	}
+}
+
+// pendingNotifications reads the due rows and closes the cursor before the
+// caller writes, for the same single-connection reason as staleAlertIDs.
+func pendingNotifications(ctx context.Context, db *sql.DB) ([]outboxRow, error) {
+	rows, err := db.QueryContext(ctx, `
+		SELECT id, incident_id, user_id, topic, kind, attempts
+		FROM   notifications
+		WHERE  sent_at IS NULL
+		  AND  send_after <= ?
+		  AND  attempts < ?
+		ORDER  BY id
+		LIMIT  ?`, time.Now().Unix(), notifyMaxAttempts, notifyBatch)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var batch []outboxRow
+	for rows.Next() {
+		var n outboxRow
+		if err := rows.Scan(&n.id, &n.incidentID, &n.userID, &n.topic, &n.kind, &n.attempts); err != nil {
+			return nil, err
+		}
+		batch = append(batch, n)
+	}
+	return batch, rows.Err()
+}
+
+// markFailed bumps the attempt count and pushes the row out to its next retry.
+func markFailed(ctx context.Context, db *sql.DB, n outboxRow, cause error) {
+	next := time.Now().Add(retryDelay(n.attempts)).Unix()
+	if _, err := db.ExecContext(ctx,
+		"UPDATE notifications SET attempts = attempts + 1, send_after = ?, last_error = ? WHERE id = ?",
+		next, cause.Error(), n.id); err != nil {
+		log.Printf("notifier: mark failed %d: %v", n.id, err)
+	}
+}
+
+// retryDelay doubles the wait per attempt, up to notifyRetryMax.
+func retryDelay(attempts int) time.Duration {
+	d := notifyRetryBase << attempts
+	if d > notifyRetryMax || d <= 0 {
+		return notifyRetryMax
+	}
+	return d
+}
+
+// deliver renders one notification against the incident's *current* state and
+// publishes it. Rendering happens here rather than at enqueue time so a message
+// that waited in the queue while its incident escalated goes out at the
+// severity the incident has now.
+func deliver(ctx context.Context, db *sql.DB, cfg NotifyConfig, n outboxRow) error {
+	inc, err := fetchIncident(ctx, db, n.incidentID)
+	if err != nil {
+		return fmt.Errorf("load incident: %w", err)
+	}
+
+	var firing int
+	if err := db.QueryRowContext(ctx, `
+		SELECT COUNT(*)
+		FROM   incident_alerts ia
+		JOIN   alerts a ON a.id = ia.alert_id
+		WHERE  ia.incident_id = ? AND a.status = 'firing'`, n.incidentID).Scan(&firing); err != nil {
+		return fmt.Errorf("count firing: %w", err)
+	}
+
+	msg := renderNotification(inc, n, firing, cfg)
+
+	// An Acknowledge button needs both a user to attribute the acknowledgement
+	// to and a URL the phone can reach. Minted per delivery, so every push
+	// carries its own short-lived token rather than reusing one.
+	if n.kind != notifyResolved && n.userID != nil && cfg.PublicURL != "" {
+		raw, err := issueAckToken(ctx, db, n.incidentID, *n.userID)
+		if err != nil {
+			return fmt.Errorf("issue ack token: %w", err)
+		}
+		msg.Actions = append(msg.Actions, ntfyAction{
+			Action: "http",
+			Label:  "Acknowledge",
+			URL:    strings.TrimSuffix(cfg.PublicURL, "/") + "/api/notify/ack/" + raw,
+			Method: "POST",
+			Clear:  true,
+		})
+	}
+
+	return publish(ctx, cfg, msg)
+}
+
+// ntfyMessage is ntfy's JSON publish format. Using it rather than the X-Actions
+// header avoids that header's comma and quote escaping rules, which are easy to
+// break with a title that happens to contain a comma.
+type ntfyMessage struct {
+	Topic    string       `json:"topic"`
+	Title    string       `json:"title,omitempty"`
+	Message  string       `json:"message"`
+	Priority int          `json:"priority,omitempty"`
+	Tags     []string     `json:"tags,omitempty"`
+	Click    string       `json:"click,omitempty"`
+	Actions  []ntfyAction `json:"actions,omitempty"`
+}
+
+type ntfyAction struct {
+	Action string `json:"action"`
+	Label  string `json:"label"`
+	URL    string `json:"url"`
+	Method string `json:"method,omitempty"`
+	Clear  bool   `json:"clear,omitempty"`
+}
+
+// renderNotification builds the message body for one queued notification.
+func renderNotification(inc models.Incident, n outboxRow, firing int, cfg NotifyConfig) ntfyMessage {
+	msg := ntfyMessage{Topic: n.topic}
+
+	if cfg.PublicURL != "" {
+		msg.Click = fmt.Sprintf("%s/api/incidents/%d",
+			strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID)
+	}
+
+	switch n.kind {
+	case notifyResolved:
+		msg.Title = "Resolved: " + inc.Title
+		msg.Message = "All alerts stopped firing after " +
+			humanDuration(time.Since(inc.TriggeredAt))
+		msg.Priority = ntfyPriorityLow
+		msg.Tags = []string{"white_check_mark"}
+		return msg
+
+	case notifyReminder:
+		msg.Title = "Still unacknowledged: " + inc.Title
+	default:
+		msg.Title = inc.Title
+	}
+
+	severity := derefString(inc.Severity)
+
+	parts := []string{fmt.Sprintf("%d alert%s firing", firing, plural(firing))}
+	if severity != "" {
+		parts = append(parts, "severity "+severity)
+	}
+	if assignee := derefString(inc.AssignedToUser); assignee != "" {
+		parts = append(parts, "on call: "+assignee)
+	}
+	if n.kind == notifyReminder {
+		parts = append(parts, "open "+humanDuration(time.Since(inc.TriggeredAt)))
+	}
+
+	msg.Message = strings.Join(parts, " · ")
+	msg.Priority = ntfyPriority(severity)
+	msg.Tags = []string{severityTag(severity)}
+	return msg
+}
+
+// ntfy's priority scale. Max is the one that overrides the phone's quiet
+// settings, which is the whole point of paging on critical.
+const (
+	ntfyPriorityLow     = 2
+	ntfyPriorityDefault = 3
+	ntfyPriorityHigh    = 4
+	ntfyPriorityMax     = 5
+)
+
+// ntfyPriority maps an incident's severity onto ntfy's scale, following the
+// same ordering severityRank uses. An unrecognised severity gets the default
+// rather than being silenced.
+func ntfyPriority(severity string) int {
+	switch severityRank(severity) {
+	case 4:
+		return ntfyPriorityMax
+	case 3:
+		return ntfyPriorityHigh
+	case 2:
+		return ntfyPriorityDefault
+	case 1:
+		return ntfyPriorityLow
+	default:
+		return ntfyPriorityDefault
+	}
+}
+
+func severityTag(severity string) string {
+	switch severityRank(severity) {
+	case 4:
+		return "rotating_light"
+	case 3:
+		return "red_circle"
+	case 2:
+		return "warning"
+	case 1:
+		return "information_source"
+	default:
+		return "bell"
+	}
+}
+
+// publish POSTs one message to ntfy.
+func publish(ctx context.Context, cfg NotifyConfig, msg ntfyMessage) error {
+	body, err := json.Marshal(msg)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		strings.TrimSuffix(cfg.BaseURL, "/"), bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if cfg.Token != "" {
+		req.Header.Set("Authorization", "Bearer "+cfg.Token)
+	}
+
+	resp, err := notifyClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("ntfy returned %s", resp.Status)
+	}
+	return nil
+}
+
+// enqueueNotification adds one row to the outbox, due immediately.
+func enqueueNotification(ctx context.Context, q querier, incidentID int64, userID *int64, topic, kind string) error {
+	now := time.Now().Unix()
+	_, err := q.ExecContext(ctx, `
+		INSERT INTO notifications (incident_id, user_id, topic, kind, created_at, send_after)
+		VALUES (?, ?, ?, ?, ?, ?)`, incidentID, userID, topic, kind, now, now)
+	return err
+}
+
+// notifyTarget decides where a newly opened incident's notification goes.
+//
+// The on-call user's own topic when they have one, otherwise the fallback
+// topic with no user attached. Deliberately not "the fallback topic, attributed
+// to the on-call user": the fallback is shared, and an Acknowledge button on a
+// shared topic would let any subscriber acknowledge as somebody else.
+func notifyTarget(ctx context.Context, q querier, cfg NotifyConfig, onCall *int64) (topic string, userID *int64) {
+	if onCall != nil {
+		var t *string
+		err := q.QueryRowContext(ctx,
+			"SELECT ntfy_topic FROM users WHERE id = ?", *onCall).Scan(&t)
+		if err == nil && t != nil && *t != "" {
+			return *t, onCall
+		}
+	}
+	return cfg.FallbackTopic, nil
+}
+
+// enqueueOpened queues the notification for a freshly opened incident. It is the
+// only enqueue point that has to resolve a topic from scratch; every later
+// notification for the incident reuses what this one chose.
+func enqueueOpened(ctx context.Context, q querier, cfg NotifyConfig, incidentID int64, onCall *int64) error {
+	if !cfg.enabled() {
+		return nil
+	}
+	topic, userID := notifyTarget(ctx, q, cfg, onCall)
+	if topic == "" {
+		// Nobody on call has a topic and there is no fallback: there is nowhere
+		// to send this, and queueing it would only accumulate undeliverable rows.
+		return nil
+	}
+	return enqueueNotification(ctx, q, incidentID, userID, topic, notifyTriggered)
+}
+
+// enqueueResolved queues the all-clear, reusing the topic the incident's last
+// notification went to. That needs no configuration to reach this function, and
+// it gives the right rule for free: you only hear that something resolved if you
+// were told it started.
+func enqueueResolved(ctx context.Context, q querier, incidentID int64) error {
+	var topic string
+	var userID *int64
+	err := q.QueryRowContext(ctx, `
+		SELECT topic, user_id FROM notifications
+		 WHERE incident_id = ? ORDER BY id DESC LIMIT 1`, incidentID).Scan(&topic, &userID)
+	if err == sql.ErrNoRows {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	return enqueueNotification(ctx, q, incidentID, userID, topic, notifyResolved)
+}
+
+// humanDuration renders an age the way a person reads it at 3am: coarse, and
+// never more than two units.
+func humanDuration(d time.Duration) string {
+	if d < time.Minute {
+		return "less than a minute"
+	}
+	if d < time.Hour {
+		return fmt.Sprintf("%dm", int(d.Minutes()))
+	}
+	h := int(d.Hours())
+	m := int(d.Minutes()) - h*60
+	if m == 0 {
+		return fmt.Sprintf("%dh", h)
+	}
+	return fmt.Sprintf("%dh%dm", h, m)
+}
+
+func plural(n int) string {
+	if n == 1 {
+		return ""
+	}
+	return "s"
+}
+
+// derefString reads a nullable text column as a plain string.
+func derefString(s *string) string {
+	if s == nil {
+		return ""
+	}
+	return *s
+}

internal/api/notify_ack.go

@@ -0,0 +1,98 @@
+package api
+
+import (
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+)
+
+// issueAckToken mints the secret behind one notification's Acknowledge button
+// and returns the raw value to embed in its URL. Only the hash is stored, the
+// same way api_keys works.
+//
+// A fresh token per delivery rather than one per incident: the raw value only
+// exists for as long as it takes to build the message, so there is nothing to
+// look up and reuse later, and a reminder that supersedes an earlier page
+// carries its own credential.
+func issueAckToken(ctx context.Context, q querier, incidentID, userID int64) (string, error) {
+	raw, hash, err := randomToken()
+	if err != nil {
+		return "", err
+	}
+	now := time.Now()
+	if _, err := q.ExecContext(ctx, `
+		INSERT INTO incident_ack_tokens (token_hash, incident_id, user_id, created_at, expires_at)
+		VALUES (?, ?, ?, ?, ?)`,
+		hash, incidentID, userID, now.Unix(), now.Add(ackTokenTTL).Unix()); err != nil {
+		return "", err
+	}
+	return raw, nil
+}
+
+// handleNotifyAck acknowledges an incident from the Acknowledge button in a
+// push notification.
+//
+// It is deliberately outside AuthMiddleware: the caller is a phone acting on a
+// notification, not a client holding an API key. What stands in for the key is
+// the token in the path — 256 bits of entropy, valid for one incident, one
+// action, and one day. It must stay publicly reachable for the button to work
+// when the responder is off the cluster network.
+func handleNotifyAck(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		h := sha256.Sum256([]byte(chi.URLParam(r, "token")))
+		hash := hex.EncodeToString(h[:])
+
+		var incidentID, userID int64
+		err := db.QueryRowContext(r.Context(), `
+			SELECT incident_id, user_id FROM incident_ack_tokens
+			 WHERE token_hash = ? AND expires_at > ?`,
+			hash, time.Now().Unix()).Scan(&incidentID, &userID)
+		if err != nil {
+			// Unknown and expired get the same answer, so the endpoint cannot be
+			// used to probe which tokens once existed.
+			respond(w, http.StatusNotFound, errResp("invalid or expired token"))
+			return
+		}
+
+		acked, err := acknowledgeIncident(r.Context(), db, incidentID, userID)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if !acked {
+			// The incident closed between the page and the tap. Nothing to do,
+			// and nothing the responder did wrong — report the state, not an error,
+			// so ntfy shows a success toast rather than a failure.
+			respond(w, http.StatusOK, map[string]any{
+				"incident_id": incidentID,
+				"status":      "resolved",
+			})
+			return
+		}
+		respond(w, http.StatusOK, map[string]any{
+			"incident_id": incidentID,
+			"status":      "acknowledged",
+		})
+	}
+}
+
+// purgeAckTokens drops tokens whose notifications are long past. Nothing else
+// deletes them: incidents are archived rather than removed, so the cascade never
+// fires in practice.
+func purgeAckTokens(ctx context.Context, db *sql.DB) {
+	res, err := db.ExecContext(ctx,
+		"DELETE FROM incident_ack_tokens WHERE expires_at < ?", time.Now().Unix())
+	if err != nil {
+		log.Printf("sweeper: purge ack tokens: %v", err)
+		return
+	}
+	if n, _ := res.RowsAffected(); n > 0 {
+		log.Printf("sweeper: purged %d expired ack token(s)", n)
+	}
+}

internal/api/notify_test.go

@@ -0,0 +1,580 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/yeniklas/terdut-server/internal/api"
+)
+
+// ---------------------------------------------------------------------------
+// Fake ntfy
+// ---------------------------------------------------------------------------
+
+// pushed is one message the fake ntfy received, in ntfy's JSON publish shape.
+type pushed struct {
+	Topic    string   `json:"topic"`
+	Title    string   `json:"title"`
+	Message  string   `json:"message"`
+	Priority int      `json:"priority"`
+	Tags     []string `json:"tags"`
+	Click    string   `json:"click"`
+	Actions  []struct {
+		Action string `json:"action"`
+		Label  string `json:"label"`
+		URL    string `json:"url"`
+		Method string `json:"method"`
+		Clear  bool   `json:"clear"`
+	} `json:"actions"`
+}
+
+// fakeNtfy records what the notifier published. status controls the reply, so a
+// test can make delivery fail.
+type fakeNtfy struct {
+	*httptest.Server
+	mu     sync.Mutex
+	got    []pushed
+	status int
+}
+
+func newFakeNtfy(t *testing.T) *fakeNtfy {
+	t.Helper()
+	f := &fakeNtfy{status: http.StatusOK}
+	f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var msg pushed
+		if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
+			http.Error(w, "bad json", http.StatusBadRequest)
+			return
+		}
+		f.mu.Lock()
+		f.got = append(f.got, msg)
+		status := f.status
+		f.mu.Unlock()
+		w.WriteHeader(status)
+	}))
+	t.Cleanup(f.Close)
+	return f
+}
+
+func (f *fakeNtfy) messages() []pushed {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]pushed(nil), f.got...)
+}
+
+func (f *fakeNtfy) failWith(status int) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.status = status
+}
+
+// ---------------------------------------------------------------------------
+// Harness
+// ---------------------------------------------------------------------------
+
+// notifyTS builds a server with notifications enabled, the admin on call today,
+// and a topic on the admin — the setup every delivery test needs.
+func notifyTS(t *testing.T, cfg api.NotifyConfig) (*ts, *fakeNtfy) {
+	t.Helper()
+	f := newFakeNtfy(t)
+	cfg.BaseURL = f.URL
+	s := newTS(t, cfg)
+
+	putOnCall(t, s, 1)
+	setTopic(t, s, 1, "terdut-admin")
+	return s, f
+}
+
+func putOnCall(t *testing.T, s *ts, userID int) {
+	t.Helper()
+	today := time.Now().UTC().Format("2006-01-02")
+	resp := s.req(t, http.MethodPost, "/api/schedule",
+		map[string]any{"user_id": userID, "dates": []string{today}})
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("schedule assignment returned %d", resp.StatusCode)
+	}
+}
+
+func setTopic(t *testing.T, s *ts, userID int, topic string) {
+	t.Helper()
+	resp := s.req(t, http.MethodPut,
+		fmt.Sprintf("/api/users/%d/notify", userID), map[string]any{"ntfy_topic": topic})
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("set notify topic returned %d", resp.StatusCode)
+	}
+}
+
+func (s *ts) sweepNotify(t *testing.T) {
+	t.Helper()
+	api.NotifySweep(context.Background(), s.db, s.notify)
+}
+
+// countNotifications reports how many outbox rows exist, optionally of one kind.
+func (s *ts) countNotifications(t *testing.T, kind string) int {
+	t.Helper()
+	var n int
+	query := "SELECT COUNT(*) FROM notifications"
+	args := []any{}
+	if kind != "" {
+		query += " WHERE kind = ?"
+		args = append(args, kind)
+	}
+	if err := s.db.QueryRow(query, args...).Scan(&n); err != nil {
+		t.Fatalf("count notifications: %v", err)
+	}
+	return n
+}
+
+// fireCritical posts a single critical alert, which opens one incident.
+func fireCritical(t *testing.T, s *ts) {
+	t.Helper()
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-notify", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime,
+			map[string]string{"severity": "critical"}),
+	}, "{}:{alertname=\"DiskFull\"}")
+}
+
+// ---------------------------------------------------------------------------
+// Delivery
+// ---------------------------------------------------------------------------
+
+func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+
+	if got := s.countNotifications(t, "triggered"); got != 1 {
+		t.Fatalf("expected 1 queued notification, got %d", got)
+	}
+	s.sweepNotify(t)
+
+	msgs := f.messages()
+	if len(msgs) != 1 {
+		t.Fatalf("expected 1 push, got %d", len(msgs))
+	}
+	m := msgs[0]
+
+	if m.Topic != "terdut-admin" {
+		t.Errorf("expected the on-call user's topic, got %q", m.Topic)
+	}
+	if m.Priority != 5 {
+		t.Errorf("expected max priority for a critical incident, got %d", m.Priority)
+	}
+	if !strings.Contains(m.Title, "DiskFull") {
+		t.Errorf("expected the incident title in %q", m.Title)
+	}
+	if !strings.Contains(m.Message, "severity critical") {
+		t.Errorf("expected the severity in %q", m.Message)
+	}
+	if m.Click != "https://terdut.example.com/api/incidents/1" {
+		t.Errorf("unexpected click target %q", m.Click)
+	}
+	if len(m.Actions) != 1 || m.Actions[0].Label != "Acknowledge" {
+		t.Fatalf("expected an Acknowledge action, got %+v", m.Actions)
+	}
+	if m.Actions[0].Method != http.MethodPost {
+		t.Errorf("expected the action to POST, got %q", m.Actions[0].Method)
+	}
+}
+
+// A delivered row must not be delivered again on the next pass.
+func TestNotify_DeliveredOnlyOnce(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected 1 push across two passes, got %d", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Acknowledging from the notification
+// ---------------------------------------------------------------------------
+
+func TestNotify_AckButtonAcknowledgesIncident(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	ackURL := f.messages()[0].Actions[0].URL
+	// The action URL is built for the public hostname; point it at the test
+	// server, which is the same handler.
+	path := ackURL[strings.Index(ackURL, "/api/notify/ack/"):]
+
+	resp, err := http.Post(s.URL+path, "application/json", nil)
+	if err != nil {
+		t.Fatalf("ack: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("expected 200 from the ack button, got %d", resp.StatusCode)
+	}
+
+	inc := getIncident(t, s, 1)
+	if inc["status"] != "acknowledged" {
+		t.Errorf("expected the incident acknowledged, got %v", inc["status"])
+	}
+	if inc["acknowledged_by"] != "admin" {
+		t.Errorf("expected the ack attributed to the token's user, got %v", inc["acknowledged_by"])
+	}
+
+	// The timeline must record it the same way the authenticated route would.
+	events := timeline(t, s, 1)
+	if !contains(eventTypes(events), "acknowledged") {
+		t.Errorf("expected an acknowledged event, got %v", eventTypes(events))
+	}
+	for _, e := range events {
+		if e["type"] == "acknowledged" && e["username"] != "admin" {
+			t.Errorf("expected the acknowledged event attributed to admin, got %v", e["username"])
+		}
+	}
+}
+
+func TestNotify_AckRejectsUnknownToken(t *testing.T) {
+	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+	fireCritical(t, s)
+
+	resp, err := http.Post(s.URL+"/api/notify/ack/deadbeef", "application/json", nil)
+	if err != nil {
+		t.Fatalf("ack: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 for an unknown token, got %d", resp.StatusCode)
+	}
+	if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
+		t.Errorf("expected the incident untouched, got %v", inc["status"])
+	}
+}
+
+func TestNotify_AckRejectsExpiredToken(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	ackURL := f.messages()[0].Actions[0].URL
+	path := ackURL[strings.Index(ackURL, "/api/notify/ack/"):]
+
+	// Age the token past its TTL. The token's inputs are wall-clock timestamps,
+	// so this is the same trick the sweeper tests use.
+	s.exec(t, "UPDATE incident_ack_tokens SET expires_at = ?", time.Now().Add(-time.Minute).Unix())
+
+	resp, err := http.Post(s.URL+path, "application/json", nil)
+	if err != nil {
+		t.Fatalf("ack: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("expected 404 for an expired token, got %d", resp.StatusCode)
+	}
+	if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
+		t.Errorf("expected the incident untouched, got %v", inc["status"])
+	}
+}
+
+// The sweeper is what stops expired tokens accumulating forever.
+func TestNotify_SweepPurgesExpiredAckTokens(t *testing.T) {
+	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+	s.exec(t, "UPDATE incident_ack_tokens SET expires_at = ?", time.Now().Add(-time.Minute).Unix())
+
+	api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour)
+
+	var n int
+	if err := s.db.QueryRow("SELECT COUNT(*) FROM incident_ack_tokens").Scan(&n); err != nil {
+		t.Fatalf("count tokens: %v", err)
+	}
+	if n != 0 {
+		t.Errorf("expected expired tokens purged, %d left", n)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Reminders
+// ---------------------------------------------------------------------------
+
+// ageNotifications backdates every sent notification so the next pass sees the
+// reminder as due.
+func (s *ts) ageNotifications(t *testing.T, by time.Duration) {
+	t.Helper()
+	s.exec(t, "UPDATE notifications SET created_at = ? WHERE sent_at IS NOT NULL",
+		time.Now().Add(-by).Unix())
+}
+
+func TestNotify_UnacknowledgedIncidentIsRenotified(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{
+		PublicURL:   "https://terdut.example.com",
+		RepeatEvery: 15 * time.Minute,
+	})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+	s.ageNotifications(t, 20*time.Minute)
+	s.sweepNotify(t)
+
+	msgs := f.messages()
+	if len(msgs) != 2 {
+		t.Fatalf("expected a reminder push, got %d message(s)", len(msgs))
+	}
+	if !strings.Contains(msgs[1].Title, "Still unacknowledged") {
+		t.Errorf("expected the reminder to say so, got %q", msgs[1].Title)
+	}
+	if msgs[1].Topic != "terdut-admin" {
+		t.Errorf("expected the reminder on the same topic, got %q", msgs[1].Topic)
+	}
+	// Each page carries its own credential.
+	if len(msgs[1].Actions) != 1 || msgs[1].Actions[0].URL == msgs[0].Actions[0].URL {
+		t.Errorf("expected the reminder to carry a fresh ack token")
+	}
+}
+
+func TestNotify_AcknowledgedIncidentStopsReminders(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{
+		PublicURL:   "https://terdut.example.com",
+		RepeatEvery: 15 * time.Minute,
+	})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
+	resp.Body.Close()
+
+	s.ageNotifications(t, 20*time.Minute)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected no reminder once acknowledged, got %d message(s)", got)
+	}
+}
+
+// Snooze is the deliberate "not now", and it is what mutes the pager.
+func TestNotify_SnoozedIncidentStopsReminders(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{
+		PublicURL:   "https://terdut.example.com",
+		RepeatEvery: 15 * time.Minute,
+	})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]any{"duration": "1h"})
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("snooze returned %d", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	s.ageNotifications(t, 20*time.Minute)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected no reminder while snoozed, got %d message(s)", got)
+	}
+}
+
+func TestNotify_ZeroRepeatDisablesReminders(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+	s.ageNotifications(t, 24*time.Hour)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected reminders off, got %d message(s)", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Resolution
+// ---------------------------------------------------------------------------
+
+func TestNotify_ResolvedIncidentSendsAllClear(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-notify", "DiskFull", "resolved", "2026-05-20T10:00:00Z",
+			"2026-05-20T11:00:00Z", map[string]string{"severity": "critical"}),
+	}, "{}:{alertname=\"DiskFull\"}")
+	s.sweepNotify(t)
+
+	msgs := f.messages()
+	if len(msgs) != 2 {
+		t.Fatalf("expected an all-clear push, got %d message(s)", len(msgs))
+	}
+	if !strings.HasPrefix(msgs[1].Title, "Resolved:") {
+		t.Errorf("expected a resolved title, got %q", msgs[1].Title)
+	}
+	if msgs[1].Priority != 2 {
+		t.Errorf("expected the all-clear at low priority, got %d", msgs[1].Priority)
+	}
+	// Nothing to acknowledge on a closed incident.
+	if len(msgs[1].Actions) != 0 {
+		t.Errorf("expected no actions on the all-clear, got %+v", msgs[1].Actions)
+	}
+}
+
+// Closing an incident by hand sends nothing: the person who did it knows.
+func TestNotify_ManualResolveSendsNothing(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
+	resp.Body.Close()
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected no push for a manual resolve, got %d message(s)", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Routing and configuration
+// ---------------------------------------------------------------------------
+
+// With nobody on call the page goes to the shared fallback, and carries no
+// Acknowledge button — there is no user to attribute the acknowledgement to.
+func TestNotify_FallbackTopicHasNoAckButton(t *testing.T) {
+	f := newFakeNtfy(t)
+	s := newTS(t, api.NotifyConfig{
+		BaseURL:       f.URL,
+		FallbackTopic: "terdut-oncall",
+		PublicURL:     "https://terdut.example.com",
+	})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	msgs := f.messages()
+	if len(msgs) != 1 {
+		t.Fatalf("expected 1 push, got %d", len(msgs))
+	}
+	if msgs[0].Topic != "terdut-oncall" {
+		t.Errorf("expected the fallback topic, got %q", msgs[0].Topic)
+	}
+	if len(msgs[0].Actions) != 0 {
+		t.Errorf("expected no ack button on a shared topic, got %+v", msgs[0].Actions)
+	}
+}
+
+// Nobody on call and no fallback means there is nowhere to send: queueing would
+// only pile up rows that can never be delivered.
+func TestNotify_NoTargetQueuesNothing(t *testing.T) {
+	f := newFakeNtfy(t)
+	s := newTS(t, api.NotifyConfig{BaseURL: f.URL})
+
+	fireCritical(t, s)
+
+	if got := s.countNotifications(t, ""); got != 0 {
+		t.Errorf("expected nothing queued without a target, got %d", got)
+	}
+	s.sweepNotify(t)
+	if got := len(f.messages()); got != 0 {
+		t.Errorf("expected no push, got %d", got)
+	}
+}
+
+// The zero NotifyConfig is what every pre-existing test runs under.
+func TestNotify_DisabledQueuesNothing(t *testing.T) {
+	s := newTS(t)
+	putOnCall(t, s, 1)
+	setTopic(t, s, 1, "terdut-admin")
+
+	fireCritical(t, s)
+
+	if got := s.countNotifications(t, ""); got != 0 {
+		t.Errorf("expected nothing queued with notifications off, got %d", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Retries
+// ---------------------------------------------------------------------------
+
+func TestNotify_FailedDeliveryRetriesWithBackoff(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+	f.failWith(http.StatusInternalServerError)
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	var attempts int
+	var sentAt *int64
+	var sendAfter int64
+	var lastError *string
+	if err := s.db.QueryRow(
+		"SELECT attempts, sent_at, send_after, last_error FROM notifications WHERE id = 1").
+		Scan(&attempts, &sentAt, &sendAfter, &lastError); err != nil {
+		t.Fatalf("read notification: %v", err)
+	}
+	if attempts != 1 {
+		t.Errorf("expected 1 attempt recorded, got %d", attempts)
+	}
+	if sentAt != nil {
+		t.Errorf("expected the row unsent, got sent_at %v", *sentAt)
+	}
+	if sendAfter <= time.Now().Unix() {
+		t.Errorf("expected the retry pushed into the future, got %d", sendAfter)
+	}
+	if lastError == nil || !strings.Contains(*lastError, "500") {
+		t.Errorf("expected the failure recorded, got %v", lastError)
+	}
+
+	// Backing off means the next pass leaves it alone until it is due.
+	s.sweepNotify(t)
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected no immediate retry, got %d attempt(s)", got)
+	}
+
+	// Once due and once ntfy recovers, it goes out.
+	f.failWith(http.StatusOK)
+	s.exec(t, "UPDATE notifications SET send_after = ? WHERE id = 1", time.Now().Add(-time.Second).Unix())
+	s.sweepNotify(t)
+
+	if err := s.db.QueryRow("SELECT sent_at FROM notifications WHERE id = 1").Scan(&sentAt); err != nil {
+		t.Fatalf("read notification: %v", err)
+	}
+	if sentAt == nil {
+		t.Error("expected the retry to succeed once ntfy recovered")
+	}
+}
+
+// An ntfy outage must not produce a reminder backlog that all lands at once
+// when it comes back: the previous page has to have been sent first.
+func TestNotify_UnsentNotificationBlocksReminders(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{
+		PublicURL:   "https://terdut.example.com",
+		RepeatEvery: 15 * time.Minute,
+	})
+	f.failWith(http.StatusInternalServerError)
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+	s.exec(t, "UPDATE notifications SET created_at = ?", time.Now().Add(-time.Hour).Unix())
+	s.sweepNotify(t)
+
+	if got := s.countNotifications(t, "reminder"); got != 0 {
+		t.Errorf("expected no reminders queued behind an undelivered page, got %d", got)
+	}
+}

internal/api/router.go

@@ -8,7 +8,10 @@ import (
 	"github.com/go-chi/chi/v5/middleware"
 )
 
-func NewRouter(db *sql.DB) http.Handler {
+// NewRouter builds the HTTP surface. notify is passed through to the webhook,
+// the only handler that has to decide where a new incident's page goes; a zero
+// value disables notifications.
+func NewRouter(db *sql.DB, notify NotifyConfig) http.Handler {
 	r := chi.NewRouter()
 	r.Use(middleware.Logger)
 	r.Use(middleware.Recoverer)
@@ -17,9 +20,13 @@ func NewRouter(db *sql.DB) http.Handler {
 		respond(w, http.StatusOK, map[string]string{"status": "ok"})
 	})
 
-	// Unauthenticated: bootstrap and Alertmanager webhook receiver.
+	// Unauthenticated: bootstrap, the Alertmanager webhook receiver, and the
+	// Acknowledge button in a push notification. The last one is authorised by
+	// the single-use token in its path rather than an API key, and has to stay
+	// reachable from outside the cluster for the button to work.
 	r.Post("/api/bootstrap", handleBootstrap(db))
-	r.Post("/api/alertmanager/webhook", handleAlertmanagerWebhook(db))
+	r.Post("/api/alertmanager/webhook", handleAlertmanagerWebhook(db, notify))
+	r.Post("/api/notify/ack/{token}", handleNotifyAck(db))
 
 	// All other /api routes require a valid API key.
 	r.Group(func(r chi.Router) {
@@ -28,22 +35,36 @@ func NewRouter(db *sql.DB) http.Handler {
 		r.Get("/api/users", handleListUsers(db))
 		r.Post("/api/users", handleCreateUser(db))
 		r.Delete("/api/users/{id}", handleDeleteUser(db))
+		r.Put("/api/users/{id}/notify", handleSetNotifyTarget(db))
 		r.Post("/api/users/{id}/api-keys", handleCreateAPIKey(db))
 		r.Delete("/api/users/{id}/api-keys/{keyID}", handleDeleteAPIKey(db))
 
+		// Alerts are read-only: they are Alertmanager's record, not a work
+		// queue. Everything a person does happens on the incident instead.
 		r.Get("/api/alerts", handleListAlerts(db))
 		r.Get("/api/alerts/{id}", handleGetAlert(db))
-		r.Post("/api/alerts/{id}/acknowledge", handleAcknowledge(db))
+
-		r.Delete("/api/alerts/{id}/acknowledge", handleUnacknowledge(db))
+		r.Get("/api/incidents", handleListIncidents(db))
-		r.Get("/api/alerts/{id}/comments", handleListComments(db))
+		r.Get("/api/incidents/{id}", handleGetIncident(db))
-		r.Post("/api/alerts/{id}/comments", handleCreateComment(db))
+		r.Get("/api/incidents/{id}/alerts", handleIncidentAlerts(db))
-		r.Delete("/api/alerts/{id}/comments/{commentID}", handleDeleteComment(db))
+		r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db))
+		r.Post("/api/incidents/{id}/acknowledge", handleIncidentAcknowledge(db))
+		r.Delete("/api/incidents/{id}/acknowledge", handleIncidentUnacknowledge(db))
+		r.Post("/api/incidents/{id}/resolve", handleIncidentResolve(db))
+		r.Post("/api/incidents/{id}/assign", handleIncidentAssign(db))
+		r.Post("/api/incidents/{id}/snooze", handleIncidentSnooze(db))
+		r.Delete("/api/incidents/{id}/snooze", handleIncidentUnsnooze(db))
+		r.Post("/api/incidents/{id}/archive", handleIncidentArchive(db))
+		r.Delete("/api/incidents/{id}/archive", handleIncidentUnarchive(db))
+		r.Post("/api/incidents/{id}/notes", handleCreateNote(db))
+		r.Delete("/api/incidents/{id}/notes/{eventID}", handleDeleteNote(db))
 
 		r.Post("/api/schedule", handleCreateSchedule(db))
 		r.Get("/api/schedule/current", handleCurrentSchedule(db)) // must be before /{id}
 		r.Get("/api/schedule", handleListSchedule(db))
 		r.Delete("/api/schedule/{id}", handleDeleteSchedule(db))
 
+		r.Get("/api/stats/incidents", handleStatsIncidents(db))
 		r.Get("/api/stats/alerts", handleStatsAlerts(db))
 		r.Get("/api/stats/alerts/top", handleStatsTop(db))
 		r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db))

internal/api/stats.go

@@ -11,7 +11,7 @@ import (
 
 func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query())
+		where, args := statsFilter(r.URL.Query(), "received_at")
 
 		var total, firing, resolved int64
 		err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
@@ -34,7 +34,7 @@ func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
 
 func handleStatsTop(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query())
+		where, args := statsFilter(r.URL.Query(), "received_at")
 
 		limit := 10
 		if l := r.URL.Query().Get("limit"); l != "" {
@@ -78,7 +78,7 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 
 func handleStatsByHour(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query())
+		where, args := statsFilter(r.URL.Query(), "received_at")
 
 		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
 			SELECT CAST(strftime('%%H', datetime(received_at, 'unixepoch')) AS INTEGER) AS hr,
@@ -118,7 +118,7 @@ func handleStatsByHour(db *sql.DB) http.HandlerFunc {
 
 func handleStatsByDay(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query())
+		where, args := statsFilter(r.URL.Query(), "received_at")
 
 		// SQLite strftime('%w') → 0=Sunday … 6=Saturday
 		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
@@ -159,23 +159,61 @@ func handleStatsByDay(db *sql.DB) http.HandlerFunc {
 	}
 }
 
-// statsFilter builds a WHERE clause and args from optional ?from and ?to query params.
+// handleStatsIncidents reports the queue and the two numbers a rota actually
-func statsFilter(q url.Values) (where string, args []any) {
+// cares about: how long it takes someone to pick work up, and how long it takes
-	clauses := []string{}
+// to finish. Neither was computable before incidents existed — alert rows are
+// mutated in place and carry no acknowledgement or closure time.
+func handleStatsIncidents(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		where, args := statsFilter(r.URL.Query(), "triggered_at")
+
+		var total, triggered, acknowledged, resolved int64
+		var mtta, mttr *float64
+		err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
+			SELECT COUNT(*),
+			       SUM(CASE WHEN status = 'triggered'    THEN 1 ELSE 0 END),
+			       SUM(CASE WHEN status = 'acknowledged' THEN 1 ELSE 0 END),
+			       SUM(CASE WHEN status = 'resolved'     THEN 1 ELSE 0 END),
+			       AVG(CASE WHEN acknowledged_at IS NOT NULL
+			                THEN acknowledged_at - triggered_at END),
+			       AVG(CASE WHEN resolved_at IS NOT NULL
+			                THEN resolved_at - triggered_at END)
+			FROM incidents WHERE %s`, where), args...,
+		).Scan(&total, &triggered, &acknowledged, &resolved, &mtta, &mttr)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+
+		respond(w, http.StatusOK, map[string]any{
+			"total":        total,
+			"triggered":    triggered,
+			"acknowledged": acknowledged,
+			"resolved":     resolved,
+			// Null until something has actually been acknowledged or resolved —
+			// zero would read as "instant", which is a different claim.
+			"mtta_seconds": mtta,
+			"mttr_seconds": mttr,
+		})
+	}
+}
+
+// statsFilter builds a WHERE clause and args from optional ?from and ?to query
+// params, filtering on timeCol. Archived rows are always excluded, matching the
+// default list views.
+func statsFilter(q url.Values, timeCol string) (where string, args []any) {
+	clauses := []string{"archived_at IS NULL"}
 	if from := q.Get("from"); from != "" {
 		if t, err := time.Parse("2006-01-02", from); err == nil {
-			clauses = append(clauses, "received_at >= ?")
+			clauses = append(clauses, timeCol+" >= ?")
 			args = append(args, t.UTC().Unix())
 		}
 	}
 	if to := q.Get("to"); to != "" {
 		if t, err := time.Parse("2006-01-02", to); err == nil {
-			clauses = append(clauses, "received_at < ?")
+			clauses = append(clauses, timeCol+" < ?")
 			args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
 		}
 	}
-	if len(clauses) == 0 {
-		return "1=1", args
-	}
 	return strings.Join(clauses, " AND "), args
 }

internal/api/users.go

@@ -48,7 +48,7 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
 		}
 		userID, _ := res.LastInsertId()
 
-		raw, hash, err := newAPIKey()
+		raw, hash, err := randomToken()
 		if err != nil {
 			respond(w, http.StatusInternalServerError, errResp("internal error"))
 			return
@@ -70,7 +70,7 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
 func handleListUsers(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		rows, err := db.QueryContext(r.Context(),
-			"SELECT id, username, email, created_at FROM users ORDER BY id")
+			"SELECT id, username, email, created_at, ntfy_topic FROM users ORDER BY id")
 		if err != nil {
 			respond(w, http.StatusInternalServerError, errResp("internal error"))
 			return
@@ -81,7 +81,7 @@ func handleListUsers(db *sql.DB) http.HandlerFunc {
 		for rows.Next() {
 			var u models.User
 			var ts int64
-			if err := rows.Scan(&u.ID, &u.Username, &u.Email, &ts); err != nil {
+			if err := rows.Scan(&u.ID, &u.Username, &u.Email, &ts, &u.NtfyTopic); err != nil {
 				respond(w, http.StatusInternalServerError, errResp("internal error"))
 				return
 			}
@@ -123,6 +123,50 @@ func handleCreateUser(db *sql.DB) http.HandlerFunc {
 	}
 }
 
+// handleSetNotifyTarget points a user's push notifications at an ntfy topic, or
+// clears it with an empty string. The topic is a shared secret with the ntfy
+// server — anyone who knows it can publish to it — so pick an unguessable one
+// unless your ntfy enforces access control.
+func handleSetNotifyTarget(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid user id"))
+			return
+		}
+		var req struct {
+			NtfyTopic string `json:"ntfy_topic"`
+		}
+		if err := decodeJSON(r, &req); err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid request body"))
+			return
+		}
+
+		var topic *string
+		if t := strings.TrimSpace(req.NtfyTopic); t != "" {
+			topic = &t
+		}
+
+		res, err := db.ExecContext(r.Context(),
+			"UPDATE users SET ntfy_topic = ? WHERE id = ?", topic, id)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			respond(w, http.StatusNotFound, errResp("user not found"))
+			return
+		}
+
+		user, err := fetchUser(r.Context(), db, id)
+		if err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		respond(w, http.StatusOK, user)
+	}
+}
+
 func handleDeleteUser(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
@@ -170,7 +214,7 @@ func handleCreateAPIKey(db *sql.DB) http.HandlerFunc {
 			return
 		}
 
-		raw, hash, err := newAPIKey()
+		raw, hash, err := randomToken()
 		if err != nil {
 			respond(w, http.StatusInternalServerError, errResp("internal error"))
 			return
@@ -215,8 +259,9 @@ func handleDeleteAPIKey(db *sql.DB) http.HandlerFunc {
 	}
 }
 
-// newAPIKey generates a random 32-byte key encoded as hex, plus its SHA-256 hash for storage.
+// randomToken generates a random 32-byte secret encoded as hex, plus its SHA-256
-func newAPIKey() (raw, hash string, err error) {
+// hash for storage. Used for API keys and for notification acknowledge tokens.
+func randomToken() (raw, hash string, err error) {
 	b := make([]byte, 32)
 	if _, err = rand.Read(b); err != nil {
 		return
@@ -230,8 +275,9 @@ func newAPIKey() (raw, hash string, err error) {
 func fetchUser(ctx context.Context, db *sql.DB, id int64) (models.User, error) {
 	var u models.User
 	var ts int64
-	err := db.QueryRowContext(ctx, "SELECT id, username, email, created_at FROM users WHERE id = ?", id).
+	err := db.QueryRowContext(ctx,
-		Scan(&u.ID, &u.Username, &u.Email, &ts)
+		"SELECT id, username, email, created_at, ntfy_topic FROM users WHERE id = ?", id).
+		Scan(&u.ID, &u.Username, &u.Email, &ts, &u.NtfyTopic)
 	if err != nil {
 		return u, err
 	}

internal/config/config.go

@@ -1,10 +1,38 @@
 package config
 
-import "os"
+import (
+	"os"
+	"time"
+)
 
 type Config struct {
-	Addr   string
+	Addr         string
-	DBPath string
+	DBPath       string
+	ArchiveAfter time.Duration
+
+	// StaleAfter is how long a firing alert may go without a refreshing webhook
+	// before the sweeper treats it as resolved. It must exceed Alertmanager's
+	// repeat_interval (default 4h), which is what refreshes the alert.
+	StaleAfter time.Duration
+
+	// NtfyURL is the ntfy server push notifications are published to. Empty
+	// disables notifications entirely.
+	NtfyURL string
+
+	// NtfyToken is an optional bearer token for an access-controlled ntfy.
+	NtfyToken string
+
+	// NtfyFallbackTopic receives incidents that open with nobody on call.
+	NtfyFallbackTopic string
+
+	// PublicURL is the base URL a phone uses to reach this server, used for the
+	// link and the Acknowledge button inside a notification. Without it
+	// notifications carry neither.
+	PublicURL string
+
+	// NotifyRepeat is how long an incident may sit unacknowledged before it is
+	// notified again. Zero disables reminders.
+	NotifyRepeat time.Duration
 }
 
 func Load() Config {
@@ -16,5 +44,28 @@ func Load() Config {
 	if dbPath == "" {
 		dbPath = "terdut.db"
 	}
-	return Config{Addr: addr, DBPath: dbPath}
+	return Config{
+		Addr:         addr,
+		DBPath:       dbPath,
+		ArchiveAfter: duration("TERDUT_ARCHIVE_AFTER", 7*24*time.Hour),
+		StaleAfter:   duration("TERDUT_STALE_AFTER", 6*time.Hour),
+
+		NtfyURL:           os.Getenv("TERDUT_NTFY_URL"),
+		NtfyToken:         os.Getenv("TERDUT_NTFY_TOKEN"),
+		NtfyFallbackTopic: os.Getenv("TERDUT_NTFY_FALLBACK_TOPIC"),
+		PublicURL:         os.Getenv("TERDUT_PUBLIC_URL"),
+		NotifyRepeat:      duration("TERDUT_NOTIFY_REPEAT", 15*time.Minute),
+	}
+}
+
+// duration reads a time.ParseDuration-formatted env var. An unset or
+// unparseable value falls back to def rather than failing startup: a typo in one
+// tuning knob should not take the server down.
+func duration(env string, def time.Duration) time.Duration {
+	if s := os.Getenv(env); s != "" {
+		if d, err := time.ParseDuration(s); err == nil {
+			return d
+		}
+	}
+	return def
 }

internal/db/migrations/006_alert_archive.sql

@@ -0,0 +1,2 @@
+ALTER TABLE alerts ADD COLUMN archived_at INTEGER;
+CREATE INDEX alerts_archived_at_idx ON alerts(archived_at);

internal/db/migrations/007_alert_resolution_source.sql

@@ -0,0 +1,4 @@
+-- Records why an alert left the firing state: 'alertmanager' when a resolved
+-- webhook set it, 'expiry' when the sweeper inferred it from staleness.
+-- NULL for firing alerts and for rows that predate this migration.
+ALTER TABLE alerts ADD COLUMN resolution_source TEXT;

internal/db/migrations/008_incidents.sql

@@ -0,0 +1,123 @@
+-- Splits the single alerts row into two objects, the way an incident management
+-- tool needs them: alerts stay the machine-owned signal record that Alertmanager
+-- writes, and incidents become the human work item people acknowledge, assign,
+-- snooze, discuss and resolve.
+--
+-- Correlation uses Alertmanager's own groupKey, so incidents follow the group_by
+-- routing tree the operator already tuned rather than a second grouping scheme
+-- invented here.
+
+CREATE TABLE incidents (
+    id                INTEGER PRIMARY KEY AUTOINCREMENT,
+    group_key         TEXT    NOT NULL,              -- Alertmanager groupKey, opaque
+    title             TEXT    NOT NULL,              -- rendered from group_labels
+    group_labels      TEXT    NOT NULL DEFAULT '{}', -- JSON
+    status            TEXT    NOT NULL CHECK(status IN ('triggered', 'acknowledged', 'resolved')),
+    severity          TEXT,                          -- highest `severity` label across firing members
+    triggered_at      INTEGER NOT NULL,
+    acknowledged_by   INTEGER REFERENCES users(id) ON DELETE SET NULL,
+    acknowledged_at   INTEGER,
+    assigned_to       INTEGER REFERENCES users(id) ON DELETE SET NULL,
+    snoozed_until     INTEGER,
+    resolved_at       INTEGER,
+    resolution_source TEXT,                          -- 'alerts' | 'manual'
+    archived_at       INTEGER
+);
+
+-- Load-bearing: at most one OPEN incident per group_key. This is what makes
+-- "resolved incident + a new alert occurrence = a new incident" work, and it is
+-- the constraint the webhook's find-or-open lookup relies on.
+CREATE UNIQUE INDEX incidents_open_group_key_idx ON incidents(group_key) WHERE resolved_at IS NULL;
+CREATE INDEX incidents_status_idx       ON incidents(status);
+CREATE INDEX incidents_triggered_at_idx ON incidents(triggered_at DESC);
+CREATE INDEX incidents_archived_at_idx  ON incidents(archived_at);
+
+-- Membership is historical, not a pointer on alerts: one alert row (one
+-- fingerprint) resolves and re-fires over time and belongs to a different
+-- incident each occurrence.
+CREATE TABLE incident_alerts (
+    incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+    alert_id    INTEGER NOT NULL REFERENCES alerts(id)    ON DELETE CASCADE,
+    added_at    INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
+    PRIMARY KEY (incident_id, alert_id)
+);
+
+CREATE INDEX incident_alerts_alert_id_idx ON incident_alerts(alert_id);
+
+-- The timeline. Append-only, and the only history this server keeps: alert rows
+-- are mutated in place, so without this there is no record that anything
+-- happened. Notes are events too, so one query renders the whole story.
+CREATE TABLE incident_events (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+    -- triggered | alert_added | alert_resolved | acknowledged | unacknowledged
+    -- | assigned | snoozed | unsnoozed | resolved | note
+    type        TEXT    NOT NULL,
+    user_id     INTEGER REFERENCES users(id)  ON DELETE SET NULL, -- NULL = the server acted
+    alert_id    INTEGER REFERENCES alerts(id) ON DELETE SET NULL,
+    detail      TEXT,
+    created_at  INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
+);
+
+CREATE INDEX incident_events_incident_idx ON incident_events(incident_id, created_at);
+
+-- ---------------------------------------------------------------------------
+-- Backfill
+--
+-- Every pre-existing alert gets its own incident, archived ones included, so no
+-- acknowledgement and no comment is orphaned. There is no historical groupKey to
+-- correlate on, hence one incident per fingerprint under a 'backfill:' prefix
+-- that can never collide with a real Alertmanager groupKey.
+-- ---------------------------------------------------------------------------
+
+INSERT INTO incidents (group_key, title, group_labels, status, severity, triggered_at,
+                       acknowledged_by, acknowledged_at, assigned_to,
+                       resolved_at, resolution_source, archived_at)
+SELECT 'backfill:' || a.fingerprint,
+       a.name,
+       json_object('alertname', a.name),
+       CASE WHEN a.status = 'resolved'          THEN 'resolved'
+            WHEN a.acknowledged_by IS NOT NULL  THEN 'acknowledged'
+            ELSE 'triggered' END,
+       json_extract(a.labels, '$.severity'),
+       a.starts_at,
+       a.acknowledged_by,
+       a.acknowledged_at,
+       a.acknowledged_by,
+       CASE WHEN a.status = 'resolved' THEN COALESCE(a.ends_at, a.received_at) END,
+       CASE WHEN a.status = 'resolved' THEN 'alerts' END,
+       a.archived_at
+FROM alerts a;
+
+INSERT INTO incident_alerts (incident_id, alert_id, added_at)
+SELECT i.id, a.id, a.starts_at
+FROM alerts a
+JOIN incidents i ON i.group_key = 'backfill:' || a.fingerprint;
+
+INSERT INTO incident_events (incident_id, type, alert_id, created_at)
+SELECT i.id, 'triggered', ia.alert_id, i.triggered_at
+FROM incidents i JOIN incident_alerts ia ON ia.incident_id = i.id;
+
+INSERT INTO incident_events (incident_id, type, user_id, created_at)
+SELECT i.id, 'acknowledged', i.acknowledged_by, i.acknowledged_at
+FROM incidents i WHERE i.acknowledged_at IS NOT NULL;
+
+INSERT INTO incident_events (incident_id, type, created_at)
+SELECT i.id, 'resolved', i.resolved_at
+FROM incidents i WHERE i.resolved_at IS NOT NULL;
+
+INSERT INTO incident_events (incident_id, type, user_id, alert_id, detail, created_at)
+SELECT ia.incident_id, 'note', c.user_id, c.alert_id, c.content, c.created_at
+FROM alert_comments c
+JOIN incident_alerts ia ON ia.alert_id = c.alert_id;
+
+-- ---------------------------------------------------------------------------
+-- Workflow state now lives on incidents only. Leaving these behind would keep
+-- the bug they caused: the webhook upsert owns the alerts row and never cleared
+-- the acknowledgement, so a re-fire days later still read as acknowledged.
+-- ---------------------------------------------------------------------------
+
+DROP TABLE alert_comments;
+
+ALTER TABLE alerts DROP COLUMN acknowledged_by;
+ALTER TABLE alerts DROP COLUMN acknowledged_at;

internal/db/migrations/009_notifications.sql

@@ -0,0 +1,44 @@
+-- Adds push notification delivery, so an incident reaches the person on call
+-- instead of waiting to be discovered.
+--
+-- Delivery is an outbox rather than an inline HTTP call: the pool is limited to
+-- a single connection, so a POST made while holding the webhook's transaction
+-- would stall every other request behind it. The webhook inserts a row; the
+-- notifier goroutine delivers it.
+
+CREATE TABLE notifications (
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+    -- Nullable: a notification sent to the fallback topic belongs to nobody,
+    -- because nobody was on call when the incident opened.
+    user_id     INTEGER REFERENCES users(id) ON DELETE SET NULL,
+    topic       TEXT    NOT NULL,              -- resolved at enqueue: who was on call then
+    kind        TEXT    NOT NULL CHECK(kind IN ('triggered', 'reminder', 'resolved')),
+    created_at  INTEGER NOT NULL,
+    send_after  INTEGER NOT NULL,              -- retry backoff watermark
+    attempts    INTEGER NOT NULL DEFAULT 0,
+    sent_at     INTEGER,
+    last_error  TEXT                           -- kept after the last attempt, for debugging
+);
+
+-- The delivery loop's only query: what is due and still unsent.
+CREATE INDEX notifications_pending_idx  ON notifications(send_after) WHERE sent_at IS NULL;
+-- Reminders and resolved notices both look up an incident's newest row.
+CREATE INDEX notifications_incident_idx ON notifications(incident_id, id DESC);
+
+-- A notification body is stored on the ntfy server and cached on the device, so
+-- a real API key must never appear in one. Each delivery mints its own token
+-- instead: one incident, one action, one day.
+CREATE TABLE incident_ack_tokens (
+    token_hash  TEXT    PRIMARY KEY,           -- SHA-256 of the raw token, as with api_keys
+    incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
+    user_id     INTEGER NOT NULL REFERENCES users(id)     ON DELETE CASCADE,
+    created_at  INTEGER NOT NULL,
+    expires_at  INTEGER NOT NULL
+);
+
+CREATE INDEX incident_ack_tokens_expires_idx ON incident_ack_tokens(expires_at);
+
+-- Where this user's notifications go. NULL means they get none; incidents
+-- assigned to them fall back to the configured fallback topic.
+ALTER TABLE users ADD COLUMN ntfy_topic TEXT;

internal/models/alert.go

@@ -2,6 +2,10 @@ package models
 
 import "time"
 
+// Alert is the machine-owned signal record: what Alertmanager told us, and
+// nothing else. It has two states, firing and resolved, and no human ever writes
+// to it — acknowledgement, assignment, notes and closure all live on the
+// Incident an alert belongs to.
 type Alert struct {
 	ID           int64             `json:"id"`
 	Fingerprint  string            `json:"fingerprint"`
@@ -12,10 +16,39 @@ type Alert struct {
 	StartsAt     time.Time         `json:"starts_at"`
 	EndsAt       *time.Time        `json:"ends_at,omitempty"`
 	GeneratorURL string            `json:"generator_url"`
-	ReceivedAt   time.Time         `json:"received_at"`
 
-	// Populated when the alert has been acknowledged.
+	// ReceivedAt is when the server last accepted a webhook for this
-	AcknowledgedByID   *int64     `json:"acknowledged_by_id,omitempty"`
+	// fingerprint, including the unchanged firing notifications Alertmanager
-	AcknowledgedByUser *string    `json:"acknowledged_by,omitempty"`
+	// re-sends every repeat_interval.
-	AcknowledgedAt     *time.Time `json:"acknowledged_at,omitempty"`
+	//
+	// This is a documented part of the public API, not an internal ingest
+	// detail: StartsAt never changes for an alert instance, so ReceivedAt is
+	// the only signal a client has that a firing alert is still being
+	// refreshed. The sweeper stale-dates against it (see expireStale), API
+	// clients render it, and GET /api/alerts is ordered by it. Anything that
+	// stops the webhook handler from advancing it on a re-send is a breaking
+	// change — see "received_at is a liveness heartbeat" in the README and
+	// TestWebhook_ResendBumpsReceivedAt.
+	ReceivedAt time.Time `json:"received_at"`
+
+	// IncidentID is the most recent incident this alert belongs to. An alert row
+	// is reused across occurrences of the same fingerprint, so over its life it
+	// belongs to a series of incidents; incident_alerts keeps the full history
+	// and this is only the newest link.
+	IncidentID *int64 `json:"incident_id,omitempty"`
+
+	// ResolutionSource records why a resolved alert left the firing state:
+	// "alertmanager" for a real resolved webhook, "expiry" when the sweeper
+	// inferred it after the alert stopped being refreshed. Nil while firing, and
+	// cleared again by a re-fire under the same fingerprint.
+	//
+	// Also public API: it is how a client knows whether EndsAt was observed or
+	// inferred. Under "expiry" nothing ever reported an end, so EndsAt is only
+	// an upper bound (see expireStale) and ReceivedAt is the more truthful
+	// signal. Treat the value set as open — see "resolution_source says how much
+	// to trust ends_at" in the README, and TestWebhook_ResolvedSetsSource /
+	// TestExpiry_StaleFiringAlert.
+	ResolutionSource *string `json:"resolution_source,omitempty"`
+
+	ArchivedAt *time.Time `json:"archived_at,omitempty"`
 }

internal/models/comment.go

@@ -1,12 +0,0 @@
-package models
-
-import "time"
-
-type Comment struct {
-	ID        int64     `json:"id"`
-	AlertID   int64     `json:"alert_id"`
-	UserID    int64     `json:"user_id"`
-	Username  string    `json:"username"`
-	Content   string    `json:"content"`
-	CreatedAt time.Time `json:"created_at"`
-}

internal/models/incident.go

@@ -0,0 +1,68 @@
+package models
+
+import "time"
+
+// Incident is the human work item: the thing that gets acknowledged, assigned,
+// snoozed, discussed and resolved. Alerts are the machine-owned signal records
+// underneath it — many alerts map to one incident, correlated by the groupKey
+// Alertmanager already computed from the operator's group_by configuration.
+//
+// Nothing here is ever written by the Alertmanager webhook except Status, which
+// the webhook and the sweeper may flip to "resolved" once every member alert has
+// stopped firing.
+type Incident struct {
+	ID          int64             `json:"id"`
+	GroupKey    string            `json:"group_key"`
+	Title       string            `json:"title"`
+	GroupLabels map[string]string `json:"group_labels"`
+
+	// Status is "triggered", "acknowledged" or "resolved".
+	Status string `json:"status"`
+
+	// Severity is the highest `severity` label across the alerts that were
+	// firing when it was last recomputed. It is deliberately not cleared when an
+	// incident resolves — a resolved incident should still say how bad it was.
+	Severity *string `json:"severity,omitempty"`
+
+	TriggeredAt time.Time `json:"triggered_at"`
+
+	AcknowledgedByID   *int64     `json:"acknowledged_by_id,omitempty"`
+	AcknowledgedByUser *string    `json:"acknowledged_by,omitempty"`
+	AcknowledgedAt     *time.Time `json:"acknowledged_at,omitempty"`
+
+	AssignedToID   *int64  `json:"assigned_to_id,omitempty"`
+	AssignedToUser *string `json:"assigned_to,omitempty"`
+
+	// SnoozedUntil hides the incident from the default queue without closing it.
+	// A timestamp in the past reads as "not snoozed"; nothing sweeps it.
+	SnoozedUntil *time.Time `json:"snoozed_until,omitempty"`
+
+	ResolvedAt *time.Time `json:"resolved_at,omitempty"`
+
+	// ResolutionSource is "alerts" when every member alert stopped firing, or
+	// "manual" when a human closed it. Manual resolution is terminal: a later
+	// occurrence opens a new incident rather than reopening this one.
+	ResolutionSource *string `json:"resolution_source,omitempty"`
+
+	ArchivedAt *time.Time `json:"archived_at,omitempty"`
+
+	// Alerts is populated by GET /api/incidents/{id} only.
+	Alerts []Alert `json:"alerts,omitempty"`
+}
+
+// IncidentEvent is one entry in an incident's timeline. The table is append-only
+// and is the only history this server keeps — alert rows are mutated in place.
+//
+// Type is one of: triggered, alert_added, alert_resolved, acknowledged,
+// unacknowledged, assigned, snoozed, unsnoozed, resolved, note. A nil UserID
+// means the server acted rather than a person.
+type IncidentEvent struct {
+	ID         int64     `json:"id"`
+	IncidentID int64     `json:"incident_id"`
+	Type       string    `json:"type"`
+	UserID     *int64    `json:"user_id,omitempty"`
+	Username   *string   `json:"username,omitempty"`
+	AlertID    *int64    `json:"alert_id,omitempty"`
+	Detail     *string   `json:"detail,omitempty"`
+	CreatedAt  time.Time `json:"created_at"`
+}

internal/models/user.go

@@ -7,6 +7,11 @@ type User struct {
 	Username  string    `json:"username"`
 	Email     string    `json:"email"`
 	CreatedAt time.Time `json:"created_at"`
+
+	// NtfyTopic is where this user's push notifications go. Nil means they get
+	// none of their own; incidents assigned to them fall back to the configured
+	// fallback topic instead.
+	NtfyTopic *string `json:"ntfy_topic,omitempty"`
 }
 
 type APIKey struct {