Set the chart's placeholder version to 0.47.0

Cosmetic: make helm-package passes --version and --app-version from the
tag, so these fields decide nothing about what is published. They are
updated so the tree heading for v0.47.0 does not say 0.46.1, as in
a314083 and 81197a3.

Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN

README.md

@@ -16,9 +16,9 @@ answers, and tracked to resolution. One binary, one Postgres.
   cross-talk. [Details](./docs/incidents.md)
 - **On-call rota.** Each team keeps its own rota, and new incidents go to whoever is on call.
   [Web UI](./docs/web-ui.md)
-- **Pages that escalate.** Push notifications through [ntfy](https://ntfy.sh), with an
+- **Pages that escalate.** Notifications through pluggable channels ([ntfy](https://ntfy.sh)
-  Acknowledge button right in the notification, and escalation ladders that move on to the next
+  today), with an Acknowledge button right in the notification where the channel can show one,
-  level when nobody answers. [Notifications](./docs/notifications.md) ·
+  and escalation ladders that move on to the next level when nobody answers. [Notifications](./docs/notifications.md) ·
   [Escalation](./docs/escalation.md)
 - **Notices when the alerts stop.** Dead man's switches turn the absence of a heartbeat such as
   `Watchdog` into an incident. [Details](./docs/dead-mans-switch.md)

charts/terdut-server/Chart.yaml

@@ -15,5 +15,5 @@ type: application
 # appVersion and image.tag in values.yaml no longer agree, and that is not an oversight:
 # image.tag stays "latest", which is what a local install actually pulls. appVersion is
 # metadata and drives nothing.
-version: 0.44.1
+version: 0.47.0
-appVersion: "v0.44.1"
+appVersion: "v0.47.0"

charts/terdut-server/templates/bootstrap-job.yaml

@@ -13,6 +13,10 @@ metadata:
     helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 spec:
   backoffLimit: 3
+  # Bounds the whole Job, retries included, so a database that never comes up ends the
+  # hook instead of leaving it pending. Each attempt waits up to bootstrap.waitSeconds
+  # for the server, and the allowance covers every attempt plus the pod backoff between.
+  activeDeadlineSeconds: {{ add (mul (int .Values.bootstrap.waitSeconds) 4) 300 }}
   template:
     metadata:
       labels:
@@ -42,60 +46,131 @@ spec:
             - |
               SERVICE_URL="http://{{ include "terdut-server.fullname" . }}:{{ .Values.service.port }}"
               SECRET_NAME="{{ include "terdut-server.bootstrapSecretName" . }}"
+              USERNAME="{{ .Values.bootstrap.username }}"
+              EMAIL="{{ .Values.bootstrap.email }}"
               K8S_API="https://kubernetes.default.svc"
               SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
               CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
               NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)"
+              SECRETS="$K8S_API/api/v1/namespaces/$NAMESPACE/secrets"
 
-              echo "Waiting for terdut-server to be ready..."
+              # kapi METHOD URL [BODY]: sets CODE and BODY from the Kubernetes API.
-              RETRIES=60
+              kapi() {
-              while [ "$RETRIES" -gt 0 ]; do
+                _ctype="application/json"
-                curl -sf "$SERVICE_URL/healthz" > /dev/null 2>&1 && break
+                [ "$1" = "PATCH" ] && _ctype="application/merge-patch+json"
-                RETRIES=$((RETRIES - 1))
+                if [ -n "$3" ]; then
-                sleep 2
+                  _resp=$(printf '%s' "$3" | curl -s -w "\n%{http_code}" -X "$1" "$2" \
-              done
+                    --cacert "$CA_CERT" -H "Authorization: Bearer $SA_TOKEN" \
-              if [ "$RETRIES" -eq 0 ]; then
+                    -H "Content-Type: $_ctype" -d @-)
-                echo "Timed out waiting for server to be ready."
+                else
+                  _resp=$(curl -s -w "\n%{http_code}" -X "$1" "$2" \
+                    --cacert "$CA_CERT" -H "Authorization: Bearer $SA_TOKEN")
+                fi
+                CODE=$(echo "$_resp" | tail -1)
+                BODY=$(echo "$_resp" | sed '$d')
+              }
+
+              # The server exits when the database does not answer within ~10 seconds, so
+              # while the database is down the pod crash-loops, with a restart backoff that
+              # grows to five minutes, and /healthz only answers once it has connected. Wait
+              # for that for as long as bootstrap.waitSeconds allows, not a fixed two minutes:
+              # a failed hook fails the release and nothing runs it again until the next
+              # upgrade, so a database that was merely slow would leave no administrator.
+              WAIT={{ int .Values.bootstrap.waitSeconds }}
+              echo "Waiting up to ${WAIT}s for terdut-server to be ready (it needs the database)..."
+              WAITED=0
+              while ! curl -sf "$SERVICE_URL/healthz" > /dev/null 2>&1; do
+                if [ "$WAITED" -ge "$WAIT" ]; then
+                  echo "Timed out after ${WAITED}s waiting for the server to be ready."
+                  echo "Check the database and the terdut-server pod's logs, then run helm upgrade again."
                   exit 1
                 fi
-              echo "Server is ready."
+                [ $((WAITED % 30)) -eq 0 ] && [ "$WAITED" -gt 0 ] && echo "Still waiting (${WAITED}s)..."
+                sleep 2
+                WAITED=$((WAITED + 2))
+              done
+              echo "Server is ready after ${WAITED}s."
 
-              RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$SERVICE_URL/api/bootstrap" \
+              # The Secret is the source of truth for the administrator's password: reuse
-                -H "Content-Type: application/json" \
+              # the one it holds, so recreating the database brings the same account back.
-                -d '{"username":"{{ .Values.bootstrap.username }}","email":"{{ .Values.bootstrap.email }}"}')
+              PASSWORD=""
+              EXISTS=0
+              kapi GET "$SECRETS/$SECRET_NAME"
+              if [ "$CODE" = "200" ]; then
+                EXISTS=1
+                PASSWORD=$(echo "$BODY" | grep -o '"password": *"[^"]*"' | head -1 | cut -d'"' -f4 | base64 -d)
+              elif [ "$CODE" != "404" ]; then
+                echo "Failed to read secret '$SECRET_NAME' (HTTP $CODE)."
+                exit 1
+              fi
 
+              # Written BEFORE the server is asked to create the account, so a crash in
+              # between cannot leave an administrator whose password nobody has.
+              GENERATED=0
+              if [ -z "$PASSWORD" ]; then
+                PASSWORD=$(head -c 256 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | head -c 32)
+                if [ "${#PASSWORD}" -lt 32 ]; then
+                  echo "Failed to generate a password."
+                  exit 1
+                fi
+                GENERATED=1
+                DATA=$(printf '"username":"%s","password":"%s"' "$USERNAME" "$PASSWORD")
+                if [ "$EXISTS" = "1" ]; then
+                  kapi PATCH "$SECRETS/$SECRET_NAME" "{\"stringData\":{$DATA}}"
+                else
+                  kapi POST "$SECRETS" "{\"apiVersion\":\"v1\",\"kind\":\"Secret\",\"metadata\":{\"name\":\"$SECRET_NAME\"},\"stringData\":{$DATA}}"
+                fi
+                case "$CODE" in 200|201) ;; *)
+                  echo "Failed to store the password in secret '$SECRET_NAME' (HTTP $CODE)."
+                  exit 1 ;;
+                esac
+              fi
+
+              RESPONSE=$(printf '{"username":"%s","email":"%s","password":"%s"}' "$USERNAME" "$EMAIL" "$PASSWORD" \
+                | curl -s -w "\n%{http_code}" -X POST "$SERVICE_URL/api/bootstrap" \
+                  -H "Content-Type: application/json" -d @-)
               HTTP_CODE=$(echo "$RESPONSE" | tail -1)
-              BODY=$(echo "$RESPONSE" | head -1)
+              RESP_BODY=$(echo "$RESPONSE" | head -1)
 
               if [ "$HTTP_CODE" = "403" ]; then
-                echo "Server already bootstrapped, nothing to do."
+                # Somebody else made the first account. The Secret is only worth keeping
+                # if its password signs in.
+                LOGIN=$(printf '{"username":"%s","password":"%s"}' "$USERNAME" "$PASSWORD" \
+                  | curl -s -o /dev/null -w "%{http_code}" -X POST "$SERVICE_URL/api/login" \
+                    -H "Content-Type: application/json" -d @-)
+                if [ "$LOGIN" = "200" ]; then
+                  echo "Server already bootstrapped; the password in '$SECRET_NAME' signs in."
+                  exit 0
+                fi
+                echo "Server already bootstrapped by something else; '$USERNAME' does not sign in with the password in '$SECRET_NAME'."
+                if [ "$GENERATED" = "1" ]; then
+                  if [ "$EXISTS" = "1" ]; then
+                    kapi PATCH "$SECRETS/$SECRET_NAME" '{"data":{"username":null,"password":null}}'
+                  else
+                    kapi DELETE "$SECRETS/$SECRET_NAME"
+                  fi
+                  echo "Removed the password this run generated."
+                fi
                 exit 0
               fi
 
               if [ "$HTTP_CODE" != "201" ]; then
-                echo "Bootstrap failed (HTTP $HTTP_CODE): $BODY"
+                echo "Bootstrap failed (HTTP $HTTP_CODE): $RESP_BODY"
                 exit 1
               fi
 
-              API_KEY=$(echo "$BODY" | grep -o '"key":"[^"]*"' | cut -d'"' -f4)
+              API_KEY=$(echo "$RESP_BODY" | grep -o '"key":"[^"]*"' | cut -d'"' -f4)
               if [ -z "$API_KEY" ]; then
                 echo "Failed to extract API key from response."
                 exit 1
               fi
 
-              echo "Bootstrap succeeded. Storing API key in secret '$SECRET_NAME'."
+              echo "Bootstrap succeeded. Storing the API key in secret '$SECRET_NAME'."
-
+              kapi PATCH "$SECRETS/$SECRET_NAME" "{\"stringData\":{\"api-key\":\"$API_KEY\"}}"
-              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
+              if [ "$CODE" != "200" ]; then
-                -X POST "$K8S_API/api/v1/namespaces/$NAMESPACE/secrets" \
+                echo "Failed to store the API key (HTTP $CODE)."
-                --cacert "$CA_CERT" \
-                -H "Authorization: Bearer $SA_TOKEN" \
-                -H "Content-Type: application/json" \
-                -d "$(printf '{"apiVersion":"v1","kind":"Secret","metadata":{"name":"%s"},"stringData":{"api-key":"%s"}}' "$SECRET_NAME" "$API_KEY")")
-
-              if [ "$HTTP_CODE" != "201" ]; then
-                echo "Failed to create secret (HTTP $HTTP_CODE)."
                 exit 1
               fi
 
-              echo "Secret '$SECRET_NAME' created successfully."
+              echo "Secret '$SECRET_NAME' holds username, password and api-key."
 {{- end }}

charts/terdut-server/templates/bootstrap-rbac.yaml

@@ -27,6 +27,12 @@ rules:
   - apiGroups: [""]
     resources: ["secrets"]
     verbs: ["create"]
+  # Reading, filling in and (when the account turns out not to be ours) cleaning up the
+  # one Secret by name; `create` cannot be restricted to a name.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["{{ include "terdut-server.bootstrapSecretName" . }}"]
+    verbs: ["get", "patch", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

charts/terdut-server/values.yaml

@@ -160,9 +160,20 @@ oidc:
 # Backups are not this chart's business: Postgres is backed up where it runs,
 # through a k8up.io/backupcommand pg_dump annotation on the database pod itself.
 
+# Creates the first administrator after install/upgrade, with a generated password, and keeps
+# the credentials in a Secret. Set enabled: false to create the first user yourself with
+# POST /api/bootstrap (or on an SSO-only install that wants no local account).
 bootstrap:
   enabled: true
   username: admin
   email: admin@example.com
-  # secretName overrides the default of <fullname>-admin-key
+  # How long each attempt waits for the server to answer /healthz, which it only does once
+  # the database is reachable. Raise it when the database starts after the release (a fresh
+  # cluster, a database operator) -- and keep Flux's HelmRelease spec.timeout / helm
+  # --timeout above it, because Helm waits for this hook and fails the release when its own
+  # timeout comes first (the Job still carries on and can finish the bootstrap).
+  waitSeconds: 600
+  # secretName overrides the default of <fullname>-admin-key. It holds username, password and
+  # api-key. The password is generated once and then reused: delete the Secret and the
+  # database to start over.
   secretName: ""

cmd/terdut/main.go

@@ -11,6 +11,7 @@ import (
 	"git.ryuvia.com/niklas/terdut-server/internal/api"
 	"git.ryuvia.com/niklas/terdut-server/internal/config"
 	"git.ryuvia.com/niklas/terdut-server/internal/db"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 )
 
 var version = "dev"
@@ -31,10 +32,9 @@ func main() {
 		log.Fatalf("migrate: %v", err)
 	}
 
-	notify := api.NotifyConfig{
+	notifyCfg := api.NotifyConfig{
-		BaseURL:       cfg.NtfyURL,
+		Providers:   notifyProviders(cfg),
-		Token:         cfg.NtfyToken,
+		Fallback:    notifyFallback(cfg),
-		FallbackTopic: cfg.NtfyFallbackTopic,
 		PublicURL:   cfg.PublicURL,
 		RepeatEvery: cfg.NotifyRepeat,
 	}
@@ -49,7 +49,7 @@ func main() {
 		log.Fatalf("%v", err)
 	}
 
-	router := api.NewRouter(database, notify, cfg, version)
+	router := api.NewRouter(database, notifyCfg, cfg, version)
 
 	srv := &http.Server{
 		Addr:         cfg.Addr,
@@ -62,8 +62,8 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notify)
+	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notifyCfg)
-	go api.StartNotifier(ctx, database, notify)
+	go api.StartNotifier(ctx, database, notifyCfg)
 
 	go func() {
 		log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
@@ -80,3 +80,23 @@ func main() {
 		log.Printf("shutdown: %v", err)
 	}
 }
+
+// notifyProviders registers the notification providers this install has
+// configuration for. A provider with no configuration is simply absent, which is
+// what keeps it out of the web UI's list of channel types.
+func notifyProviders(cfg config.Config) *notify.Registry {
+	var providers []notify.Provider
+	if cfg.NtfyURL != "" {
+		providers = append(providers, notify.NewNtfy(cfg.NtfyURL, cfg.NtfyToken))
+	}
+	return notify.NewRegistry(providers...)
+}
+
+// notifyFallback is where an incident goes when nobody on call can be reached.
+// Only ntfy has a fallback setting today; a second provider adds its own.
+func notifyFallback(cfg config.Config) notify.Target {
+	if cfg.NtfyURL != "" && cfg.NtfyFallbackTopic != "" {
+		return notify.Target{Type: notify.TypeNtfy, Address: cfg.NtfyFallbackTopic}
+	}
+	return notify.Target{}
+}

docs/README.md

@@ -11,8 +11,8 @@ The [README](../README.md) is the short tour. These pages hold the detail.
 - [The web UI](./web-ui.md): sessions, the Team and Admin tabs.
 - [Alertmanager configuration](./alertmanager.md): routes, integration keys and webhooks.
 - [Alerts and incidents](./incidents.md): correlation, lifecycle, on-call assignment, stale-alert expiry.
-- [Push notifications](./notifications.md): ntfy pages and acknowledging from them.
+- [Notifications](./notifications.md): channels and providers, ntfy pages, and acknowledging from them.
-- [Escalation](./escalation.md): ladders, repeats and the fallback topic.
+- [Escalation](./escalation.md): ladders, repeats and the fallback.
 - [Dead man's switches](./dead-mans-switch.md): noticing that alerts stopped arriving.
 
 **Integrating and contributing**

docs/api.md

@@ -116,7 +116,12 @@ on anybody's.
 | `DELETE` | `/api/users/{id}` | **admin** | Delete user (cascades to keys). `409` for yourself or the last administrator |
 | `PUT` | `/api/users/{id}/admin` | **admin** | Grant or revoke the administrator flag `{"is_admin"}`. `409` for yourself, the last administrator, or an administrator granted by single sign-on |
 | `PUT` | `/api/users/{id}/disabled` | **admin** | Take an account out of use, or put it back `{"disabled"}`. `409` for yourself or the last administrator |
-| `PUT` | `/api/users/{id}/notify` | self or admin | Set push notification target `{"ntfy_topic"}` — empty string clears it |
+| `GET` | `/api/users/{id}/channels` | self or admin | The user's [notification channels](./notifications.md#channels) `[{id, type, address, enabled, created_at}]`. `address` is only sent to the owner: it can be a secret |
+| `POST` | `/api/users/{id}/channels` | self or admin | Add a channel `{"type","address"}`. `400` for a type this server has no provider for or an address the provider rejects; `409` for a duplicate or at the limit of 10 |
+| `PATCH` | `/api/users/{id}/channels/{channelID}` | self or admin | Turn it on or off, or change its address `{"enabled"?,"address"?}` |
+| `DELETE` | `/api/users/{id}/channels/{channelID}` | self or admin | Remove it |
+| `POST` | `/api/users/{id}/channels/{channelID}/test` | **self** | Send a test message through it. `502` with the provider's reason when it does not land; an administrator cannot send one to somebody else |
+| `GET` | `/api/notify/providers` | any | Which kinds of channel this server can deliver `[{type, label, address_label, address_hint}]`. Empty when notifications are off |
 | `PUT` | `/api/users/{id}/password` | self or admin | Set web UI password `{"password","current_password"}`. `current_password` is required only when changing your own existing password. Ends the user's other sessions |
 | `POST` | `/api/users/{id}/api-keys` | self or admin | Issue API key `{"name"}` — key shown once |
 | `DELETE` | `/api/users/{id}/api-keys/{keyID}` | self or admin | Revoke API key |
@@ -197,7 +202,7 @@ the team gets the same `404` as anybody else.
 | `POST` | `/api/teams` | any | Create a team `{"name"}`; a human creator becomes its first owner. An instance-scoped [service account](#service-accounts) may also create one, and it gets no owner at all — expected for a team an operator is about to hand a team-scoped credential to, not an orphaned team a human made |
 | `PUT` | `/api/teams/{teamID}` | **owner** | Rename it `{"name"}`. `409` if the name is taken |
 | `DELETE` | `/api/teams/{teamID}` | **owner** | Delete a team and everything under it. `409` while it has open incidents |
-| `GET` | `/api/teams/{teamID}/members` | member | Who is in the team, with `status` (`oncall` if the rota has them today, `unpageable` when a page to them would go nowhere — even if they are on call — else `reachable`), `on_call`, `next_shift` (first rota day after today), `pageable` and `problem` (`has no ntfy topic` / `account is disabled`; never the topic itself) and `last_active_at` (their newest session or API-key use). Every member sees the same list |
+| `GET` | `/api/teams/{teamID}/members` | member | Who is in the team, with `status` (`oncall` if the rota has them today, `unpageable` when a page to them would go nowhere — even if they are on call — else `reachable`), `on_call`, `next_shift` (first rota day after today), `pageable` and `problem` (`has no notification channel` / `account is disabled`; never a channel's address) and `last_active_at` (their newest session or API-key use). Every member sees the same list |
 | `POST` | `/api/teams/{teamID}/members` | **owner** | Add a member, or change their role `{"user_id","role"}`. `409` when it would demote the last owner, or the membership is managed by single sign-on |
 | `DELETE` | `/api/teams/{teamID}/members/{userID}` | **owner** | Remove a member. `409` for the last owner, or a membership managed by single sign-on |
 | `GET` | `/api/teams/{teamID}/oidc-groups` | member | Which groups control this team's membership: `{"member_group","owner_group"}`. An empty string means no group grants that role here |
@@ -209,8 +214,8 @@ the team gets the same `404` as anybody else.
 | `GET` | `/api/teams/{teamID}/invites` | **owner** | The team's invite links, with their uses and expiry. Never the tokens |
 | `POST` | `/api/teams/{teamID}/invites` | **owner** | Mint one `{"role","max_uses"}` — the full URL is returned once |
 | `DELETE` | `/api/teams/{teamID}/invites/{inviteID}` | **owner** | Revoke a link before it expires |
-| `GET` | `/api/teams/{teamID}/escalation` | member | The team's [escalation ladder](./escalation.md#escalation) `{repeat_count, fallback_topic, levels[], last_escalated_at?, last_escalated_incident_id?}`. Empty levels means the team has none. Each level also carries `status` (`ready`, `escalating` when an unanswered incident has climbed to it, `unreachable` when nobody on it could be woken), `waiting` (ids of the open incidents on it) and, per target, `username` (who it means today — the person on call, for a rota target), `reachable` and `problem`. The extra fields are output only; `PUT` takes the plain shape |
+| `GET` | `/api/teams/{teamID}/escalation` | member | The team's [escalation ladder](./escalation.md#escalation) `{repeat_count, fallback: {type, address}, levels[], last_escalated_at?, last_escalated_incident_id?}`. Empty levels means the team has none. Each level also carries `status` (`ready`, `escalating` when an unanswered incident has climbed to it, `unreachable` when nobody on it could be woken), `waiting` (ids of the open incidents on it) and, per target, `username` (who it means today — the person on call, for a rota target), `reachable` and `problem`. The extra fields are output only; `PUT` takes the plain shape |
-| `PUT` | `/api/teams/{teamID}/escalation` | **owner** | Replace it wholesale. `400` for a level with no targets or no timeout — a rung that pages nobody is a silence with a number on it |
+| `PUT` | `/api/teams/{teamID}/escalation` | **owner** | Replace it wholesale (`fallback` is `{"type","address"}`, both empty for none). `400` for a level with no targets or no timeout, or a fallback no provider here can deliver — a rung that pages nobody is a silence with a number on it |
 | `GET` | `/api/teams/{teamID}/deadman/switches` | member | The team's [dead man's switches](./dead-mans-switch.md), each `{id, name, matcher, timeout_seconds, severity, status, last_heartbeat_at, last_triggered_at, open_incident_id, sources[]}`. `status` is `healthy`, `dead` or `dormant`; `sources` has one entry per heartbeat fingerprint. Empty when the team watches nothing |
 | `POST` | `/api/teams/{teamID}/deadman/switches` | **owner** | Add one: `{name?, matcher, timeout_seconds, severity?}`. `400` when the matcher names no `alertname` or holds several, or the timeout is not positive — a switch that silently watches nothing is the failure this feature exists to prevent |
 | `PUT` | `/api/teams/{teamID}/deadman/switches/{switchID}` | **owner** | Replace one in place, same body and validation as create. Its id is unchanged — for an automated caller reconciling a spec change, unlike delete-and-recreate |
@@ -292,11 +297,12 @@ Types written today: `triggered`, `alert_added`, `alert_resolved`,
 (absent on assignments made before they were recorded). New types may be added; render
 unknown ones generically rather than dropping them.
 
-On `notified` and `notify_failed`, `detail` carries the notification kind
+On `notified` and `notify_failed`, `detail` is `<kind> via <channel type>` —
-(`triggered` | `reminder` | `resolved`), and on a failure the reason after it.
+kind is `triggered` | `reminder` | `escalated` | `resolved`, e.g. `triggered via ntfy` — and on a
+failure the reason follows it. A person with two channels has one event per channel per page.
 `user_id` is who was paged — absent means the page went to the shared fallback
-topic and so belongs to nobody. The topic itself is never written to the
+and so belongs to nobody. A channel's address is never written to the
-timeline: it is a shared secret with the ntfy server, and every API key can read
+timeline: it can be a shared secret (an ntfy topic is), and every API key can read
 this.
 
 ## Alerts
@@ -421,13 +427,15 @@ Every figure counts the caller's own teams only: a report that counted other
 teams' incidents would leak their volume, and their alert names through the
 top-alerts list, and would not be a number about the reader's work anyway.
 
-All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats filter on `triggered_at`.
+All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats, and the top, heatmap and per-alert endpoints (an occurrence is an incident), filter on `triggered_at`.
 
 | Method | Path | Description |
 |---|---|---|
 | `GET` | `/api/stats/incidents` | `{total, triggered, acknowledged, resolved, mtta_seconds, mttr_seconds}` |
 | `GET` | `/api/stats/alerts` | `{total, firing, resolved}` counts |
-| `GET` | `/api/stats/alerts/top` | Most frequent alert names. `?limit=` (default 10, max 100) |
+| `GET` | `/api/stats/alerts/top` | Alert names ranked by incidents opened (filters on `triggered_at`). Each entry: `{name, count, last_seen, mean_resolve_seconds, daily}`; `daily` is 30 per-day counts ending at `?to` (or today), regardless of `?from`. `?limit=` (default 10, max 100) |
+| `GET` | `/api/stats/alerts/heatmap` | `{grid}`: incidents by UTC weekday (0=Sunday) × hour, always 7×24 |
+| `GET` | `/api/stats/alerts/{name}` | One alert's drilldown over its incidents: `{name, count, first_seen, last_seen, mean_gap_seconds, mean/median_ack_seconds, mean/median_resolve_seconds, longest_open_seconds, by_hour[24], by_weekday[7], occurrences[]}` (newest 200; means are `null` when unmeasurable). 404 when the alert opened no incident in scope |
 | `GET` | `/api/stats/alerts/by-hour` | Count per hour-of-day (UTC), all 24 slots returned |
 | `GET` | `/api/stats/alerts/by-day` | Count per day-of-week, all 7 slots with names returned |
 

docs/configuration.md

@@ -21,9 +21,9 @@ over an administrator's edit.
 | `TERDUT_DB_DSN` | — | **Required.** Postgres connection string, e.g. `postgres://terdut:secret@localhost:5432/terdut?sslmode=require` |
 | `TERDUT_ARCHIVE_AFTER` | `168h` (7d) | **seed.** How long a resolved alert or incident stays in the default list before being auto-archived |
 | `TERDUT_STALE_AFTER` | `6h` | **seed.** How long a firing alert may go without a refreshing webhook before it is treated as resolved — **must exceed your Alertmanager `repeat_interval`** |
-| `TERDUT_NTFY_URL` | — | ntfy server to publish push notifications to. Empty disables notifications entirely |
+| `TERDUT_NTFY_URL` | — | ntfy server to publish push notifications to; configures the `ntfy` [notification provider](./notifications.md). With no provider configured, notifications are off entirely |
 | `TERDUT_NTFY_TOKEN` | — | Bearer token for an access-controlled ntfy |
-| `TERDUT_NTFY_FALLBACK_TOPIC` | — | Topic used when nobody is on call |
+| `TERDUT_NTFY_FALLBACK_TOPIC` | — | ntfy topic used when nobody on call can be reached |
 | `TERDUT_PUBLIC_URL` | — | Base URL a phone uses to reach this server: the notification's link into the web UI, its Acknowledge button, and whether the session cookie is `Secure` |
 | `TERDUT_NOTIFY_REPEAT` | `15m` | **seed.** How long an incident may sit unacknowledged before it is paged again. `0` notifies once and never repeats |
 | `TERDUT_PASSWORD_LOGIN` | `true` | `false` refuses password login and password sign-up (`403`), leaving single sign-on the only way in. Refused at startup unless SSO is configured |

docs/deployment.md

@@ -35,7 +35,10 @@ than an `Ingress`. TLS is terminated at the gateway, so the server itself never
 | `networking.hostname` | `terdut.example.com` | Hostname the `HTTPRoute` serves |
 | `networking.listener` | `""` | Gateway listener (`sectionName`) to bind to. Empty attaches to every matching listener, **including plaintext HTTP** — set it to the HTTPS listener's name to serve TLS only |
 | `networking.servicePort` | `8080` | Port the route forwards to; keep in sync with `service.port` |
-| `bootstrap.enabled` | `true` | Runs a post-install hook that creates the first user and stores its API key in the `<release>-admin-key` Secret. Already-bootstrapped servers are left alone |
+| `bootstrap.enabled` | `true` | Runs a post-install/upgrade hook that creates the first administrator with a generated password and stores `username`, `password` and `api-key` in the `<release>-admin-key` Secret. The password is generated once and reused, so recreating the database brings the same account back. Already-bootstrapped servers are left alone. Set `false` to create the first user yourself with `POST /api/bootstrap` |
+| `bootstrap.waitSeconds` | `600` | How long each attempt of the hook waits for the server to answer `/healthz`, which it does only once the database is reachable. Keep Helm's `--timeout` (Flux `spec.timeout`) above it; if the timeout comes first the release is marked failed, but the Job keeps running and can still create the administrator |
+| `bootstrap.username` / `bootstrap.email` | `admin` / `admin@example.com` | Account the hook creates |
+| `bootstrap.secretName` | `""` | Secret to keep the credentials in; empty means `<release>-admin-key` |
 | `database.dsn` | `""` | **Required.** Postgres DSN, with no password in it. The chart provisions no database |
 | `database.passwordSecret.name` | `""` | Secret supplying `PGPASSWORD`. With the Zalando postgres operator, the Secret it generates for the role |
 | `database.passwordSecret.key` | `password` | Key within that Secret |

docs/escalation.md

@@ -15,12 +15,12 @@ changes and nobody remembers to edit the policy.
 level 1   5m    oncall            the rota gets first refusal
 level 2   5m    user:bob          then a named second
                                   then repeat_count more rounds
-                                  then the team's fallback topic, once
+                                  then the team's fallback, once
 ```
 
 When a level's timeout passes with the incident still `triggered`, the next
 level is paged. Off the end of the ladder the whole thing runs again
-`repeat_count` times, and after that the team's `fallback_topic` is paged once
+`repeat_count` times, and after that the team's `fallback` is paged once
 as the end of the line. The incident stays open throughout: running out of
 people to wake is not the same as somebody answering.
 
@@ -31,7 +31,7 @@ it is, and it resumes when the snooze runs out.
 
 Every step is on the incident's timeline with the level and the names it woke,
 so somebody reading it afterwards can tell why their phone rang at 04:00. A
-level whose targets are all unreachable — no ntfy topic, a disabled account, an
+level whose targets are all unreachable — no notification channel, a disabled account, an
 empty rota — is recorded as `nobody reachable` and the ladder moves on rather
 than stalling on a rung that cannot ring.
 
@@ -39,7 +39,8 @@ than stalling on a rung that cannot ring.
 escalation; a team without keeps the reminder behaviour exactly as it was. Two
 pages for one silence is the surest way to get a tool muted.
 
-The ladder's `fallback_topic` is per team, unlike `TERDUT_NTFY_FALLBACK_TOPIC`,
+The ladder's `fallback` is per team and names a provider and an address
-which is the install-wide topic used when an incident opens with nobody on call.
+(`{"type": "ntfy", "address": "..."}`), unlike `TERDUT_NTFY_FALLBACK_TOPIC`,
+which is the install-wide ntfy topic used when an incident opens with nobody on call.
 They answer different questions: one is "nobody was scheduled", the other is
 "everybody scheduled has been tried".

docs/notifications.md

@@ -1,27 +1,56 @@
-# Push notifications
+# Notifications
 
-_Pages through ntfy, who gets them and how to acknowledge from the notification._ Back to the [README](../README.md) and the [documentation index](./README.md).
+_How pages reach people: channels and providers, ntfy, who gets them and how to acknowledge from the notification._ Back to the [README](../README.md) and the [documentation index](./README.md).
 
-With `TERDUT_NTFY_URL` set, an incident that opens is pushed to the on-call
+A page reaches a person through their **channels**. A channel is one way of
-person's phone through [ntfy](https://ntfy.sh). Everybody sets their own topic
+reaching them: a *provider* (the kind of thing that delivers — `ntfy` today)
-under *Account* in the web UI, where a **Send a test push** button proves it
+and an *address* that provider understands (an ntfy topic). A person can have
-before an incident has to; `PUT /api/users/{id}/notify` is the same thing over
+several, and a page goes to **every channel that is switched on**, each queued
-the API, and an administrator may set somebody else's. A user with no topic
+and retried independently, so one provider being down does not hold back the
-falls back to `TERDUT_NTFY_FALLBACK_TOPIC`, as does an incident that opens with
+others.
-nobody on call. If neither yields a topic, nothing is queued.
 
-The **server** is the install's one ntfy, from `TERDUT_NTFY_URL`, and is not
+## Providers
-something a user picks. Only the topic is per-person.
 
-A topic is a shared secret with the ntfy server: anyone who knows it can both
+A provider is configured by the **server**, in its deployment, and is not
-read the pages and publish to it, so an unguessable one is worth the trouble.
+something a user picks. Only the kinds the server has been configured with can
-That is also why the topic never appears in an incident's timeline, which every
+be added as a channel: the Account page asks the server (`GET
-API key can read.
+/api/notify/providers`) and offers just those. With none configured,
+notifications are off entirely: no goroutine, and nothing is ever queued.
 
-Three things get pushed:
+| Provider | Configured by | Address | Acknowledge button |
+|---|---|---|---|
+| `ntfy` | `TERDUT_NTFY_URL` (+ `TERDUT_NTFY_TOKEN`) | a topic | yes |
 
-- **triggered** — an incident opened. Priority follows severity (`critical` maps
+A provider that cannot show a button gets the incident's link in the message
-  to ntfy's max priority, the one that overrides the phone's quiet settings).
+body instead. Adding a provider is one Go type in `internal/notify` and a line
+in `cmd/terdut/main.go`; the notifier, queue, API and web UI do not change.
+
+## Channels
+
+Everybody manages their own under *Account* in the web UI: add a channel, send
+a **test** through it (the point is that a phone buzzes, and only the person
+holding it can tell), turn it off, remove it. `GET|POST
+/api/users/{id}/channels` and `PATCH|DELETE .../{channelID}` are the same thing
+over the API, and an administrator may manage somebody else's, but never sees
+where they go: a channel's **address** is only sent to its owner, because it can
+be a secret. An account holds at most 10 channels.
+
+Turning a channel off is the off switch for it: pages skip it, including the
+reminders of an incident that already paged it. A user with no usable channel
+falls back to the install's fallback target (`TERDUT_NTFY_FALLBACK_TOPIC`), as
+does an incident that opens with nobody on call. If neither yields a target,
+nothing is queued.
+
+An ntfy topic is a shared secret with the ntfy server: anyone who knows it can
+both read the pages and publish to it, so an unguessable one is worth the
+trouble. That is also why an address never appears in an incident's timeline,
+which every API key can read — the timeline names the channel *type* only.
+
+Three things get sent:
+
+- **triggered** — an incident opened. Urgency follows severity (on ntfy,
+  `critical` maps to max priority, the one that overrides the phone's quiet
+  settings).
 - **reminder** — the incident is still `triggered` after `TERDUT_NOTIFY_REPEAT`.
   Repeats until somebody acts. Acknowledging, snoozing, resolving or archiving
   all stop it — snooze is the mute button.
@@ -29,12 +58,12 @@ Three things get pushed:
   whoever was paged in the first place, and only for the automatic cascade:
   resolving by hand pushes nothing, since the person who did it already knows.
 
-Notifications carry an **Acknowledge** button that acknowledges the incident
+Notifications on a provider that can show one carry an **Acknowledge** button
-without opening anything. It POSTs to `/api/notify/ack/{token}`, an
+that acknowledges the incident without opening anything. It POSTs to `/api/notify/ack/{token}`, an
 unauthenticated route authorised by the 256-bit token in its path — minted fresh
 per notification, scoped to one incident and one action, and valid for 24 hours.
 A real API key is never put in a notification, because the message is stored on
-the ntfy server and cached on the device.
+the provider's server and cached on the device.
 
 The token is **not** consumed by use. Acknowledging is idempotent, so a token
 stays valid for its full 24 hours and a second tap is a no-op that reports the
@@ -47,16 +76,19 @@ Two consequences worth planning for:
 
 - `/api/notify/ack/{token}` **must stay publicly reachable**, or the button will
   not work when the responder is off your network.
-- Notifications sent to the fallback topic carry **no** Acknowledge button. The
+- Notifications sent to the fallback target carry **no** Acknowledge button. The
-  topic is shared, and a button on it would let any subscriber acknowledge as
+  target is shared, and a button on it would let any recipient acknowledge as
   somebody else.
 
-Delivery is a queue, not an inline call: the webhook writes a row and a
+Delivery is a queue, not an inline call: the webhook writes a row per channel
-background notifier sends it within 30 seconds, retrying with exponential
+and a background notifier sends it within 30 seconds, retrying with exponential
-backoff up to 8 attempts. Nothing about ingestion blocks on ntfy being reachable.
+backoff up to 8 attempts. Nothing about ingestion blocks on a provider being
+reachable. Each row keeps a snapshot of its address, so a message already
+queued still goes out if the channel is edited or removed; later reminders and
+the all-clear follow the channels as they are then.
 
 Every delivery is recorded on the incident's timeline: a `notified` event once
-ntfy accepts the publish, and a `notify_failed` event when a notification
+the provider accepts it (`triggered via ntfy`), and a `notify_failed` event when
-exhausts its retries. Written from the result rather than at enqueue, so the
+a notification exhausts its retries. Written from the result rather than at enqueue, so the
 timeline says what actually happened — and a page that never landed is visible
 instead of looking the same as one that did.

docs/single-sign-on.md

@@ -101,5 +101,5 @@ A login expires after 10 minutes. `GET /api/auth/config` reports `device_login`.
 use) and password login is the way in. With `TERDUT_PASSWORD_LOGIN=false` that way
 is closed: set it back to `true`. The first administrator comes from the bootstrap
 endpoint, and stays a manual administrator that no group can revoke; on an SSO-only
-install set `bootstrap.enabled: false` in the chart if you don't want that account,
+install set `bootstrap.enabled: false` in the chart if you don't want that account;
-or keep it and never give it a password.
+left on, the chart creates it with a generated password kept in the `<release>-admin-key` Secret.

docs/web-ui.md

@@ -5,7 +5,7 @@ _What the web UI offers, how sign-in and sessions work, and the Team and Admin t
 The server serves a web UI at `/`: the incident queue, each incident's alerts
 and timeline with every action (acknowledge, assign, snooze, note, resolve,
 archive), who is on call, the alert feed, and an *Account* tab for your own
-password and the ntfy topic your pages go to. It is built for a phone first. On a phone
+password and the notification channels your pages go to. It is built for a phone first. On a phone
 it navigates through a hamburger menu and has a sticky action bar, it follows the
 system's dark mode, and it can be added to the home screen. From 900px wide it switches
 to a sidebar with the queue and the incident side by side. The Stats page shows

internal/api/admin_test.go

@@ -7,6 +7,8 @@ import (
 	"net/http"
 	"strconv"
 	"testing"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/api"
 )
 
 // The bootstrap user is an administrator; everybody it creates afterwards is
@@ -96,8 +98,9 @@ func TestAdmin_MemberIsRefusedAdministration(t *testing.T) {
 			map[string]string{"password": "hunter2-hunter2"}},
 		{"mint a key for the admin", http.MethodPost, "/api/users/1/api-keys",
 			map[string]string{"name": "borrowed"}},
-		{"retarget the admin's notifications", http.MethodPut, "/api/users/1/notify",
+		{"add a channel to the admin's notifications", http.MethodPost, "/api/users/1/channels",
-			map[string]string{"ntfy_topic": "attacker-topic"}},
+			map[string]string{"type": "ntfy", "address": "attacker-topic"}},
+		{"read the admin's channels", http.MethodGet, "/api/users/1/channels", nil},
 	}
 
 	for _, c := range cases {
@@ -111,14 +114,14 @@ func TestAdmin_MemberIsRefusedAdministration(t *testing.T) {
 
 // Being refused other people's accounts must not cost a user their own.
 func TestAdmin_MemberKeepsTheirOwnAccount(t *testing.T) {
-	s := newTS(t)
+	s := newTS(t, api.NotifyConfig{Providers: newFakeNtfy(t).providers()})
 	memberID, call := member(t, s, "member")
 	self := "/api/users/" + id64(memberID)
 
-	resp := call(http.MethodPut, self+"/notify", map[string]string{"ntfy_topic": "terdut-member"})
+	resp := call(http.MethodPost, self+"/channels", map[string]string{"type": "ntfy", "address": "terdut-member"})
 	resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode != http.StatusCreated {
-		t.Errorf("own notify target: %d", resp.StatusCode)
+		t.Errorf("own notification channel: %d", resp.StatusCode)
 	}
 
 	resp = call(http.MethodPut, self+"/password", map[string]string{"password": "correct-horse-battery"})

internal/api/auth.go

@@ -345,6 +345,10 @@ func handleMe(db *sql.DB) http.HandlerFunc {
 			serverError(w, r, err)
 			return
 		}
+		if user.Channels, err = loadChannels(r.Context(), db, caller.ID, true); err != nil {
+			serverError(w, r, err)
+			return
+		}
 		var hash sql.NullString
 		var dismissed *int64
 		if err := db.QueryRowContext(r.Context(),

internal/api/authz_scope_test.go

@@ -163,7 +163,11 @@ func TestAuthzScope_SelfOrAdminRoutesRefuseAnotherNonAdminUser(t *testing.T) {
 		method, path string
 	}{
 		{http.MethodGet, "/api/users/" + bUserID + "/teams"},
-		{http.MethodPut, "/api/users/" + bUserID + "/notify"},
+		{http.MethodGet, "/api/users/" + bUserID + "/channels"},
+		{http.MethodPost, "/api/users/" + bUserID + "/channels"},
+		{http.MethodPatch, "/api/users/" + bUserID + "/channels/1"},
+		{http.MethodDelete, "/api/users/" + bUserID + "/channels/1"},
+		{http.MethodPost, "/api/users/" + bUserID + "/channels/1/test"},
 		{http.MethodPut, "/api/users/" + bUserID + "/password"},
 		{http.MethodGet, "/api/users/" + bUserID + "/api-keys"},
 		{http.MethodPost, "/api/users/" + bUserID + "/api-keys"},

internal/api/channels.go

@@ -0,0 +1,390 @@
+package api
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/models"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
+)
+
+// maxChannelsPerUser bounds how many channels one account can hold. Nobody needs
+// more than a handful, and every enabled one is a delivery per page.
+const maxChannelsPerUser = 10
+
+// loadChannels reads a user's channels. withAddress is false for anyone but the
+// owner: an address can be a secret (an ntfy topic lets anyone holding it
+// publish to it), and an administrator has no need of it to run the install.
+func loadChannels(ctx context.Context, q querier, userID int64, withAddress bool) ([]models.NotificationChannel, error) {
+	rows, err := q.QueryContext(ctx, `
+		SELECT id, type, address, enabled, created_at
+		  FROM notification_channels WHERE user_id = $1 ORDER BY id`, userID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	out := []models.NotificationChannel{}
+	for rows.Next() {
+		var c models.NotificationChannel
+		var ts int64
+		if err := rows.Scan(&c.ID, &c.Type, &c.Address, &c.Enabled, &ts); err != nil {
+			return nil, err
+		}
+		c.CreatedAt = time.Unix(ts, 0).UTC()
+		if !withAddress {
+			c.Address = ""
+		}
+		out = append(out, c)
+	}
+	return out, rows.Err()
+}
+
+// hasReachableChannel reports whether a page to this user would go anywhere: an
+// enabled channel of a type this server can deliver. It is the one definition of
+// "pageable" the team and escalation pages share with the notifier, so they
+// cannot promise a page that the notifier would skip.
+func hasReachableChannel(ctx context.Context, q querier, providers *notify.Registry, userID int64) (bool, error) {
+	rows, err := q.QueryContext(ctx,
+		"SELECT type FROM notification_channels WHERE user_id = $1 AND enabled", userID)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var typ string
+		if err := rows.Scan(&typ); err != nil {
+			return false, err
+		}
+		if providers.Has(typ) {
+			return true, nil
+		}
+	}
+	return false, rows.Err()
+}
+
+// validateTarget checks a fallback target. Both halves empty is "none", which is
+// fine; one half alone is a mistake; and the type must be a provider this server
+// has, since a fallback nobody can deliver would look configured and page
+// nobody.
+func validateTarget(providers *notify.Registry, t notify.Target) (notify.Target, error) {
+	if t.IsZero() {
+		return notify.Target{}, nil
+	}
+	if t.Type == "" || t.Address == "" {
+		return notify.Target{}, errors.New("type and address go together: set both, or neither")
+	}
+	p, ok := providers.Get(t.Type)
+	if !ok {
+		return notify.Target{}, errors.New("this server has no " + strconv.Quote(t.Type) + " provider")
+	}
+	address, err := p.Validate(t.Address)
+	if err != nil {
+		return notify.Target{}, err
+	}
+	return notify.Target{Type: t.Type, Address: address}, nil
+}
+
+// providerTypes lists the registered provider types, as a query parameter for
+// "channels this server can deliver". Never nil: a nil slice is NULL to Postgres,
+// and `= ANY(NULL)` is not false.
+func providerTypes(providers *notify.Registry) []string {
+	types := []string{}
+	for _, p := range providers.List() {
+		types = append(types, p.Type())
+	}
+	return types
+}
+
+// providerView is what the UI needs to offer a channel type.
+type providerView struct {
+	Type         string `json:"type"`
+	Label        string `json:"label"`
+	AddressLabel string `json:"address_label"`
+	AddressHint  string `json:"address_hint"`
+}
+
+// handleListProviders says which kinds of channel this server can deliver, so
+// the UI offers only those. Readable by anyone signed in: it names no secret.
+func handleListProviders(providers *notify.Registry) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		out := []providerView{}
+		for _, p := range providers.List() {
+			out = append(out, providerView{
+				Type: p.Type(), Label: p.Label(),
+				AddressLabel: p.AddressLabel(), AddressHint: p.AddressHint(),
+			})
+		}
+		respond(w, http.StatusOK, out)
+	}
+}
+
+// channelOwner parses {id} and checks the caller may act on that user's
+// channels: themselves, or an administrator.
+func channelOwner(w http.ResponseWriter, r *http.Request) (int64, bool) {
+	id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
+	if err != nil {
+		respond(w, http.StatusBadRequest, errResp("invalid user id"))
+		return 0, false
+	}
+	if !requireSelfOrAdmin(w, r, id) {
+		return 0, false
+	}
+	return id, true
+}
+
+func isOwner(r *http.Request, userID int64) bool {
+	caller, ok := userFromContext(r.Context())
+	return ok && caller.ID == userID
+}
+
+// handleListChannels returns a user's channels. The owner sees the addresses; an
+// administrator sees the types and whether each is on.
+func handleListChannels(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := channelOwner(w, r)
+		if !ok {
+			return
+		}
+		channels, err := loadChannels(r.Context(), db, id, isOwner(r, id))
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		respond(w, http.StatusOK, channels)
+	}
+}
+
+// handleCreateChannel adds a channel to a user. The same address twice on one
+// account is a 409 rather than two pages for one phone.
+func handleCreateChannel(db *sql.DB, providers *notify.Registry) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := channelOwner(w, r)
+		if !ok {
+			return
+		}
+		var req struct {
+			Type    string `json:"type"`
+			Address string `json:"address"`
+		}
+		if err := decodeJSON(r, &req); err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid request body"))
+			return
+		}
+		p, ok := providers.Get(req.Type)
+		if !ok {
+			respond(w, http.StatusBadRequest,
+				errResp("this server has no "+strconv.Quote(req.Type)+" provider"))
+			return
+		}
+		address, err := p.Validate(req.Address)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp(err.Error()))
+			return
+		}
+
+		var count int
+		if err := db.QueryRowContext(r.Context(),
+			"SELECT COUNT(*) FROM notification_channels WHERE user_id = $1", id).Scan(&count); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		if count >= maxChannelsPerUser {
+			respond(w, http.StatusConflict,
+				errResp("an account can have at most "+strconv.Itoa(maxChannelsPerUser)+" notification channels"))
+			return
+		}
+
+		var channelID int64
+		err = db.QueryRowContext(r.Context(), `
+			INSERT INTO notification_channels (user_id, type, address)
+			VALUES ($1, $2, $3) RETURNING id`, id, p.Type(), address).Scan(&channelID)
+		if err != nil {
+			switch {
+			case isUniqueViolation(err):
+				respond(w, http.StatusConflict, errResp("that channel already exists"))
+			case isForeignKeyViolation(err):
+				respond(w, http.StatusNotFound, errResp("user not found"))
+			default:
+				serverError(w, r, err)
+			}
+			return
+		}
+		respondChannel(w, r, db, id, channelID, http.StatusCreated)
+	}
+}
+
+// handleUpdateChannel turns a channel on or off, or changes its address. Off is
+// the mute button for one channel: it stays in the list, and pages skip it.
+func handleUpdateChannel(db *sql.DB, providers *notify.Registry) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := channelOwner(w, r)
+		if !ok {
+			return
+		}
+		channelID, err := strconv.ParseInt(chi.URLParam(r, "channelID"), 10, 64)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid channel id"))
+			return
+		}
+		var req struct {
+			Enabled *bool   `json:"enabled"`
+			Address *string `json:"address"`
+		}
+		if err := decodeJSON(r, &req); err != nil || (req.Enabled == nil && req.Address == nil) {
+			respond(w, http.StatusBadRequest, errResp("enabled or address is required"))
+			return
+		}
+
+		var typ string
+		if err := db.QueryRowContext(r.Context(),
+			"SELECT type FROM notification_channels WHERE id = $1 AND user_id = $2",
+			channelID, id).Scan(&typ); errors.Is(err, sql.ErrNoRows) {
+			respond(w, http.StatusNotFound, errResp("channel not found"))
+			return
+		} else if err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		if req.Address != nil {
+			p, ok := providers.Get(typ)
+			if !ok {
+				respond(w, http.StatusConflict,
+					errResp("this server no longer has a "+strconv.Quote(typ)+" provider, so its address cannot be checked"))
+				return
+			}
+			address, err := p.Validate(*req.Address)
+			if err != nil {
+				respond(w, http.StatusBadRequest, errResp(err.Error()))
+				return
+			}
+			if _, err := db.ExecContext(r.Context(),
+				"UPDATE notification_channels SET address = $1 WHERE id = $2", address, channelID); err != nil {
+				if isUniqueViolation(err) {
+					respond(w, http.StatusConflict, errResp("that channel already exists"))
+					return
+				}
+				serverError(w, r, err)
+				return
+			}
+		}
+		if req.Enabled != nil {
+			if _, err := db.ExecContext(r.Context(),
+				"UPDATE notification_channels SET enabled = $1 WHERE id = $2", *req.Enabled, channelID); err != nil {
+				serverError(w, r, err)
+				return
+			}
+		}
+		respondChannel(w, r, db, id, channelID, http.StatusOK)
+	}
+}
+
+// handleDeleteChannel removes a channel. Notifications already queued for it
+// keep their address snapshot and are still delivered; later reminders skip it.
+func handleDeleteChannel(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, ok := channelOwner(w, r)
+		if !ok {
+			return
+		}
+		channelID, err := strconv.ParseInt(chi.URLParam(r, "channelID"), 10, 64)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid channel id"))
+			return
+		}
+		res, err := db.ExecContext(r.Context(),
+			"DELETE FROM notification_channels WHERE id = $1 AND user_id = $2", channelID, id)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		if n, _ := res.RowsAffected(); n == 0 {
+			respond(w, http.StatusNotFound, errResp("channel not found"))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+// respondChannel writes one channel back, with the address only to its owner.
+func respondChannel(w http.ResponseWriter, r *http.Request, db *sql.DB, userID, channelID int64, status int) {
+	channels, err := loadChannels(r.Context(), db, userID, isOwner(r, userID))
+	if err != nil {
+		serverError(w, r, err)
+		return
+	}
+	for _, c := range channels {
+		if c.ID == channelID {
+			respond(w, status, c)
+			return
+		}
+	}
+	respond(w, http.StatusNotFound, errResp("channel not found"))
+}
+
+// handleTestChannel sends one test message through a channel.
+//
+// The point of the first-run checklist's notification step is not that an
+// address has been typed but that a phone buzzes, and only the person holding it
+// can tell whether it did. Sent directly rather than through the outbox: the
+// outbox row requires an incident, and this deliberately belongs to no incident.
+//
+// Owner only, not an administrator: a test is a page, and nobody should be woken
+// by somebody else's click.
+func handleTestChannel(db *sql.DB, cfg NotifyConfig) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid user id"))
+			return
+		}
+		if !isOwner(r, id) {
+			respond(w, http.StatusForbidden, errResp("a test can only be sent to your own channel"))
+			return
+		}
+		channelID, err := strconv.ParseInt(chi.URLParam(r, "channelID"), 10, 64)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid channel id"))
+			return
+		}
+
+		var typ, address string
+		if err := db.QueryRowContext(r.Context(),
+			"SELECT type, address FROM notification_channels WHERE id = $1 AND user_id = $2",
+			channelID, id).Scan(&typ, &address); errors.Is(err, sql.ErrNoRows) {
+			respond(w, http.StatusNotFound, errResp("channel not found"))
+			return
+		} else if err != nil {
+			serverError(w, r, err)
+			return
+		}
+
+		p, ok := cfg.Providers.Get(typ)
+		if !ok {
+			respond(w, http.StatusServiceUnavailable,
+				errResp("this server has no "+strconv.Quote(typ)+" provider configured, so it can send nothing there"))
+			return
+		}
+
+		if err := p.Send(r.Context(), notify.Message{
+			Address: address,
+			Kind:    notify.KindTest,
+			Title:   "terdut test",
+			Body:    "If this arrived, your notifications work.",
+		}); err != nil {
+			// The failure is the useful part here: a wrong address, a credential
+			// the provider rejects, or a provider that is down all look the same
+			// from the phone, which is silence.
+			respond(w, http.StatusBadGateway, errResp(p.Label()+" rejected the test: "+err.Error()))
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}

internal/api/channels_test.go

@@ -0,0 +1,529 @@
+package api_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/api"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
+)
+
+// ---------------------------------------------------------------------------
+// A second provider
+// ---------------------------------------------------------------------------
+
+// fakeMail is a provider that cannot render action buttons, which is what the
+// notifier's capability handling is for. Registering it next to ntfy is also the
+// proof that nothing in the notifier still assumes there is only one.
+type fakeMail struct {
+	mu   sync.Mutex
+	got  []notify.Message
+	fail error
+}
+
+func (m *fakeMail) Type() string         { return "mail" }
+func (m *fakeMail) Label() string        { return "Mail" }
+func (m *fakeMail) AddressLabel() string { return "Email address" }
+func (m *fakeMail) AddressHint() string  { return "Where to send pages." }
+
+func (m *fakeMail) Capabilities() notify.Capabilities {
+	return notify.Capabilities{ActionButtons: false}
+}
+
+func (m *fakeMail) Validate(address string) (string, error) {
+	address = strings.TrimSpace(address)
+	if !strings.Contains(address, "@") {
+		return "", errors.New("that does not look like an email address")
+	}
+	return address, nil
+}
+
+func (m *fakeMail) Send(_ context.Context, msg notify.Message) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.fail != nil {
+		return m.fail
+	}
+	m.got = append(m.got, msg)
+	return nil
+}
+
+func (m *fakeMail) messages() []notify.Message {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return append([]notify.Message(nil), m.got...)
+}
+
+func (m *fakeMail) failWith(err error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.fail = err
+}
+
+// twoProviderTS is notifyTS with ntfy and the fake mail provider both
+// registered, the admin on call, and a channel of each kind on the admin.
+func twoProviderTS(t *testing.T, cfg api.NotifyConfig) (*ts, *fakeNtfy, *fakeMail) {
+	t.Helper()
+	f := newFakeNtfy(t)
+	mail := &fakeMail{}
+	cfg.Providers = notify.NewRegistry(notify.NewNtfy(f.URL, ""), mail)
+	s := newTS(t, cfg)
+
+	putOnCall(t, s, 1)
+	addNtfyChannel(t, s, 1, "terdut-admin")
+	addChannel(t, s, 1, "mail", "admin@example.com")
+	return s, f, mail
+}
+
+func readBody(t *testing.T, resp *http.Response) string {
+	t.Helper()
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	return string(b)
+}
+
+func addChannel(t *testing.T, s *ts, userID int, typ, address string) {
+	t.Helper()
+	resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/users/%d/channels", userID),
+		map[string]any{"type": typ, "address": address})
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("add %s channel returned %d", typ, resp.StatusCode)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Fan-out
+// ---------------------------------------------------------------------------
+
+// A page goes to every enabled channel the person has, each in its own format.
+func TestChannels_PageFansOutToEveryEnabledChannel(t *testing.T) {
+	s, f, mail := twoProviderTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Fatalf("expected 1 ntfy push, got %d", got)
+	}
+	if got := len(mail.messages()); got != 1 {
+		t.Fatalf("expected 1 mail message, got %d", got)
+	}
+	if to := mail.messages()[0].Address; to != "admin@example.com" {
+		t.Errorf("mail went to %q", to)
+	}
+
+	// One event per delivery, naming the channel type but never the address.
+	events := notifyEvents(t, s, 1)
+	if len(events) != 2 {
+		t.Fatalf("expected an event per channel, got %v", events)
+	}
+	seen := map[string]bool{}
+	for _, e := range events {
+		detail, _ := e["detail"].(string)
+		seen[detail] = true
+		if strings.Contains(detail, "terdut-admin") || strings.Contains(detail, "example.com") {
+			t.Errorf("the timeline must not carry an address, got %q", detail)
+		}
+	}
+	if !seen["triggered via ntfy"] || !seen["triggered via mail"] {
+		t.Errorf("expected one event per channel type, got %v", seen)
+	}
+}
+
+// A provider that cannot render a button still gets a way to acknowledge,
+// as text; the one that can gets the button.
+func TestChannels_AckIsAButtonOrAnInlineLinkByCapability(t *testing.T) {
+	s, f, mail := twoProviderTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	if m := f.messages()[0]; len(m.Actions) != 1 || m.Actions[0].Label != "Acknowledge" {
+		t.Errorf("ntfy should carry the Acknowledge button, got %+v", m.Actions)
+	}
+	m := mail.messages()[0]
+	if m.AckURL != "" {
+		t.Errorf("a provider without action buttons must not be handed an AckURL, got %q", m.AckURL)
+	}
+	if !strings.Contains(m.Body, "Acknowledge: https://terdut.example.com/incidents/1") {
+		t.Errorf("expected the acknowledge link in the body, got %q", m.Body)
+	}
+	if m.ClickURL != "https://terdut.example.com/incidents/1" {
+		t.Errorf("expected the incident link as click URL, got %q", m.ClickURL)
+	}
+
+	// Only the button needs a token; minting one for text nobody can use is waste.
+	var tokens int
+	if err := s.db.QueryRow("SELECT COUNT(*) FROM incident_ack_tokens").Scan(&tokens); err != nil {
+		t.Fatal(err)
+	}
+	if tokens != 1 {
+		t.Errorf("expected one ack token (for the ntfy button), got %d", tokens)
+	}
+}
+
+// One channel failing must not hold back the other, and its failure is recorded
+// with the channel it happened on.
+func TestChannels_OneFailingChannelDoesNotBlockTheOther(t *testing.T) {
+	s, f, mail := twoProviderTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+	mail.failWith(errors.New("smtp is down"))
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 1 {
+		t.Fatalf("ntfy should have delivered, got %d", got)
+	}
+	var lastError string
+	if err := s.db.QueryRow(
+		"SELECT last_error FROM notifications WHERE channel_type = 'mail'").Scan(&lastError); err != nil {
+		t.Fatalf("read the mail row: %v", err)
+	}
+	if !strings.Contains(lastError, "smtp is down") {
+		t.Errorf("expected the failure kept on the row, got %q", lastError)
+	}
+
+	// Back up: the retry (made due) lands without re-sending the ntfy page.
+	mail.failWith(nil)
+	s.exec(t, "UPDATE notifications SET send_after = 0 WHERE sent_at IS NULL")
+	s.sweepNotify(t)
+	if got := len(mail.messages()); got != 1 {
+		t.Errorf("expected the retry to deliver the mail, got %d", got)
+	}
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("ntfy must not be paged twice by the other channel's retry, got %d", got)
+	}
+}
+
+// Reminders and the all-clear go to every channel the original page went to.
+func TestChannels_RemindersAndAllClearReachEveryChannel(t *testing.T) {
+	s, f, mail := twoProviderTS(t, api.NotifyConfig{
+		PublicURL:   "https://terdut.example.com",
+		RepeatEvery: 15 * time.Minute,
+	})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+	s.ageNotifications(t, 20*time.Minute)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 2 {
+		t.Errorf("expected a reminder on ntfy, got %d message(s)", got)
+	}
+	if got := len(mail.messages()); got != 2 {
+		t.Errorf("expected a reminder on mail, got %d message(s)", got)
+	}
+
+	// The incident resolves by itself: the all-clear follows the last batch.
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-notify", "DiskFull", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T10:30:00Z",
+			map[string]string{"severity": "critical"}),
+	}, "{}:{alertname=\"DiskFull\"}")
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 3 {
+		t.Errorf("expected the all-clear on ntfy, got %d message(s)", got)
+	}
+	if got := len(mail.messages()); got != 3 {
+		t.Errorf("expected the all-clear on mail, got %d message(s)", got)
+	}
+}
+
+// Turning a channel off is the off switch for it: pages skip it, including the
+// reminders of an incident that already paged it.
+func TestChannels_DisabledChannelIsSkipped(t *testing.T) {
+	s, f, mail := twoProviderTS(t, api.NotifyConfig{
+		PublicURL:   "https://terdut.example.com",
+		RepeatEvery: 15 * time.Minute,
+	})
+
+	fireCritical(t, s)
+	s.sweepNotify(t)
+
+	var mailChannel int
+	if err := s.db.QueryRow(
+		"SELECT id FROM notification_channels WHERE type = 'mail'").Scan(&mailChannel); err != nil {
+		t.Fatal(err)
+	}
+	resp := s.req(t, http.MethodPatch, fmt.Sprintf("/api/users/1/channels/%d", mailChannel),
+		map[string]any{"enabled": false})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("disable returned %d", resp.StatusCode)
+	}
+
+	s.ageNotifications(t, 20*time.Minute)
+	s.sweepNotify(t)
+
+	if got := len(f.messages()); got != 2 {
+		t.Errorf("ntfy should still be reminded, got %d message(s)", got)
+	}
+	if got := len(mail.messages()); got != 1 {
+		t.Errorf("a disabled channel must not be reminded, got %d message(s)", got)
+	}
+
+	// And a new incident does not page it at all.
+	postWebhook(t, s, []map[string]any{
+		amAlert("fp-second", "OtherAlert", "firing", "2026-05-20T11:00:00Z", zeroTime,
+			map[string]string{"severity": "critical"}),
+	}, "{}:{alertname=\"OtherAlert\"}")
+	s.sweepNotify(t)
+	if got := len(mail.messages()); got != 1 {
+		t.Errorf("a disabled channel must not be paged, got %d message(s)", got)
+	}
+}
+
+// A channel whose provider this server no longer has is skipped at enqueue rather
+// than queued to fail eight times.
+func TestChannels_UnconfiguredProviderIsSkipped(t *testing.T) {
+	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
+	s.exec(t, "INSERT INTO notification_channels (user_id, type, address) VALUES (1, 'mail', 'admin@example.com')")
+
+	fireCritical(t, s)
+
+	if got := s.countNotifications(t, ""); got != 1 {
+		t.Errorf("expected only the ntfy row queued, got %d", got)
+	}
+	s.sweepNotify(t)
+	if got := len(f.messages()); got != 1 {
+		t.Errorf("expected the ntfy page, got %d", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// The channel API
+// ---------------------------------------------------------------------------
+
+func TestChannelsAPI_CreateValidatesTypeAndAddress(t *testing.T) {
+	s, _, _ := twoProviderTS(t, api.NotifyConfig{})
+
+	cases := []struct {
+		name string
+		body map[string]any
+		want int
+	}{
+		{"unknown type", map[string]any{"type": "carrier-pigeon", "address": "x"}, http.StatusBadRequest},
+		{"address the provider rejects", map[string]any{"type": "mail", "address": "not-an-address"}, http.StatusBadRequest},
+		{"empty topic", map[string]any{"type": "ntfy", "address": "  "}, http.StatusBadRequest},
+		{"topic with a slash", map[string]any{"type": "ntfy", "address": "a/b"}, http.StatusBadRequest},
+		{"duplicate", map[string]any{"type": "ntfy", "address": "terdut-admin"}, http.StatusConflict},
+		{"fine", map[string]any{"type": "ntfy", "address": "another-topic"}, http.StatusCreated},
+	}
+	for _, c := range cases {
+		resp := s.req(t, http.MethodPost, "/api/users/1/channels", c.body)
+		resp.Body.Close()
+		if resp.StatusCode != c.want {
+			t.Errorf("%s: expected %d, got %d", c.name, c.want, resp.StatusCode)
+		}
+	}
+}
+
+func TestChannelsAPI_CapsChannelsPerAccount(t *testing.T) {
+	s, _, _ := twoProviderTS(t, api.NotifyConfig{})
+	for i := 0; i < 8; i++ { // the two from setup make ten
+		addNtfyChannel(t, s, 1, fmt.Sprintf("topic-%d", i))
+	}
+	resp := s.req(t, http.MethodPost, "/api/users/1/channels",
+		map[string]any{"type": "ntfy", "address": "one-too-many"})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusConflict {
+		t.Errorf("expected 409 at the cap, got %d", resp.StatusCode)
+	}
+}
+
+func TestChannelsAPI_ListUpdateDelete(t *testing.T) {
+	s, _, _ := twoProviderTS(t, api.NotifyConfig{})
+
+	var channels []struct {
+		ID      int64  `json:"id"`
+		Type    string `json:"type"`
+		Address string `json:"address"`
+		Enabled bool   `json:"enabled"`
+	}
+	decode(t, s.req(t, http.MethodGet, "/api/users/1/channels", nil), &channels)
+	if len(channels) != 2 || channels[0].Type != "ntfy" || channels[0].Address != "terdut-admin" || !channels[0].Enabled {
+		t.Fatalf("expected the two channels in creation order, got %+v", channels)
+	}
+
+	path := fmt.Sprintf("/api/users/1/channels/%d", channels[0].ID)
+	resp := s.req(t, http.MethodPatch, path, map[string]any{"address": "renamed-topic", "enabled": false})
+	var updated struct {
+		Address string `json:"address"`
+		Enabled bool   `json:"enabled"`
+	}
+	decode(t, resp, &updated)
+	if updated.Address != "renamed-topic" || updated.Enabled {
+		t.Errorf("expected the update applied, got %+v", updated)
+	}
+
+	if resp := s.req(t, http.MethodPatch, path, map[string]any{}); resp.StatusCode != http.StatusBadRequest {
+		t.Errorf("an empty patch should be a 400, got %d", resp.StatusCode)
+	}
+
+	resp = s.req(t, http.MethodDelete, path, nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		t.Errorf("delete returned %d", resp.StatusCode)
+	}
+	resp = s.req(t, http.MethodDelete, path, nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusNotFound {
+		t.Errorf("deleting twice should be a 404, got %d", resp.StatusCode)
+	}
+}
+
+// Somebody else's channels are not yours to read or change, and an administrator
+// manages them without ever seeing where they go.
+func TestChannelsAPI_Authorization(t *testing.T) {
+	s, _, _ := twoProviderTS(t, api.NotifyConfig{})
+	memberID, asMember := member(t, s, "member")
+	memberPath := "/api/users/" + id64(memberID) + "/channels"
+
+	// The member owns a channel; the admin can see that it exists, not its address.
+	resp := asMember(http.MethodPost, memberPath, map[string]any{"type": "ntfy", "address": "member-secret"})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("a member adds their own channel: %d", resp.StatusCode)
+	}
+
+	var own []struct {
+		Address string `json:"address"`
+	}
+	decode(t, asMember(http.MethodGet, memberPath, nil), &own)
+	if len(own) != 1 || own[0].Address != "member-secret" {
+		t.Errorf("the owner should see their address, got %+v", own)
+	}
+
+	var seenByAdmin []struct {
+		Type    string `json:"type"`
+		Address string `json:"address"`
+	}
+	decode(t, s.req(t, http.MethodGet, memberPath, nil), &seenByAdmin)
+	if len(seenByAdmin) != 1 || seenByAdmin[0].Type != "ntfy" || seenByAdmin[0].Address != "" {
+		t.Errorf("an administrator should see the channel but not its address, got %+v", seenByAdmin)
+	}
+
+	// A member cannot touch the admin's.
+	for _, c := range []struct{ method, path string }{
+		{http.MethodGet, "/api/users/1/channels"},
+		{http.MethodPost, "/api/users/1/channels"},
+		{http.MethodPatch, "/api/users/1/channels/1"},
+		{http.MethodDelete, "/api/users/1/channels/1"},
+		{http.MethodPost, "/api/users/1/channels/1/test"},
+	} {
+		resp := asMember(c.method, c.path, map[string]any{"type": "ntfy", "address": "x", "enabled": false})
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusForbidden {
+			t.Errorf("%s %s: expected 403, got %d", c.method, c.path, resp.StatusCode)
+		}
+	}
+
+	// The administrator may manage the member's channels but not page them.
+	var list []struct {
+		ID int64 `json:"id"`
+	}
+	decode(t, s.req(t, http.MethodGet, memberPath, nil), &list)
+	resp = s.req(t, http.MethodPost, fmt.Sprintf("%s/%d/test", memberPath, list[0].ID), nil)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusForbidden {
+		t.Errorf("an administrator must not be able to test someone else's channel, got %d", resp.StatusCode)
+	}
+}
+
+func TestChannelsAPI_TestSendsThroughTheProvider(t *testing.T) {
+	s, f, mail := twoProviderTS(t, api.NotifyConfig{})
+
+	var channels []struct {
+		ID   int64  `json:"id"`
+		Type string `json:"type"`
+	}
+	decode(t, s.req(t, http.MethodGet, "/api/users/1/channels", nil), &channels)
+
+	for _, c := range channels {
+		resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/users/1/channels/%d/test", c.ID), nil)
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusNoContent {
+			t.Errorf("test of %s returned %d", c.Type, resp.StatusCode)
+		}
+	}
+	if msgs := f.messages(); len(msgs) != 1 || msgs[0].Topic != "terdut-admin" || msgs[0].Title != "terdut test" {
+		t.Errorf("expected a test push on the admin's topic, got %+v", msgs)
+	}
+	if msgs := mail.messages(); len(msgs) != 1 || msgs[0].Kind != notify.KindTest {
+		t.Errorf("expected a test message on mail, got %+v", msgs)
+	}
+
+	// The provider's complaint is the answer: it is why nothing arrived.
+	mail.failWith(errors.New("relay denied"))
+	resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/users/1/channels/%d/test", channels[1].ID), nil)
+	body := readBody(t, resp)
+	if resp.StatusCode != http.StatusBadGateway || !strings.Contains(body, "relay denied") {
+		t.Errorf("expected a 502 carrying the reason, got %d %q", resp.StatusCode, body)
+	}
+}
+
+func TestChannelsAPI_ProvidersListsOnlyWhatIsConfigured(t *testing.T) {
+	s, _, _ := twoProviderTS(t, api.NotifyConfig{})
+	var providers []struct {
+		Type         string `json:"type"`
+		Label        string `json:"label"`
+		AddressLabel string `json:"address_label"`
+		AddressHint  string `json:"address_hint"`
+	}
+	decode(t, s.req(t, http.MethodGet, "/api/notify/providers", nil), &providers)
+	if len(providers) != 2 || providers[0].Type != "mail" || providers[1].Type != "ntfy" {
+		t.Fatalf("expected mail and ntfy in type order, got %+v", providers)
+	}
+	if providers[1].AddressLabel != "Topic" || providers[1].AddressHint == "" {
+		t.Errorf("a provider should describe its address field, got %+v", providers[1])
+	}
+
+	none := newTS(t)
+	providers = nil
+	decode(t, none.req(t, http.MethodGet, "/api/notify/providers", nil), &providers)
+	if len(providers) != 0 {
+		t.Errorf("a server with no provider should list none, got %+v", providers)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Fallback target
+// ---------------------------------------------------------------------------
+
+func TestChannelsAPI_FallbackMustBeADeliverableTarget(t *testing.T) {
+	s, _, _ := twoProviderTS(t, api.NotifyConfig{})
+	put := func(fallback map[string]string) int {
+		resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", map[string]any{
+			"repeat_count": 0, "fallback": fallback, "levels": []any{},
+		})
+		resp.Body.Close()
+		return resp.StatusCode
+	}
+
+	if got := put(map[string]string{"type": "ntfy", "address": "team-fallback"}); got != http.StatusOK {
+		t.Errorf("a configured provider should be accepted, got %d", got)
+	}
+	if got := put(map[string]string{"type": "", "address": ""}); got != http.StatusOK {
+		t.Errorf("no fallback should be accepted, got %d", got)
+	}
+	if got := put(map[string]string{"type": "ntfy"}); got != http.StatusBadRequest {
+		t.Errorf("a type without an address should be rejected, got %d", got)
+	}
+	if got := put(map[string]string{"type": "carrier-pigeon", "address": "x"}); got != http.StatusBadRequest {
+		t.Errorf("a provider this server lacks should be rejected, got %d", got)
+	}
+	if got := put(map[string]string{"type": "mail", "address": "nope"}); got != http.StatusBadRequest {
+		t.Errorf("an address the provider rejects should be rejected, got %d", got)
+	}
+}

internal/api/deadman_test.go

@@ -29,12 +29,12 @@ func deadmanTS(t *testing.T, cfg api.DeadmanConfig) (*ts, *fakeNtfy) {
 	t.Helper()
 	f := newFakeNtfy(t)
 	s := newDeadmanTS(t, cfg, api.NotifyConfig{
-		BaseURL:   f.URL,
+		Providers: f.providers(),
 		PublicURL: "https://terdut.example.com",
 	})
 
 	putOnCall(t, s, 1)
-	setTopic(t, s, 1, "terdut-admin")
+	addNtfyChannel(t, s, 1, "terdut-admin")
 	return s, f
 }
 

internal/api/escalation.go

@@ -9,6 +9,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 )
 
 // evEscalated records a rung of the ladder on the incident's timeline: which
@@ -21,7 +23,7 @@ const evEscalated = "escalated"
 type escalationPolicy struct {
 	teamID      int64
 	repeatCount int64
-	fallbackTopic string
+	fallback    notify.Target
 	levels      []escalationLevel
 }
 
@@ -39,7 +41,7 @@ type escalationTarget struct {
 
 // configured reports whether this team has anything to escalate through. A
 // policy row with no levels is the same as no policy: the team gets the
-// pre-escalation behaviour, which is reminders on the assignee's topic.
+// pre-escalation behaviour, which is reminders to the assignee's channels.
 func (p *escalationPolicy) configured() bool { return p != nil && len(p.levels) > 0 }
 
 // level returns the level at a 1-based position.
@@ -58,8 +60,8 @@ func (p *escalationPolicy) level(pos int64) (escalationLevel, bool) {
 func loadEscalationPolicy(ctx context.Context, q querier, teamID int64) (*escalationPolicy, error) {
 	p := &escalationPolicy{teamID: teamID}
 	err := q.QueryRowContext(ctx,
-		"SELECT repeat_count, fallback_topic FROM escalation_policies WHERE team_id = $1",
+		"SELECT repeat_count, fallback_type, fallback_address FROM escalation_policies WHERE team_id = $1",
-		teamID).Scan(&p.repeatCount, &p.fallbackTopic)
+		teamID).Scan(&p.repeatCount, &p.fallback.Type, &p.fallback.Address)
 	if err == sql.ErrNoRows {
 		return nil, nil
 	}
@@ -196,7 +198,7 @@ func advanceEscalation(ctx context.Context, db *sql.DB, cfg NotifyConfig, policy
 		if round < policy.repeatCount {
 			next, nextRound = 1, round+1
 		} else {
-			if err := escalationExhausted(ctx, tx, policy, incidentID, now); err != nil {
+			if err := escalationExhausted(ctx, tx, cfg, policy, incidentID, now); err != nil {
 				return err
 			}
 			return tx.Commit()
@@ -236,18 +238,23 @@ func advanceEscalation(ctx context.Context, db *sql.DB, cfg NotifyConfig, policy
 	return tx.Commit()
 }
 
-// escalationExhausted is the end of the line: the fallback topic, once, and a
+// escalationExhausted is the end of the line: the fallback target, once, and a
 // timeline entry saying the ladder is finished. The incident stays triggered —
 // escalation running out is not the same as somebody answering.
-func escalationExhausted(ctx context.Context, tx *sql.Tx, policy *escalationPolicy, incidentID int64, now time.Time) error {
+func escalationExhausted(ctx context.Context, tx *sql.Tx, cfg NotifyConfig, policy *escalationPolicy, incidentID int64, now time.Time) error {
 	detail := "escalation exhausted"
-	if policy.fallbackTopic != "" {
+	// The address is left out of the detail: it can be a shared secret, and the
-		if err := enqueueNotification(ctx, tx, incidentID, nil, policy.fallbackTopic, notifyEscalated); err != nil {
+	// timeline is readable by every API key.
+	switch recipients := fallbackRecipients(cfg, policy.fallback); {
+	case len(recipients) > 0:
+		if err := enqueueNotifications(ctx, tx, incidentID, recipients, notifyEscalated); err != nil {
 			return err
 		}
-		detail += ": paged " + policy.fallbackTopic
+		detail += ": paged the team fallback (" + policy.fallback.Type + ")"
-	} else {
+	case !policy.fallback.IsZero():
-		detail += ": no fallback topic configured"
+		detail += ": the team fallback cannot be delivered (" + policy.fallback.Type + " is not configured)"
+	default:
+		detail += ": no fallback configured"
 	}
 
 	// Level 0 again, so the sweep stops considering it. The round counter is
@@ -267,6 +274,7 @@ func escalationExhausted(ctx context.Context, tx *sql.Tx, policy *escalationPoli
 // phone, not as whoever was paged first.
 func pageLevel(ctx context.Context, tx *sql.Tx, cfg NotifyConfig, policy *escalationPolicy, incidentID int64, level escalationLevel) ([]string, error) {
 	var paged []string
+	var recipients []recipient
 	seen := map[int64]bool{}
 
 	for _, t := range level.targets {
@@ -286,24 +294,31 @@ func pageLevel(ctx context.Context, tx *sql.Tx, cfg NotifyConfig, policy *escala
 		}
 		seen[*userID] = true
 
-		var topic *string
 		var username string
 		if err := tx.QueryRowContext(ctx,
-			"SELECT ntfy_topic, username FROM users WHERE id = $1 AND disabled_at IS NULL",
+			"SELECT username FROM users WHERE id = $1 AND disabled_at IS NULL",
-			*userID).Scan(&topic, &username); err != nil {
+			*userID).Scan(&username); err != nil {
 			// A disabled or deleted account is not an error in the middle of an
 			// escalation: it is a target that cannot be woken, and the next
 			// level is the answer to that.
 			continue
 		}
-		if topic == nil || *topic == "" {
+		rs, err := userRecipients(ctx, tx, cfg, *userID)
-			continue
+		if err != nil {
-		}
-		if err := enqueueNotification(ctx, tx, incidentID, userID, *topic, notifyEscalated); err != nil {
 			return nil, err
 		}
+		if len(rs) == 0 {
+			continue
+		}
+		recipients = append(recipients, rs...)
 		paged = append(paged, username)
 	}
+
+	// One call for the whole level, so its rows share a created_at and are
+	// reminded and resolved as the one batch they are.
+	if err := enqueueNotifications(ctx, tx, incidentID, recipients, notifyEscalated); err != nil {
+		return nil, err
+	}
 	return paged, nil
 }
 
@@ -332,7 +347,7 @@ func stopEscalation(ctx context.Context, q querier, incidentID int64) error {
 }
 
 // handleGetEscalation returns a team's ladder.
-func handleGetEscalation(db *sql.DB) http.HandlerFunc {
+func handleGetEscalation(db *sql.DB, providers *notify.Registry) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		teamID, ok := teamParam(w, r)
 		if !ok {
@@ -347,7 +362,7 @@ func handleGetEscalation(db *sql.DB) http.HandlerFunc {
 			serverError(w, r, err)
 			return
 		}
-		view, err := escalationStatus(r.Context(), db, teamID, policy)
+		view, err := escalationStatus(r.Context(), db, providers, teamID, policy)
 		if err != nil {
 			serverError(w, r, err)
 			return
@@ -397,7 +412,7 @@ type escalationLevelView struct {
 type escalationView struct {
 	TeamID      int64                 `json:"team_id"`
 	RepeatCount int64                 `json:"repeat_count"`
-	FallbackTopic string                `json:"fallback_topic"`
+	Fallback    notify.Target         `json:"fallback"`
 	Levels      []escalationLevelView `json:"levels"`
 
 	// LastEscalatedAt is when an incident of this team last moved up the ladder,
@@ -410,10 +425,10 @@ type escalationView struct {
 // escalationStatus is a team's ladder together with what it would do right now
 // and what it has been doing. The resolution follows pageLevel's rules, so the
 // page cannot promise a page that the notifier would skip.
-func escalationStatus(ctx context.Context, db *sql.DB, teamID int64, policy *escalationPolicy) (escalationView, error) {
+func escalationStatus(ctx context.Context, db *sql.DB, providers *notify.Registry, teamID int64, policy *escalationPolicy) (escalationView, error) {
 	base := escalationResponse(policy, teamID)
 	out := escalationView{
-		TeamID: teamID, RepeatCount: base.RepeatCount, FallbackTopic: base.FallbackTopic,
+		TeamID: teamID, RepeatCount: base.RepeatCount, Fallback: base.Fallback,
 		Levels: []escalationLevelView{},
 	}
 	if !policy.configured() {
@@ -427,7 +442,7 @@ func escalationStatus(ctx context.Context, db *sql.DB, teamID int64, policy *esc
 
 	type account struct {
 		username string
-		topic    bool
+		channel  bool
 		disabled bool
 	}
 	accounts := map[int64]account{}
@@ -436,14 +451,17 @@ func escalationStatus(ctx context.Context, db *sql.DB, teamID int64, policy *esc
 			return a, nil
 		}
 		var a account
-		var topic *string
 		var disabledAt *int64
 		if err := db.QueryRowContext(ctx,
-			"SELECT username, ntfy_topic, disabled_at FROM users WHERE id = $1", id).
+			"SELECT username, disabled_at FROM users WHERE id = $1", id).
-			Scan(&a.username, &topic, &disabledAt); err != nil {
+			Scan(&a.username, &disabledAt); err != nil {
 			return a, err
 		}
-		a.topic = topic != nil && *topic != ""
+		reachable, err := hasReachableChannel(ctx, db, providers, id)
+		if err != nil {
+			return a, err
+		}
+		a.channel = reachable
 		a.disabled = disabledAt != nil
 		accounts[id] = a
 		return a, nil
@@ -497,8 +515,8 @@ func escalationStatus(ctx context.Context, db *sql.DB, teamID int64, policy *esc
 					view.Problem = "account not found"
 				case a.disabled:
 					view.Username, view.Problem = a.username, "account is disabled"
-				case !a.topic:
+				case !a.channel:
-					view.Username, view.Problem = a.username, "has no ntfy topic"
+					view.Username, view.Problem = a.username, "has no notification channel"
 				default:
 					view.Username, view.Reachable = a.username, true
 				}
@@ -553,7 +571,7 @@ type escalationTargetJSON struct {
 type escalationJSON struct {
 	TeamID      int64                 `json:"team_id"`
 	RepeatCount int64                 `json:"repeat_count"`
-	FallbackTopic string                `json:"fallback_topic"`
+	Fallback    notify.Target         `json:"fallback"`
 	Levels      []escalationLevelJSON `json:"levels"`
 }
 
@@ -563,7 +581,7 @@ func escalationResponse(p *escalationPolicy, teamID int64) escalationJSON {
 		return out
 	}
 	out.RepeatCount = p.repeatCount
-	out.FallbackTopic = p.fallbackTopic
+	out.Fallback = p.fallback
 	for _, l := range p.levels {
 		level := escalationLevelJSON{
 			Position:       l.position,
@@ -585,7 +603,7 @@ func escalationResponse(p *escalationPolicy, teamID int64) escalationJSON {
 // whole ladder makes the order the client's to decide and the server's to
 // store, and makes an edit atomic — there is no moment where level 2 exists
 // twice.
-func handleSetEscalation(db *sql.DB) http.HandlerFunc {
+func handleSetEscalation(db *sql.DB, providers *notify.Registry) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		teamID, ok := teamParam(w, r)
 		if !ok {
@@ -604,6 +622,11 @@ func handleSetEscalation(db *sql.DB) http.HandlerFunc {
 			respond(w, http.StatusBadRequest, errResp("repeat_count must be between 0 and 10"))
 			return
 		}
+		fallback, err := validateTarget(providers, req.Fallback)
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("fallback: "+err.Error()))
+			return
+		}
 		for i, l := range req.Levels {
 			for j, t := range l.Targets {
 				if t.Username == "" {
@@ -664,13 +687,14 @@ func handleSetEscalation(db *sql.DB) http.HandlerFunc {
 		defer tx.Rollback() //nolint:errcheck
 
 		if _, err := tx.ExecContext(r.Context(), `
-			INSERT INTO escalation_policies (team_id, repeat_count, fallback_topic, updated_at)
+			INSERT INTO escalation_policies (team_id, repeat_count, fallback_type, fallback_address, updated_at)
-			VALUES ($1, $2, $3, `+nowEpoch+`)
+			VALUES ($1, $2, $3, $4, `+nowEpoch+`)
 			ON CONFLICT (team_id) DO UPDATE SET
 				repeat_count = excluded.repeat_count,
-				fallback_topic = excluded.fallback_topic,
+				fallback_type = excluded.fallback_type,
+				fallback_address = excluded.fallback_address,
 				updated_at = excluded.updated_at`,
-			teamID, req.RepeatCount, req.FallbackTopic); err != nil {
+			teamID, req.RepeatCount, fallback.Type, fallback.Address); err != nil {
 			serverError(w, r, err)
 			return
 		}

internal/api/escalation_test.go

@@ -9,8 +9,8 @@ import (
 	"git.ryuvia.com/niklas/terdut-server/internal/api"
 )
 
-// teamUser creates a user in the default team with an ntfy topic, so they can
+// teamUser creates a user in the default team with an ntfy channel, so they can
-// actually be paged.
+// actually be paged. An empty topic leaves them with none.
 func teamUser(t *testing.T, s *ts, username, topic string) int64 {
 	t.Helper()
 	var user struct {
@@ -21,7 +21,9 @@ func teamUser(t *testing.T, s *ts, username, topic string) int64 {
 	resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/members",
 		map[string]any{"user_id": user.ID, "role": "member"})
 	resp.Body.Close()
-	setTopic(t, s, int(user.ID), topic)
+	if topic != "" {
+		addNtfyChannel(t, s, int(user.ID), topic)
+	}
 	return user.ID
 }
 
@@ -30,13 +32,21 @@ func teamUser(t *testing.T, s *ts, username, topic string) int64 {
 // man's switch tests use on received_at: the sweeper reads a stored timestamp,
 // so moving the timestamp is moving the clock.
 
+// fallbackTarget is the JSON for a team's fallback: an ntfy topic, or none.
+func fallbackTarget(topic string) map[string]string {
+	if topic == "" {
+		return map[string]string{"type": "", "address": ""}
+	}
+	return map[string]string{"type": "ntfy", "address": topic}
+}
+
 // ladder configures the default team with two levels: the rota first, then a
 // named person, then the fallback topic.
 func ladder(t *testing.T, s *ts, secondUserID int64, repeat int64, fallback string) {
 	t.Helper()
 	resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/escalation", map[string]any{
 		"repeat_count": repeat,
-		"fallback_topic": fallback,
+		"fallback":     fallbackTarget(fallback),
 		"levels": []map[string]any{
 			{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "oncall"}}},
 			{"timeout_seconds": 300, "targets": []map[string]any{{"kind": "user", "user_id": secondUserID}}},
@@ -442,14 +452,14 @@ func TestEscalation_StatusFlagsUnreachableLevels(t *testing.T) {
 
 	// Nobody on call today, and the named person loses their topic.
 	s.exec(t, "DELETE FROM schedule_entries")
-	s.exec(t, "UPDATE users SET ntfy_topic = NULL WHERE id = $1", silent)
+	s.exec(t, "DELETE FROM notification_channels WHERE user_id = $1", silent)
 
 	v := readLadder(t, s)
 	if v.Levels[0].Status != "unreachable" || v.Levels[0].Targets[0].Problem != "nobody is on call today" {
 		t.Errorf("an empty rota should make level 1 unreachable, got %+v", v.Levels[0])
 	}
-	if v.Levels[1].Status != "unreachable" || v.Levels[1].Targets[0].Problem != "has no ntfy topic" {
+	if v.Levels[1].Status != "unreachable" || v.Levels[1].Targets[0].Problem != "has no notification channel" {
-		t.Errorf("a person with no topic should make level 2 unreachable, got %+v", v.Levels[1])
+		t.Errorf("a person with no channel should make level 2 unreachable, got %+v", v.Levels[1])
 	}
 
 	s.exec(t, "UPDATE users SET disabled_at = 1 WHERE id = $1", silent)

internal/api/helpers.go

@@ -63,6 +63,13 @@ func isUniqueViolation(err error) bool {
 	return errors.As(err, &pgErr) && pgErr.Code == pgerrcode.UniqueViolation
 }
 
+// isForeignKeyViolation reports whether err is a reference to a row that does not
+// exist, which callers turn into 404 rather than 500.
+func isForeignKeyViolation(err error) bool {
+	var pgErr *pgconn.PgError
+	return errors.As(err, &pgErr) && pgErr.Code == pgerrcode.ForeignKeyViolation
+}
+
 func respond(w http.ResponseWriter, status int, v any) {
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(status)

internal/api/members_test.go

@@ -40,23 +40,23 @@ func TestMembers_StatusReflectsRotaAndPageability(t *testing.T) {
 	s, _ := notifyTS(t, testNotify()) // admin is on call today, with a topic
 	teamUser(t, s, "reachable", "terdut-reachable")
 	silent := teamUser(t, s, "silent", "terdut-silent")
-	s.exec(t, "UPDATE users SET ntfy_topic = NULL WHERE id = $1", silent)
+	s.exec(t, "DELETE FROM notification_channels WHERE user_id = $1", silent)
 
 	got := readMembers(t, s)
 	if m := got["admin"]; m.Status != "oncall" || !m.OnCall || !m.Pageable {
 		t.Errorf("the person on call should read on call, got %+v", m)
 	}
 	if m := got["reachable"]; m.Status != "reachable" || m.OnCall {
-		t.Errorf("a member with a topic who is off the rota is reachable, got %+v", m)
+		t.Errorf("a member with a channel who is off the rota is reachable, got %+v", m)
 	}
-	if m := got["silent"]; m.Status != "unpageable" || m.Problem != "has no ntfy topic" {
+	if m := got["silent"]; m.Status != "unpageable" || m.Problem != "has no notification channel" {
-		t.Errorf("no topic means they cannot be paged, got %+v", m)
+		t.Errorf("no channel means they cannot be paged, got %+v", m)
 	}
 
 	// Being on call does not rescue an account that cannot be woken.
-	s.exec(t, "UPDATE users SET ntfy_topic = NULL WHERE username = 'admin'")
+	s.exec(t, "DELETE FROM notification_channels WHERE user_id = (SELECT id FROM users WHERE username = 'admin')")
 	if m := readMembers(t, s)["admin"]; m.Status != "unpageable" || !m.OnCall {
-		t.Errorf("an on-call person with no topic is the red case, got %+v", m)
+		t.Errorf("an on-call person with no channel is the red case, got %+v", m)
 	}
 
 	s.exec(t, "UPDATE users SET disabled_at = 1 WHERE id = $1", silent)

internal/api/middleware.go

@@ -123,8 +123,8 @@ func AdminOnly(next http.Handler) http.Handler {
 }
 
 // requireSelfOrAdmin guards the endpoints that are self-service for your own
-// account and administration for anybody else's: your password, your ntfy
+// account and administration for anybody else's: your password, your notification
-// topic, your API keys. Reports whether the request may proceed, and answers it
+// channels, your API keys. Reports whether the request may proceed, and answers it
 // if not.
 //
 // An API key is not an escalation: it carries exactly the rights of the user it

internal/api/notifier.go

@@ -1,18 +1,16 @@
 package api
 
 import (
-	"bytes"
 	"context"
 	"database/sql"
-	"encoding/json"
 	"fmt"
 	"log"
-	"net/http"
 	"regexp"
 	"strings"
 	"time"
 
 	"git.ryuvia.com/niklas/terdut-server/internal/models"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 )
 
 const (
@@ -20,8 +18,8 @@ const (
 	// 15 minute tick is far too coarse for something that has to wake a person.
 	notifyInterval = 30 * time.Second
 
-	// notifyRetryBase and notifyRetryMax bound the delivery backoff. ntfy being
+	// notifyRetryBase and notifyRetryMax bound the delivery backoff. A provider
-	// briefly unreachable should not lose the page.
+	// being briefly unreachable should not lose the page.
 	notifyRetryBase = 30 * time.Second
 	notifyRetryMax  = 15 * time.Minute
 
@@ -48,43 +46,41 @@ const notifierLockKey int64 = 7265_0002
 
 // Notification kinds, recording why a push was sent.
 const (
-	notifyTriggered = "triggered"
+	notifyTriggered = notify.KindTriggered
-	notifyReminder  = "reminder"
+	notifyReminder  = notify.KindReminder
-	notifyResolved  = "resolved"
+	notifyResolved  = notify.KindResolved
 
 	// notifyEscalated is a page that went out because nobody answered the last
 	// one. Told apart from a reminder because it goes to somebody else.
-	notifyEscalated = "escalated"
+	notifyEscalated = notify.KindEscalated
 )
 
 // Timeline event types the notifier writes, so an incident's history says who
 // was paged and whether the page landed. Written from the delivery result
 // rather than at enqueue: a queued notification is an intention, and claiming
-// somebody was told before ntfy accepted it would be a lie the timeline keeps.
+// somebody was told before the provider accepted it would be a lie the timeline
+// keeps.
 //
-// The topic is deliberately absent from both. It is a shared secret with the
+// The channel's address is deliberately absent from both; only its type is
-// ntfy server — anyone holding it can publish to it — and the timeline is
+// named. An address can be a shared secret (an ntfy topic lets anyone holding it
-// readable by every API key.
+// publish to it) and the timeline is readable by every API key.
 const (
 	eventNotified     = "notified"
 	eventNotifyFailed = "notify_failed"
 )
 
-// NotifyConfig is everything the notifier needs to reach ntfy and to build URLs
+// NotifyConfig is everything the notifier needs to reach people and to build
-// a phone can follow back to this server.
+// URLs they can follow back to this server.
 type NotifyConfig struct {
-	// BaseURL is the ntfy server. Empty disables notifications entirely: no
+	// Providers are the ways this server can deliver a notification. Empty
-	// goroutine, and nothing is ever enqueued.
+	// disables notifications entirely: no goroutine, and nothing is ever enqueued.
-	BaseURL string
+	Providers *notify.Registry
 
-	// Token is an optional bearer token for an access-controlled ntfy.
+	// Fallback receives incidents that open with nobody on call who can be
-	Token string
+	// reached. Those notifications carry no Acknowledge button — there is no user
-
+	// to attribute the acknowledgement to, and putting one on a shared target
-	// FallbackTopic receives incidents that open with nobody on call. Those
+	// would let any recipient acknowledge as somebody else.
-	// notifications carry no Acknowledge button — there is no user to attribute
+	Fallback notify.Target
-	// the acknowledgement to, and putting one on a shared topic would let any
-	// subscriber acknowledge as somebody else.
-	FallbackTopic string
 
 	// PublicURL is the base URL a phone uses to reach this server, for the
 	// notification's click target and its Acknowledge action. Without it a
@@ -97,11 +93,7 @@ type NotifyConfig struct {
 }
 
 // enabled reports whether notifications are configured at all.
-func (c NotifyConfig) enabled() bool { return c.BaseURL != "" }
+func (c NotifyConfig) enabled() bool { return !c.Providers.Empty() }
-
-// notifyClient is shared: a page is small and infrequent, and the timeout is
-// what keeps a hung ntfy from stalling the delivery pass.
-var notifyClient = &http.Client{Timeout: 10 * time.Second}
 
 // StartNotifier delivers queued notifications until ctx is cancelled, starting
 // with an immediate pass so a restart flushes whatever the last one left behind.
@@ -111,10 +103,14 @@ var notifyClient = &http.Client{Timeout: 10 * time.Second}
 // delivers; the rest skip that tick rather than racing the same pass.
 func StartNotifier(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 	if !cfg.enabled() {
-		log.Print("notifier: disabled (no ntfy URL configured)")
+		log.Print("notifier: disabled (no notification provider configured)")
 		return
 	}
-	log.Printf("notifier: publishing to %s", cfg.BaseURL)
+	var types []string
+	for _, p := range cfg.Providers.List() {
+		types = append(types, p.Type())
+	}
+	log.Printf("notifier: delivering through %s", strings.Join(types, ", "))
 
 	ticker := time.NewTicker(notifyInterval)
 	defer ticker.Stop()
@@ -156,8 +152,13 @@ func NotifySweep(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 // why there is no separate reminder cap.
 //
 // The previous notification must have actually been sent before another is
-// queued, so an ntfy outage produces a retry backlog rather than a reminder
+// queued, so a provider outage produces a retry backlog rather than a reminder
-// backlog that all lands at once when it comes back.
+// backlog that all lands at once when it comes back. "Previous notification"
+// is the incident's latest batch: everything one fan-out queued shares a
+// created_at, and the batch counts as sent once any of its rows has landed.
+//
+// A channel the user has since disabled or removed is not reminded: that is the
+// off switch for a channel, and a reminder that ignored it would defeat it.
 func enqueueReminders(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 	// cfg.RepeatEvery is what the server started with; the settings table is
 	// what it runs on. Read per tick, so an administrator lengthening the
@@ -168,28 +169,30 @@ func enqueueReminders(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 	}
 	now := time.Now()
 
-	type due struct {
-		incidentID int64
-		userID     *int64
-		topic      string
-	}
-
 	rows, err := db.QueryContext(ctx, `
-		SELECT n.incident_id, n.user_id, n.topic
+		SELECT n.incident_id, n.user_id, n.channel_id, n.channel_type, n.address
 		FROM   notifications n
 		JOIN   incidents i ON i.id = n.incident_id
-		WHERE  n.id = (SELECT MAX(id) FROM notifications WHERE incident_id = n.incident_id)
+		LEFT   JOIN notification_channels c ON c.id = n.channel_id
-		  AND  n.sent_at IS NOT NULL
+		WHERE  n.created_at = (SELECT MAX(created_at) FROM notifications WHERE incident_id = n.incident_id)
+		  AND  EXISTS (SELECT 1 FROM notifications s
+		                WHERE s.incident_id = n.incident_id
+		                  AND s.created_at = n.created_at
+		                  AND s.sent_at IS NOT NULL)
 		  AND  n.created_at <= $1
 		  AND  i.resolved_at IS NULL
 		  AND  i.archived_at IS NULL
 		  AND  i.status = 'triggered'
 		  AND  (i.snoozed_until IS NULL OR i.snoozed_until <= $2)
+		  -- A shared fallback target has no user and no channel row; everything
+		  -- else must still point at an enabled channel.
+		  AND  (n.user_id IS NULL OR c.enabled)
 		  -- A team with an escalation ladder gets escalation instead. Both
 		  -- would mean two pages for one silence, which is how people learn to
 		  -- mute a tool.
 		  AND  NOT EXISTS (
-		           SELECT 1 FROM escalation_levels el WHERE el.team_id = i.team_id)`,
+		           SELECT 1 FROM escalation_levels el WHERE el.team_id = i.team_id)
+		ORDER  BY n.id`,
 		now.Add(-repeat).Unix(), now.Unix())
 	if err != nil {
 		log.Printf("notifier: find reminders: %v", err)
@@ -199,15 +202,20 @@ func enqueueReminders(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 	// Collected before inserting, rather than written while walking the cursor:
 	// the inserts below are what this query selects on, and a cursor reading its
 	// own writes is a hazard whatever the pool size.
-	var pending []due
+	byIncident := map[int64][]recipient{}
+	var order []int64
 	for rows.Next() {
-		var d due
+		var incidentID int64
-		if err := rows.Scan(&d.incidentID, &d.userID, &d.topic); err != nil {
+		var r recipient
+		if err := rows.Scan(&incidentID, &r.userID, &r.channelID, &r.typ, &r.address); err != nil {
 			rows.Close()
 			log.Printf("notifier: scan reminder: %v", err)
 			return
 		}
-		pending = append(pending, d)
+		if _, seen := byIncident[incidentID]; !seen {
+			order = append(order, incidentID)
+		}
+		byIncident[incidentID] = append(byIncident[incidentID], r)
 	}
 	if err := rows.Err(); err != nil {
 		rows.Close()
@@ -216,13 +224,15 @@ func enqueueReminders(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 	}
 	rows.Close()
 
-	for _, d := range pending {
+	// One batch per incident, so the reminder is itself a batch the next one
-		if err := enqueueNotification(ctx, db, d.incidentID, d.userID, d.topic, notifyReminder); err != nil {
+	// can be measured from.
-			log.Printf("notifier: queue reminder for incident %d: %v", d.incidentID, err)
+	for _, id := range order {
+		if err := enqueueNotifications(ctx, db, id, byIncident[id], notifyReminder); err != nil {
+			log.Printf("notifier: queue reminder for incident %d: %v", id, err)
 		}
 	}
-	if len(pending) > 0 {
+	if len(order) > 0 {
-		log.Printf("notifier: queued %d reminder(s)", len(pending))
+		log.Printf("notifier: queued reminders for %d incident(s)", len(order))
 	}
 }
 
@@ -231,7 +241,8 @@ type outboxRow struct {
 	id         int64
 	incidentID int64
 	userID     *int64
-	topic      string
+	channel    string // channel type; the provider that delivers it
+	address    string
 	kind       string
 	attempts   int
 }
@@ -258,7 +269,8 @@ func deliverPending(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 		}
 		// Logged, not returned: the page has already gone out, and treating a
 		// failed timeline write as a failed delivery would send it again.
-		if err := logEvent(ctx, db, n.incidentID, eventNotified, n.userID, nil, nil, &n.kind); err != nil {
+		detail := deliveryDetail(n)
+		if err := logEvent(ctx, db, n.incidentID, eventNotified, n.userID, nil, nil, &detail); err != nil {
 			log.Printf("notifier: log delivery of %d: %v", n.id, err)
 		}
 		sent++
@@ -272,7 +284,7 @@ func deliverPending(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 // caller writes, for the same single-connection reason as staleAlertIDs.
 func pendingNotifications(ctx context.Context, db *sql.DB) ([]outboxRow, error) {
 	rows, err := db.QueryContext(ctx, `
-		SELECT id, incident_id, user_id, topic, kind, attempts
+		SELECT id, incident_id, user_id, channel_type, address, kind, attempts
 		FROM   notifications
 		WHERE  sent_at IS NULL
 		  AND  send_after <= $1
@@ -287,7 +299,7 @@ func pendingNotifications(ctx context.Context, db *sql.DB) ([]outboxRow, error)
 	var batch []outboxRow
 	for rows.Next() {
 		var n outboxRow
-		if err := rows.Scan(&n.id, &n.incidentID, &n.userID, &n.topic, &n.kind, &n.attempts); err != nil {
+		if err := rows.Scan(&n.id, &n.incidentID, &n.userID, &n.channel, &n.address, &n.kind, &n.attempts); err != nil {
 			return nil, err
 		}
 		batch = append(batch, n)
@@ -312,12 +324,18 @@ func markFailed(ctx context.Context, db *sql.DB, n outboxRow, cause error) {
 	if n.attempts+1 < notifyMaxAttempts {
 		return
 	}
-	detail := fmt.Sprintf("%s: %s", n.kind, cause)
+	detail := fmt.Sprintf("%s: %s", deliveryDetail(n), cause)
 	if err := logEvent(ctx, db, n.incidentID, eventNotifyFailed, n.userID, nil, nil, &detail); err != nil {
 		log.Printf("notifier: log failure of %d: %v", n.id, err)
 	}
 }
 
+// deliveryDetail is what a timeline event says about a delivery: the kind and
+// the channel type, "triggered via ntfy". Never the address.
+func deliveryDetail(n outboxRow) string {
+	return n.kind + " via " + n.channel
+}
+
 // retryDelay doubles the wait per attempt, up to notifyRetryMax.
 func retryDelay(attempts int) time.Duration {
 	d := notifyRetryBase << attempts
@@ -328,10 +346,17 @@ func retryDelay(attempts int) time.Duration {
 }
 
 // deliver renders one notification against the incident's *current* state and
-// publishes it. Rendering happens here rather than at enqueue time so a message
+// hands it to the provider its channel names. Rendering happens here rather than
-// that waited in the queue while its incident escalated goes out at the
+// at enqueue time so a message that waited in the queue while its incident
-// severity the incident has now.
+// escalated goes out at the severity the incident has now.
 func deliver(ctx context.Context, db *sql.DB, cfg NotifyConfig, n outboxRow) error {
+	// Looked up first: a row for a provider this server no longer has can never
+	// succeed, and there is no point rendering it.
+	provider, ok := cfg.Providers.Get(n.channel)
+	if !ok {
+		return fmt.Errorf("no %q provider is configured", n.channel)
+	}
+
 	inc, err := fetchIncident(ctx, db, n.incidentID)
 	if err != nil {
 		return fmt.Errorf("load incident: %w", err)
@@ -354,59 +379,43 @@ func deliver(ctx context.Context, db *sql.DB, cfg NotifyConfig, n outboxRow) err
 	if n.kind == notifyTriggered {
 		if sim, err := similarIncidents(ctx, db, n.incidentID, 1); err == nil && len(sim) > 0 && len(sim[0].ResolutionNotes) > 0 {
 			notes := sim[0].ResolutionNotes
-			msg.Message += "\nLast time: " + shorten(derefString(notes[len(notes)-1].Detail), 160)
+			msg.Body += "\nLast time: " + shorten(derefString(notes[len(notes)-1].Detail), 160)
 		}
 	}
 
-	// An Acknowledge button needs both a user to attribute the acknowledgement
+	// An Acknowledge control needs both a user to attribute the acknowledgement
-	// to and a URL the phone can reach. Minted per delivery, so every push
+	// to and a URL the recipient can reach.
-	// carries its own short-lived token rather than reusing one.
 	if n.kind != notifyResolved && n.userID != nil && cfg.PublicURL != "" {
+		base := strings.TrimSuffix(cfg.PublicURL, "/")
+		if provider.Capabilities().ActionButtons {
+			// Minted per delivery, so every push carries its own short-lived
+			// token rather than reusing one.
 			raw, err := issueAckToken(ctx, db, n.incidentID, *n.userID)
 			if err != nil {
 				return fmt.Errorf("issue ack token: %w", err)
 			}
-		msg.Actions = append(msg.Actions, ntfyAction{
+			msg.AckURL = base + "/api/notify/ack/" + raw
-			Action: "http",
+		} else {
-			Label:  "Acknowledge",
+			// No button to put it on, so the way to acknowledge travels as text.
-			URL:    strings.TrimSuffix(cfg.PublicURL, "/") + "/api/notify/ack/" + raw,
+			// The token URL is POST-only and a link to it would do nothing, and
-			Method: "POST",
+			// minting one nobody can use is waste: this is the incident page,
-			Clear:  true,
+			// which has the Acknowledge control.
-		})
+			msg.Body += fmt.Sprintf("\nAcknowledge: %s/incidents/%d", base, n.incidentID)
+		}
 	}
 
-	return publish(ctx, cfg, msg)
+	return provider.Send(ctx, msg)
 }
 
-// ntfyMessage is ntfy's JSON publish format. Using it rather than the X-Actions
+// renderNotification builds the provider-neutral message for one queued
-// header avoids that header's comma and quote escaping rules, which are easy to
+// notification.
-// break with a title that happens to contain a comma.
+func renderNotification(inc models.Incident, n outboxRow, firing int, cfg NotifyConfig) notify.Message {
-type ntfyMessage struct {
+	msg := notify.Message{Address: n.address, Kind: n.kind}
-	Topic    string       `json:"topic"`
-	Title    string       `json:"title,omitempty"`
-	Message  string       `json:"message"`
-	Priority int          `json:"priority,omitempty"`
-	Tags     []string     `json:"tags,omitempty"`
-	Click    string       `json:"click,omitempty"`
-	Actions  []ntfyAction `json:"actions,omitempty"`
-}
-
-type ntfyAction struct {
-	Action string `json:"action"`
-	Label  string `json:"label"`
-	URL    string `json:"url"`
-	Method string `json:"method,omitempty"`
-	Clear  bool   `json:"clear,omitempty"`
-}
-
-// renderNotification builds the message body for one queued notification.
-func renderNotification(inc models.Incident, n outboxRow, firing int, cfg NotifyConfig) ntfyMessage {
-	msg := ntfyMessage{Topic: n.topic}
 
 	if cfg.PublicURL != "" {
 		// The web UI's page for the incident, so tapping the notification
 		// opens something a browser can use.
-		msg.Click = fmt.Sprintf("%s/incidents/%d",
+		msg.ClickURL = fmt.Sprintf("%s/incidents/%d",
 			strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID)
 	}
 
@@ -415,10 +424,8 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
 	switch n.kind {
 	case notifyResolved:
 		msg.Title = "Resolved: " + title
-		msg.Message = "All alerts stopped firing after " +
+		msg.Body = "All alerts stopped firing after " +
 			humanDuration(time.Since(inc.TriggeredAt))
-		msg.Priority = ntfyPriorityLow
-		msg.Tags = []string{"white_check_mark"}
 		return msg
 
 	case notifyReminder:
@@ -440,9 +447,8 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
 		parts = append(parts, "open "+humanDuration(time.Since(inc.TriggeredAt)))
 	}
 
-	msg.Message = strings.Join(parts, " · ")
+	msg.Body = strings.Join(parts, " · ")
-	msg.Priority = ntfyPriority(severity)
+	msg.Level = severityRank(severity)
-	msg.Tags = []string{severityTag(severity)}
 	return msg
 }
 
@@ -488,137 +494,139 @@ func titleWithoutLabel(title, key, value string) string {
 	return m[1] + " (" + strings.Join(rest, ", ") + ")"
 }
 
-// ntfy's priority scale. Max is the one that overrides the phone's quiet
+// recipient is one place a notification goes: a user's channel, or the shared
-// settings, which is the whole point of paging on critical.
+// fallback target. channelID and userID are nil for the fallback, which belongs
-const (
+// to nobody. A batch may hold several users (an escalation level with more than
-	ntfyPriorityLow     = 2
+// one target), which is why the user is here and not on the batch.
-	ntfyPriorityDefault = 3
+type recipient struct {
-	ntfyPriorityHigh    = 4
+	userID    *int64
-	ntfyPriorityMax     = 5
+	channelID *int64
-)
+	typ       string
-
+	address   string
-// ntfyPriority maps an incident's severity onto ntfy's scale, following the
-// same ordering severityRank uses. An unrecognised severity gets the default
-// rather than being silenced.
-func ntfyPriority(severity string) int {
-	switch severityRank(severity) {
-	case 4:
-		return ntfyPriorityMax
-	case 3:
-		return ntfyPriorityHigh
-	case 2:
-		return ntfyPriorityDefault
-	case 1:
-		return ntfyPriorityLow
-	default:
-		return ntfyPriorityDefault
-	}
 }
 
-func severityTag(severity string) string {
+// enqueueNotifications queues one row per recipient for a single notification.
-	switch severityRank(severity) {
+//
-	case 4:
+// The rows share one created_at. That is what makes them a batch: the reminder
-		return "rotating_light"
+// and all-clear paths find "who was told last" as the rows carrying an
-	case 3:
+// incident's newest created_at, so a fan-out to three channels is reminded and
-		return "red_circle"
+// resolved as one unit rather than as whichever row happened to land last.
-	case 2:
+func enqueueNotifications(ctx context.Context, q querier, incidentID int64, recipients []recipient, kind string) error {
-		return "warning"
+	now := time.Now().Unix()
-	case 1:
+	for _, r := range recipients {
-		return "information_source"
+		if _, err := q.ExecContext(ctx, `
-	default:
+			INSERT INTO notifications (incident_id, user_id, channel_id, channel_type, address, kind, created_at, send_after)
-		return "bell"
+			VALUES ($1, $2, $3, $4, $5, $6, $7, $7)`,
-	}
+			incidentID, r.userID, r.channelID, r.typ, r.address, kind, now); err != nil {
-}
-
-// publish POSTs one message to ntfy.
-func publish(ctx context.Context, cfg NotifyConfig, msg ntfyMessage) error {
-	body, err := json.Marshal(msg)
-	if err != nil {
 			return err
 		}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
-		strings.TrimSuffix(cfg.BaseURL, "/"), bytes.NewReader(body))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/json")
-	if cfg.Token != "" {
-		req.Header.Set("Authorization", "Bearer "+cfg.Token)
-	}
-
-	resp, err := notifyClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("ntfy returned %s", resp.Status)
 	}
 	return nil
 }
 
-// enqueueNotification adds one row to the outbox, due immediately.
+// userRecipients is the channels a notification to this user should go to: the
-func enqueueNotification(ctx context.Context, q querier, incidentID int64, userID *int64, topic, kind string) error {
+// ones they have enabled, of a type this server can deliver. A channel whose
-	now := time.Now().Unix()
+// provider is not configured is skipped rather than queued, since it could only
-	_, err := q.ExecContext(ctx, `
+// fail eight times and then sit in the table.
-		INSERT INTO notifications (incident_id, user_id, topic, kind, created_at, send_after)
+func userRecipients(ctx context.Context, q querier, cfg NotifyConfig, userID int64) ([]recipient, error) {
-		VALUES ($1, $2, $3, $4, $5, $6)`, incidentID, userID, topic, kind, now, now)
+	uid := userID
-	return err
+	rows, err := q.QueryContext(ctx, `
+		SELECT id, type, address FROM notification_channels
+		 WHERE user_id = $1 AND enabled
+		 ORDER BY id`, userID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []recipient
+	for rows.Next() {
+		var id int64
+		var r recipient
+		if err := rows.Scan(&id, &r.typ, &r.address); err != nil {
+			return nil, err
+		}
+		if !cfg.Providers.Has(r.typ) {
+			continue
+		}
+		r.channelID, r.userID = &id, &uid
+		out = append(out, r)
+	}
+	return out, rows.Err()
 }
 
 // notifyTarget decides where a newly opened incident's notification goes.
 //
-// The on-call user's own topic when they have one, otherwise the fallback
+// The on-call user's own channels when they have any, otherwise the fallback
-// topic with no user attached. Deliberately not "the fallback topic, attributed
+// target with no user attached. Deliberately not "the fallback, attributed to
-// to the on-call user": the fallback is shared, and an Acknowledge button on a
+// the on-call user": the fallback is shared, and an Acknowledge button on a
-// shared topic would let any subscriber acknowledge as somebody else.
+// shared target would let any recipient acknowledge as somebody else.
-func notifyTarget(ctx context.Context, q querier, cfg NotifyConfig, onCall *int64) (topic string, userID *int64) {
+func notifyTarget(ctx context.Context, q querier, cfg NotifyConfig, onCall *int64) []recipient {
 	if onCall != nil {
-		var t *string
+		if rs, err := userRecipients(ctx, q, cfg, *onCall); err == nil && len(rs) > 0 {
-		err := q.QueryRowContext(ctx,
+			return rs
-			"SELECT ntfy_topic FROM users WHERE id = $1", *onCall).Scan(&t)
-		if err == nil && t != nil && *t != "" {
-			return *t, onCall
 		}
 	}
-	return cfg.FallbackTopic, nil
+	return fallbackRecipients(cfg, cfg.Fallback)
+}
+
+// fallbackRecipients turns a fallback target into the zero or one recipients it
+// stands for. A target whose provider this server does not have is nobody.
+func fallbackRecipients(cfg NotifyConfig, t notify.Target) []recipient {
+	if t.Address == "" || !cfg.Providers.Has(t.Type) {
+		return nil
+	}
+	return []recipient{{typ: t.Type, address: t.Address}}
 }
 
 // enqueueOpened queues the notification for a freshly opened incident. It is the
-// only enqueue point that has to resolve a topic from scratch; every later
+// only enqueue point that has to resolve recipients from scratch; every later
 // notification for the incident reuses what this one chose.
 func enqueueOpened(ctx context.Context, q querier, cfg NotifyConfig, incidentID int64, onCall *int64) error {
 	if !cfg.enabled() {
 		return nil
 	}
-	topic, userID := notifyTarget(ctx, q, cfg, onCall)
+	recipients := notifyTarget(ctx, q, cfg, onCall)
-	if topic == "" {
+	if len(recipients) == 0 {
-		// Nobody on call has a topic and there is no fallback: there is nowhere
+		// Nobody on call has a channel and there is no fallback: there is nowhere
 		// to send this, and queueing it would only accumulate undeliverable rows.
 		return nil
 	}
-	return enqueueNotification(ctx, q, incidentID, userID, topic, notifyTriggered)
+	return enqueueNotifications(ctx, q, incidentID, recipients, notifyTriggered)
 }
 
-// enqueueResolved queues the all-clear, reusing the topic the incident's last
+// enqueueResolved queues the all-clear, reusing the recipients of the incident's
-// notification went to. That needs no configuration to reach this function, and
+// last batch. That needs no configuration to reach this function, and it gives
-// it gives the right rule for free: you only hear that something resolved if you
+// the right rule for free: you only hear that something resolved if you were
-// were told it started.
+// told it started. A channel since disabled or removed is not told.
 func enqueueResolved(ctx context.Context, q querier, incidentID int64) error {
-	var topic string
+	rows, err := q.QueryContext(ctx, `
-	var userID *int64
+		SELECT n.user_id, n.channel_id, n.channel_type, n.address
-	err := q.QueryRowContext(ctx, `
+		  FROM notifications n
-		SELECT topic, user_id FROM notifications
+		  LEFT JOIN notification_channels c ON c.id = n.channel_id
-		 WHERE incident_id = $1 ORDER BY id DESC LIMIT 1`, incidentID).Scan(&topic, &userID)
+		 WHERE n.incident_id = $1
-	if err == sql.ErrNoRows {
+		   AND n.created_at = (SELECT MAX(created_at) FROM notifications WHERE incident_id = $1)
-		return nil
+		   AND (n.user_id IS NULL OR c.enabled)
-	}
+		 ORDER BY n.id`, incidentID)
 	if err != nil {
 		return err
 	}
-	return enqueueNotification(ctx, q, incidentID, userID, topic, notifyResolved)
+
+	var recipients []recipient
+	for rows.Next() {
+		var r recipient
+		if err := rows.Scan(&r.userID, &r.channelID, &r.typ, &r.address); err != nil {
+			rows.Close()
+			return err
+		}
+		recipients = append(recipients, r)
+	}
+	err = rows.Err()
+	rows.Close()
+	if err != nil || len(recipients) == 0 {
+		return err
+	}
+	return enqueueNotifications(ctx, q, incidentID, recipients, notifyResolved)
 }
 
 // humanDuration renders an age the way a person reads it at 3am: coarse, and

internal/api/notify_ack.go

@@ -70,7 +70,7 @@ func handleNotifyAck(db *sql.DB) http.HandlerFunc {
 			// already acknowledged (e.g. from the web UI, or an earlier tap of
 			// the same button) — either way nothing the responder did wrong, so
 			// report the actual state rather than assuming "resolved", and let
-			// ntfy show a success toast rather than a failure.
+			// the provider show a success toast rather than a failure.
 			inc, err := fetchIncident(r.Context(), db, incidentID)
 			if err != nil {
 				serverError(w, r, err)

internal/api/notify_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"git.ryuvia.com/niklas/terdut-server/internal/api"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 )
 
 // ---------------------------------------------------------------------------
@@ -91,6 +92,15 @@ func (f *fakeNtfy) forget() {
 	f.got = nil
 }
 
+// providers registers the fake as the install's one ntfy provider.
+func (f *fakeNtfy) providers() *notify.Registry {
+	return notify.NewRegistry(notify.NewNtfy(f.URL, ""))
+}
+
+func ntfyTarget(topic string) notify.Target {
+	return notify.Target{Type: notify.TypeNtfy, Address: topic}
+}
+
 func (f *fakeNtfy) failWith(status int) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -106,11 +116,11 @@ func (f *fakeNtfy) failWith(status int) {
 func notifyTS(t *testing.T, cfg api.NotifyConfig) (*ts, *fakeNtfy) {
 	t.Helper()
 	f := newFakeNtfy(t)
-	cfg.BaseURL = f.URL
+	cfg.Providers = f.providers()
 	s := newTS(t, cfg)
 
 	putOnCall(t, s, 1)
-	setTopic(t, s, 1, "terdut-admin")
+	addNtfyChannel(t, s, 1, "terdut-admin")
 	return s, f
 }
 
@@ -125,13 +135,14 @@ func putOnCall(t *testing.T, s *ts, userID int) {
 	}
 }
 
-func setTopic(t *testing.T, s *ts, userID int, topic string) {
+// addNtfyChannel gives a user an ntfy channel, the way the Account page does.
+func addNtfyChannel(t *testing.T, s *ts, userID int, topic string) {
 	t.Helper()
-	resp := s.req(t, http.MethodPut,
+	resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/users/%d/channels", userID),
-		fmt.Sprintf("/api/users/%d/notify", userID), map[string]any{"ntfy_topic": topic})
+		map[string]any{"type": "ntfy", "address": topic})
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode != http.StatusCreated {
-		t.Fatalf("set notify topic returned %d", resp.StatusCode)
+		t.Fatalf("add notification channel returned %d", resp.StatusCode)
 	}
 }
 
@@ -339,8 +350,8 @@ func TestNotify_DeliveryIsRecordedOnTheTimeline(t *testing.T) {
 	if e["type"] != "notified" {
 		t.Errorf("expected a notified event, got %v", e["type"])
 	}
-	if e["detail"] != "triggered" {
+	if e["detail"] != "triggered via ntfy" {
-		t.Errorf("expected the kind in detail, got %v", e["detail"])
+		t.Errorf("expected the kind and the channel type in detail, got %v", e["detail"])
 	}
 	if e["username"] != "admin" {
 		t.Errorf("expected the paged user attached, got %v", e["username"])
@@ -370,7 +381,7 @@ func TestNotify_TimelineRecordsOneEventPerDelivery(t *testing.T) {
 	if len(events) != 2 {
 		t.Fatalf("expected one event per delivery, got %v", events)
 	}
-	if events[0]["detail"] != "triggered" || events[1]["detail"] != "reminder" {
+	if events[0]["detail"] != "triggered via ntfy" || events[1]["detail"] != "reminder via ntfy" {
 		t.Errorf("expected triggered then reminder, got %v and %v",
 			events[0]["detail"], events[1]["detail"])
 	}
@@ -392,7 +403,7 @@ func TestNotify_AllClearIsRecordedOnTheTimeline(t *testing.T) {
 	if len(events) != 2 {
 		t.Fatalf("expected the all-clear recorded, got %v", events)
 	}
-	if events[1]["detail"] != "resolved" {
+	if events[1]["detail"] != "resolved via ntfy" {
 		t.Errorf("expected a resolved event, got %v", events[1]["detail"])
 	}
 }
@@ -402,8 +413,8 @@ func TestNotify_AllClearIsRecordedOnTheTimeline(t *testing.T) {
 func TestNotify_FallbackDeliveryHasNoUser(t *testing.T) {
 	f := newFakeNtfy(t)
 	s := newTS(t, api.NotifyConfig{
-		BaseURL:       f.URL,
+		Providers: f.providers(),
-		FallbackTopic: "terdut-oncall",
+		Fallback:  ntfyTarget("terdut-oncall"),
 		PublicURL: "https://terdut.example.com",
 	})
 
@@ -441,7 +452,7 @@ func TestNotify_ExhaustedRetriesAreRecordedOnce(t *testing.T) {
 		t.Errorf("expected notify_failed, got %v", events[0]["type"])
 	}
 	detail, _ := events[0]["detail"].(string)
-	if !strings.HasPrefix(detail, "triggered: ") || !strings.Contains(detail, "500") {
+	if !strings.HasPrefix(detail, "triggered via ntfy: ") || !strings.Contains(detail, "500") {
 		t.Errorf("expected the kind and the reason in %q", detail)
 	}
 }
@@ -743,11 +754,11 @@ func TestNotify_ManualResolveSendsNothing(t *testing.T) {
 
 // With nobody on call the page goes to the shared fallback, and carries no
 // Acknowledge button — there is no user to attribute the acknowledgement to.
-func TestNotify_FallbackTopicHasNoAckButton(t *testing.T) {
+func TestNotify_FallbackHasNoAckButton(t *testing.T) {
 	f := newFakeNtfy(t)
 	s := newTS(t, api.NotifyConfig{
-		BaseURL:       f.URL,
+		Providers: f.providers(),
-		FallbackTopic: "terdut-oncall",
+		Fallback:  ntfyTarget("terdut-oncall"),
 		PublicURL: "https://terdut.example.com",
 	})
 
@@ -770,7 +781,7 @@ func TestNotify_FallbackTopicHasNoAckButton(t *testing.T) {
 // only pile up rows that can never be delivered.
 func TestNotify_NoTargetQueuesNothing(t *testing.T) {
 	f := newFakeNtfy(t)
-	s := newTS(t, api.NotifyConfig{BaseURL: f.URL})
+	s := newTS(t, api.NotifyConfig{Providers: f.providers()})
 
 	fireCritical(t, s)
 
@@ -787,7 +798,9 @@ func TestNotify_NoTargetQueuesNothing(t *testing.T) {
 func TestNotify_DisabledQueuesNothing(t *testing.T) {
 	s := newTS(t)
 	putOnCall(t, s, 1)
-	setTopic(t, s, 1, "terdut-admin")
+	// Inserted rather than added through the API, which refuses a type this
+	// server has no provider for.
+	s.exec(t, "INSERT INTO notification_channels (user_id, type, address) VALUES (1, 'ntfy', 'terdut-admin')")
 
 	fireCritical(t, s)
 

internal/api/router.go

@@ -95,9 +95,9 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 			r.Post("/api/oidc/device/deny", handleDeviceDecision(db, false))
 		}
 		r.Put("/api/me/onboarding", handleDismissOnboarding(db))
-		// Proves the topic works, which is the only part of "notifications are
+		// Which kinds of notification channel this server can deliver, so the
-		// set up" that the person holding the phone can confirm.
+		// UI offers only those.
-		r.Post("/api/me/notify/test", handleTestNotification(notify, db))
+		r.Get("/api/notify/providers", handleListProviders(notify.Providers))
 
 		// Readable by anyone signed in: the queue's assignment control and the
 		// on-call schedule both need to name people.
@@ -107,7 +107,13 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		// requireSelfOrAdmin rather than sitting behind AdminOnly, because
 		// which rule applies depends on the {id} in the path.
 		r.Get("/api/users/{id}/teams", handleUserTeams(db))
-		r.Put("/api/users/{id}/notify", handleSetNotifyTarget(db))
+		r.Get("/api/users/{id}/channels", handleListChannels(db))
+		r.Post("/api/users/{id}/channels", handleCreateChannel(db, notify.Providers))
+		r.Patch("/api/users/{id}/channels/{channelID}", handleUpdateChannel(db, notify.Providers))
+		r.Delete("/api/users/{id}/channels/{channelID}", handleDeleteChannel(db))
+		// Proves the channel works, which is the only part of "notifications are
+		// set up" that the person holding the phone can confirm.
+		r.Post("/api/users/{id}/channels/{channelID}/test", handleTestChannel(db, notify))
 		r.Put("/api/users/{id}/password", handleSetPassword(db))
 		r.Get("/api/users/{id}/api-keys", handleListAPIKeys(db))
 		r.Post("/api/users/{id}/api-keys", handleCreateAPIKey(db))
@@ -130,7 +136,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 			// an administrator from outside; this is a different
 			// question, so it is a different endpoint.
 			r.Get("/api/admin/teams/{teamID}", handleAdminGetTeam(db))
-			r.Get("/api/admin/settings", handleGetSettings(db, cfg))
+			r.Get("/api/admin/settings", handleGetSettings(db, cfg, notify.Providers))
 			r.Put("/api/admin/settings", handleSetSettings(db))
 		})
 
@@ -175,7 +181,7 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.With(opMode).Post("/api/teams", handleCreateTeam(db))
 		r.With(opMode).Put("/api/teams/{teamID}", handleRenameTeam(db))
 		r.With(opMode).Delete("/api/teams/{teamID}", handleDeleteTeam(db))
-		r.Get("/api/teams/{teamID}/members", handleListTeamMembers(db))
+		r.Get("/api/teams/{teamID}/members", handleListTeamMembers(db, notify.Providers))
 		r.Post("/api/teams/{teamID}/members", handleAddTeamMember(db))
 		r.Delete("/api/teams/{teamID}/members/{userID}", handleRemoveTeamMember(db))
 
@@ -192,8 +198,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Delete("/api/teams/{teamID}/invites/{inviteID}", handleRevokeInvite(db))
 
 		// A team's escalation ladder: who is paged when nobody answers.
-		r.Get("/api/teams/{teamID}/escalation", handleGetEscalation(db))
+		r.Get("/api/teams/{teamID}/escalation", handleGetEscalation(db, notify.Providers))
-		r.With(opMode).Put("/api/teams/{teamID}/escalation", handleSetEscalation(db))
+		r.With(opMode).Put("/api/teams/{teamID}/escalation", handleSetEscalation(db, notify.Providers))
 
 		// A team's own dead man's switches: which of its alerts are heartbeats,
 		// and how long a silence has to last before somebody is paged.
@@ -221,6 +227,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Get("/api/stats/alerts/top", handleStatsTop(db))
 		r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db))
 		r.Get("/api/stats/alerts/by-day", handleStatsByDay(db))
+		r.Get("/api/stats/alerts/heatmap", handleStatsHeatmap(db))
+		r.Get("/api/stats/alerts/{name}", handleStatsAlert(db))
 	})
 
 	// Anything else under /api is a mistake in a client, and should say so in

internal/api/service_accounts_test.go

@@ -141,7 +141,7 @@ func TestServiceAccount_TeamScopeIsBoundToItsOwnTeam(t *testing.T) {
 	teamB := createTeamAs(t, s, instanceKey, "team-b")
 	keyA := createServiceAccount(t, s, instanceKey, "team-a-sa", models.ServiceAccountScopeTeam, teamA)
 
-	policy := map[string]any{"repeat_count": 0, "fallback_topic": "", "levels": []any{}}
+	policy := map[string]any{"repeat_count": 0, "fallback": map[string]string{"type": "", "address": ""}, "levels": []any{}}
 
 	resp := s.reqAs(t, keyA, http.MethodPut, "/api/teams/"+id64(teamA)+"/escalation", policy)
 	if resp.StatusCode != http.StatusOK {

internal/api/settings.go

@@ -11,6 +11,7 @@ import (
 
 	"git.ryuvia.com/niklas/terdut-server/internal/config"
 	"git.ryuvia.com/niklas/terdut-server/internal/models"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 	"github.com/go-chi/chi/v5"
 )
 
@@ -111,7 +112,7 @@ type settingValue struct {
 	MaxSeconds  int64  `json:"max_seconds"`
 }
 
-func handleGetSettings(db *sql.DB, cfg config.Config) http.HandlerFunc {
+func handleGetSettings(db *sql.DB, cfg config.Config, providers *notify.Registry) http.HandlerFunc {
 	settings := NewSettings(db)
 	return func(w http.ResponseWriter, r *http.Request) {
 		out := settingsResponse{
@@ -128,6 +129,7 @@ func handleGetSettings(db *sql.DB, cfg config.Config) http.HandlerFunc {
 				// Never the ntfy token or the DSN: both are credentials, and an
 				// admin page that renders them turns a browser tab into a place
 				// they leak from.
+				"notification_providers": strings.Join(providerTypes(providers), ", "),
 				"ntfy_url":               cfg.NtfyURL,
 				"ntfy_configured":        strconv.FormatBool(cfg.NtfyURL != ""),
 				"ntfy_token_set":         strconv.FormatBool(cfg.NtfyToken != ""),

internal/api/signup.go

@@ -427,53 +427,6 @@ func handleRevokeInvite(db *sql.DB) http.HandlerFunc {
 // Onboarding
 // ---------------------------------------------------------------------------
 
-// handleTestNotification publishes one push to the caller's own topic.
-//
-// The point of the first-run checklist's notification step is not that a topic
-// string has been typed but that a phone buzzes, and only the person holding it
-// can tell whether it did. Published directly rather than through the outbox:
-// the outbox row requires an incident, and this deliberately belongs to no
-// incident.
-func handleTestNotification(cfg NotifyConfig, db *sql.DB) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if cfg.BaseURL == "" {
-			respond(w, http.StatusServiceUnavailable,
-				errResp("this server has no ntfy configured, so it can send nothing"))
-			return
-		}
-		caller, ok := userFromContext(r.Context())
-		if !ok {
-			respond(w, http.StatusForbidden, errResp("this endpoint is for human accounts only"))
-			return
-		}
-
-		var topic *string
-		if err := db.QueryRowContext(r.Context(),
-			"SELECT ntfy_topic FROM users WHERE id = $1", caller.ID).Scan(&topic); err != nil {
-			serverError(w, r, err)
-			return
-		}
-		if topic == nil || *topic == "" {
-			respond(w, http.StatusBadRequest, errResp("set a notification topic first"))
-			return
-		}
-
-		if err := publish(r.Context(), cfg, ntfyMessage{
-			Topic:   *topic,
-			Title:   "terdut test",
-			Message: "If this arrived, your notifications work.",
-			Tags:    []string{"white_check_mark"},
-		}); err != nil {
-			// The failure is the useful part here: a wrong topic, a token the
-			// ntfy server rejects, or an ntfy that is down all look the same
-			// from the phone, which is silence.
-			respond(w, http.StatusBadGateway, errResp("ntfy rejected the test: "+err.Error()))
-			return
-		}
-		w.WriteHeader(http.StatusNoContent)
-	}
-}
-
 // handleDismissOnboarding hides the first-run checklist, or brings it back.
 // Stored per user rather than in the browser: somebody who finishes setting up
 // on a laptop should not be nagged again on their phone.

internal/api/signup_test.go

@@ -10,6 +10,7 @@ import (
 
 	"git.ryuvia.com/niklas/terdut-server/internal/api"
 	"git.ryuvia.com/niklas/terdut-server/internal/models"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 )
 
 // signup posts to the unauthenticated sign-up endpoint, the way the form does,
@@ -301,18 +302,18 @@ func TestSignup_InviteCreatedByAServiceAccountLeavesCreatedByNull(t *testing.T)
 }
 
 // Neither of these has a real user_id to act on behalf of; both must 403 a
-// service account explicitly rather than 500 (handleTestNotification, which
+// service account explicitly rather than 500 (handleTestChannel, which once
-// used to query ntfy_topic for user id 0) or silently no-op (handleDismissOnboarding,
+// queried the notification target of user id 0) or silently no-op (handleDismissOnboarding,
 // which used to UPDATE ... WHERE id = 0, affecting nothing and still
 // returning 204).
 func TestServiceAccount_HumanOnlyEndpointsRefuseExplicitly(t *testing.T) {
-	// BaseURL set (even to a fake, unreachable address) so handleTestNotification
+	// A provider set (even one pointing at an unreachable address) so the test
-	// reaches its AsHuman() check instead of short-circuiting on "ntfy not
+	// endpoint reaches its owner check instead of short-circuiting on "no
-	// configured" first — this test is about the human-only check, not ntfy.
+	// provider configured" first — this test is about the human-only check.
-	s := newTS(t, api.NotifyConfig{BaseURL: "http://ntfy.invalid"})
+	s := newTS(t, api.NotifyConfig{Providers: notify.NewRegistry(notify.NewNtfy("http://ntfy.invalid", ""))})
 	instanceKey := createServiceAccount(t, s, s.key, "terdut-operator", models.ServiceAccountScopeInstance, 0)
 
-	if resp := s.reqAs(t, instanceKey, http.MethodPost, "/api/me/notify/test", nil); resp.StatusCode != http.StatusForbidden {
+	if resp := s.reqAs(t, instanceKey, http.MethodPost, "/api/users/1/channels/1/test", nil); resp.StatusCode != http.StatusForbidden {
 		t.Errorf("expected 403 for a service account testing notifications, got %d", resp.StatusCode)
 	} else {
 		resp.Body.Close()

internal/api/stats.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strings"
 	"time"
+
+	"github.com/go-chi/chi/v5"
 )
 
 func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
@@ -35,12 +38,24 @@ func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
 	}
 }
 
+// alertNameSQL is the alert-name half of an incident's signature (see
+// incidentSignature: "name|stable,labels"). An occurrence of an alert is an
+// incident, because alert rows are reused per fingerprint and carry no history.
+const alertNameSQL = "split_part(signature, '|', 1)"
+
+const sparkDays = 30
+
+// handleStatsTop ranks alert names by how many incidents they opened in the
+// window, with what a list row needs besides the count: when it last fired, how
+// long it takes to fix, and a per-day series for the last sparkDays days.
 func handleStatsTop(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))
+		q := r.URL.Query()
+		teams := callerTeamIDs(r.Context())
+		where, args := statsFilter(q, "triggered_at", teams)
 
 		limit := 10
-		if l := r.URL.Query().Get("limit"); l != "" {
+		if l := q.Get("limit"); l != "" {
 			var n int
 			fmt.Sscanf(l, "%d", &n)
 			if n > 0 && n <= 100 {
@@ -48,12 +63,13 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 			}
 		}
 		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
-			SELECT name, COUNT(*) AS cnt
+			SELECT %[1]s AS name, COUNT(*) AS cnt, MAX(triggered_at),
-			FROM   alerts
+			       AVG(CASE WHEN resolved_at IS NOT NULL THEN resolved_at - triggered_at END)
-			WHERE  %s
+			FROM   incidents
+			WHERE  %[2]s
 			GROUP  BY name
-			ORDER  BY cnt DESC
+			ORDER  BY cnt DESC, name ASC
-			LIMIT  %s`, where, args.add(limit)), args.all()...)
+			LIMIT  %[3]s`, alertNameSQL, where, args.add(limit)), args.all()...)
 		if err != nil {
 			serverError(w, r, err)
 			return
@@ -63,24 +79,245 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 		type entry struct {
 			Name        string   `json:"name"`
 			Count       int64    `json:"count"`
+			LastSeen    int64    `json:"last_seen"`
+			MeanResolve *float64 `json:"mean_resolve_seconds"`
+			Daily       []int64  `json:"daily"`
 		}
 		result := []entry{}
+		names := []string{}
 		for rows.Next() {
 			var e entry
-			if err := rows.Scan(&e.Name, &e.Count); err != nil {
+			if err := rows.Scan(&e.Name, &e.Count, &e.LastSeen, &e.MeanResolve); err != nil {
 				serverError(w, r, err)
 				return
 			}
+			e.Daily = make([]int64, sparkDays)
 			result = append(result, e)
+			names = append(names, e.Name)
 		}
 		if err := rows.Err(); err != nil {
 			serverError(w, r, err)
 			return
 		}
+
+		// The sparkline always covers the last sparkDays days ending at ?to (or
+		// today), whatever ?from says, so the rows stay comparable.
+		end := time.Now().UTC().Truncate(24 * time.Hour)
+		if to, err := time.Parse("2006-01-02", q.Get("to")); err == nil {
+			end = to.UTC()
+		}
+		first := end.AddDate(0, 0, -(sparkDays - 1))
+		sq := url.Values{}
+		sq.Set("from", first.Format("2006-01-02"))
+		sq.Set("to", end.Format("2006-01-02"))
+		swhere, sargs := statsFilter(sq, "triggered_at", teams)
+		if len(names) > 0 {
+			swhere += " AND " + alertNameSQL + " = ANY(" + sargs.add(names) + ")"
+			srows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+				SELECT %s AS name, ((triggered_at - %s) / 86400)::int AS day, COUNT(*)
+				FROM   incidents
+				WHERE  %s
+				GROUP  BY name, day`, alertNameSQL, sargs.add(first.Unix()), swhere), sargs.all()...)
+			if err != nil {
+				serverError(w, r, err)
+				return
+			}
+			defer srows.Close()
+			idx := map[string]int{}
+			for i, e := range result {
+				idx[e.Name] = i
+			}
+			for srows.Next() {
+				var name string
+				var day int
+				var cnt int64
+				if err := srows.Scan(&name, &day, &cnt); err != nil {
+					serverError(w, r, err)
+					return
+				}
+				if day >= 0 && day < sparkDays {
+					result[idx[name]].Daily[day] = cnt
+				}
+			}
+			if err := srows.Err(); err != nil {
+				serverError(w, r, err)
+				return
+			}
+		}
 		respond(w, http.StatusOK, result)
 	}
 }
 
+// handleStatsHeatmap counts incidents by UTC weekday (0=Sunday) and hour:
+// always 7 rows of 24 cells, so a client never has to fill gaps.
+func handleStatsHeatmap(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		where, args := statsFilter(r.URL.Query(), "triggered_at", callerTeamIDs(r.Context()))
+		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+			SELECT EXTRACT(DOW  FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
+			       EXTRACT(HOUR FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
+			       COUNT(*)
+			FROM   incidents
+			WHERE  %s
+			GROUP  BY 1, 2`, where), args.all()...)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		defer rows.Close()
+		grid := make([][]int64, 7)
+		for d := range grid {
+			grid[d] = make([]int64, 24)
+		}
+		for rows.Next() {
+			var d, h int
+			var c int64
+			if err := rows.Scan(&d, &h, &c); err != nil {
+				serverError(w, r, err)
+				return
+			}
+			grid[d][h] = c
+		}
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		respond(w, http.StatusOK, map[string]any{"grid": grid})
+	}
+}
+
+// maxOccurrences caps the drilldown's table; the aggregates still cover all.
+const maxOccurrences = 200
+
+// handleStatsAlert is the per-alert drilldown. Every figure is computed over
+// the incidents that alert opened in the window, so it agrees with the top list.
+func handleStatsAlert(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// chi hands back the still-escaped segment when the path has %2F in it.
+		name, err := url.PathUnescape(chi.URLParam(r, "name"))
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid alert name"))
+			return
+		}
+		where, args := statsFilter(r.URL.Query(), "i.triggered_at", callerTeamIDs(r.Context()))
+		where = strings.ReplaceAll(where, "archived_at", "i.archived_at")
+		where = strings.ReplaceAll(where, "team_id =", "i.team_id =")
+		where += " AND split_part(i.signature, '|', 1) = " + args.add(name)
+
+		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+			SELECT i.id, i.triggered_at, i.acknowledged_at, i.resolved_at, i.status, i.severity,
+			       COALESCE(u.username, sa.name)
+			FROM   incidents i
+			LEFT   JOIN users u ON u.id = i.acknowledged_by
+			LEFT   JOIN service_accounts sa ON sa.id = i.acknowledged_by_service_account_id
+			WHERE  %s
+			ORDER  BY i.triggered_at DESC`, where), args.all()...)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		defer rows.Close()
+
+		type occurrence struct {
+			IncidentID     int64   `json:"incident_id"`
+			TriggeredAt    int64   `json:"triggered_at"`
+			AckSeconds     *int64  `json:"ack_seconds"`
+			ResolveSeconds *int64  `json:"resolve_seconds"`
+			Status         string  `json:"status"`
+			Severity       *string `json:"severity,omitempty"`
+			AcknowledgedBy *string `json:"acknowledged_by,omitempty"`
+		}
+		all := []occurrence{}
+		for rows.Next() {
+			var o occurrence
+			var ack, res *int64
+			if err := rows.Scan(&o.IncidentID, &o.TriggeredAt, &ack, &res, &o.Status, &o.Severity, &o.AcknowledgedBy); err != nil {
+				serverError(w, r, err)
+				return
+			}
+			if ack != nil {
+				v := *ack - o.TriggeredAt
+				o.AckSeconds = &v
+			}
+			if res != nil {
+				v := *res - o.TriggeredAt
+				o.ResolveSeconds = &v
+			}
+			all = append(all, o)
+		}
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		if len(all) == 0 {
+			respond(w, http.StatusNotFound, errResp("no incidents for this alert"))
+			return
+		}
+
+		byHour := make([]int64, 24)
+		byDay := make([]int64, 7)
+		var acks, fixes []int64
+		var longest int64
+		for _, o := range all {
+			t := time.Unix(o.TriggeredAt, 0).UTC()
+			byHour[t.Hour()]++
+			byDay[int(t.Weekday())]++
+			if o.AckSeconds != nil {
+				acks = append(acks, *o.AckSeconds)
+			}
+			if o.ResolveSeconds != nil {
+				fixes = append(fixes, *o.ResolveSeconds)
+				longest = max(longest, *o.ResolveSeconds)
+			}
+		}
+		// all is newest first, so the gap runs from the last row back to the first.
+		var meanGap *float64
+		if n := len(all); n > 1 {
+			g := float64(all[0].TriggeredAt-all[n-1].TriggeredAt) / float64(n-1)
+			meanGap = &g
+		}
+		stat := func(xs []int64) (mean, median *float64) {
+			if len(xs) == 0 {
+				return nil, nil
+			}
+			s := append([]int64(nil), xs...)
+			sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
+			var sum int64
+			for _, x := range s {
+				sum += x
+			}
+			m, md := float64(sum)/float64(len(s)), float64(s[len(s)/2])
+			return &m, &md
+		}
+		meanAck, medAck := stat(acks)
+		meanFix, medFix := stat(fixes)
+
+		occ := all
+		if len(occ) > maxOccurrences {
+			occ = occ[:maxOccurrences]
+		}
+		resp := map[string]any{
+			"name":                   name,
+			"count":                  len(all),
+			"first_seen":             all[len(all)-1].TriggeredAt,
+			"last_seen":              all[0].TriggeredAt,
+			"mean_gap_seconds":       meanGap,
+			"mean_ack_seconds":       meanAck,
+			"median_ack_seconds":     medAck,
+			"mean_resolve_seconds":   meanFix,
+			"median_resolve_seconds": medFix,
+			"longest_open_seconds":   nil,
+			"by_hour":                byHour,
+			"by_weekday":             byDay,
+			"occurrences":            occ,
+		}
+		if len(fixes) > 0 {
+			resp["longest_open_seconds"] = longest
+		}
+		respond(w, http.StatusOK, resp)
+	}
+}
+
 func handleStatsByHour(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))

internal/api/stats_alert_test.go

@@ -0,0 +1,88 @@
+package api_test
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestStatsTop_CountsIncidentsWithSparkAndMeans(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	postToIntegration(t, s, red.key, "fp-1", "DiskFull")
+	postToIntegration(t, s, red.key, "fp-2", "DiskFull") // same group key: one incident
+	postToIntegration(t, s, red.key, "fp-3", "Memory")
+
+	top := list(t, red.call(http.MethodGet, "/api/stats/alerts/top", nil))
+	if len(top) != 2 {
+		t.Fatalf("expected 2 names, got %d", len(top))
+	}
+	for _, row := range top {
+		if row["count"].(float64) != 1 {
+			t.Errorf("%v: count %v, want 1 incident", row["name"], row["count"])
+		}
+		daily := row["daily"].([]any)
+		if len(daily) != 30 {
+			t.Fatalf("daily has %d days, want 30", len(daily))
+		}
+		if row["mean_resolve_seconds"] != nil {
+			t.Errorf("unresolved incident must have null mean, got %v", row["mean_resolve_seconds"])
+		}
+	}
+}
+
+func TestStatsHeatmap_ShapeAndScope(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	blue := newTeam(t, s, "blue")
+	postToIntegration(t, s, red.key, "fp-r", "RedOnly")
+	postToIntegration(t, s, blue.key, "fp-b", "BlueOnly")
+
+	var out struct {
+		Grid [][]int64 `json:"grid"`
+	}
+	decode(t, red.call(http.MethodGet, "/api/stats/alerts/heatmap", nil), &out)
+	if len(out.Grid) != 7 || len(out.Grid[0]) != 24 {
+		t.Fatalf("grid is %dx?, want 7x24", len(out.Grid))
+	}
+	var sum int64
+	for _, row := range out.Grid {
+		for _, c := range row {
+			sum += c
+		}
+	}
+	if sum != 1 {
+		t.Errorf("red's heatmap counted %d incidents, want 1", sum)
+	}
+}
+
+func TestStatsAlert_DrilldownAndScope(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	blue := newTeam(t, s, "blue")
+	postToIntegration(t, s, red.key, "fp-r", "DiskFull")
+	postToIntegration(t, s, blue.key, "fp-b", "BlueOnly")
+
+	var d map[string]any
+	decode(t, red.call(http.MethodGet, "/api/stats/alerts/DiskFull", nil), &d)
+	if d["count"].(float64) != 1 {
+		t.Errorf("count = %v, want 1", d["count"])
+	}
+	if len(d["by_hour"].([]any)) != 24 || len(d["by_weekday"].([]any)) != 7 {
+		t.Errorf("by_hour/by_weekday have wrong lengths")
+	}
+	if occ := d["occurrences"].([]any); len(occ) != 1 {
+		t.Errorf("occurrences = %d, want 1", len(occ))
+	}
+	if d["mean_ack_seconds"] != nil || d["longest_open_seconds"] != nil {
+		t.Errorf("nothing acked or resolved yet, means must be null: %v", d)
+	}
+
+	// Another team's alert, and a name nobody has, are both a plain 404.
+	for _, name := range []string{"BlueOnly", "Nope"} {
+		resp := red.call(http.MethodGet, "/api/stats/alerts/"+name, nil)
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusNotFound {
+			t.Errorf("%s: expected 404, got %d", name, resp.StatusCode)
+		}
+	}
+}

internal/api/teams.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"git.ryuvia.com/niklas/terdut-server/internal/models"
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
 	"github.com/go-chi/chi/v5"
 )
 
@@ -283,8 +284,9 @@ type memberStatus struct {
 	// NextShift is the first day after today the rota has them (YYYY-MM-DD).
 	NextShift *string `json:"next_shift,omitempty"`
 
-	// Pageable is whether they have an ntfy topic and an enabled account — the
+	// Pageable is whether they have an enabled channel of a type this server can
-	// conditions pageLevel and the notifier skip on. Never the topic itself.
+	// deliver, and an enabled account — the conditions pageLevel and the notifier
+	// skip on. Never the channel's address.
 	Pageable bool   `json:"pageable"`
 	Problem  string `json:"problem,omitempty"`
 
@@ -294,7 +296,7 @@ type memberStatus struct {
 
 // handleListTeamMembers names everybody in a team, with their status. Visible to
 // any member: you can see who else is on the rota you are on.
-func handleListTeamMembers(db *sql.DB) http.HandlerFunc {
+func handleListTeamMembers(db *sql.DB, providers *notify.Registry) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		teamID, ok := teamParam(w, r)
 		if !ok {
@@ -306,7 +308,8 @@ func handleListTeamMembers(db *sql.DB) http.HandlerFunc {
 
 		rows, err := db.QueryContext(r.Context(), `
 			SELECT m.team_id, m.user_id, u.username, m.role, m.joined_at, m.source,
-			       u.ntfy_topic IS NOT NULL AND u.ntfy_topic <> '',
+			       EXISTS (SELECT 1 FROM notification_channels c
+			                WHERE c.user_id = u.id AND c.enabled AND c.type = ANY($3::text[])),
 			       u.disabled_at IS NOT NULL,
 			       GREATEST(
 			         COALESCE((SELECT MAX(last_seen_at) FROM sessions WHERE user_id = u.id), 0),
@@ -318,7 +321,7 @@ func handleListTeamMembers(db *sql.DB) http.HandlerFunc {
 			FROM   team_members m
 			JOIN   users u ON u.id = m.user_id
 			WHERE  m.team_id = $1
-			ORDER  BY u.username`, teamID, todayUTC())
+			ORDER  BY u.username`, teamID, todayUTC(), providerTypes(providers))
 		if err != nil {
 			serverError(w, r, err)
 			return
@@ -329,9 +332,9 @@ func handleListTeamMembers(db *sql.DB) http.HandlerFunc {
 		for rows.Next() {
 			var m memberStatus
 			var joined, lastActive int64
-			var hasTopic, disabled bool
+			var hasChannel, disabled bool
 			if err := rows.Scan(&m.TeamID, &m.UserID, &m.Username, &m.Role, &joined, &m.Source,
-				&hasTopic, &disabled, &lastActive, &m.OnCall, &m.NextShift); err != nil {
+				&hasChannel, &disabled, &lastActive, &m.OnCall, &m.NextShift); err != nil {
 				serverError(w, r, err)
 				return
 			}
@@ -343,8 +346,8 @@ func handleListTeamMembers(db *sql.DB) http.HandlerFunc {
 			switch {
 			case disabled:
 				m.Problem = "account is disabled"
-			case !hasTopic:
+			case !hasChannel:
-				m.Problem = "has no ntfy topic"
+				m.Problem = "has no notification channel"
 			}
 			m.Pageable = m.Problem == ""
 			switch {

internal/api/teams_test.go

@@ -459,18 +459,38 @@ func TestTeams_OutsiderSeesNothing(t *testing.T) {
 // ---------------------------------------------------------------------------
 
 // Everybody signed in can list users to name them, but only an admin (or the
-// row's owner) sees an email or an ntfy topic, which is a publish secret.
+// row's owner) sees an email, and only the owner sees a channel's address, which
-func TestListUsers_RedactsEmailAndTopicForOthers(t *testing.T) {
+// can be a publish secret. An admin sees that a channel exists, not where it goes.
+func TestListUsers_RedactsEmailAndChannelsForOthers(t *testing.T) {
 	s := newTS(t)
 	red := newTeam(t, s, "red")
 	_ = newTeam(t, s, "blue")
-	s.exec(t, "UPDATE users SET ntfy_topic = 'secret-topic'")
+	s.exec(t, `INSERT INTO notification_channels (user_id, type, address)
+	           SELECT id, 'ntfy', 'secret-topic' FROM users`)
+
+	channelAddresses := func(u map[string]any) []string {
+		var out []string
+		list, _ := u["channels"].([]any)
+		for _, c := range list {
+			addr, _ := c.(map[string]any)["address"].(string) // omitted when blank
+			out = append(out, addr)
+		}
+		return out
+	}
 
 	var asAdmin []map[string]any
 	decode(t, s.req(t, http.MethodGet, "/api/users", nil), &asAdmin)
 	for _, u := range asAdmin {
-		if u["email"] == "" || u["ntfy_topic"] != "secret-topic" {
+		if u["email"] == "" {
-			t.Errorf("admin should see everything, got %v", u)
+			t.Errorf("admin should see every email, got %v", u)
+		}
+		addrs := channelAddresses(u)
+		if len(addrs) != 1 {
+			t.Errorf("admin should see that every user has a channel, got %v", u)
+			continue
+		}
+		if own := u["username"] == "admin"; own != (addrs[0] != "") {
+			t.Errorf("admin should see only their own channel's address, got %v", u)
 		}
 	}
 
@@ -480,8 +500,15 @@ func TestListUsers_RedactsEmailAndTopicForOthers(t *testing.T) {
 	}
 	for _, u := range asMember {
 		own := u["username"] == "red-user"
-		if own != (u["email"] != "") || own != (u["ntfy_topic"] != nil) {
+		if own != (u["email"] != "") {
-			t.Errorf("only red-user's own row should keep email and topic, got %v", u)
+			t.Errorf("only red-user's own row should keep its email, got %v", u)
+		}
+		addrs := channelAddresses(u)
+		if own && (len(addrs) != 1 || addrs[0] != "secret-topic") {
+			t.Errorf("red-user should see their own channel and address, got %v", u)
+		}
+		if !own && len(addrs) != 0 {
+			t.Errorf("a member must not see anybody else's channels, got %v", u)
 		}
 	}
 }

internal/api/users.go

@@ -8,7 +8,6 @@ import (
 	"encoding/hex"
 	"net/http"
 	"strconv"
-	"strings"
 	"time"
 
 	"git.ryuvia.com/niklas/terdut-server/internal/models"
@@ -109,15 +108,16 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
 
 // handleListUsers is readable by anyone signed in, because the assignment
 // control and the schedule need to name people. What it returns about other
-// people is therefore only what naming them takes: email and ntfy_topic are
+// people is therefore only what naming them takes: email is blanked unless the
-// blanked unless the caller is an admin or the row is their own. The topic in
+// caller is an admin or the row is their own, and channels are listed only for
-// particular is a publish secret.
+// the caller themselves (with addresses) and for an admin (without, since an
+// address can be a secret).
 func handleListUsers(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		caller, _ := userFromContext(r.Context())
 		seeAll := caller.IsAdmin
 		rows, err := db.QueryContext(r.Context(),
-			"SELECT id, username, email, created_at, ntfy_topic, is_admin, admin_source, disabled_at FROM users ORDER BY id")
+			"SELECT id, username, email, created_at, is_admin, admin_source, disabled_at FROM users ORDER BY id")
 		if err != nil {
 			serverError(w, r, err)
 			return
@@ -129,7 +129,7 @@ func handleListUsers(db *sql.DB) http.HandlerFunc {
 			var u models.User
 			var ts int64
 			var disabled *int64
-			if err := rows.Scan(&u.ID, &u.Username, &u.Email, &ts, &u.NtfyTopic, &u.IsAdmin, &u.AdminSource, &disabled); err != nil {
+			if err := rows.Scan(&u.ID, &u.Username, &u.Email, &ts, &u.IsAdmin, &u.AdminSource, &disabled); err != nil {
 				serverError(w, r, err)
 				return
 			}
@@ -137,7 +137,6 @@ func handleListUsers(db *sql.DB) http.HandlerFunc {
 			u.DisabledAt = unixPtr(disabled)
 			if !seeAll && u.ID != caller.ID {
 				u.Email = ""
-				u.NtfyTopic = nil
 			}
 			users = append(users, u)
 		}
@@ -145,6 +144,20 @@ func handleListUsers(db *sql.DB) http.HandlerFunc {
 			serverError(w, r, err)
 			return
 		}
+		rows.Close()
+
+		// After the user cursor is closed: the pool may be a single connection.
+		for i := range users {
+			if !seeAll && users[i].ID != caller.ID {
+				continue
+			}
+			channels, err := loadChannels(r.Context(), db, users[i].ID, users[i].ID == caller.ID)
+			if err != nil {
+				serverError(w, r, err)
+				return
+			}
+			users[i].Channels = channels
+		}
 		respond(w, http.StatusOK, users)
 	}
 }
@@ -180,53 +193,6 @@ func handleCreateUser(db *sql.DB) http.HandlerFunc {
 	}
 }
 
-// handleSetNotifyTarget points a user's push notifications at an ntfy topic, or
-// clears it with an empty string. The topic is a shared secret with the ntfy
-// server — anyone who knows it can publish to it — so pick an unguessable one
-// unless your ntfy enforces access control.
-func handleSetNotifyTarget(db *sql.DB) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
-		if err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid user id"))
-			return
-		}
-		if !requireSelfOrAdmin(w, r, id) {
-			return
-		}
-		var req struct {
-			NtfyTopic string `json:"ntfy_topic"`
-		}
-		if err := decodeJSON(r, &req); err != nil {
-			respond(w, http.StatusBadRequest, errResp("invalid request body"))
-			return
-		}
-
-		var topic *string
-		if t := strings.TrimSpace(req.NtfyTopic); t != "" {
-			topic = &t
-		}
-
-		res, err := db.ExecContext(r.Context(),
-			"UPDATE users SET ntfy_topic = $1 WHERE id = $2", topic, id)
-		if err != nil {
-			serverError(w, r, err)
-			return
-		}
-		if n, _ := res.RowsAffected(); n == 0 {
-			respond(w, http.StatusNotFound, errResp("user not found"))
-			return
-		}
-
-		user, err := fetchUser(r.Context(), db, id)
-		if err != nil {
-			serverError(w, r, err)
-			return
-		}
-		respond(w, http.StatusOK, user)
-	}
-}
-
 func handleDeleteUser(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
@@ -431,8 +397,8 @@ func fetchUser(ctx context.Context, db *sql.DB, id int64) (models.User, error) {
 	var ts int64
 	var disabled *int64
 	err := db.QueryRowContext(ctx,
-		"SELECT id, username, email, created_at, ntfy_topic, is_admin, admin_source, disabled_at FROM users WHERE id = $1", id).
+		"SELECT id, username, email, created_at, is_admin, admin_source, disabled_at FROM users WHERE id = $1", id).
-		Scan(&u.ID, &u.Username, &u.Email, &ts, &u.NtfyTopic, &u.IsAdmin, &u.AdminSource, &disabled)
+		Scan(&u.ID, &u.Username, &u.Email, &ts, &u.IsAdmin, &u.AdminSource, &disabled)
 	if err != nil {
 		return u, err
 	}

internal/db/migrations/001_schema.sql

@@ -65,7 +65,8 @@ CREATE TABLE escalation_levels (
 CREATE TABLE escalation_policies (
     team_id bigint NOT NULL,
     repeat_count bigint DEFAULT 0 NOT NULL,
-    fallback_topic text DEFAULT ''::text NOT NULL,
+    fallback_type text DEFAULT ''::text NOT NULL,
+    fallback_address text DEFAULT ''::text NOT NULL,
     updated_at bigint DEFAULT (floor(EXTRACT(epoch FROM now())))::bigint NOT NULL,
     CONSTRAINT escalation_policies_repeat_count_check CHECK (((repeat_count >= 0) AND (repeat_count <= 10)))
 );
@@ -159,11 +160,22 @@ CREATE TABLE invites (
     CONSTRAINT invites_role_check CHECK ((role = ANY (ARRAY['owner'::text, 'member'::text])))
 );
 
+CREATE TABLE notification_channels (
+    id bigint GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+    user_id bigint NOT NULL,
+    type text NOT NULL,
+    address text NOT NULL,
+    enabled boolean DEFAULT true NOT NULL,
+    created_at bigint DEFAULT (floor(EXTRACT(epoch FROM now())))::bigint NOT NULL
+);
+
 CREATE TABLE notifications (
     id bigint GENERATED BY DEFAULT AS IDENTITY NOT NULL,
     incident_id bigint NOT NULL,
     user_id bigint,
-    topic text NOT NULL,
+    channel_id bigint,
+    channel_type text NOT NULL,
+    address text NOT NULL,
     kind text NOT NULL,
     created_at bigint NOT NULL,
     send_after bigint NOT NULL,
@@ -268,7 +280,6 @@ CREATE TABLE users (
     username text NOT NULL,
     email text NOT NULL,
     created_at bigint DEFAULT (floor(EXTRACT(epoch FROM now())))::bigint NOT NULL,
-    ntfy_topic text,
     password_hash text,
     is_admin boolean DEFAULT false NOT NULL,
     disabled_at bigint,
@@ -332,6 +343,12 @@ ALTER TABLE invites
 ALTER TABLE invites
     ADD CONSTRAINT invites_token_hash_key UNIQUE (token_hash);
 
+ALTER TABLE notification_channels
+    ADD CONSTRAINT notification_channels_pkey PRIMARY KEY (id);
+
+ALTER TABLE notification_channels
+    ADD CONSTRAINT notification_channels_user_type_address_key UNIQUE (user_id, type, address);
+
 ALTER TABLE notifications
     ADD CONSTRAINT notifications_pkey PRIMARY KEY (id);
 
@@ -452,6 +469,8 @@ CREATE UNIQUE INDEX deadman_switches_team_name_key ON deadman_switches (team_id,
 
 CREATE INDEX invites_team_idx ON invites USING btree (team_id);
 
+CREATE INDEX notification_channels_user_idx ON notification_channels USING btree (user_id);
+
 CREATE INDEX notifications_incident_idx ON notifications USING btree (incident_id, id DESC);
 
 CREATE INDEX notifications_pending_idx ON notifications USING btree (send_after) WHERE (sent_at IS NULL);
@@ -550,6 +569,12 @@ ALTER TABLE invites
 ALTER TABLE invites
     ADD CONSTRAINT invites_team_id_fkey FOREIGN KEY (team_id) REFERENCES teams(id) ON DELETE CASCADE;
 
+ALTER TABLE notification_channels
+    ADD CONSTRAINT notification_channels_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
+
+ALTER TABLE notifications
+    ADD CONSTRAINT notifications_channel_id_fkey FOREIGN KEY (channel_id) REFERENCES notification_channels(id) ON DELETE SET NULL;
+
 ALTER TABLE notifications
     ADD CONSTRAINT notifications_incident_id_fkey FOREIGN KEY (incident_id) REFERENCES incidents(id) ON DELETE CASCADE;
 

internal/models/user.go

@@ -8,10 +8,11 @@ type User struct {
 	Email     string    `json:"email"`
 	CreatedAt time.Time `json:"created_at"`
 
-	// NtfyTopic is where this user's push notifications go. Nil means they get
+	// Channels are where this user's notifications go. Empty means they get none
-	// none of their own; incidents assigned to them fall back to the configured
+	// of their own; incidents assigned to them fall back to the configured
-	// fallback topic instead.
+	// fallback target instead. Only filled in for the user themselves and for an
-	NtfyTopic *string `json:"ntfy_topic,omitempty"`
+	// administrator, and the address only for the user themselves.
+	Channels []NotificationChannel `json:"channels,omitempty"`
 
 	// DisabledAt is when the account was taken out of use, or nil. A disabled
 	// user cannot authenticate by either credential, and keeps their name on
@@ -31,6 +32,19 @@ type User struct {
 	AdminSource string `json:"admin_source,omitempty"`
 }
 
+// NotificationChannel is one way of reaching a user: a provider type and an
+// address that provider understands.
+type NotificationChannel struct {
+	ID   int64  `json:"id"`
+	Type string `json:"type"`
+
+	// Address is an ntfy topic, later an email address. Blank unless the caller is
+	// the owner: it can be a secret, and an administrator has no need of it.
+	Address   string    `json:"address,omitempty"`
+	Enabled   bool      `json:"enabled"`
+	CreatedAt time.Time `json:"created_at"`
+}
+
 type APIKey struct {
 	ID         int64      `json:"id"`
 	UserID     int64      `json:"user_id"`

internal/notify/notify.go

@@ -0,0 +1,138 @@
+// Package notify is how Terdut reaches a person: a Provider per way of doing it
+// (ntfy today), and a Registry of the ones this server has been configured with.
+//
+// The package knows nothing about incidents. The notifier renders a Message from
+// an incident and hands it to whichever Provider the recipient's channel names,
+// so a new way of paging someone is one new Provider and nothing else.
+package notify
+
+import (
+	"context"
+	"sort"
+)
+
+// Message kinds. Providers may use them to shape a message (ntfy sends a
+// "resolved" quietly), but a Provider that ignores them is correct.
+const (
+	KindTriggered = "triggered"
+	KindReminder  = "reminder"
+	KindEscalated = "escalated"
+	KindResolved  = "resolved"
+	KindTest      = "test"
+)
+
+// Message is one notification, in no provider's format.
+type Message struct {
+	// Address is where this provider delivers it: an ntfy topic, later an email
+	// address. Opaque to everything except the Provider that owns it.
+	Address string
+
+	Kind  string
+	Title string
+	Body  string
+
+	// Level is the incident's severity on a 0-4 scale: 0 unknown, 1 info,
+	// 2 warning, 3 error, 4 critical. A provider with a notion of urgency maps it.
+	Level int
+
+	// ClickURL opens the incident in the web UI. Empty when the server does not
+	// know its own public URL.
+	ClickURL string
+
+	// AckURL acknowledges the incident when POSTed. Only set for a provider whose
+	// Capabilities say it can render an action; for the rest the notifier puts
+	// the link in Body instead.
+	AckURL string
+}
+
+// Capabilities says what a Provider can render, so the notifier does not have to
+// know which provider it is talking to.
+type Capabilities struct {
+	// ActionButtons is whether the provider can show a button that POSTs to a URL
+	// without opening anything, which is what the Acknowledge button is.
+	ActionButtons bool
+}
+
+// Provider is one way of delivering a Message.
+type Provider interface {
+	// Type is the stable identifier stored in a channel row and shown in the
+	// API: "ntfy". Lower case, never changes once released.
+	Type() string
+
+	// Label is the name a person sees: "ntfy".
+	Label() string
+
+	// AddressLabel and AddressHint describe the address field for the UI: a
+	// "Topic" labelled field with a sentence on what to put in it.
+	AddressLabel() string
+	AddressHint() string
+
+	Capabilities() Capabilities
+
+	// Validate checks an address a user typed and returns it normalised. An error
+	// is shown to that user as is.
+	Validate(address string) (string, error)
+
+	// Send delivers one message. It must honour ctx, and an error means the
+	// message did not land, so the notifier will retry.
+	Send(ctx context.Context, msg Message) error
+}
+
+// Target names a provider and an address without belonging to a user: the
+// fallback for an incident nobody is on call for.
+type Target struct {
+	Type    string `json:"type"`
+	Address string `json:"address"`
+}
+
+// IsZero reports whether no target is set.
+func (t Target) IsZero() bool { return t.Type == "" && t.Address == "" }
+
+// Registry is the providers a server was configured with. The zero value and
+// nil are both an empty registry, so a server with nothing configured needs no
+// special case at its call sites.
+type Registry struct {
+	byType map[string]Provider
+}
+
+// NewRegistry registers providers by Type. A later provider of the same type
+// replaces the earlier one.
+func NewRegistry(providers ...Provider) *Registry {
+	r := &Registry{byType: make(map[string]Provider, len(providers))}
+	for _, p := range providers {
+		r.byType[p.Type()] = p
+	}
+	return r
+}
+
+// Get returns the provider for a type.
+func (r *Registry) Get(typ string) (Provider, bool) {
+	if r == nil {
+		return nil, false
+	}
+	p, ok := r.byType[typ]
+	return p, ok
+}
+
+// Has reports whether a provider of this type is configured.
+func (r *Registry) Has(typ string) bool {
+	_, ok := r.Get(typ)
+	return ok
+}
+
+// Empty reports whether no provider is configured, which disables notifications.
+func (r *Registry) Empty() bool { return r == nil || len(r.byType) == 0 }
+
+// List returns the providers ordered by type, so the UI and the tests see them
+// in a stable order.
+func (r *Registry) List() []Provider {
+	if r == nil {
+		return nil
+	}
+	out := make([]Provider, 0, len(r.byType))
+	for _, p := range r.byType {
+		out = append(out, p)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Type() < out[j].Type() })
+	return out
+}

internal/notify/notify_test.go

@@ -0,0 +1,111 @@
+package notify_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"git.ryuvia.com/niklas/terdut-server/internal/notify"
+)
+
+func TestRegistry_NilAndEmptyAreEmpty(t *testing.T) {
+	var nilReg *notify.Registry
+	for name, r := range map[string]*notify.Registry{"nil": nilReg, "empty": notify.NewRegistry()} {
+		if !r.Empty() || r.Has("ntfy") || len(r.List()) != 0 {
+			t.Errorf("%s registry should be empty", name)
+		}
+		if _, ok := r.Get("ntfy"); ok {
+			t.Errorf("%s registry should have no ntfy", name)
+		}
+	}
+}
+
+func TestRegistry_ListIsOrderedByType(t *testing.T) {
+	r := notify.NewRegistry(notify.NewNtfy("http://x", ""), fakeProvider{"email"})
+	list := r.List()
+	if len(list) != 2 || list[0].Type() != "email" || list[1].Type() != "ntfy" {
+		t.Errorf("expected providers ordered by type, got %v", list)
+	}
+}
+
+type fakeProvider struct{ typ string }
+
+func (f fakeProvider) Type() string                               { return f.typ }
+func (f fakeProvider) Label() string                              { return f.typ }
+func (f fakeProvider) AddressLabel() string                       { return "" }
+func (f fakeProvider) AddressHint() string                        { return "" }
+func (f fakeProvider) Capabilities() notify.Capabilities          { return notify.Capabilities{} }
+func (f fakeProvider) Validate(a string) (string, error)          { return a, nil }
+func (f fakeProvider) Send(context.Context, notify.Message) error { return nil }
+
+func TestNtfy_ValidateTopic(t *testing.T) {
+	n := notify.NewNtfy("http://x", "")
+	for _, bad := range []string{"", "  ", "a b", "a/b", "a?b", "a#b"} {
+		if _, err := n.Validate(bad); err == nil {
+			t.Errorf("topic %q should be rejected", bad)
+		}
+	}
+	if got, err := n.Validate("  my-topic "); err != nil || got != "my-topic" {
+		t.Errorf("a good topic should be trimmed and accepted, got %q, %v", got, err)
+	}
+}
+
+func TestNtfy_SendEncodesTheMessage(t *testing.T) {
+	var got map[string]any
+	var auth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		auth = r.Header.Get("Authorization")
+		_ = json.NewDecoder(r.Body).Decode(&got)
+	}))
+	defer srv.Close()
+
+	n := notify.NewNtfy(srv.URL+"/", "tok")
+	err := n.Send(context.Background(), notify.Message{
+		Address: "t", Kind: notify.KindTriggered, Title: "T", Body: "B", Level: 4,
+		ClickURL: "https://x/incidents/1", AckURL: "https://x/ack",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if auth != "Bearer tok" {
+		t.Errorf("expected the bearer token, got %q", auth)
+	}
+	if got["topic"] != "t" || got["title"] != "T" || got["message"] != "B" || got["priority"] != float64(5) {
+		t.Errorf("unexpected body %v", got)
+	}
+	actions, _ := got["actions"].([]any)
+	if len(actions) != 1 || actions[0].(map[string]any)["url"] != "https://x/ack" {
+		t.Errorf("expected the Acknowledge action, got %v", got["actions"])
+	}
+}
+
+func TestNtfy_ResolvedIsQuietAndTestHasNoPriority(t *testing.T) {
+	var got map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = nil
+		_ = json.NewDecoder(r.Body).Decode(&got)
+	}))
+	defer srv.Close()
+	n := notify.NewNtfy(srv.URL, "")
+
+	_ = n.Send(context.Background(), notify.Message{Address: "t", Kind: notify.KindResolved, Level: 4})
+	if got["priority"] != float64(2) {
+		t.Errorf("a resolved page should be low priority whatever the severity, got %v", got["priority"])
+	}
+	_ = n.Send(context.Background(), notify.Message{Address: "t", Kind: notify.KindTest})
+	if _, ok := got["priority"]; ok {
+		t.Errorf("a test should carry no priority, got %v", got["priority"])
+	}
+}
+
+func TestNtfy_SendReportsARejection(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	defer srv.Close()
+	if err := notify.NewNtfy(srv.URL, "").Send(context.Background(), notify.Message{Address: "t"}); err == nil {
+		t.Error("a 403 from ntfy must be an error so the notifier retries")
+	}
+}

internal/notify/ntfy.go

@@ -0,0 +1,177 @@
+package notify
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// TypeNtfy is the Type of the ntfy provider.
+const TypeNtfy = "ntfy"
+
+// ntfy's priority scale. Max is the one that overrides the phone's quiet
+// settings, which is the whole point of paging on critical.
+const (
+	ntfyPriorityLow     = 2
+	ntfyPriorityDefault = 3
+	ntfyPriorityHigh    = 4
+	ntfyPriorityMax     = 5
+)
+
+// Ntfy publishes to an ntfy server. One server per install, from its
+// configuration; only the topic is per person.
+type Ntfy struct {
+	baseURL string
+	token   string
+	client  *http.Client
+}
+
+// NewNtfy builds the provider. token is an optional bearer token for an
+// access-controlled server. The client's timeout is what keeps a hung ntfy from
+// stalling a delivery pass: a page is small and infrequent.
+func NewNtfy(baseURL, token string) *Ntfy {
+	return &Ntfy{
+		baseURL: strings.TrimSuffix(baseURL, "/"),
+		token:   token,
+		client:  &http.Client{Timeout: 10 * time.Second},
+	}
+}
+
+func (n *Ntfy) Type() string         { return TypeNtfy }
+func (n *Ntfy) Label() string        { return "ntfy" }
+func (n *Ntfy) AddressLabel() string { return "Topic" }
+func (n *Ntfy) AddressHint() string {
+	return "Subscribe to this topic in the ntfy app. A topic is a shared secret: anyone who knows it can read " +
+		"your pages and publish to it, so pick one that is hard to guess."
+}
+
+// Capabilities: ntfy renders an http action as a button on the notification.
+func (n *Ntfy) Capabilities() Capabilities { return Capabilities{ActionButtons: true} }
+
+func (n *Ntfy) Validate(address string) (string, error) {
+	topic := strings.TrimSpace(address)
+	if topic == "" {
+		return "", errors.New("a topic is required")
+	}
+	if strings.ContainsAny(topic, " \t\r\n/?#") {
+		return "", errors.New("a topic cannot contain spaces, slashes, '?' or '#'")
+	}
+	return topic, nil
+}
+
+// ntfyMessage is ntfy's JSON publish format. Using it rather than the X-Actions
+// header avoids that header's comma and quote escaping rules, which are easy to
+// break with a title that happens to contain a comma.
+type ntfyMessage struct {
+	Topic    string       `json:"topic"`
+	Title    string       `json:"title,omitempty"`
+	Message  string       `json:"message"`
+	Priority int          `json:"priority,omitempty"`
+	Tags     []string     `json:"tags,omitempty"`
+	Click    string       `json:"click,omitempty"`
+	Actions  []ntfyAction `json:"actions,omitempty"`
+}
+
+type ntfyAction struct {
+	Action string `json:"action"`
+	Label  string `json:"label"`
+	URL    string `json:"url"`
+	Method string `json:"method,omitempty"`
+	Clear  bool   `json:"clear,omitempty"`
+}
+
+// Send POSTs one message to ntfy.
+func (n *Ntfy) Send(ctx context.Context, msg Message) error {
+	body, err := json.Marshal(n.encode(msg))
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, n.baseURL, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if n.token != "" {
+		req.Header.Set("Authorization", "Bearer "+n.token)
+	}
+
+	resp, err := n.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("ntfy returned %s", resp.Status)
+	}
+	return nil
+}
+
+func (n *Ntfy) encode(msg Message) ntfyMessage {
+	out := ntfyMessage{
+		Topic:   msg.Address,
+		Title:   msg.Title,
+		Message: msg.Body,
+		Click:   msg.ClickURL,
+	}
+
+	switch msg.Kind {
+	case KindResolved:
+		out.Priority = ntfyPriorityLow
+		out.Tags = []string{"white_check_mark"}
+	case KindTest:
+		out.Tags = []string{"white_check_mark"}
+	default:
+		out.Priority = ntfyPriority(msg.Level)
+		out.Tags = []string{ntfyTag(msg.Level)}
+	}
+
+	if msg.AckURL != "" {
+		out.Actions = append(out.Actions, ntfyAction{
+			Action: "http",
+			Label:  "Acknowledge",
+			URL:    msg.AckURL,
+			Method: "POST",
+			Clear:  true,
+		})
+	}
+	return out
+}
+
+// ntfyPriority maps a severity level onto ntfy's scale. An unrecognised level
+// gets the default rather than being silenced.
+func ntfyPriority(level int) int {
+	switch level {
+	case 4:
+		return ntfyPriorityMax
+	case 3:
+		return ntfyPriorityHigh
+	case 2:
+		return ntfyPriorityDefault
+	case 1:
+		return ntfyPriorityLow
+	default:
+		return ntfyPriorityDefault
+	}
+}
+
+func ntfyTag(level int) string {
+	switch level {
+	case 4:
+		return "rotating_light"
+	case 3:
+		return "red_circle"
+	case 2:
+		return "warning"
+	case 1:
+		return "information_source"
+	default:
+		return "bell"
+	}
+}

internal/web/static/app.css

@@ -826,6 +826,16 @@ details[open] > summary { margin-bottom: 8px; }
 .pw-form { display: grid; gap: 12px; padding: 16px; }
 .account-fold { border-top: 1px solid var(--border); padding-top: 12px; }
 .account-fold .stacked-form { margin-top: 8px; }
+
+/* Notification channels on the Account page: a row per channel, the kind and
+   address on the left and its actions under or beside them. Wrapping rather
+   than a fixed grid, because an address can be as long as a topic is secret. */
+.channels { list-style: none; margin: 0; padding: 0; display: grid; gap: 12px; }
+.channel { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 8px; }
+.channel.off .channel-main { opacity: 0.6; }
+.channel-main { display: flex; flex-wrap: wrap; align-items: baseline; gap: 8px; min-width: 0; }
+.channel-main code { overflow-wrap: anywhere; }
+.channel .row-actions { margin-top: 0; gap: 6px; }
 .form-ok {
   margin: 0; padding: 10px 12px;
   background: var(--ok-soft); color: var(--ok);
@@ -998,6 +1008,8 @@ kbd {
 .admin-settings .setting-value { width: 5.5em; margin-right: 6px; }
 .admin-settings .setting-unit { max-width: 8em; }
 .admin-settings button[type="submit"] { margin-top: 12px; }
+/* The environment section follows the form in the same card; h3 has no margin of its own. */
+.admin-settings + h3 { margin-top: 24px; }
 .small { font-size: 13px; }
 
 /* A name in an admin table is the way to that row's own page -- a person's or
@@ -1308,9 +1320,71 @@ button.rota-week:hover { background: var(--surface-2); color: var(--text); }
 .columns .col-bar { fill: var(--accent); }
 .columns .col:hover .col-bar { opacity: 0.75; }
 .columns .col-label { fill: var(--faint); font-size: 10px; font-family: var(--font); }
+.chart-head { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: baseline; gap: 4px 8px; margin-bottom: 10px; }
+.chart-head .chart-title { margin-bottom: 0; }
+.chart-note { font-size: 12px; color: var(--muted); }
+.stat-note { margin-top: 2px; font-size: 12px; color: var(--muted); }
+.stats-head { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 12px; }
+.stats-head .chips { padding-left: 0; padding-right: 0; flex: 1 1 100%; }
+.stats-alert-name { font-family: var(--mono, ui-monospace, monospace); font-size: 20px; overflow-wrap: anywhere; }
+.stats-split { display: grid; gap: 14px; }
+.pattern-strip { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 20px; font-size: 14px; }
+.pattern-strip .chart-title { margin-bottom: 0; }
+.patterns { display: grid; gap: 14px; }
+.pattern-title { font-size: 15px; font-weight: 600; }
+.pattern-sub { margin-top: 2px; font-size: 13px; color: var(--muted); }
+
+/* Top alerts: each row is a link to the alert's drilldown. */
+.toplist { display: grid; overflow-x: auto; }
+.top-row {
+  display: grid; align-items: center; gap: 12px; min-width: 640px; min-height: 44px; padding: 6px 4px;
+  grid-template-columns: minmax(140px, 2.4fr) minmax(130px, 1.4fr) 100px 100px 70px 18px;
+  border-top: 1px solid var(--border); color: inherit; text-decoration: none; font-size: 13px;
+}
+a.top-row:hover { background: var(--surface-2); }
+.top-head { min-height: 0; border-top: 0; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); grid-template-columns: minmax(140px, 2.4fr) minmax(130px, 1.4fr) 100px 100px 70px 18px; }
+.top-count { display: flex; align-items: center; gap: 8px; }
+.top-count .hbar-track { flex: 1; height: 6px; }
+.top-count .hbar-count { order: -1; min-width: 2ch; color: var(--text); font-weight: 600; }
+.top-pct, .top-cell { color: var(--muted); font-variant-numeric: tabular-nums; }
+.spark { display: flex; align-items: flex-end; gap: 1px; height: 24px; }
+.spark-bar { width: 2px; border-radius: 1px; background: var(--border-strong); }
+.spark-bar.on { background: var(--accent); }
+
+/* Heatmap: weekday × hour. */
+.heatmap { display: grid; gap: 3px; overflow-x: auto; min-width: 0; }
+.heatmap > * { min-width: 520px; }
+.heat-row { display: flex; align-items: center; gap: 8px; }
+.heat-day { width: 30px; flex: none; font-size: 12px; color: var(--muted); }
+.heat-cells { flex: 1; display: grid; grid-template-columns: repeat(24, minmax(0, 1fr)); gap: 3px; }
+.heat-cell { height: 24px; border-radius: 4px; background: var(--surface-2); }
+.heat-cell.lv1 { background: color-mix(in srgb, var(--accent) 30%, var(--surface-2)); }
+.heat-cell.lv2 { background: color-mix(in srgb, var(--accent) 55%, var(--surface-2)); }
+.heat-cell.lv3 { background: color-mix(in srgb, var(--accent) 80%, var(--surface-2)); }
+.heat-cell.lv4 { background: var(--accent); }
+.heat-hour { font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; }
+
+/* Drilldown: timeline, resolve bars, occurrences. */
+.tl { position: relative; height: 56px; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; background: var(--surface-2); }
+.tl-mark { position: absolute; height: 18px; min-width: 8px; border-radius: 5px; background: var(--accent); }
+.tl-axis { display: flex; justify-content: space-between; margin-top: 6px; font-size: 11px; color: var(--muted); }
+.fixbars { display: grid; gap: 8px; }
+.fix-row { display: flex; align-items: center; gap: 10px; font-size: 12.5px; }
+.fix-label { width: 104px; flex: none; color: var(--muted); font-variant-numeric: tabular-nums; }
+.fix-track { flex: 1; min-width: 0; display: flex; height: 16px; border-radius: 4px; background: var(--surface-2); overflow: hidden; }
+.fix-ack { background: var(--warn); }
+.fix-work { background: var(--accent); }
+.fix-total { width: 56px; flex: none; text-align: right; font-weight: 600; font-variant-numeric: tabular-nums; }
+.occ-wrap { overflow-x: auto; }
+.occ { width: 100%; min-width: 560px; border-collapse: collapse; font-size: 13px; font-variant-numeric: tabular-nums; }
+.occ th { text-align: left; padding: 6px 8px; font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
+.occ td { padding: 10px 8px; border-top: 1px solid var(--border); }
 @media (min-width: 900px) {
   .stat-tiles { grid-template-columns: repeat(3, 1fr); }
 }
+@media (min-width: 900px) {
+  .stats-split { grid-template-columns: 3fr 1fr; align-items: stretch; }
+}
 
 /* On-call, wide: the hero and the week on the left, your shifts on the right.
    .view-page > * caps its children at 760px, so the grid lifts that. */

internal/web/static/js/account.js

@@ -10,11 +10,17 @@ const view = () => document.getElementById('view-more');
 
 // Rendered once per visit rather than on every poll, so a half-typed password
 // is never wiped out from under you.
-export function show() {
+export async function show() {
-  render();
+  let providers = [];
+  try {
+    providers = await api.notifyProviders();
+  } catch {
+    /* the card says the server can send nothing, which is the safe reading */
+  }
+  render(providers);
 }
 
-function render() {
+function render(providers) {
   const { user, has_password: hasPassword } = state.me;
   clear(view(),
     h('div', { class: 'card account-card' },
@@ -24,7 +30,7 @@ function render() {
         h('div', { class: 'account-email', text: user.email }))),
 
     h('div', { class: 'page-head' }, h('h2', { text: 'Notifications' })),
-    notifyForm(user),
+    notifyCard(user, providers),
 
     ...passwordSection(user, hasPassword),
 
@@ -59,79 +65,132 @@ function themePicker() {
     h('p', { class: 'muted small', text: 'System follows your device. This applies to this browser only.' }));
 }
 
-// Where this user's pages go. The onboarding checklist's first step sends
+// Where this user's pages go. Each channel is one way of reaching them — an ntfy
-// people here for it, and until now there was nothing here to send them to:
+// topic today, other kinds as the server grows them — and a page goes to every
-// the topic could only be set with curl or by an administrator.
+// channel that is switched on. Which kinds exist is the server's to say: the
-//
+// list comes from /notify/providers, so a deployment with one kind configured
-// The topic is the whole address — the server it is published to is the
+// shows no picker.
-// install's one ntfy, set in the deployment and not something a user picks.
+function notifyCard(user, providers) {
-function notifyStatus(topic) {
+  const root = h('div', { class: 'card pw-form' });
-  return topic ? `Topic: ${topic}` : 'No topic set — pages go to the team’s fallback topic.';
+  let channels = user.channels || [];
-}
+  const label = (type) => providers.find((p) => p.type === type)?.label || type;
 
-function notifyForm(user) {
+  // Keep the cached user in step, so the onboarding checklist stops asking for
-  const err = h('p', { class: 'form-error', role: 'alert', hidden: true });
+  // this and the test button works without a reload.
-  const topic = h('input', {
+  const commit = (next) => {
-    name: 'ntfy_topic', type: 'text', autocomplete: 'off',
+    channels = next;
-    autocapitalize: 'none', spellcheck: false,
+    state.me.user.channels = next;
-    value: user.ntfy_topic || '',
+    draw();
-    placeholder: 'terdut-a7f3c91e',
+  };
-  });
-  const submit = h('button', { class: 'btn btn-primary', type: 'submit', text: 'Save topic' });
 
-  const form = h('form', { class: 'stacked-form' },
+  const row = (c) => {
-    h('label', {},
+    const known = providers.some((p) => p.type === c.type);
-      h('span', { text: 'ntfy topic' }),
+    return h('li', { class: c.enabled ? 'channel' : 'channel off' },
-      topic),
+      h('div', { class: 'channel-main' },
-    h('p', { class: 'muted small' },
+        h('strong', { text: label(c.type) }),
-      'Subscribe to this topic in the ntfy app and incidents assigned to you ',
+        h('code', { text: c.address }),
-      'reach your phone. Leave it empty and they page the team’s fallback ',
+        !c.enabled && h('span', { class: 'muted small', text: 'off' }),
-      'topic instead.'),
+        !known && h('span', { class: 'load-error small', text: 'this server cannot deliver this kind' })),
-    // Worth saying plainly: people reach for their own name, and the topic is
+      h('div', { class: 'row-actions' },
-    // the only thing standing between a stranger and their pages.
+        h('button', {
-    h('p', { class: 'muted small' },
+          class: 'btn-sm', type: 'button', text: 'Send a test',
-      'Anyone who knows the topic can read your pages and publish to it, so ',
+          disabled: !known,
-      'pick something unguessable rather than your name.'),
+          onclick: async (e) => {
-    err,
+            e.target.disabled = true;
-    submit,
-  );
-
-  const status = h('p', { class: 'muted', text: notifyStatus(user.ntfy_topic) });
-  const summary = h('summary', { text: user.ntfy_topic ? 'Change topic' : 'Set a topic' });
-  const details = h('details', { class: 'account-fold' }, summary, form);
-
-  // Only offered once a topic is saved: the test publishes to whatever the
-  // server has stored, not to whatever is half-typed in the field.
-  const test = h('button', {
-    class: 'btn', type: 'button', text: 'Send a test push',
-    hidden: !user.ntfy_topic,
-    onclick: async () => {
-      test.disabled = true;
             try {
-        await api.testNotification();
+              await api.testChannel(user.id, c.id);
-        toast('Sent. If nothing arrives, the topic is wrong or ntfy is not reachable.');
+              toast(`Sent. If nothing arrives, check the ${label(c.type)} address.`);
             } catch (ex) {
               toast(ex.message, 'error');
             } finally {
-        test.disabled = false;
+              e.target.disabled = !known;
             }
           },
+        }),
+        h('button', {
+          class: 'btn-sm', type: 'button', text: c.enabled ? 'Turn off' : 'Turn on',
+          onclick: async () => {
+            try {
+              const updated = await api.updateChannel(user.id, c.id, { enabled: !c.enabled });
+              commit(channels.map((x) => (x.id === c.id ? updated : x)));
+            } catch (ex) {
+              toast(ex.message, 'error');
+            }
+          },
+        }),
+        h('button', {
+          class: 'btn-sm danger', type: 'button', text: 'Remove',
+          onclick: async () => {
+            try {
+              await api.removeChannel(user.id, c.id);
+              commit(channels.filter((x) => x.id !== c.id));
+            } catch (ex) {
+              toast(ex.message, 'error');
+            }
+          },
+        })));
+  };
+
+  const draw = () => {
+    const enabled = channels.filter((c) => c.enabled).length;
+    clear(root,
+      h('p', { class: 'muted', text: channelStatus(channels.length, enabled, providers.length) }),
+      channels.length > 0 && h('ul', { class: 'channels' }, ...channels.map(row)),
+      providers.length > 0 && addChannelForm(user, providers, (created) => commit([...channels, created])));
+  };
+  draw();
+  return root;
+}
+
+function channelStatus(total, enabled, providerCount) {
+  if (providerCount === 0) {
+    return 'This server has no way of sending notifications set up, so it cannot page anyone. ' +
+      'An administrator configures that in the deployment.';
+  }
+  if (total === 0) return 'No channels yet — pages go to the team’s fallback instead of you.';
+  if (enabled === 0) return 'Every channel is off — pages go to the team’s fallback instead of you.';
+  return `Pages reach you on ${enabled} channel${enabled === 1 ? '' : 's'}.`;
+}
+
+function addChannelForm(user, providers, onAdded) {
+  const err = h('p', { class: 'form-error', role: 'alert', hidden: true });
+  const type = h('select', { name: 'type', 'aria-label': 'Kind of channel' },
+    ...providers.map((p) => h('option', { value: p.type, text: p.label })));
+  const address = h('input', {
+    name: 'address', type: 'text', autocomplete: 'off',
+    autocapitalize: 'none', spellcheck: false, required: true,
   });
+  const labelText = h('span');
+  const hint = h('p', { class: 'muted small' });
+  const submit = h('button', { class: 'btn btn-primary', type: 'submit', text: 'Add channel' });
+
+  // The field's name and the advice under it belong to the provider: an ntfy
+  // topic is a secret, an email address is not, and the page should not guess.
+  const describe = () => {
+    const p = providers.find((x) => x.type === type.value);
+    labelText.textContent = p?.address_label || 'Address';
+    hint.textContent = p?.address_hint || '';
+  };
+  type.addEventListener('change', describe);
+  describe();
+
+  const form = h('form', { class: 'stacked-form' },
+    providers.length > 1 && h('label', {}, h('span', { text: 'Kind' }), type),
+    h('label', {}, labelText, address),
+    hint,
+    err,
+    submit);
 
   form.addEventListener('submit', async (e) => {
     e.preventDefault();
     err.hidden = true;
     submit.disabled = true;
     try {
-      const updated = await api.setNotifyTarget(user.id, topic.value.trim());
+      const created = await api.addChannel(user.id, type.value, address.value.trim());
-      // Keep the cached user in step, so the onboarding checklist stops
+      address.value = '';
-      // asking for this and the test button appears without a reload.
-      state.me.user = updated;
-      status.textContent = notifyStatus(updated.ntfy_topic);
-      summary.textContent = updated.ntfy_topic ? 'Change topic' : 'Set a topic';
-      test.hidden = !updated.ntfy_topic;
       details.open = false;
-      toast(updated.ntfy_topic ? 'Topic saved' : 'Topic cleared. Your pages go to the team’s fallback topic.');
+      onAdded(created);
+      toast('Channel added. Send a test to make sure it arrives.');
     } catch (ex) {
       err.textContent = ex.message;
       err.hidden = false;
@@ -140,10 +199,8 @@ function notifyForm(user) {
     }
   });
 
-  return h('div', { class: 'card pw-form' },
+  const details = h('details', { class: 'account-fold' }, h('summary', { text: 'Add a channel' }), form);
-    status,
+  return details;
-    h('div', { class: 'row-actions' }, test),
-    details);
 }
 
 // With password login switched off a password opens nothing, so somebody who

internal/web/static/js/adminuser.js

@@ -23,6 +23,13 @@ let data = null; // { user, teams, allTeams }
 let error = null;
 let busy = false;
 
+// Which kinds of channel the person has, never where they go: an address can be
+// a secret, and the server does not send it to anybody but its owner.
+function channelSummary(channels) {
+  if (!channels || !channels.length) return 'None of their own';
+  return channels.map((c) => (c.enabled ? c.type : `${c.type} (off)`)).join(', ');
+}
+
 export function show(route) {
   const next = route && route.user != null ? route.user : null;
   if (next !== userID) {
@@ -104,7 +111,7 @@ function identityCard() {
     h('dl', { class: 'user-facts' },
       fact('Email', u.email),
       fact('Joined', when(u.created_at)),
-      fact('Notifications', u.ntfy_topic ? `ntfy: ${u.ntfy_topic}` : 'None of their own'),
+      fact('Notifications', channelSummary(u.channels)),
       u.disabled_at && fact('Disabled', when(u.disabled_at)),
     ),
     // Both of these refuse your own account, and the last administrator's. An

internal/web/static/js/api.js

@@ -76,10 +76,22 @@ export const users = () => call('GET', '/users');
 // about anybody else, which is what the admin page's per-user view needs.
 export const userTeams = (id) => call('GET', `/users/${id}/teams`);
 
-// Where this user's pages go. An empty topic clears it, which the server
+// Where this user's pages go: a list of channels, each a provider type and an
-// treats as "no topic of their own" rather than an error.
+// address that provider understands. Only the owner is sent the addresses.
-export const setNotifyTarget = (id, ntfyTopic) =>
+export const channels = (userID) => call('GET', `/users/${userID}/channels`);
-  call('PUT', `/users/${id}/notify`, { body: { ntfy_topic: ntfyTopic } });
+export const addChannel = (userID, type, address) =>
+  call('POST', `/users/${userID}/channels`, { body: { type, address } });
+// patch is { enabled } and/or { address }.
+export const updateChannel = (userID, channelID, patch) =>
+  call('PATCH', `/users/${userID}/channels/${channelID}`, { body: patch });
+export const removeChannel = (userID, channelID) =>
+  call('DELETE', `/users/${userID}/channels/${channelID}`);
+export const testChannel = (userID, channelID) =>
+  call('POST', `/users/${userID}/channels/${channelID}/test`);
+
+// Which kinds of channel this server can deliver: [{ type, label, address_label,
+// address_hint }]. Empty when the server has nothing configured.
+export const notifyProviders = () => call('GET', '/notify/providers');
 
 // incidents
 export const incidents = (query, opts) => call('GET', '/incidents', { query, ...opts });
@@ -103,6 +115,8 @@ export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/note
 export const statsIncidents = (query) => call('GET', '/stats/incidents', { query });
 export const statsTop = (query) => call('GET', '/stats/alerts/top', { query });
 export const statsByHour = (query) => call('GET', '/stats/alerts/by-hour', { query });
+export const statsHeatmap = (query) => call('GET', '/stats/alerts/heatmap', { query });
+export const statsAlert = (name, query) => call('GET', `/stats/alerts/${encodeURIComponent(name)}`, { query });
 export const statsByDay = (query) => call('GET', '/stats/alerts/by-day', { query });
 
 // alerts
@@ -119,7 +133,6 @@ export const createInvite = (id, role, maxUses) =>
   call('POST', `/teams/${id}/invites`, { body: { role, max_uses: maxUses } });
 export const revokeInvite = (id, inviteID) => call('DELETE', `/teams/${id}/invites/${inviteID}`);
 
-export const testNotification = () => call('POST', '/me/notify/test');
 export const dismissOnboarding = (dismissed) =>
   call('PUT', '/me/onboarding', { body: { dismissed } });
 

internal/web/static/js/app.js

@@ -59,6 +59,8 @@ function parseRoute(pathname) {
   // /admin/teams/7 fall through to the queue.
   const g = pathname.match(/^\/admin\/teams\/(\d+)\/?$/);
   if (g) return { section: 'adminteam', team: Number(g[1]) };
+  const sa = pathname.match(/^\/stats\/alert\/([^/]+)\/?$/);
+  if (sa) return { section: 'stats', alert: decodeURIComponent(sa[1]) };
   const name = pathname.replace(/^\/|\/$/g, '');
   // The Admin and Team tabs' sub-sections are routes of their own. Each view
   // owns the table of its own, since each also builds the strip that links to
@@ -147,7 +149,7 @@ function render() {
   // the settings should not land you halfway down them. So does a changed
   // subject — one team to the next is two pages, not one scrolled page.
   else if (prev.section !== route.section || prev.tab !== route.tab
-    || prev.user !== route.user || prev.team !== route.team) window.scrollTo(0, 0);
+    || prev.user !== route.user || prev.team !== route.team || prev.alert !== route.alert) window.scrollTo(0, 0);
 
   updateTitle();
 }

internal/web/static/js/charts.js

@@ -0,0 +1,117 @@
+// Shared by the stats overview and the per-alert drilldown: the time-range
+// chips, the chart cards and the SVG column chart.
+
+import { h } from './ui.js';
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+// `days` counts back from today, inclusive; the server reads from/to as UTC
+// dates, so these are too.
+export const RANGES = [
+  { id: 'today', label: 'Today', days: 1 },
+  { id: '7d', label: '7d', days: 7 },
+  { id: '30d', label: '30d', days: 30 },
+  { id: '90d', label: '90d', days: 90 },
+  { id: 'all', label: 'All', days: null },
+];
+
+// The chosen range outlives navigating between the overview and a drilldown.
+let range = '30d';
+export const getRange = () => range;
+export const setRangeId = (id) => { range = id; };
+
+const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10);
+
+export function rangeQuery() {
+  const r = RANGES.find((x) => x.id === range);
+  if (!r.days) return {};
+  const now = Date.now();
+  return { from: utcDate(now - (r.days - 1) * DAY_MS), to: utcDate(now) };
+}
+
+export function chips(onPick, ranges = RANGES) {
+  return h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Time range' },
+    ranges.map((r) => h('button', {
+      class: 'chip',
+      type: 'button',
+      role: 'tab',
+      'aria-selected': String(r.id === range),
+      onclick: () => onPick(r.id),
+      text: r.label,
+    })));
+}
+
+export function card(title, content, note) {
+  return h('section', { class: 'chart-card card card-pad' },
+    h('div', { class: 'chart-head' },
+      h('h3', { class: 'chart-title', text: title }),
+      note && h('span', { class: 'chart-note', text: note })),
+    content);
+}
+
+export const pad2 = (n) => String(n).padStart(2, '0');
+export const WEEKDAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
+// Monday first, as the mockup has it; server weekdays are 0=Sunday.
+export const MONDAY_FIRST = [1, 2, 3, 4, 5, 6, 0];
+
+const SVG_NS = 'http://www.w3.org/2000/svg';
+
+function svg(tag, attrs = {}, text) {
+  const el = document.createElementNS(SVG_NS, tag);
+  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
+  if (text != null) el.textContent = text;
+  return el;
+}
+
+// A column chart: one bar per item, the value in a tooltip, and a label under
+// the items marked `tick`.
+export function columns(items, label) {
+  const W = 480;
+  const H = 140;
+  const base = H - 18;
+  const step = W / items.length;
+  const max = Math.max(...items.map((x) => x.value), 1);
+  const root = svg('svg', {
+    class: 'columns', viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': label,
+  });
+  root.appendChild(svg('line', { class: 'axis', x1: 0, x2: W, y1: base, y2: base }));
+  items.forEach((it, i) => {
+    const bh = it.value ? Math.max(2, (it.value / max) * (base - 6)) : 0;
+    const x = i * step + step * 0.15;
+    const g = svg('g', { class: 'col' });
+    g.appendChild(svg('title', {}, `${it.label}: ${it.value}`));
+    // A full-height transparent hit area, so a tiny bar is still hoverable.
+    g.appendChild(svg('rect', { class: 'col-hit', x: i * step, y: 0, width: step, height: base }));
+    if (bh) g.appendChild(svg('rect', { class: 'col-bar', x, y: base - bh, width: step * 0.7, height: bh, rx: 2 }));
+    root.appendChild(g);
+    if (it.tick) {
+      root.appendChild(svg('text', { class: 'col-label', x: i * step + step / 2, y: H - 4, 'text-anchor': 'middle' }, it.label));
+    }
+  });
+  return root;
+}
+
+// A 7×24 weekday-by-hour grid from `grid[weekday][hour]` (0=Sunday); cell
+// opacity steps with the count relative to the busiest cell.
+export function heatmap(grid) {
+  const max = Math.max(...grid.flat(), 1);
+  const level = (c) => (c === 0 ? 0 : Math.min(4, Math.ceil((c / max) * 4)));
+  const rows = MONDAY_FIRST.map((w) => h('div', { class: 'heat-row' },
+    h('span', { class: 'heat-day', text: WEEKDAYS[w] }),
+    h('div', { class: 'heat-cells' }, grid[w].map((c, hr) => h('div', {
+      class: `heat-cell lv${level(c)}`,
+      title: `${WEEKDAYS[w]} ${pad2(hr)}:00 UTC: ${c}`,
+    })))));
+  const axis = h('div', { class: 'heat-row' },
+    h('span', { class: 'heat-day' }),
+    h('div', { class: 'heat-cells' }, Array.from({ length: 24 }, (_, hr) =>
+      h('span', { class: 'heat-hour', text: hr % 3 === 0 ? pad2(hr) : '' }))));
+  const peak = grid.flatMap((row, d) => row.map((c, hr) => ({ d, hr, c }))).reduce((a, b) => (b.c > a.c ? b : a));
+  return h('div', {
+    class: 'heatmap',
+    role: 'img',
+    'aria-label': peak.c
+      ? `Heatmap of incidents by weekday and hour in UTC. The busiest slot is ${WEEKDAYS[peak.d]} at ${pad2(peak.hr)}:00 with ${peak.c}.`
+      : 'Heatmap of incidents by weekday and hour in UTC. Nothing in this range.',
+  }, rows, axis);
+}

internal/web/static/js/incident.js

@@ -265,14 +265,18 @@ function eventText(ev, named = false) {
     case 'note': return [strong(person), ' added a note'];
     case 'resolution_note': return [strong(person), ' noted what fixed it'];
     case 'notified': {
-      const to = person ? strong(person) : 'the fallback topic';
+      const to = person ? strong(person) : 'the fallback';
-      if (ev.detail === 'reminder') return ['Reminder sent to ', to];
+      // The detail is "<kind> via <channel type>"; a person with two channels
-      if (ev.detail === 'resolved') return ['Resolution sent to ', to];
+      // has two of these per page, so the channel is what tells them apart.
+      const [kind, via] = (ev.detail || '').split(' via ');
+      const on = via ? ` on ${via}` : '';
+      if (kind === 'reminder') return ['Reminder sent to ', to, on];
+      if (kind === 'resolved') return ['Resolution sent to ', to, on];
       // 'escalated' is a second, later page — the next level firing, not the
       // same page landing twice — so it reads as a bug unless told apart
       // from the initial 'triggered' page below.
-      if (ev.detail === 'escalated') return ['Escalation paged ', to];
+      if (kind === 'escalated') return ['Escalation paged ', to, on];
-      return ['Paged ', to];
+      return ['Paged ', to, on];
     }
     case 'notify_failed': return ['Notification failed', ev.detail ? `: ${ev.detail}` : ''];
     case 'deadman_silent': return ['Heartbeat went silent', ev.detail ? ` (${ev.detail})` : ''];

internal/web/static/js/onboarding.js

@@ -36,10 +36,10 @@ export async function load() {
     ]);
     steps = [
       {
-        id: 'topic',
+        id: 'channel',
         title: 'Set where your pages go',
-        text: 'An ntfy topic on your account. Without one, incidents assigned to you page the team’s fallback topic instead of your phone.',
+        text: 'A notification channel on your account. Without one, incidents assigned to you page the team’s fallback instead of you.',
-        done: Boolean(state.me?.user?.ntfy_topic),
+        done: Boolean(firstChannel()),
         action: { label: 'Account', go: '/more' },
       },
       {
@@ -115,10 +115,10 @@ function stepRow(step) {
           class: 'btn-sm', type: 'button', text: step.action.label,
           onclick: () => navigate(step.action.go),
         }),
-        // The topic step is the only one this page can finish by itself, and
+        // The channel step is the only one this page can finish by itself, and
         // the only proof that matters is a phone buzzing.
-        step.id === 'topic' && state.me?.user?.ntfy_topic && h('button', {
+        step.id === 'channel' && firstChannel() && h('button', {
-          class: 'btn-sm', type: 'button', text: 'Send a test push',
+          class: 'btn-sm', type: 'button', text: 'Send a test',
           disabled: busy,
           onclick: sendTest,
         }),
@@ -126,11 +126,15 @@ function stepRow(step) {
   );
 }
 
+// The first channel that is switched on: the one a page would reach first.
+const firstChannel = () => (state.me?.user?.channels || []).find((c) => c.enabled);
+
 async function sendTest() {
   busy = true;
   try {
-    await api.testNotification();
+    const channel = firstChannel();
-    testResult = { ok: true, text: 'Sent. If nothing arrives, the topic is wrong or ntfy is not reachable.' };
+    await api.testChannel(state.me.user.id, channel.id);
+    testResult = { ok: true, text: 'Sent. If nothing arrives, the address is wrong or the server cannot reach the provider.' };
   } catch (err) {
     testResult = { ok: false, text: err.message };
   } finally {

internal/web/static/js/stats.js

@@ -1,99 +1,87 @@
 // Statistics: how many incidents, how fast they are answered, and when and
 // what the alerts are. The same figures the TUI's Stats tab shows, over a
 // range picked with the chips. The server scopes them to the caller's teams.
+// /stats/alert/{name} is the drilldown behind a row of the top-alerts list.
 
 import * as api from './api.js';
-import { h, clear, emptyState, spinner } from './ui.js';
+import { h, clear, emptyState, spinner, icon } from './ui.js';
 import { duration } from './format.js';
-
+import { chips, card, heatmap, rangeQuery, getRange, setRangeId, pad2 } from './charts.js';
-const DAY_MS = 24 * 60 * 60 * 1000;
+import * as drill from './statsalert.js';
-
-// `days` counts back from today, inclusive; the server reads from/to as UTC
-// dates, so these are too.
-const RANGES = [
-  { id: 'today', label: 'Today', days: 1 },
-  { id: '7d', label: '7d', days: 7 },
-  { id: '30d', label: '30d', days: 30 },
-  { id: '90d', label: '90d', days: 90 },
-  { id: 'all', label: 'All', days: null },
-];
 
 const view = () => document.getElementById('view-stats');
 
-let range = '30d';
+// What the page shows: the overview (null) or one alert's drilldown.
+let alertName = null;
 let data = null;
 let error = null;
 
-const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10);
+export function show(route) {
-
+  const next = route?.alert ?? null;
-function query(r) {
+  if (next !== alertName) data = null;
-  if (!r.days) return {};
+  alertName = next;
-  const now = Date.now();
+  error = null;
-  return { from: utcDate(now - (r.days - 1) * DAY_MS), to: utcDate(now) };
-}
-
-export function show() {
   render();
   refresh();
 }
 
 export async function refresh() {
-  const requested = range;
+  const requested = getRange();
-  const q = query(RANGES.find((x) => x.id === range));
+  const name = alertName;
+  const q = rangeQuery();
   try {
-    const [incidents, top, byHour, byDay] = await Promise.all([
+    if (name != null) {
+      try {
+        data = await drill.load(name);
+      } catch (err) {
+        if (err.status !== 404) throw err;
+        data = 'none';
+      }
+    } else {
+      const [incidents, top, heat] = await Promise.all([
         api.statsIncidents(q),
         api.statsTop({ ...q, limit: 10 }),
-      api.statsByHour(q),
+        api.statsHeatmap(q),
-      api.statsByDay(q),
       ]);
-    if (requested !== range) return;
+      data = { incidents, top, heat };
-    data = { incidents, top, byHour, byDay };
+    }
+    if (requested !== getRange() || name !== alertName) return;
     error = null;
   } catch (err) {
-    if (requested !== range) return;
+    if (requested !== getRange() || name !== alertName) return;
     error = err.message;
   }
   render();
 }
 
 function setRange(id) {
-  if (id === range) return;
+  if (id === getRange()) return;
-  range = id;
+  setRangeId(id);
   data = null;
   render();
   refresh();
 }
 
 function render() {
-  const chips = h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Time range' },
+  if (alertName != null) {
-    RANGES.map((r) => h('button', {
+    clear(view(), drill.body(alertName, data, error, setRange));
-      class: 'chip',
+    return;
-      type: 'button',
+  }
-      role: 'tab',
-      'aria-selected': String(r.id === range),
-      onclick: () => setRange(r.id),
-      text: r.label,
-    })));
-
   let body;
   if (error && !data) body = h('div', { class: 'load-error', text: error });
   else if (!data) body = spinner();
-  else if (!data.incidents.total && !data.byHour.some((x) => x.count)) {
+  else if (!data.incidents.total) {
     body = emptyState('No data in this range', 'Nothing happened in this window.', 'chart');
   } else {
     body = h('div', { class: 'stats' },
       error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
       tiles(data.incidents),
-      data.top.length > 0 && card('Top alerts', topAlerts(data.top)),
+      data.top.length > 0 && card('Top alerts', topAlerts(data.top), 'Select an alert to see when and how often it fires'),
-      card('Alerts by hour (UTC)', columns(
+      h('div', { class: 'stats-split' },
-        data.byHour.map((x) => ({ label: String(x.hour), value: x.count, tick: x.hour % 6 === 0 })),
+        card('When incidents open', heatmap(data.heat.grid), 'Weekday × hour (UTC)'),
-        'Alerts by hour of day')),
+        patternsCard(data)));
-      card('Alerts by day', columns(
-        data.byDay.map((x) => ({ label: x.day_name.slice(0, 3), value: x.count, tick: true })),
-        'Alerts by day of week')));
   }
-  clear(view(), h('div', {}, chips, body));
+  clear(view(), h('div', {}, chips(setRange), body));
 }
 
 // A missing mean means nothing has been acknowledged or resolved yet.
@@ -112,57 +100,65 @@ function tiles(s) {
     tile('Mean time to resolve', mean(s.mttr_seconds)));
 }
 
-function card(title, content) {
+function spark(daily) {
-  return h('section', { class: 'chart-card card card-pad' },
+  const max = Math.max(...daily, 1);
-    h('h3', { class: 'chart-title', text: title }), content);
+  return h('span', { class: 'spark', 'aria-hidden': 'true' }, daily.map((c) => {
-}
+    const b = h('span', { class: c ? 'spark-bar on' : 'spark-bar' });
-
+    b.style.height = `${c ? 6 + (c / max) * 16 : 2}px`;
-// Ranked names with a bar scaled to the busiest one.
+    return b;
-function topAlerts(items) {
-  const max = Math.max(...items.map((x) => x.count), 1);
-  return h('ol', { class: 'hbars' }, items.map((x) => {
-    const fill = h('span', { class: 'hbar-fill' });
-    fill.style.width = `${Math.max(2, (x.count / max) * 100)}%`;
-    return h('li', { class: 'hbar' },
-      h('span', { class: 'hbar-name', title: x.name, text: x.name }),
-      h('span', { class: 'hbar-track' }, fill),
-      h('span', { class: 'hbar-count', text: String(x.count) }));
   }));
 }
 
-const SVG_NS = 'http://www.w3.org/2000/svg';
+const lastSeen = (unix) => {
+  const d = new Date(unix * 1000);
+  return `${d.toLocaleString('en', { month: 'short', timeZone: 'UTC' })} ${d.getUTCDate()}, ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}`;
+};
 
-function svg(tag, attrs = {}, text) {
+// Ranked names, each a link to its drilldown: a bar scaled to the busiest, the
-  const el = document.createElementNS(SVG_NS, tag);
+// last 30 days as a sparkline, when it last fired and how long it takes to fix.
-  for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v));
+function topAlerts(items) {
-  if (text != null) el.textContent = text;
+  const max = Math.max(...items.map((x) => x.count), 1);
-  return el;
+  const total = items.reduce((n, x) => n + x.count, 0);
+  return h('div', { class: 'toplist' },
+    h('div', { class: 'top-row top-head', 'aria-hidden': 'true' },
+      ['Alert', 'Count', 'Last 30 days', 'Last seen', 'Mean fix'].map((t) => h('span', { text: t }))),
+    items.map((x) => {
+      const fill = h('span', { class: 'hbar-fill' });
+      fill.style.width = `${Math.max(2, (x.count / max) * 100)}%`;
+      return h('a', { class: 'top-row', href: drill.path(x.name) },
+        h('span', { class: 'hbar-name', title: x.name, text: x.name }),
+        h('span', { class: 'top-count' },
+          h('span', { class: 'hbar-count', text: String(x.count) }),
+          h('span', { class: 'hbar-track' }, fill),
+          h('span', { class: 'top-pct', text: `${Math.round((x.count / total) * 100)}%` })),
+        spark(x.daily),
+        h('span', { class: 'top-cell', text: lastSeen(x.last_seen) }),
+        h('span', { class: 'top-cell', text: mean(x.mean_resolve_seconds) }),
+        icon('chevronRight'));
+    }));
 }
 
-// A column chart: one bar per item, the value in a tooltip, and a label under
+// Observations drawn from the figures already on the page.
-// the items marked `tick`.
+function patternsCard(d) {
-function columns(items, label) {
+  const flat = d.heat.grid;
-  const W = 480;
+  const total = flat.flat().reduce((a, b) => a + b, 0);
-  const H = 140;
+  const days = flat.map((row) => row.reduce((a, b) => a + b, 0));
-  const base = H - 18;
+  const hours = Array.from({ length: 24 }, (_, hr) => flat.reduce((a, row) => a + row[hr], 0));
-  const step = W / items.length;
+  const topDay = days.indexOf(Math.max(...days));
-  const max = Math.max(...items.map((x) => x.value), 1);
+  const topHour = hours.indexOf(Math.max(...hours));
-  const root = svg('svg', {
+  const topAlert = d.top[0];
-    class: 'columns', viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': label,
+  const full = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'];
-  });
+  const items = [];
-  root.appendChild(svg('line', { class: 'axis', x1: 0, x2: W, y1: base, y2: base }));
+  if (total > 1) {
-  items.forEach((it, i) => {
+    items.push([`${full[topDay]}s are the noisiest day`, `${days[topDay]} of ${total} incidents opened on a ${full[topDay]}.`]);
-    const bh = it.value ? Math.max(2, (it.value / max) * (base - 6)) : 0;
+    items.push([`Peak hour is ${pad2(topHour)}:00 UTC`, `${hours[topHour]} incidents opened in that single hour.`]);
-    const x = i * step + step * 0.15;
-    const g = svg('g', { class: 'col' });
-    g.appendChild(svg('title', {}, `${it.label}: ${it.value}`));
-    // A full-height transparent hit area, so a tiny bar is still hoverable.
-    g.appendChild(svg('rect', { class: 'col-hit', x: i * step, y: 0, width: step, height: base }));
-    if (bh) g.appendChild(svg('rect', { class: 'col-bar', x, y: base - bh, width: step * 0.7, height: bh, rx: 2 }));
-    root.appendChild(g);
-    if (it.tick) {
-      root.appendChild(svg('text', { class: 'col-label', x: i * step + step / 2, y: H - 4, 'text-anchor': 'middle' }, it.label));
   }
-  });
+  if (topAlert && total > 1 && topAlert.count / total >= 0.4) {
-  return root;
+    items.push(['One alert drives most of the volume', `${topAlert.name} is ${Math.round((topAlert.count / total) * 100)}% of everything that fired.`]);
+  }
+  const body = items.length
+    ? items.map(([t, s]) => h('div', { class: 'pattern' }, h('div', { class: 'pattern-title', text: t }), h('div', { class: 'pattern-sub', text: s })))
+    : h('p', { class: 'chart-note', text: 'Not enough incidents yet to see a pattern.' });
+  return card('Patterns', h('div', { class: 'patterns' }, body,
+    topAlert && h('a', { class: 'btn btn-ghost', href: drill.path(topAlert.name), text: `Open ${topAlert.name} →` })));
 }

internal/web/static/js/statsalert.js

@@ -0,0 +1,136 @@
+// The drilldown behind a row of the stats page's top-alerts list: when one
+// alert fired, how fast it was picked up and fixed. An occurrence is an
+// incident that alert opened (alert rows carry no history of their own).
+
+import * as api from './api.js';
+import { h, emptyState, spinner } from './ui.js';
+import { duration } from './format.js';
+import { chips, card, columns, rangeQuery, pad2, WEEKDAYS, MONDAY_FIRST } from './charts.js';
+
+const sec = (s) => (s == null ? '—' : duration(s * 1000));
+const fmtTime = (unix) => {
+  const d = new Date(unix * 1000);
+  const mon = d.toLocaleString('en', { month: 'short', timeZone: 'UTC' });
+  return `${mon} ${d.getUTCDate()}, ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}`;
+};
+
+export const path = (name) => `/stats/alert/${encodeURIComponent(name)}`;
+
+export async function load(name) {
+  return api.statsAlert(name, rangeQuery());
+}
+
+function tiles(d) {
+  const tile = (label, value, note) => h('div', { class: 'stat-tile' },
+    h('div', { class: 'stat-value', text: String(value) }),
+    h('div', { class: 'stat-label', text: label }),
+    note && h('div', { class: 'stat-note', text: note }));
+  return h('div', { class: 'stat-tiles' },
+    tile('Occurrences', d.count),
+    tile('Last seen', fmtTime(d.last_seen), 'UTC'),
+    tile('Average gap', d.mean_gap_seconds == null ? '—' : sec(d.mean_gap_seconds), 'between occurrences'),
+    tile('Mean time to acknowledge', sec(d.mean_ack_seconds), d.median_ack_seconds == null ? '' : `median ${sec(d.median_ack_seconds)}`),
+    tile('Mean time to resolve', sec(d.mean_resolve_seconds), d.median_resolve_seconds == null ? '' : `median ${sec(d.median_resolve_seconds)}`),
+    tile('Longest open', sec(d.longest_open_seconds)));
+}
+
+// Plain-language observations, derived from the same occurrences.
+function patterns(d) {
+  const n = d.count;
+  const out = [];
+  const sunday = d.by_weekday[0];
+  const top = d.by_weekday.indexOf(Math.max(...d.by_weekday));
+  if (n > 1 && d.by_weekday[top] > n / 2) out.push(`${d.by_weekday[top]} of ${n} fired on a ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][top]}`);
+  else if (sunday > 0 && n > 1) out.push(`${sunday} of ${n} fired on a Sunday`);
+  const peak = d.by_hour.indexOf(Math.max(...d.by_hour));
+  if (n > 1 && d.by_hour[peak] > 1) out.push(`Peak hour is ${pad2(peak)}:00 UTC (${d.by_hour[peak]} of ${n})`);
+  const times = d.occurrences.map((o) => o.triggered_at).sort((a, b) => a - b);
+  const repeats = times.filter((t, i) => i > 0 && t - times[i - 1] < 86400).length;
+  if (repeats) out.push(`${repeats} of ${n} fired again within 24 hours of the previous one`);
+  return out;
+}
+
+function patternStrip(d) {
+  const items = patterns(d);
+  if (!items.length) return null;
+  return h('div', { class: 'pattern-strip card card-pad' },
+    h('span', { class: 'chart-title', text: 'Pattern' }),
+    items.map((t) => h('span', { text: t })));
+}
+
+// One block per occurrence along a time axis from the first to the last, as wide
+// as the incident stayed open. Overlaps drop to a second lane.
+function timeline(d) {
+  const occ = [...d.occurrences].sort((a, b) => a.triggered_at - b.triggered_at);
+  const start = occ[0].triggered_at;
+  const end = Math.max(...occ.map((o) => o.triggered_at + (o.resolve_seconds || 0)), start + 3600);
+  const span = end - start;
+  let prev = -100;
+  const marks = occ.map((o) => {
+    const left = ((o.triggered_at - start) / span) * 100;
+    const lane = left - prev < 1.5 ? 1 : 0;
+    prev = left;
+    const m = h('div', {
+      class: 'tl-mark',
+      title: `${fmtTime(o.triggered_at)} UTC · ${o.resolve_seconds == null ? 'still open' : `open for ${sec(o.resolve_seconds)}`}`,
+    });
+    m.style.left = `${left}%`;
+    m.style.width = `${((o.resolve_seconds || 0) / span) * 100}%`;
+    m.style.top = `${8 + lane * 22}px`;
+    return m;
+  });
+  return h('div', {},
+    h('div', { class: 'tl', role: 'img', 'aria-label': `Timeline of ${occ.length} occurrences between ${fmtTime(start)} and ${fmtTime(end)} UTC.` }, marks),
+    h('div', { class: 'tl-axis' },
+      h('span', { text: fmtTime(start) }), h('span', { text: fmtTime(end) })));
+}
+
+// Per occurrence: the wait for an acknowledgement, then the time being worked.
+function fixBars(d) {
+  const rows = d.occurrences.filter((o) => o.resolve_seconds != null).slice(0, 12);
+  if (!rows.length) return h('p', { class: 'chart-note', text: 'Nothing resolved in this range.' });
+  const max = Math.max(...rows.map((o) => o.resolve_seconds), 1);
+  return h('div', { class: 'fixbars' }, rows.map((o) => {
+    const ack = Math.min(o.ack_seconds ?? 0, o.resolve_seconds);
+    const a = h('span', { class: 'fix-ack', title: `Waiting for acknowledgement: ${sec(ack)}` });
+    const w = h('span', { class: 'fix-work', title: `Being worked: ${sec(o.resolve_seconds - ack)}` });
+    a.style.width = `${(ack / max) * 100}%`;
+    w.style.width = `${((o.resolve_seconds - ack) / max) * 100}%`;
+    return h('div', { class: 'fix-row' },
+      h('span', { class: 'fix-label', text: fmtTime(o.triggered_at) }),
+      h('span', { class: 'fix-track' }, a, w),
+      h('span', { class: 'fix-total', text: sec(o.resolve_seconds) }));
+  }));
+}
+
+function table(d) {
+  return h('div', { class: 'occ-wrap' },
+    h('table', { class: 'occ' },
+      h('thead', {}, h('tr', {}, ['Fired (UTC)', 'Time to ack', 'Duration', 'Acknowledged by', 'Status'].map((t) => h('th', { text: t })))),
+      h('tbody', {}, d.occurrences.map((o) => h('tr', {},
+        h('td', {}, h('a', { href: `/incidents/${o.incident_id}`, text: fmtTime(o.triggered_at) }),
+          ` ${WEEKDAYS[new Date(o.triggered_at * 1000).getUTCDay()]}`),
+        h('td', { text: sec(o.ack_seconds) }),
+        h('td', { text: sec(o.resolve_seconds) }),
+        h('td', { text: o.acknowledged_by || '—' }),
+        h('td', { text: o.status }))))));
+}
+
+export function body(name, d, error, onRange) {
+  const head = h('div', { class: 'stats-head' },
+    h('a', { class: 'btn btn-ghost', href: '/stats', text: '← Stats' }),
+    h('h2', { class: 'stats-alert-name', text: name }),
+    chips(onRange));
+  if (error && !d) return h('div', {}, head, h('div', { class: 'load-error', text: error }));
+  if (!d) return h('div', {}, head, spinner());
+  if (d === 'none') return h('div', {}, head, emptyState('No occurrences in this range', 'This alert did not open an incident in this window.', 'chart'));
+  return h('div', {}, head, h('div', { class: 'stats' },
+    error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
+    tiles(d),
+    patternStrip(d),
+    card('When it fired', timeline(d), 'Each block spans how long the incident stayed open'),
+    card('By hour (UTC)', columns(d.by_hour.map((v, hr) => ({ label: pad2(hr), value: v, tick: hr % 6 === 0 })), 'Occurrences by hour of day')),
+    card('By weekday', columns(MONDAY_FIRST.map((w) => ({ label: WEEKDAYS[w], value: d.by_weekday[w], tick: true })), 'Occurrences by weekday')),
+    card('Time to resolve', fixBars(d), 'Amber waited for an acknowledgement; blue was worked'),
+    card('All occurrences', table(d), `Newest first${d.count > d.occurrences.length ? `, latest ${d.occurrences.length} of ${d.count}` : ''}`)));
+}

internal/web/static/js/team.js

@@ -139,7 +139,7 @@ function render() {
   if (!data) {
     clear(view(), error
       ? h('div', { class: 'load-error', text: error })
-      : h('div', { class: 'card' }, h('p', { class: 'muted', text: 'You are not in a team yet.' })));
+      : h('div', { class: 'card card-pad' }, h('p', { class: 'muted', text: 'You are not in a team yet.' })));
     return;
   }
   clear(view(),
@@ -213,7 +213,7 @@ function overview() {
       owners === 1 ? '1 owner, full access.' : `${owners} owners, full access.`),
     menuCard('/team/escalation', 'Escalation', levels || null,
       levels
-        ? `${levels} escalation level${levels === 1 ? '' : 's'}${data.escalation.fallback_topic ? ', then a fallback topic.' : '.'}`
+        ? `${levels} escalation level${levels === 1 ? '' : 's'}${data.escalation.fallback?.address ? ', then a fallback.' : '.'}`
         : 'No ladder — nobody but the first person is woken.'),
     menuCard('/team/sources', 'Alert sources', keys || null,
       keys
@@ -630,16 +630,16 @@ function escalationCard() {
 
   if (levels.length) {
     steps.push(h('li', { class: 'step' },
-      h('div', { class: 'step-rail' }, h('span', { class: `step-node end${esc.fallback_topic ? '' : ' warn'}` },
+      h('div', { class: 'step-rail' }, h('span', { class: `step-node end${esc.fallback?.address ? '' : ' warn'}` },
-        icon(esc.fallback_topic ? 'bell' : 'flag', 'icon'))),
+        icon(esc.fallback?.address ? 'bell' : 'flag', 'icon'))),
       h('div', { class: 'step-body' },
         h('div', { class: 'step-top' },
-          h('strong', { text: esc.fallback_topic ? 'Fallback topic' : 'End of the chain' })),
+          h('strong', { text: esc.fallback?.address ? 'Fallback' : 'End of the chain' })),
-        esc.fallback_topic
+        esc.fallback?.address
-          ? h('div', { class: 'step-pages' }, h('code', { text: esc.fallback_topic }), ' is paged once.')
+          ? h('div', { class: 'step-pages' }, `${esc.fallback.type}: `, h('code', { text: esc.fallback.address }), ' is paged once.')
           : h('div', { class: 'callout', role: 'status' }, icon('alertTriangle', 'icon'),
             h('div', {},
-              h('strong', { text: 'No fallback topic' }),
+              h('strong', { text: 'No fallback' }),
               h('p', { text: 'After the last level the chain just ends and nobody else is woken.' }),
               isOwner() && h('button', {
                 class: 'btn btn-sm', type: 'button', onclick: () => openLadderEditor({ focusFallback: true }),
@@ -682,11 +682,17 @@ function escalationCard() {
 // it wholesale: the levels are an order, and patching one rung would leave the
 // numbering of the others undecided. The draft lives in the sheet, so a poll of
 // the page underneath cannot throw away half an edit.
-function openLadderEditor({ focusFallback = false } = {}) {
+async function openLadderEditor({ focusFallback = false } = {}) {
   const esc = data.escalation || {};
+  // Which kinds of fallback this server can deliver. Failing to ask leaves the
+  // fallback uneditable rather than the whole ladder.
+  const providers = await api.notifyProviders().catch(() => []);
   const draft = {
     repeat_count: esc.repeat_count || 0,
-    fallback_topic: esc.fallback_topic || '',
+    fallback: {
+      type: esc.fallback?.type || providers[0]?.type || '',
+      address: esc.fallback?.address || '',
+    },
     levels: (esc.levels || []).map((l) => ({
       timeout_seconds: l.timeout_seconds,
       targets: (l.targets || []).map((t) => ({ kind: t.kind, user_id: t.user_id })),
@@ -734,12 +740,23 @@ function openLadderEditor({ focusFallback = false } = {}) {
       value: String(draft.repeat_count),
       oninput: (e) => { draft.repeat_count = Number(e.target.value); },
     });
-    const fallback = fallbackInput = h('input', {
-      type: 'text', value: draft.fallback_topic, placeholder: 'terdut-oncall-all',
-      oninput: (e) => { draft.fallback_topic = e.target.value; },
-    });
     parts.push(h('label', {}, 'Repeat the whole ladder ', repeat, ' more times'));
-    parts.push(h('label', {}, 'Then page this ntfy topic once ', fallback));
+    if (providers.length) {
+      const fallbackKind = h('select', {
+        'aria-label': 'Kind of fallback',
+        onchange: (e) => { draft.fallback.type = e.target.value; },
+      }, ...providers.map((p) => h('option', {
+        value: p.type, text: p.label, selected: p.type === draft.fallback.type,
+      })));
+      const fallback = fallbackInput = h('input', {
+        type: 'text', value: draft.fallback.address, placeholder: 'terdut-oncall-all',
+        autocomplete: 'off', autocapitalize: 'none', spellcheck: false,
+        oninput: (e) => { draft.fallback.address = e.target.value; },
+      });
+      parts.push(h('label', {}, 'Then page ',
+        providers.length > 1 ? fallbackKind : providers[0].label, ' ', fallback, ' once'));
+      parts.push(h('p', { class: 'muted small', text: 'Leave it empty for no fallback. Whoever knows this address is paged in place of a person.' }));
+    }
     clear(body, ...parts);
   };
   paint();
@@ -747,7 +764,12 @@ function openLadderEditor({ focusFallback = false } = {}) {
   const save = h('button', { class: 'btn btn-primary', type: 'button', text: 'Save ladder' });
   save.addEventListener('click', async () => {
     try {
-      await api.setEscalation(teamID, draft);
+      // An address with no type, or a type with no address, is refused; an empty
+      // address means "no fallback", so send it as such.
+      const fallback = draft.fallback.address.trim()
+        ? { type: draft.fallback.type, address: draft.fallback.address.trim() }
+        : { type: '', address: '' };
+      await api.setEscalation(teamID, { ...draft, fallback });
     } catch (err) {
       problem.textContent = err.message;
       problem.hidden = false;
@@ -1164,7 +1186,7 @@ function openSwitchDetail(sw) {
 
 const MEMBER_STATUS = {
   oncall: { label: 'On call', hint: 'The rota has them today.' },
-  reachable: { label: 'Reachable', hint: 'Has an ntfy topic, so a page would reach them.' },
+  reachable: { label: 'Reachable', hint: 'Has a notification channel, so a page would reach them.' },
   unpageable: { label: 'Can’t be paged', hint: 'A page to them would go nowhere.' },
 };
 

internal/web/stats_test.go

@@ -20,7 +20,9 @@ func TestStatsPageIsEmbedded(t *testing.T) {
 			t.Errorf("index.html lacks %s", want)
 		}
 	}
-	if _, err := fs.Stat(sub, "js/stats.js"); err != nil {
+	for _, f := range []string{"js/stats.js", "js/statsalert.js", "js/charts.js"} {
-		t.Errorf("js/stats.js not embedded: %v", err)
+		if _, err := fs.Stat(sub, f); err != nil {
+			t.Errorf("%s not embedded: %v", f, err)
+		}
 	}
 }