Page the on-call person when an incident opens

An incident opened, got assigned to whoever held today's schedule entry,
and then sat there silently until somebody thought to look. The schedule
and the incident model were both built; nothing reached the person
holding the pager.

Notifications go out through ntfy, over plain HTTP with no new
dependencies. Delivery is an outbox rather than an inline call: the pool
is limited to a single connection, so a POST made while holding the
webhook's transaction would stall every other request behind it. The
webhook inserts a row and a notifier goroutine sends it within a tick,
retrying with exponential backoff.

Only opening an incident has to resolve a topic from scratch. Reminders
and all-clears reuse whatever that first notification chose, which keeps
configuration out of resolveIfSettled and gives the right rule for free:
you only hear that something resolved if you were told it started.

Each push carries an Acknowledge button, because the useful thing to do
at 3am is stop the pager without unlocking anything. It POSTs to an
unauthenticated /api/notify/ack/{token} — a notification body lives on
the ntfy server and in the device cache, so a real API key must never
appear in one. The token is minted per delivery, scoped to one incident
and one action, and expires in a day.

Reminders repeat until the incident stops being untouched. The stop
conditions are the states that already mean somebody has it: acknowledged,
snoozed, resolved, archived. Snooze is the mute button, so there is no
separate reminder cap.

Notifications sent to the fallback topic carry no Acknowledge button. The
topic is shared, and a button on it would let any subscriber acknowledge
as somebody else.

internal/api/alertmanager.go

@@ -58,7 +58,7 @@ type ingested struct {
 	justResolved bool
 }
 
-func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
+func handleAlertmanagerWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var payload amPayload
 		if err := decodeJSON(r, &payload); err != nil {
@@ -69,7 +69,7 @@ func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
 		// Alertmanager retries anything that is not 2xx, and a retry of a payload
 		// we failed to store is more useful than an error it cannot act on — so
 		// failures are logged, not surfaced.
-		if err := ingest(r.Context(), db, payload); err != nil {
+		if err := ingest(r.Context(), db, notify, payload); err != nil {
 			log.Printf("webhook ingest (group %q): %v", payload.GroupKey, err)
 		}
 
@@ -80,7 +80,7 @@ func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
 // ingest stores a payload's alerts and reconciles the incident for its group.
 // The whole payload is one transaction: an incident that opened but whose alerts
 // failed to link would be a work item nobody could act on.
-func ingest(ctx context.Context, db *sql.DB, payload amPayload) error {
+func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, payload amPayload) error {
 	tx, err := db.BeginTx(ctx, nil)
 	if err != nil {
 		return err
@@ -96,7 +96,7 @@ func ingest(ctx context.Context, db *sql.DB, payload amPayload) error {
 	// resolution cascade are recomputed once per incident at the end.
 	touched := map[int64]bool{}
 
-	incidentID, err := incidentForGroup(ctx, tx, payload, accepted)
+	incidentID, err := incidentForGroup(ctx, tx, notify, payload, accepted)
 	if err != nil {
 		return err
 	}
@@ -258,7 +258,7 @@ func upsertAlerts(ctx context.Context, tx *sql.Tx, alerts []amAlert) ([]ingested
 // something actually started firing. Without that, a manually resolved incident
 // would reappear on the next repeat_interval re-send of an alert that never
 // stopped, and manual resolution would be meaningless.
-func incidentForGroup(ctx context.Context, tx *sql.Tx, payload amPayload, accepted []ingested) (int64, error) {
+func incidentForGroup(ctx context.Context, tx *sql.Tx, notify NotifyConfig, payload amPayload, accepted []ingested) (int64, error) {
 	var firstName string
 	anyFiring, anyNew := false, false
 	for _, a := range accepted {
@@ -297,12 +297,12 @@ func incidentForGroup(ctx context.Context, tx *sql.Tx, payload amPayload, accept
 	if !anyNew {
 		return 0, nil
 	}
-	return openIncident(ctx, tx, groupKey, payload.GroupLabels, firstName)
+	return openIncident(ctx, tx, notify, groupKey, payload.GroupLabels, firstName)
 }
 
 // openIncident creates an incident for a group and assigns it to whoever is on
 // call today, which is the point at which the schedule stops being decorative.
-func openIncident(ctx context.Context, tx *sql.Tx, groupKey string, groupLabels map[string]string, fallbackName string) (int64, error) {
+func openIncident(ctx context.Context, tx *sql.Tx, notify NotifyConfig, groupKey string, groupLabels map[string]string, fallbackName string) (int64, error) {
 	onCall, err := currentOnCall(ctx, tx)
 	if err != nil {
 		return 0, err
@@ -335,6 +335,13 @@ func openIncident(ctx context.Context, tx *sql.Tx, groupKey string, groupLabels
 			return 0, err
 		}
 	}
+
+	// Queue the page, but do not send it here: this runs inside the webhook's
+	// transaction on a single-connection pool, so an HTTP call would hold up
+	// every other request. The notifier picks the row up within a tick.
+	if err := enqueueOpened(ctx, tx, notify, id, onCall); err != nil {
+		return 0, err
+	}
 	return id, nil
 }