Record notification delivery on the incident timeline

An incident's history went quiet after "Incident opened": nothing said
that anybody had been paged, reminded, or told it resolved. Delivery
lived only in the notifications outbox, which no API exposes, so when a
page failed to arrive there was nothing in the product that said whether
it had been sent.

The notifier now writes two event types. A notified event once ntfy
accepts the publish, carrying the kind in detail and the paged user in
user_id — absent when the page went to the shared fallback topic, which
belongs to nobody. And a notify_failed event when a notification
exhausts its retries, which is the one worth having: without it a page
that never landed leaves the timeline identical to one that did.

Both are written from the delivery result rather than at enqueue. A
queued notification is an intention, and the timeline is append-only, so
claiming somebody was told before ntfy accepted it would be a lie that
stays there. A failed timeline write is logged rather than returned, so
it cannot make a delivered row look unsent and send the page twice.

The topic is deliberately in neither: it is a shared secret with the
ntfy server, and every API key can read the timeline.

No migration — incident_events.type is free text, unlike
notifications.kind.

internal/api/notifier.go

@@ -45,6 +45,19 @@ const (
 	notifyResolved  = "resolved"
 )
 
+// Timeline event types the notifier writes, so an incident's history says who
+// was paged and whether the page landed. Written from the delivery result
+// rather than at enqueue: a queued notification is an intention, and claiming
+// somebody was told before ntfy accepted it would be a lie the timeline keeps.
+//
+// The topic is deliberately absent from both. It is a shared secret with the
+// ntfy server — anyone holding it can publish to it — and the timeline is
+// readable by every API key.
+const (
+	eventNotified     = "notified"
+	eventNotifyFailed = "notify_failed"
+)
+
 // NotifyConfig is everything the notifier needs to reach ntfy and to build URLs
 // a phone can follow back to this server.
 type NotifyConfig struct {
@@ -208,6 +221,11 @@ func deliverPending(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
 			time.Now().Unix(), n.id); err != nil {
 			log.Printf("notifier: mark sent %d: %v", n.id, err)
 		}
+		// Logged, not returned: the page has already gone out, and treating a
+		// failed timeline write as a failed delivery would send it again.
+		if err := logEvent(ctx, db, n.incidentID, eventNotified, n.userID, nil, &n.kind); err != nil {
+			log.Printf("notifier: log delivery of %d: %v", n.id, err)
+		}
 		sent++
 	}
 	if sent > 0 {
@@ -243,6 +261,11 @@ func pendingNotifications(ctx context.Context, db *sql.DB) ([]outboxRow, error)
 }
 
 // markFailed bumps the attempt count and pushes the row out to its next retry.
+//
+// The attempt that exhausts the budget also writes a timeline event. Without it
+// a page that never landed leaves the incident's history identical to one that
+// did, which is the failure most worth seeing: nobody was told, and nothing
+// says so.
 func markFailed(ctx context.Context, db *sql.DB, n outboxRow, cause error) {
 	next := time.Now().Add(retryDelay(n.attempts)).Unix()
 	if _, err := db.ExecContext(ctx,
@@ -250,6 +273,14 @@ func markFailed(ctx context.Context, db *sql.DB, n outboxRow, cause error) {
 		next, cause.Error(), n.id); err != nil {
 		log.Printf("notifier: mark failed %d: %v", n.id, err)
 	}
+
+	if n.attempts+1 < notifyMaxAttempts {
+		return
+	}
+	detail := fmt.Sprintf("%s: %s", n.kind, cause)
+	if err := logEvent(ctx, db, n.incidentID, eventNotifyFailed, n.userID, nil, &detail); err != nil {
+		log.Printf("notifier: log failure of %d: %v", n.id, err)
+	}
 }
 
 // retryDelay doubles the wait per attempt, up to notifyRetryMax.