terdut-server/internal/api/notify_test.go

package api_test

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"

	"git.ryuvia.com/niklas/terdut-server/internal/api"
)

// ---------------------------------------------------------------------------
// Fake ntfy
// ---------------------------------------------------------------------------

// pushed is one message the fake ntfy received, in ntfy's JSON publish shape.
type pushed struct {
	Topic    string   `json:"topic"`
	Title    string   `json:"title"`
	Message  string   `json:"message"`
	Priority int      `json:"priority"`
	Tags     []string `json:"tags"`
	Click    string   `json:"click"`
	Actions  []struct {
		Action string `json:"action"`
		Label  string `json:"label"`
		URL    string `json:"url"`
		Method string `json:"method"`
		Clear  bool   `json:"clear"`
	} `json:"actions"`
}

// fakeNtfy records what the notifier published. status controls the reply, so a
// test can make delivery fail.
type fakeNtfy struct {
	*httptest.Server
	mu     sync.Mutex
	got    []pushed
	status int
}

func newFakeNtfy(t *testing.T) *fakeNtfy {
	t.Helper()
	f := &fakeNtfy{status: http.StatusOK}
	f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var msg pushed
		if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
			http.Error(w, "bad json", http.StatusBadRequest)
			return
		}
		f.mu.Lock()
		f.got = append(f.got, msg)
		status := f.status
		f.mu.Unlock()
		w.WriteHeader(status)
	}))
	t.Cleanup(f.Close)
	return f
}

func (f *fakeNtfy) messages() []pushed {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]pushed(nil), f.got...)
}

// topicsSince lists the topics published to since the last forget, which is how
// the escalation tests ask "who did this tick wake".
func (f *fakeNtfy) topicsSince(t *testing.T) []string {
	t.Helper()
	f.mu.Lock()
	defer f.mu.Unlock()
	out := make([]string, 0, len(f.got))
	for _, m := range f.got {
		out = append(out, m.Topic)
	}
	return out
}

// forget drops what has been published so far, so the next assertion is about
// this tick rather than the whole test.
func (f *fakeNtfy) forget() {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.got = nil
}

func (f *fakeNtfy) failWith(status int) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.status = status
}

// ---------------------------------------------------------------------------
// Harness
// ---------------------------------------------------------------------------

// notifyTS builds a server with notifications enabled, the admin on call today,
// and a topic on the admin — the setup every delivery test needs.
func notifyTS(t *testing.T, cfg api.NotifyConfig) (*ts, *fakeNtfy) {
	t.Helper()
	f := newFakeNtfy(t)
	cfg.BaseURL = f.URL
	s := newTS(t, cfg)

	putOnCall(t, s, 1)
	setTopic(t, s, 1, "terdut-admin")
	return s, f
}

func putOnCall(t *testing.T, s *ts, userID int) {
	t.Helper()
	today := time.Now().UTC().Format("2006-01-02")
	resp := s.req(t, http.MethodPost, "/api/teams/"+defaultTeam+"/schedule",
		map[string]any{"user_id": userID, "dates": []string{today}})
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		t.Fatalf("schedule assignment returned %d", resp.StatusCode)
	}
}

func setTopic(t *testing.T, s *ts, userID int, topic string) {
	t.Helper()
	resp := s.req(t, http.MethodPut,
		fmt.Sprintf("/api/users/%d/notify", userID), map[string]any{"ntfy_topic": topic})
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("set notify topic returned %d", resp.StatusCode)
	}
}

func (s *ts) sweepNotify(t *testing.T) {
	t.Helper()
	api.NotifySweep(context.Background(), s.db, s.notify)
}

// countNotifications reports how many outbox rows exist, optionally of one kind.
func (s *ts) countNotifications(t *testing.T, kind string) int {
	t.Helper()
	var n int
	query := "SELECT COUNT(*) FROM notifications"
	args := []any{}
	if kind != "" {
		query += " WHERE kind = $1"
		args = append(args, kind)
	}
	if err := s.db.QueryRow(query, args...).Scan(&n); err != nil {
		t.Fatalf("count notifications: %v", err)
	}
	return n
}

// fireCritical posts a single critical alert, which opens one incident.
func fireCritical(t *testing.T, s *ts) {
	t.Helper()
	postWebhook(t, s, []map[string]any{
		amAlert("fp-notify", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime,
			map[string]string{"severity": "critical"}),
	}, "{}:{alertname=\"DiskFull\"}")
}

// ---------------------------------------------------------------------------
// Delivery
// ---------------------------------------------------------------------------

func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)

	if got := s.countNotifications(t, "triggered"); got != 1 {
		t.Fatalf("expected 1 queued notification, got %d", got)
	}
	s.sweepNotify(t)

	msgs := f.messages()
	if len(msgs) != 1 {
		t.Fatalf("expected 1 push, got %d", len(msgs))
	}
	m := msgs[0]

	if m.Topic != "terdut-admin" {
		t.Errorf("expected the on-call user's topic, got %q", m.Topic)
	}
	if m.Priority != 5 {
		t.Errorf("expected max priority for a critical incident, got %d", m.Priority)
	}
	if !strings.Contains(m.Title, "DiskFull") {
		t.Errorf("expected the incident title in %q", m.Title)
	}
	if !strings.Contains(m.Message, "severity critical") {
		t.Errorf("expected the severity in %q", m.Message)
	}
	if m.Click != "https://terdut.example.com/incidents/1" {
		t.Errorf("unexpected click target %q", m.Click)
	}
	if len(m.Actions) != 1 || m.Actions[0].Label != "Acknowledge" {
		t.Fatalf("expected an Acknowledge action, got %+v", m.Actions)
	}
	if m.Actions[0].Method != http.MethodPost {
		t.Errorf("expected the action to POST, got %q", m.Actions[0].Method)
	}
}

// A delivered row must not be delivered again on the next pass.
func TestNotify_DeliveredOnlyOnce(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)
	s.sweepNotify(t)

	if got := len(f.messages()); got != 1 {
		t.Errorf("expected 1 push across two passes, got %d", got)
	}
}

// ---------------------------------------------------------------------------
// Delivery on the timeline
// ---------------------------------------------------------------------------

// notifyEvents picks the notifier's entries out of an incident's timeline.
// Asserted through the API rather than the table: the timeline is what the
// clients read, so its shape is the contract worth covering.
func notifyEvents(t *testing.T, s *ts, id int) []map[string]any {
	t.Helper()
	var out []map[string]any
	for _, e := range timeline(t, s, id) {
		if e["type"] == "notified" || e["type"] == "notify_failed" {
			out = append(out, e)
		}
	}
	return out
}

func TestNotify_DeliveryIsRecordedOnTheTimeline(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	// Queued is not notified: nothing is on the timeline until ntfy accepts it.
	if got := notifyEvents(t, s, 1); len(got) != 0 {
		t.Fatalf("expected no event before delivery, got %v", got)
	}

	s.sweepNotify(t)

	events := notifyEvents(t, s, 1)
	if len(events) != 1 {
		t.Fatalf("expected 1 notification event, got %v", events)
	}
	e := events[0]
	if e["type"] != "notified" {
		t.Errorf("expected a notified event, got %v", e["type"])
	}
	if e["detail"] != "triggered" {
		t.Errorf("expected the kind in detail, got %v", e["detail"])
	}
	if e["username"] != "admin" {
		t.Errorf("expected the paged user attached, got %v", e["username"])
	}
	// The topic is a shared secret with ntfy; the timeline is not the place for it.
	for _, v := range e {
		if s, ok := v.(string); ok && strings.Contains(s, "terdut-admin") {
			t.Errorf("expected the topic kept out of the timeline, found it in %v", e)
		}
	}
}

// A redelivery-free pass must not double-log either.
func TestNotify_TimelineRecordsOneEventPerDelivery(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{
		PublicURL:   "https://terdut.example.com",
		RepeatEvery: 15 * time.Minute,
	})

	fireCritical(t, s)
	s.sweepNotify(t)
	s.sweepNotify(t)
	s.ageNotifications(t, 20*time.Minute)
	s.sweepNotify(t)

	events := notifyEvents(t, s, 1)
	if len(events) != 2 {
		t.Fatalf("expected one event per delivery, got %v", events)
	}
	if events[0]["detail"] != "triggered" || events[1]["detail"] != "reminder" {
		t.Errorf("expected triggered then reminder, got %v and %v",
			events[0]["detail"], events[1]["detail"])
	}
}

func TestNotify_AllClearIsRecordedOnTheTimeline(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)

	postWebhook(t, s, []map[string]any{
		amAlert("fp-notify", "DiskFull", "resolved", "2026-05-20T10:00:00Z",
			"2026-05-20T11:00:00Z", map[string]string{"severity": "critical"}),
	}, "{}:{alertname=\"DiskFull\"}")
	s.sweepNotify(t)

	events := notifyEvents(t, s, 1)
	if len(events) != 2 {
		t.Fatalf("expected the all-clear recorded, got %v", events)
	}
	if events[1]["detail"] != "resolved" {
		t.Errorf("expected a resolved event, got %v", events[1]["detail"])
	}
}

// A page to the shared fallback belongs to nobody, and the timeline has to say
// so rather than attributing it to whoever happens to be on call now.
func TestNotify_FallbackDeliveryHasNoUser(t *testing.T) {
	f := newFakeNtfy(t)
	s := newTS(t, api.NotifyConfig{
		BaseURL:       f.URL,
		FallbackTopic: "terdut-oncall",
		PublicURL:     "https://terdut.example.com",
	})

	fireCritical(t, s)
	s.sweepNotify(t)

	events := notifyEvents(t, s, 1)
	if len(events) != 1 {
		t.Fatalf("expected 1 notification event, got %v", events)
	}
	if got, ok := events[0]["username"]; ok && got != nil && got != "" {
		t.Errorf("expected no user on a fallback-topic page, got %v", got)
	}
}

// The failure worth seeing: nobody was paged, and the timeline says so instead
// of looking exactly like a delivery that worked.
func TestNotify_ExhaustedRetriesAreRecordedOnce(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
	f.failWith(http.StatusInternalServerError)

	fireCritical(t, s)
	// One pass per attempt, each made due by clearing the backoff the last one set.
	for i := 0; i < 10; i++ {
		s.sweepNotify(t)
		s.exec(t, "UPDATE notifications SET send_after = $1 WHERE sent_at IS NULL",
			time.Now().Add(-time.Second).Unix())
	}

	events := notifyEvents(t, s, 1)
	if len(events) != 1 {
		t.Fatalf("expected exactly one failure event, got %v", events)
	}
	if events[0]["type"] != "notify_failed" {
		t.Errorf("expected notify_failed, got %v", events[0]["type"])
	}
	detail, _ := events[0]["detail"].(string)
	if !strings.HasPrefix(detail, "triggered: ") || !strings.Contains(detail, "500") {
		t.Errorf("expected the kind and the reason in %q", detail)
	}
}

// ---------------------------------------------------------------------------
// Acknowledging from the notification
// ---------------------------------------------------------------------------

func TestNotify_AckButtonAcknowledgesIncident(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)

	ackURL := f.messages()[0].Actions[0].URL
	// The action URL is built for the public hostname; point it at the test
	// server, which is the same handler.
	path := ackURL[strings.Index(ackURL, "/api/notify/ack/"):]

	resp, err := http.Post(s.URL+path, "application/json", nil)
	if err != nil {
		t.Fatalf("ack: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("expected 200 from the ack button, got %d", resp.StatusCode)
	}

	inc := getIncident(t, s, 1)
	if inc["status"] != "acknowledged" {
		t.Errorf("expected the incident acknowledged, got %v", inc["status"])
	}
	if inc["acknowledged_by"] != "admin" {
		t.Errorf("expected the ack attributed to the token's user, got %v", inc["acknowledged_by"])
	}

	// The timeline must record it the same way the authenticated route would.
	events := timeline(t, s, 1)
	if !contains(eventTypes(events), "acknowledged") {
		t.Errorf("expected an acknowledged event, got %v", eventTypes(events))
	}
	for _, e := range events {
		if e["type"] == "acknowledged" && e["username"] != "admin" {
			t.Errorf("expected the acknowledged event attributed to admin, got %v", e["username"])
		}
	}
}

func TestNotify_AckRejectsUnknownToken(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
	fireCritical(t, s)

	resp, err := http.Post(s.URL+"/api/notify/ack/deadbeef", "application/json", nil)
	if err != nil {
		t.Fatalf("ack: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("expected 404 for an unknown token, got %d", resp.StatusCode)
	}
	if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
		t.Errorf("expected the incident untouched, got %v", inc["status"])
	}
}

func TestNotify_AckRejectsExpiredToken(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)

	ackURL := f.messages()[0].Actions[0].URL
	path := ackURL[strings.Index(ackURL, "/api/notify/ack/"):]

	// Age the token past its TTL. The token's inputs are wall-clock timestamps,
	// so this is the same trick the sweeper tests use.
	s.exec(t, "UPDATE incident_ack_tokens SET expires_at = $1", time.Now().Add(-time.Minute).Unix())

	resp, err := http.Post(s.URL+path, "application/json", nil)
	if err != nil {
		t.Fatalf("ack: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("expected 404 for an expired token, got %d", resp.StatusCode)
	}
	if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
		t.Errorf("expected the incident untouched, got %v", inc["status"])
	}
}

// The sweeper is what stops expired tokens accumulating forever.
func TestNotify_SweepPurgesExpiredAckTokens(t *testing.T) {
	s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)
	s.exec(t, "UPDATE incident_ack_tokens SET expires_at = $1", time.Now().Add(-time.Minute).Unix())

	api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour, s.notify)

	var n int
	if err := s.db.QueryRow("SELECT COUNT(*) FROM incident_ack_tokens").Scan(&n); err != nil {
		t.Fatalf("count tokens: %v", err)
	}
	if n != 0 {
		t.Errorf("expected expired tokens purged, %d left", n)
	}
}

// ---------------------------------------------------------------------------
// Reminders
// ---------------------------------------------------------------------------

// ageNotifications backdates every sent notification so the next pass sees the
// reminder as due.
func (s *ts) ageNotifications(t *testing.T, by time.Duration) {
	t.Helper()
	s.exec(t, "UPDATE notifications SET created_at = $1 WHERE sent_at IS NOT NULL",
		time.Now().Add(-by).Unix())
}

func TestNotify_UnacknowledgedIncidentIsRenotified(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{
		PublicURL:   "https://terdut.example.com",
		RepeatEvery: 15 * time.Minute,
	})

	fireCritical(t, s)
	s.sweepNotify(t)
	s.ageNotifications(t, 20*time.Minute)
	s.sweepNotify(t)

	msgs := f.messages()
	if len(msgs) != 2 {
		t.Fatalf("expected a reminder push, got %d message(s)", len(msgs))
	}
	if !strings.Contains(msgs[1].Title, "Still unacknowledged") {
		t.Errorf("expected the reminder to say so, got %q", msgs[1].Title)
	}
	if msgs[1].Topic != "terdut-admin" {
		t.Errorf("expected the reminder on the same topic, got %q", msgs[1].Topic)
	}
	// Each page carries its own credential.
	if len(msgs[1].Actions) != 1 || msgs[1].Actions[0].URL == msgs[0].Actions[0].URL {
		t.Errorf("expected the reminder to carry a fresh ack token")
	}
}

func TestNotify_AcknowledgedIncidentStopsReminders(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{
		PublicURL:   "https://terdut.example.com",
		RepeatEvery: 15 * time.Minute,
	})

	fireCritical(t, s)
	s.sweepNotify(t)

	resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
	resp.Body.Close()

	s.ageNotifications(t, 20*time.Minute)
	s.sweepNotify(t)

	if got := len(f.messages()); got != 1 {
		t.Errorf("expected no reminder once acknowledged, got %d message(s)", got)
	}
}

// Snooze is the deliberate "not now", and it is what mutes the pager.
func TestNotify_SnoozedIncidentStopsReminders(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{
		PublicURL:   "https://terdut.example.com",
		RepeatEvery: 15 * time.Minute,
	})

	fireCritical(t, s)
	s.sweepNotify(t)

	resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]any{"duration": "1h"})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("snooze returned %d", resp.StatusCode)
	}
	resp.Body.Close()

	s.ageNotifications(t, 20*time.Minute)
	s.sweepNotify(t)

	if got := len(f.messages()); got != 1 {
		t.Errorf("expected no reminder while snoozed, got %d message(s)", got)
	}
}

func TestNotify_ZeroRepeatDisablesReminders(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)
	s.ageNotifications(t, 24*time.Hour)
	s.sweepNotify(t)

	if got := len(f.messages()); got != 1 {
		t.Errorf("expected reminders off, got %d message(s)", got)
	}
}

// ---------------------------------------------------------------------------
// Resolution
// ---------------------------------------------------------------------------

func TestNotify_ResolvedIncidentSendsAllClear(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)

	postWebhook(t, s, []map[string]any{
		amAlert("fp-notify", "DiskFull", "resolved", "2026-05-20T10:00:00Z",
			"2026-05-20T11:00:00Z", map[string]string{"severity": "critical"}),
	}, "{}:{alertname=\"DiskFull\"}")
	s.sweepNotify(t)

	msgs := f.messages()
	if len(msgs) != 2 {
		t.Fatalf("expected an all-clear push, got %d message(s)", len(msgs))
	}
	if !strings.HasPrefix(msgs[1].Title, "Resolved:") {
		t.Errorf("expected a resolved title, got %q", msgs[1].Title)
	}
	if msgs[1].Priority != 2 {
		t.Errorf("expected the all-clear at low priority, got %d", msgs[1].Priority)
	}
	// Nothing to acknowledge on a closed incident.
	if len(msgs[1].Actions) != 0 {
		t.Errorf("expected no actions on the all-clear, got %+v", msgs[1].Actions)
	}
}

// Closing an incident by hand sends nothing: the person who did it knows.
func TestNotify_ManualResolveSendsNothing(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})

	fireCritical(t, s)
	s.sweepNotify(t)

	resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
	resp.Body.Close()
	s.sweepNotify(t)

	if got := len(f.messages()); got != 1 {
		t.Errorf("expected no push for a manual resolve, got %d message(s)", got)
	}
}

// ---------------------------------------------------------------------------
// Routing and configuration
// ---------------------------------------------------------------------------

// With nobody on call the page goes to the shared fallback, and carries no
// Acknowledge button — there is no user to attribute the acknowledgement to.
func TestNotify_FallbackTopicHasNoAckButton(t *testing.T) {
	f := newFakeNtfy(t)
	s := newTS(t, api.NotifyConfig{
		BaseURL:       f.URL,
		FallbackTopic: "terdut-oncall",
		PublicURL:     "https://terdut.example.com",
	})

	fireCritical(t, s)
	s.sweepNotify(t)

	msgs := f.messages()
	if len(msgs) != 1 {
		t.Fatalf("expected 1 push, got %d", len(msgs))
	}
	if msgs[0].Topic != "terdut-oncall" {
		t.Errorf("expected the fallback topic, got %q", msgs[0].Topic)
	}
	if len(msgs[0].Actions) != 0 {
		t.Errorf("expected no ack button on a shared topic, got %+v", msgs[0].Actions)
	}
}

// Nobody on call and no fallback means there is nowhere to send: queueing would
// only pile up rows that can never be delivered.
func TestNotify_NoTargetQueuesNothing(t *testing.T) {
	f := newFakeNtfy(t)
	s := newTS(t, api.NotifyConfig{BaseURL: f.URL})

	fireCritical(t, s)

	if got := s.countNotifications(t, ""); got != 0 {
		t.Errorf("expected nothing queued without a target, got %d", got)
	}
	s.sweepNotify(t)
	if got := len(f.messages()); got != 0 {
		t.Errorf("expected no push, got %d", got)
	}
}

// The zero NotifyConfig is what every pre-existing test runs under.
func TestNotify_DisabledQueuesNothing(t *testing.T) {
	s := newTS(t)
	putOnCall(t, s, 1)
	setTopic(t, s, 1, "terdut-admin")

	fireCritical(t, s)

	if got := s.countNotifications(t, ""); got != 0 {
		t.Errorf("expected nothing queued with notifications off, got %d", got)
	}
}

// ---------------------------------------------------------------------------
// Retries
// ---------------------------------------------------------------------------

func TestNotify_FailedDeliveryRetriesWithBackoff(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
	f.failWith(http.StatusInternalServerError)

	fireCritical(t, s)
	s.sweepNotify(t)

	var attempts int
	var sentAt *int64
	var sendAfter int64
	var lastError *string
	if err := s.db.QueryRow(
		"SELECT attempts, sent_at, send_after, last_error FROM notifications WHERE id = 1").
		Scan(&attempts, &sentAt, &sendAfter, &lastError); err != nil {
		t.Fatalf("read notification: %v", err)
	}
	if attempts != 1 {
		t.Errorf("expected 1 attempt recorded, got %d", attempts)
	}
	if sentAt != nil {
		t.Errorf("expected the row unsent, got sent_at %v", *sentAt)
	}
	if sendAfter <= time.Now().Unix() {
		t.Errorf("expected the retry pushed into the future, got %d", sendAfter)
	}
	if lastError == nil || !strings.Contains(*lastError, "500") {
		t.Errorf("expected the failure recorded, got %v", lastError)
	}

	// Backing off means the next pass leaves it alone until it is due.
	s.sweepNotify(t)
	if got := len(f.messages()); got != 1 {
		t.Errorf("expected no immediate retry, got %d attempt(s)", got)
	}

	// Once due and once ntfy recovers, it goes out.
	f.failWith(http.StatusOK)
	s.exec(t, "UPDATE notifications SET send_after = $1 WHERE id = 1", time.Now().Add(-time.Second).Unix())
	s.sweepNotify(t)

	if err := s.db.QueryRow("SELECT sent_at FROM notifications WHERE id = 1").Scan(&sentAt); err != nil {
		t.Fatalf("read notification: %v", err)
	}
	if sentAt == nil {
		t.Error("expected the retry to succeed once ntfy recovered")
	}
}

// An ntfy outage must not produce a reminder backlog that all lands at once
// when it comes back: the previous page has to have been sent first.
func TestNotify_UnsentNotificationBlocksReminders(t *testing.T) {
	s, f := notifyTS(t, api.NotifyConfig{
		PublicURL:   "https://terdut.example.com",
		RepeatEvery: 15 * time.Minute,
	})
	f.failWith(http.StatusInternalServerError)

	fireCritical(t, s)
	s.sweepNotify(t)
	s.exec(t, "UPDATE notifications SET created_at = $1", time.Now().Add(-time.Hour).Unix())
	s.sweepNotify(t)

	if got := s.countNotifications(t, "reminder"); got != 0 {
		t.Errorf("expected no reminders queued behind an undelivered page, got %d", got)
	}
}