diff --git a/internal/api/notifier.go b/internal/api/notifier.go index 66a46a9..8f5b621 100644 --- a/internal/api/notifier.go +++ b/internal/api/notifier.go @@ -8,6 +8,7 @@ import ( "fmt" "log" "net/http" + "regexp" "strings" "time" @@ -409,9 +410,11 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID) } + title := pageTitle(inc) + switch n.kind { case notifyResolved: - msg.Title = "Resolved: " + inc.Title + msg.Title = "Resolved: " + title msg.Message = "All alerts stopped firing after " + humanDuration(time.Since(inc.TriggeredAt)) msg.Priority = ntfyPriorityLow @@ -419,9 +422,9 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify return msg case notifyReminder: - msg.Title = "Still unacknowledged: " + inc.Title + msg.Title = "Still unacknowledged: " + title default: - msg.Title = inc.Title + msg.Title = title } severity := derefString(inc.Severity) @@ -443,6 +446,48 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify return msg } +// originLabel is the label that says where an alert came from, for a team with +// several Kubernetes clusters behind it. It comes from Prometheus's +// externalLabels and reaches an incident through Alertmanager's group_by; the +// web UI reads the same label, and the README ("Several clusters, one team") +// explains how to set it up. +const originLabel = "cluster" + +// pageTitle is the incident's title for a notification. A phone's lock screen +// cuts a long title off at the end, and the incident title puts the grouping +// labels there, so the cluster would be the first thing lost. When the incident +// has an origin it leads instead, "[prod-eu] PodRestarting (namespace=foo)", and +// is dropped from the parenthesis so it is not said twice. A title that is not +// in incidentTitle's "name (k=v, k=v)" shape keeps its text and gains the prefix. +func pageTitle(inc models.Incident) string { + origin := inc.GroupLabels[originLabel] + if origin == "" { + return inc.Title + } + return "[" + origin + "] " + titleWithoutLabel(inc.Title, originLabel, origin) +} + +var titleShape = regexp.MustCompile(`(?s)^(.*?) \((.*)\)$`) + +// titleWithoutLabel removes "key=value" from the parenthesised tail of a title +// built by incidentTitle, and the parentheses with it if nothing else is left. +func titleWithoutLabel(title, key, value string) string { + m := titleShape.FindStringSubmatch(title) + if m == nil { + return title + } + var rest []string + for _, part := range strings.Split(m[2], ", ") { + if part != key+"="+value { + rest = append(rest, part) + } + } + if len(rest) == 0 { + return m[1] + } + return m[1] + " (" + strings.Join(rest, ", ") + ")" +} + // ntfy's priority scale. Max is the one that overrides the phone's quiet // settings, which is the whole point of paging on critical. const ( diff --git a/internal/api/notify_test.go b/internal/api/notify_test.go index 55453b3..703c077 100644 --- a/internal/api/notify_test.go +++ b/internal/api/notify_test.go @@ -1,6 +1,7 @@ package api_test import ( + "bytes" "context" "encoding/json" "fmt" @@ -164,10 +165,84 @@ func fireCritical(t *testing.T, s *ts) { }, "{}:{alertname=\"DiskFull\"}") } +// fireGrouped posts one critical alert whose Alertmanager group carries the +// given labels, the way group_by puts them on the webhook. +func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) { + t.Helper() + payload := map[string]any{ + "version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels, + "alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing", + "2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})}, + } + data, _ := json.Marshal(payload) + resp, err := http.Post(s.URL+"/api/integrations/"+s.ingestKey+"/alertmanager", + "application/json", bytes.NewReader(data)) + if err != nil { + t.Fatalf("post webhook: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("webhook returned %d", resp.StatusCode) + } +} + // --------------------------------------------------------------------------- // Delivery // --------------------------------------------------------------------------- +// A phone cuts a long title off at the end, and the incident title keeps the +// grouping labels there, so the cluster leads the page instead. +func TestNotify_ClusterLeadsTheTitle(t *testing.T) { + s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"}) + + fireGrouped(t, s, map[string]string{ + "alertname": "PodRestarting", "cluster": "prod-eu", "namespace": "shop", + }, `{}:{alertname="PodRestarting",cluster="prod-eu",namespace="shop"}`) + s.sweepNotify(t) + + msgs := f.messages() + if len(msgs) != 1 { + t.Fatalf("expected 1 push, got %d", len(msgs)) + } + if want := "[prod-eu] PodRestarting (namespace=shop)"; msgs[0].Title != want { + t.Errorf("title = %q, want %q", msgs[0].Title, want) + } +} + +// Only the cluster is the whole grouping: no parenthesis is left behind. +func TestNotify_ClusterAloneLeavesNoParenthesis(t *testing.T) { + s, f := notifyTS(t, api.NotifyConfig{}) + + fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "cluster": "prod-eu"}, + `{}:{alertname="PodRestarting",cluster="prod-eu"}`) + s.sweepNotify(t) + + msgs := f.messages() + if len(msgs) != 1 { + t.Fatalf("expected 1 push, got %d", len(msgs)) + } + if want := "[prod-eu] PodRestarting"; msgs[0].Title != want { + t.Errorf("title = %q, want %q", msgs[0].Title, want) + } +} + +// Nothing changes for a team whose alerts have no cluster label. +func TestNotify_NoClusterKeepsTheTitle(t *testing.T) { + s, f := notifyTS(t, api.NotifyConfig{}) + + fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "namespace": "shop"}, + `{}:{alertname="PodRestarting",namespace="shop"}`) + s.sweepNotify(t) + + msgs := f.messages() + if len(msgs) != 1 { + t.Fatalf("expected 1 push, got %d", len(msgs)) + } + if want := "PodRestarting (namespace=shop)"; msgs[0].Title != want { + t.Errorf("title = %q, want %q", msgs[0].Title, want) + } +} + func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) { s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})