Put the cluster first in an ntfy page title

A phone's lock screen cuts a long title off at the end, and an incident title
keeps its grouping labels there: "PodRestarting (cluster=prod-eu,
namespace=shop)". For a team with a Kubernetes cluster per alert source, the
cluster is the first thing a person wants and the first thing lost.

When the incident has a `cluster` group label, the title of the page now
leads with it, "[prod-eu] PodRestarting (namespace=shop)", and the label is
dropped from the parenthesis so it is not said twice, along with the
parenthesis itself if nothing else is left. The reminder and the resolution
use the same title, and the label is the one the web UI's chip reads. The
message body is unchanged.

An incident without a cluster label gets the same title as before, and
incidents that are already open keep theirs; only pages sent from now on
change. The constant originLabel names the label, as ORIGIN_LABEL does in the
web UI.

No endpoint or JSON shape changed, so nothing to mirror in terdut-tui.
This commit is contained in:
Niklas Ye
2026-10-08 18:07:59 +02:00
parent def0f68d00
commit a23e88c16d
2 changed files with 123 additions and 3 deletions
+48 -3
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"log"
"net/http"
"regexp"
"strings"
"time"
@@ -409,9 +410,11 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID)
}
title := pageTitle(inc)
switch n.kind {
case notifyResolved:
msg.Title = "Resolved: " + inc.Title
msg.Title = "Resolved: " + title
msg.Message = "All alerts stopped firing after " +
humanDuration(time.Since(inc.TriggeredAt))
msg.Priority = ntfyPriorityLow
@@ -419,9 +422,9 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
return msg
case notifyReminder:
msg.Title = "Still unacknowledged: " + inc.Title
msg.Title = "Still unacknowledged: " + title
default:
msg.Title = inc.Title
msg.Title = title
}
severity := derefString(inc.Severity)
@@ -443,6 +446,48 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
return msg
}
// originLabel is the label that says where an alert came from, for a team with
// several Kubernetes clusters behind it. It comes from Prometheus's
// externalLabels and reaches an incident through Alertmanager's group_by; the
// web UI reads the same label, and the README ("Several clusters, one team")
// explains how to set it up.
const originLabel = "cluster"
// pageTitle is the incident's title for a notification. A phone's lock screen
// cuts a long title off at the end, and the incident title puts the grouping
// labels there, so the cluster would be the first thing lost. When the incident
// has an origin it leads instead, "[prod-eu] PodRestarting (namespace=foo)", and
// is dropped from the parenthesis so it is not said twice. A title that is not
// in incidentTitle's "name (k=v, k=v)" shape keeps its text and gains the prefix.
func pageTitle(inc models.Incident) string {
origin := inc.GroupLabels[originLabel]
if origin == "" {
return inc.Title
}
return "[" + origin + "] " + titleWithoutLabel(inc.Title, originLabel, origin)
}
var titleShape = regexp.MustCompile(`(?s)^(.*?) \((.*)\)$`)
// titleWithoutLabel removes "key=value" from the parenthesised tail of a title
// built by incidentTitle, and the parentheses with it if nothing else is left.
func titleWithoutLabel(title, key, value string) string {
m := titleShape.FindStringSubmatch(title)
if m == nil {
return title
}
var rest []string
for _, part := range strings.Split(m[2], ", ") {
if part != key+"="+value {
rest = append(rest, part)
}
}
if len(rest) == 0 {
return m[1]
}
return m[1] + " (" + strings.Join(rest, ", ") + ")"
}
// ntfy's priority scale. Max is the one that overrides the phone's quiet
// settings, which is the whole point of paging on critical.
const (
+75
View File
@@ -1,6 +1,7 @@
package api_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -164,10 +165,84 @@ func fireCritical(t *testing.T, s *ts) {
}, "{}:{alertname=\"DiskFull\"}")
}
// fireGrouped posts one critical alert whose Alertmanager group carries the
// given labels, the way group_by puts them on the webhook.
func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) {
t.Helper()
payload := map[string]any{
"version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels,
"alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing",
"2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})},
}
data, _ := json.Marshal(payload)
resp, err := http.Post(s.URL+"/api/integrations/"+s.ingestKey+"/alertmanager",
"application/json", bytes.NewReader(data))
if err != nil {
t.Fatalf("post webhook: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("webhook returned %d", resp.StatusCode)
}
}
// ---------------------------------------------------------------------------
// Delivery
// ---------------------------------------------------------------------------
// A phone cuts a long title off at the end, and the incident title keeps the
// grouping labels there, so the cluster leads the page instead.
func TestNotify_ClusterLeadsTheTitle(t *testing.T) {
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
fireGrouped(t, s, map[string]string{
"alertname": "PodRestarting", "cluster": "prod-eu", "namespace": "shop",
}, `{}:{alertname="PodRestarting",cluster="prod-eu",namespace="shop"}`)
s.sweepNotify(t)
msgs := f.messages()
if len(msgs) != 1 {
t.Fatalf("expected 1 push, got %d", len(msgs))
}
if want := "[prod-eu] PodRestarting (namespace=shop)"; msgs[0].Title != want {
t.Errorf("title = %q, want %q", msgs[0].Title, want)
}
}
// Only the cluster is the whole grouping: no parenthesis is left behind.
func TestNotify_ClusterAloneLeavesNoParenthesis(t *testing.T) {
s, f := notifyTS(t, api.NotifyConfig{})
fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "cluster": "prod-eu"},
`{}:{alertname="PodRestarting",cluster="prod-eu"}`)
s.sweepNotify(t)
msgs := f.messages()
if len(msgs) != 1 {
t.Fatalf("expected 1 push, got %d", len(msgs))
}
if want := "[prod-eu] PodRestarting"; msgs[0].Title != want {
t.Errorf("title = %q, want %q", msgs[0].Title, want)
}
}
// Nothing changes for a team whose alerts have no cluster label.
func TestNotify_NoClusterKeepsTheTitle(t *testing.T) {
s, f := notifyTS(t, api.NotifyConfig{})
fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "namespace": "shop"},
`{}:{alertname="PodRestarting",namespace="shop"}`)
s.sweepNotify(t)
msgs := f.messages()
if len(msgs) != 1 {
t.Fatalf("expected 1 push, got %d", len(msgs))
}
if want := "PodRestarting (namespace=shop)"; msgs[0].Title != want {
t.Errorf("title = %q, want %q", msgs[0].Title, want)
}
}
func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) {
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})