Put the cluster first in an ntfy page title
A phone's lock screen cuts a long title off at the end, and an incident title keeps its grouping labels there: "PodRestarting (cluster=prod-eu, namespace=shop)". For a team with a Kubernetes cluster per alert source, the cluster is the first thing a person wants and the first thing lost. When the incident has a `cluster` group label, the title of the page now leads with it, "[prod-eu] PodRestarting (namespace=shop)", and the label is dropped from the parenthesis so it is not said twice, along with the parenthesis itself if nothing else is left. The reminder and the resolution use the same title, and the label is the one the web UI's chip reads. The message body is unchanged. An incident without a cluster label gets the same title as before, and incidents that are already open keep theirs; only pages sent from now on change. The constant originLabel names the label, as ORIGIN_LABEL does in the web UI. No endpoint or JSON shape changed, so nothing to mirror in terdut-tui.
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -409,9 +410,11 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
|
||||
strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID)
|
||||
}
|
||||
|
||||
title := pageTitle(inc)
|
||||
|
||||
switch n.kind {
|
||||
case notifyResolved:
|
||||
msg.Title = "Resolved: " + inc.Title
|
||||
msg.Title = "Resolved: " + title
|
||||
msg.Message = "All alerts stopped firing after " +
|
||||
humanDuration(time.Since(inc.TriggeredAt))
|
||||
msg.Priority = ntfyPriorityLow
|
||||
@@ -419,9 +422,9 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
|
||||
return msg
|
||||
|
||||
case notifyReminder:
|
||||
msg.Title = "Still unacknowledged: " + inc.Title
|
||||
msg.Title = "Still unacknowledged: " + title
|
||||
default:
|
||||
msg.Title = inc.Title
|
||||
msg.Title = title
|
||||
}
|
||||
|
||||
severity := derefString(inc.Severity)
|
||||
@@ -443,6 +446,48 @@ func renderNotification(inc models.Incident, n outboxRow, firing int, cfg Notify
|
||||
return msg
|
||||
}
|
||||
|
||||
// originLabel is the label that says where an alert came from, for a team with
|
||||
// several Kubernetes clusters behind it. It comes from Prometheus's
|
||||
// externalLabels and reaches an incident through Alertmanager's group_by; the
|
||||
// web UI reads the same label, and the README ("Several clusters, one team")
|
||||
// explains how to set it up.
|
||||
const originLabel = "cluster"
|
||||
|
||||
// pageTitle is the incident's title for a notification. A phone's lock screen
|
||||
// cuts a long title off at the end, and the incident title puts the grouping
|
||||
// labels there, so the cluster would be the first thing lost. When the incident
|
||||
// has an origin it leads instead, "[prod-eu] PodRestarting (namespace=foo)", and
|
||||
// is dropped from the parenthesis so it is not said twice. A title that is not
|
||||
// in incidentTitle's "name (k=v, k=v)" shape keeps its text and gains the prefix.
|
||||
func pageTitle(inc models.Incident) string {
|
||||
origin := inc.GroupLabels[originLabel]
|
||||
if origin == "" {
|
||||
return inc.Title
|
||||
}
|
||||
return "[" + origin + "] " + titleWithoutLabel(inc.Title, originLabel, origin)
|
||||
}
|
||||
|
||||
var titleShape = regexp.MustCompile(`(?s)^(.*?) \((.*)\)$`)
|
||||
|
||||
// titleWithoutLabel removes "key=value" from the parenthesised tail of a title
|
||||
// built by incidentTitle, and the parentheses with it if nothing else is left.
|
||||
func titleWithoutLabel(title, key, value string) string {
|
||||
m := titleShape.FindStringSubmatch(title)
|
||||
if m == nil {
|
||||
return title
|
||||
}
|
||||
var rest []string
|
||||
for _, part := range strings.Split(m[2], ", ") {
|
||||
if part != key+"="+value {
|
||||
rest = append(rest, part)
|
||||
}
|
||||
}
|
||||
if len(rest) == 0 {
|
||||
return m[1]
|
||||
}
|
||||
return m[1] + " (" + strings.Join(rest, ", ") + ")"
|
||||
}
|
||||
|
||||
// ntfy's priority scale. Max is the one that overrides the phone's quiet
|
||||
// settings, which is the whole point of paging on critical.
|
||||
const (
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -164,10 +165,84 @@ func fireCritical(t *testing.T, s *ts) {
|
||||
}, "{}:{alertname=\"DiskFull\"}")
|
||||
}
|
||||
|
||||
// fireGrouped posts one critical alert whose Alertmanager group carries the
|
||||
// given labels, the way group_by puts them on the webhook.
|
||||
func fireGrouped(t *testing.T, s *ts, groupLabels map[string]string, groupKey string) {
|
||||
t.Helper()
|
||||
payload := map[string]any{
|
||||
"version": "4", "status": "firing", "groupKey": groupKey, "groupLabels": groupLabels,
|
||||
"alerts": []map[string]any{amAlert("fp-grouped", "PodRestarting", "firing",
|
||||
"2026-05-20T10:00:00Z", zeroTime, map[string]string{"severity": "critical"})},
|
||||
}
|
||||
data, _ := json.Marshal(payload)
|
||||
resp, err := http.Post(s.URL+"/api/integrations/"+s.ingestKey+"/alertmanager",
|
||||
"application/json", bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("post webhook: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("webhook returned %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Delivery
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// A phone cuts a long title off at the end, and the incident title keeps the
|
||||
// grouping labels there, so the cluster leads the page instead.
|
||||
func TestNotify_ClusterLeadsTheTitle(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||
|
||||
fireGrouped(t, s, map[string]string{
|
||||
"alertname": "PodRestarting", "cluster": "prod-eu", "namespace": "shop",
|
||||
}, `{}:{alertname="PodRestarting",cluster="prod-eu",namespace="shop"}`)
|
||||
s.sweepNotify(t)
|
||||
|
||||
msgs := f.messages()
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||
}
|
||||
if want := "[prod-eu] PodRestarting (namespace=shop)"; msgs[0].Title != want {
|
||||
t.Errorf("title = %q, want %q", msgs[0].Title, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the cluster is the whole grouping: no parenthesis is left behind.
|
||||
func TestNotify_ClusterAloneLeavesNoParenthesis(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{})
|
||||
|
||||
fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "cluster": "prod-eu"},
|
||||
`{}:{alertname="PodRestarting",cluster="prod-eu"}`)
|
||||
s.sweepNotify(t)
|
||||
|
||||
msgs := f.messages()
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||
}
|
||||
if want := "[prod-eu] PodRestarting"; msgs[0].Title != want {
|
||||
t.Errorf("title = %q, want %q", msgs[0].Title, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing changes for a team whose alerts have no cluster label.
|
||||
func TestNotify_NoClusterKeepsTheTitle(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{})
|
||||
|
||||
fireGrouped(t, s, map[string]string{"alertname": "PodRestarting", "namespace": "shop"},
|
||||
`{}:{alertname="PodRestarting",namespace="shop"}`)
|
||||
s.sweepNotify(t)
|
||||
|
||||
msgs := f.messages()
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||
}
|
||||
if want := "PodRestarting (namespace=shop)"; msgs[0].Title != want {
|
||||
t.Errorf("title = %q, want %q", msgs[0].Title, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) {
|
||||
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user