Niklas Ye
60ebb75cd2
Each incident gets a signature: the alert name plus the group labels that
say what is broken, minus the ones that only say where it ran (instance,
pod, container, ...). GET /api/incidents/{id}/similar returns resolved
incidents in the same team with the same signature that have notes.
Notes can be marked as the resolution note, "what fixed it", either with a
resolution field on resolve or pinned on a note. Those lead the similar
list, show on the incident page as "Seen before", and the triggered
notification carries the latest one.
Claude-Session: https://claude.ai/code/session_01MMados3BD1oSjevHxbmVqU
100 lines
3.6 KiB
Go
100 lines
3.6 KiB
Go
package api_test
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"net/http"
|
|
"strconv"
|
|
"testing"
|
|
)
|
|
|
|
// postGrouped posts a firing webhook whose group labels are exactly the given
|
|
// map, unlike postWebhook, which only ever groups by alertname.
|
|
func postGrouped(t *testing.T, s *ts, fingerprint, startsAt string, groupLabels map[string]string) {
|
|
t.Helper()
|
|
labels := map[string]string{}
|
|
for k, v := range groupLabels {
|
|
labels[k] = v
|
|
}
|
|
payload := map[string]any{
|
|
"version": "4", "status": "firing",
|
|
"groupKey": fingerprint,
|
|
"groupLabels": groupLabels,
|
|
"alerts": []map[string]any{
|
|
amAlert(fingerprint, groupLabels["alertname"], "firing", startsAt, zeroTime, labels),
|
|
},
|
|
}
|
|
data, _ := json.Marshal(payload)
|
|
resp, err := http.Post(s.URL+"/api/integrations/"+s.ingestKey+"/alertmanager",
|
|
"application/json", bytes.NewReader(data))
|
|
if err != nil {
|
|
t.Fatalf("post webhook: %v", err)
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
func similar(t *testing.T, s *ts, id int) []map[string]any {
|
|
t.Helper()
|
|
var out []map[string]any
|
|
decode(t, s.req(t, http.MethodGet, "/api/incidents/"+strconv.Itoa(id)+"/similar", nil), &out)
|
|
return out
|
|
}
|
|
|
|
// Same alert on another instance is the same problem; a resolution note left on
|
|
// the first one is what the second one should be shown.
|
|
func TestSimilar_IgnoresVolatileLabelsAndLeadsWithResolutionNote(t *testing.T) {
|
|
s := newTS(t)
|
|
postGrouped(t, s, "fp-a", "2026-05-20T10:00:00Z",
|
|
map[string]string{"alertname": "DiskFull", "instance": "web-1", "job": "node"})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve",
|
|
map[string]string{"resolution": "rotated the logs"}).Body.Close()
|
|
|
|
postGrouped(t, s, "fp-b", "2026-05-21T10:00:00Z",
|
|
map[string]string{"alertname": "DiskFull", "instance": "web-2", "job": "node"})
|
|
|
|
got := similar(t, s, 2)
|
|
if len(got) != 1 || int(got[0]["id"].(float64)) != 1 {
|
|
t.Fatalf("expected incident 1 as the only similar one, got %v", got)
|
|
}
|
|
notes := got[0]["resolution_notes"].([]any)
|
|
if len(notes) != 1 || notes[0].(map[string]any)["detail"] != "rotated the logs" {
|
|
t.Fatalf("expected the resolution note, got %v", notes)
|
|
}
|
|
}
|
|
|
|
// A different stable label (job) is a different problem, and an incident nobody
|
|
// wrote a note on has nothing to show.
|
|
func TestSimilar_DifferentSignatureOrNoNotesIsExcluded(t *testing.T) {
|
|
s := newTS(t)
|
|
postGrouped(t, s, "fp-1", "2026-05-20T10:00:00Z",
|
|
map[string]string{"alertname": "DiskFull", "job": "node"})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/notes", map[string]string{"content": "checked"}).Body.Close()
|
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
|
|
|
postGrouped(t, s, "fp-2", "2026-05-20T11:00:00Z",
|
|
map[string]string{"alertname": "DiskFull", "job": "db"})
|
|
s.req(t, http.MethodPost, "/api/incidents/2/resolve", nil).Body.Close()
|
|
|
|
postGrouped(t, s, "fp-3", "2026-05-21T10:00:00Z",
|
|
map[string]string{"alertname": "DiskFull", "job": "db"})
|
|
|
|
// Incident 3 matches 2 by signature, but 2 has no notes.
|
|
if got := similar(t, s, 3); len(got) != 0 {
|
|
t.Fatalf("expected nothing similar to incident 3, got %v", got)
|
|
}
|
|
}
|
|
|
|
// An open incident is not "earlier experience" yet, and the incident itself is
|
|
// never its own match.
|
|
func TestSimilar_OpenIncidentsAreNotListed(t *testing.T) {
|
|
s := newTS(t)
|
|
postGrouped(t, s, "fp-o1", "2026-05-20T10:00:00Z", map[string]string{"alertname": "Flap"})
|
|
s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
|
map[string]any{"content": "still open", "pinned": true}).Body.Close()
|
|
postGrouped(t, s, "fp-o2", "2026-05-21T10:00:00Z", map[string]string{"alertname": "Flap"})
|
|
|
|
if got := similar(t, s, 2); len(got) != 0 {
|
|
t.Fatalf("expected an open incident not to be listed, got %v", got)
|
|
}
|
|
}
|