Drill into a single alert from the stats page

The stats page could only say how many alerts fired. It now answers when
one alert fires, how often, and how fast it is picked up and fixed: each
row of the top-alerts list opens /stats/alert/{name}, with a timeline,
by-hour and by-weekday charts, time-to-resolve per occurrence and a table
of every occurrence linking to its incident. The overview gains a
weekday-by-hour heatmap and a few derived observations.

An occurrence is an incident, not an alert row. Alert rows are reused per
fingerprint and mutated in place, so they carry no history; incidents are
the only record of each time something fired and was answered. That means
/api/stats/alerts/top now counts incidents opened (filtered on
triggered_at) so the list and the drilldown agree. The response keeps name
and count and only adds fields, so existing clients keep working, but the
counts can differ from before.

No migration: the lookups use incidents_signature_idx. Label breakdown,
team and severity filters and period-over-period deltas from the design
draft are left for later.

Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN

internal/api/router.go

@@ -227,6 +227,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin
 		r.Get("/api/stats/alerts/top", handleStatsTop(db))
 		r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db))
 		r.Get("/api/stats/alerts/by-day", handleStatsByDay(db))
+		r.Get("/api/stats/alerts/heatmap", handleStatsHeatmap(db))
+		r.Get("/api/stats/alerts/{name}", handleStatsAlert(db))
 	})
 
 	// Anything else under /api is a mistake in a client, and should say so in

internal/api/stats.go

@@ -5,8 +5,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strings"
 	"time"
+
+	"github.com/go-chi/chi/v5"
 )
 
 func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
@@ -35,12 +38,24 @@ func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
 	}
 }
 
+// alertNameSQL is the alert-name half of an incident's signature (see
+// incidentSignature: "name|stable,labels"). An occurrence of an alert is an
+// incident, because alert rows are reused per fingerprint and carry no history.
+const alertNameSQL = "split_part(signature, '|', 1)"
+
+const sparkDays = 30
+
+// handleStatsTop ranks alert names by how many incidents they opened in the
+// window, with what a list row needs besides the count: when it last fired, how
+// long it takes to fix, and a per-day series for the last sparkDays days.
 func handleStatsTop(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))
+		q := r.URL.Query()
+		teams := callerTeamIDs(r.Context())
+		where, args := statsFilter(q, "triggered_at", teams)
 
 		limit := 10
-		if l := r.URL.Query().Get("limit"); l != "" {
+		if l := q.Get("limit"); l != "" {
 			var n int
 			fmt.Sscanf(l, "%d", &n)
 			if n > 0 && n <= 100 {
@@ -48,12 +63,13 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 			}
 		}
 		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
-			SELECT name, COUNT(*) AS cnt
-			FROM   alerts
-			WHERE  %s
+			SELECT %[1]s AS name, COUNT(*) AS cnt, MAX(triggered_at),
+			       AVG(CASE WHEN resolved_at IS NOT NULL THEN resolved_at - triggered_at END)
+			FROM   incidents
+			WHERE  %[2]s
 			GROUP  BY name
-			ORDER  BY cnt DESC
-			LIMIT  %s`, where, args.add(limit)), args.all()...)
+			ORDER  BY cnt DESC, name ASC
+			LIMIT  %[3]s`, alertNameSQL, where, args.add(limit)), args.all()...)
 		if err != nil {
 			serverError(w, r, err)
 			return
@@ -61,26 +77,247 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
 		defer rows.Close()
 
 		type entry struct {
-			Name  string `json:"name"`
-			Count int64  `json:"count"`
+			Name        string   `json:"name"`
+			Count       int64    `json:"count"`
+			LastSeen    int64    `json:"last_seen"`
+			MeanResolve *float64 `json:"mean_resolve_seconds"`
+			Daily       []int64  `json:"daily"`
 		}
 		result := []entry{}
+		names := []string{}
 		for rows.Next() {
 			var e entry
-			if err := rows.Scan(&e.Name, &e.Count); err != nil {
+			if err := rows.Scan(&e.Name, &e.Count, &e.LastSeen, &e.MeanResolve); err != nil {
 				serverError(w, r, err)
 				return
 			}
+			e.Daily = make([]int64, sparkDays)
 			result = append(result, e)
+			names = append(names, e.Name)
 		}
 		if err := rows.Err(); err != nil {
 			serverError(w, r, err)
 			return
 		}
+
+		// The sparkline always covers the last sparkDays days ending at ?to (or
+		// today), whatever ?from says, so the rows stay comparable.
+		end := time.Now().UTC().Truncate(24 * time.Hour)
+		if to, err := time.Parse("2006-01-02", q.Get("to")); err == nil {
+			end = to.UTC()
+		}
+		first := end.AddDate(0, 0, -(sparkDays - 1))
+		sq := url.Values{}
+		sq.Set("from", first.Format("2006-01-02"))
+		sq.Set("to", end.Format("2006-01-02"))
+		swhere, sargs := statsFilter(sq, "triggered_at", teams)
+		if len(names) > 0 {
+			swhere += " AND " + alertNameSQL + " = ANY(" + sargs.add(names) + ")"
+			srows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+				SELECT %s AS name, ((triggered_at - %s) / 86400)::int AS day, COUNT(*)
+				FROM   incidents
+				WHERE  %s
+				GROUP  BY name, day`, alertNameSQL, sargs.add(first.Unix()), swhere), sargs.all()...)
+			if err != nil {
+				serverError(w, r, err)
+				return
+			}
+			defer srows.Close()
+			idx := map[string]int{}
+			for i, e := range result {
+				idx[e.Name] = i
+			}
+			for srows.Next() {
+				var name string
+				var day int
+				var cnt int64
+				if err := srows.Scan(&name, &day, &cnt); err != nil {
+					serverError(w, r, err)
+					return
+				}
+				if day >= 0 && day < sparkDays {
+					result[idx[name]].Daily[day] = cnt
+				}
+			}
+			if err := srows.Err(); err != nil {
+				serverError(w, r, err)
+				return
+			}
+		}
 		respond(w, http.StatusOK, result)
 	}
 }
 
+// handleStatsHeatmap counts incidents by UTC weekday (0=Sunday) and hour:
+// always 7 rows of 24 cells, so a client never has to fill gaps.
+func handleStatsHeatmap(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		where, args := statsFilter(r.URL.Query(), "triggered_at", callerTeamIDs(r.Context()))
+		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+			SELECT EXTRACT(DOW  FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
+			       EXTRACT(HOUR FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
+			       COUNT(*)
+			FROM   incidents
+			WHERE  %s
+			GROUP  BY 1, 2`, where), args.all()...)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		defer rows.Close()
+		grid := make([][]int64, 7)
+		for d := range grid {
+			grid[d] = make([]int64, 24)
+		}
+		for rows.Next() {
+			var d, h int
+			var c int64
+			if err := rows.Scan(&d, &h, &c); err != nil {
+				serverError(w, r, err)
+				return
+			}
+			grid[d][h] = c
+		}
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		respond(w, http.StatusOK, map[string]any{"grid": grid})
+	}
+}
+
+// maxOccurrences caps the drilldown's table; the aggregates still cover all.
+const maxOccurrences = 200
+
+// handleStatsAlert is the per-alert drilldown. Every figure is computed over
+// the incidents that alert opened in the window, so it agrees with the top list.
+func handleStatsAlert(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// chi hands back the still-escaped segment when the path has %2F in it.
+		name, err := url.PathUnescape(chi.URLParam(r, "name"))
+		if err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid alert name"))
+			return
+		}
+		where, args := statsFilter(r.URL.Query(), "i.triggered_at", callerTeamIDs(r.Context()))
+		where = strings.ReplaceAll(where, "archived_at", "i.archived_at")
+		where = strings.ReplaceAll(where, "team_id =", "i.team_id =")
+		where += " AND split_part(i.signature, '|', 1) = " + args.add(name)
+
+		rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
+			SELECT i.id, i.triggered_at, i.acknowledged_at, i.resolved_at, i.status, i.severity,
+			       COALESCE(u.username, sa.name)
+			FROM   incidents i
+			LEFT   JOIN users u ON u.id = i.acknowledged_by
+			LEFT   JOIN service_accounts sa ON sa.id = i.acknowledged_by_service_account_id
+			WHERE  %s
+			ORDER  BY i.triggered_at DESC`, where), args.all()...)
+		if err != nil {
+			serverError(w, r, err)
+			return
+		}
+		defer rows.Close()
+
+		type occurrence struct {
+			IncidentID     int64   `json:"incident_id"`
+			TriggeredAt    int64   `json:"triggered_at"`
+			AckSeconds     *int64  `json:"ack_seconds"`
+			ResolveSeconds *int64  `json:"resolve_seconds"`
+			Status         string  `json:"status"`
+			Severity       *string `json:"severity,omitempty"`
+			AcknowledgedBy *string `json:"acknowledged_by,omitempty"`
+		}
+		all := []occurrence{}
+		for rows.Next() {
+			var o occurrence
+			var ack, res *int64
+			if err := rows.Scan(&o.IncidentID, &o.TriggeredAt, &ack, &res, &o.Status, &o.Severity, &o.AcknowledgedBy); err != nil {
+				serverError(w, r, err)
+				return
+			}
+			if ack != nil {
+				v := *ack - o.TriggeredAt
+				o.AckSeconds = &v
+			}
+			if res != nil {
+				v := *res - o.TriggeredAt
+				o.ResolveSeconds = &v
+			}
+			all = append(all, o)
+		}
+		if err := rows.Err(); err != nil {
+			serverError(w, r, err)
+			return
+		}
+		if len(all) == 0 {
+			respond(w, http.StatusNotFound, errResp("no incidents for this alert"))
+			return
+		}
+
+		byHour := make([]int64, 24)
+		byDay := make([]int64, 7)
+		var acks, fixes []int64
+		var longest int64
+		for _, o := range all {
+			t := time.Unix(o.TriggeredAt, 0).UTC()
+			byHour[t.Hour()]++
+			byDay[int(t.Weekday())]++
+			if o.AckSeconds != nil {
+				acks = append(acks, *o.AckSeconds)
+			}
+			if o.ResolveSeconds != nil {
+				fixes = append(fixes, *o.ResolveSeconds)
+				longest = max(longest, *o.ResolveSeconds)
+			}
+		}
+		// all is newest first, so the gap runs from the last row back to the first.
+		var meanGap *float64
+		if n := len(all); n > 1 {
+			g := float64(all[0].TriggeredAt-all[n-1].TriggeredAt) / float64(n-1)
+			meanGap = &g
+		}
+		stat := func(xs []int64) (mean, median *float64) {
+			if len(xs) == 0 {
+				return nil, nil
+			}
+			s := append([]int64(nil), xs...)
+			sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
+			var sum int64
+			for _, x := range s {
+				sum += x
+			}
+			m, md := float64(sum)/float64(len(s)), float64(s[len(s)/2])
+			return &m, &md
+		}
+		meanAck, medAck := stat(acks)
+		meanFix, medFix := stat(fixes)
+
+		occ := all
+		if len(occ) > maxOccurrences {
+			occ = occ[:maxOccurrences]
+		}
+		resp := map[string]any{
+			"name":                   name,
+			"count":                  len(all),
+			"first_seen":             all[len(all)-1].TriggeredAt,
+			"last_seen":              all[0].TriggeredAt,
+			"mean_gap_seconds":       meanGap,
+			"mean_ack_seconds":       meanAck,
+			"median_ack_seconds":     medAck,
+			"mean_resolve_seconds":   meanFix,
+			"median_resolve_seconds": medFix,
+			"longest_open_seconds":   nil,
+			"by_hour":                byHour,
+			"by_weekday":             byDay,
+			"occurrences":            occ,
+		}
+		if len(fixes) > 0 {
+			resp["longest_open_seconds"] = longest
+		}
+		respond(w, http.StatusOK, resp)
+	}
+}
+
 func handleStatsByHour(db *sql.DB) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))

internal/api/stats_alert_test.go

@@ -0,0 +1,88 @@
+package api_test
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestStatsTop_CountsIncidentsWithSparkAndMeans(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	postToIntegration(t, s, red.key, "fp-1", "DiskFull")
+	postToIntegration(t, s, red.key, "fp-2", "DiskFull") // same group key: one incident
+	postToIntegration(t, s, red.key, "fp-3", "Memory")
+
+	top := list(t, red.call(http.MethodGet, "/api/stats/alerts/top", nil))
+	if len(top) != 2 {
+		t.Fatalf("expected 2 names, got %d", len(top))
+	}
+	for _, row := range top {
+		if row["count"].(float64) != 1 {
+			t.Errorf("%v: count %v, want 1 incident", row["name"], row["count"])
+		}
+		daily := row["daily"].([]any)
+		if len(daily) != 30 {
+			t.Fatalf("daily has %d days, want 30", len(daily))
+		}
+		if row["mean_resolve_seconds"] != nil {
+			t.Errorf("unresolved incident must have null mean, got %v", row["mean_resolve_seconds"])
+		}
+	}
+}
+
+func TestStatsHeatmap_ShapeAndScope(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	blue := newTeam(t, s, "blue")
+	postToIntegration(t, s, red.key, "fp-r", "RedOnly")
+	postToIntegration(t, s, blue.key, "fp-b", "BlueOnly")
+
+	var out struct {
+		Grid [][]int64 `json:"grid"`
+	}
+	decode(t, red.call(http.MethodGet, "/api/stats/alerts/heatmap", nil), &out)
+	if len(out.Grid) != 7 || len(out.Grid[0]) != 24 {
+		t.Fatalf("grid is %dx?, want 7x24", len(out.Grid))
+	}
+	var sum int64
+	for _, row := range out.Grid {
+		for _, c := range row {
+			sum += c
+		}
+	}
+	if sum != 1 {
+		t.Errorf("red's heatmap counted %d incidents, want 1", sum)
+	}
+}
+
+func TestStatsAlert_DrilldownAndScope(t *testing.T) {
+	s := newTS(t)
+	red := newTeam(t, s, "red")
+	blue := newTeam(t, s, "blue")
+	postToIntegration(t, s, red.key, "fp-r", "DiskFull")
+	postToIntegration(t, s, blue.key, "fp-b", "BlueOnly")
+
+	var d map[string]any
+	decode(t, red.call(http.MethodGet, "/api/stats/alerts/DiskFull", nil), &d)
+	if d["count"].(float64) != 1 {
+		t.Errorf("count = %v, want 1", d["count"])
+	}
+	if len(d["by_hour"].([]any)) != 24 || len(d["by_weekday"].([]any)) != 7 {
+		t.Errorf("by_hour/by_weekday have wrong lengths")
+	}
+	if occ := d["occurrences"].([]any); len(occ) != 1 {
+		t.Errorf("occurrences = %d, want 1", len(occ))
+	}
+	if d["mean_ack_seconds"] != nil || d["longest_open_seconds"] != nil {
+		t.Errorf("nothing acked or resolved yet, means must be null: %v", d)
+	}
+
+	// Another team's alert, and a name nobody has, are both a plain 404.
+	for _, name := range []string{"BlueOnly", "Nope"} {
+		resp := red.call(http.MethodGet, "/api/stats/alerts/"+name, nil)
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusNotFound {
+			t.Errorf("%s: expected 404, got %d", name, resp.StatusCode)
+		}
+	}
+}