diff --git a/docs/api.md b/docs/api.md index 9e4e867..3ba087d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -427,13 +427,15 @@ Every figure counts the caller's own teams only: a report that counted other teams' incidents would leak their volume, and their alert names through the top-alerts list, and would not be a number about the reader's work anyway. -All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats filter on `triggered_at`. +All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats, and the top, heatmap and per-alert endpoints (an occurrence is an incident), filter on `triggered_at`. | Method | Path | Description | |---|---|---| | `GET` | `/api/stats/incidents` | `{total, triggered, acknowledged, resolved, mtta_seconds, mttr_seconds}` | | `GET` | `/api/stats/alerts` | `{total, firing, resolved}` counts | -| `GET` | `/api/stats/alerts/top` | Most frequent alert names. `?limit=` (default 10, max 100) | +| `GET` | `/api/stats/alerts/top` | Alert names ranked by incidents opened (filters on `triggered_at`). Each entry: `{name, count, last_seen, mean_resolve_seconds, daily}`; `daily` is 30 per-day counts ending at `?to` (or today), regardless of `?from`. `?limit=` (default 10, max 100) | +| `GET` | `/api/stats/alerts/heatmap` | `{grid}`: incidents by UTC weekday (0=Sunday) × hour, always 7×24 | +| `GET` | `/api/stats/alerts/{name}` | One alert's drilldown over its incidents: `{name, count, first_seen, last_seen, mean_gap_seconds, mean/median_ack_seconds, mean/median_resolve_seconds, longest_open_seconds, by_hour[24], by_weekday[7], occurrences[]}` (newest 200; means are `null` when unmeasurable). 404 when the alert opened no incident in scope | | `GET` | `/api/stats/alerts/by-hour` | Count per hour-of-day (UTC), all 24 slots returned | | `GET` | `/api/stats/alerts/by-day` | Count per day-of-week, all 7 slots with names returned | diff --git a/internal/api/router.go b/internal/api/router.go index 5706913..ed4f402 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -227,6 +227,8 @@ func NewRouter(db *sql.DB, notify NotifyConfig, cfg config.Config, version strin r.Get("/api/stats/alerts/top", handleStatsTop(db)) r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db)) r.Get("/api/stats/alerts/by-day", handleStatsByDay(db)) + r.Get("/api/stats/alerts/heatmap", handleStatsHeatmap(db)) + r.Get("/api/stats/alerts/{name}", handleStatsAlert(db)) }) // Anything else under /api is a mistake in a client, and should say so in diff --git a/internal/api/stats.go b/internal/api/stats.go index a9ab4da..2b1bf9a 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -5,8 +5,11 @@ import ( "fmt" "net/http" "net/url" + "sort" "strings" "time" + + "github.com/go-chi/chi/v5" ) func handleStatsAlerts(db *sql.DB) http.HandlerFunc { @@ -35,12 +38,24 @@ func handleStatsAlerts(db *sql.DB) http.HandlerFunc { } } +// alertNameSQL is the alert-name half of an incident's signature (see +// incidentSignature: "name|stable,labels"). An occurrence of an alert is an +// incident, because alert rows are reused per fingerprint and carry no history. +const alertNameSQL = "split_part(signature, '|', 1)" + +const sparkDays = 30 + +// handleStatsTop ranks alert names by how many incidents they opened in the +// window, with what a list row needs besides the count: when it last fired, how +// long it takes to fix, and a per-day series for the last sparkDays days. func handleStatsTop(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context())) + q := r.URL.Query() + teams := callerTeamIDs(r.Context()) + where, args := statsFilter(q, "triggered_at", teams) limit := 10 - if l := r.URL.Query().Get("limit"); l != "" { + if l := q.Get("limit"); l != "" { var n int fmt.Sscanf(l, "%d", &n) if n > 0 && n <= 100 { @@ -48,12 +63,13 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc { } } rows, err := db.QueryContext(r.Context(), fmt.Sprintf(` - SELECT name, COUNT(*) AS cnt - FROM alerts - WHERE %s + SELECT %[1]s AS name, COUNT(*) AS cnt, MAX(triggered_at), + AVG(CASE WHEN resolved_at IS NOT NULL THEN resolved_at - triggered_at END) + FROM incidents + WHERE %[2]s GROUP BY name - ORDER BY cnt DESC - LIMIT %s`, where, args.add(limit)), args.all()...) + ORDER BY cnt DESC, name ASC + LIMIT %[3]s`, alertNameSQL, where, args.add(limit)), args.all()...) if err != nil { serverError(w, r, err) return @@ -61,26 +77,247 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc { defer rows.Close() type entry struct { - Name string `json:"name"` - Count int64 `json:"count"` + Name string `json:"name"` + Count int64 `json:"count"` + LastSeen int64 `json:"last_seen"` + MeanResolve *float64 `json:"mean_resolve_seconds"` + Daily []int64 `json:"daily"` } result := []entry{} + names := []string{} for rows.Next() { var e entry - if err := rows.Scan(&e.Name, &e.Count); err != nil { + if err := rows.Scan(&e.Name, &e.Count, &e.LastSeen, &e.MeanResolve); err != nil { serverError(w, r, err) return } + e.Daily = make([]int64, sparkDays) result = append(result, e) + names = append(names, e.Name) } if err := rows.Err(); err != nil { serverError(w, r, err) return } + + // The sparkline always covers the last sparkDays days ending at ?to (or + // today), whatever ?from says, so the rows stay comparable. + end := time.Now().UTC().Truncate(24 * time.Hour) + if to, err := time.Parse("2006-01-02", q.Get("to")); err == nil { + end = to.UTC() + } + first := end.AddDate(0, 0, -(sparkDays - 1)) + sq := url.Values{} + sq.Set("from", first.Format("2006-01-02")) + sq.Set("to", end.Format("2006-01-02")) + swhere, sargs := statsFilter(sq, "triggered_at", teams) + if len(names) > 0 { + swhere += " AND " + alertNameSQL + " = ANY(" + sargs.add(names) + ")" + srows, err := db.QueryContext(r.Context(), fmt.Sprintf(` + SELECT %s AS name, ((triggered_at - %s) / 86400)::int AS day, COUNT(*) + FROM incidents + WHERE %s + GROUP BY name, day`, alertNameSQL, sargs.add(first.Unix()), swhere), sargs.all()...) + if err != nil { + serverError(w, r, err) + return + } + defer srows.Close() + idx := map[string]int{} + for i, e := range result { + idx[e.Name] = i + } + for srows.Next() { + var name string + var day int + var cnt int64 + if err := srows.Scan(&name, &day, &cnt); err != nil { + serverError(w, r, err) + return + } + if day >= 0 && day < sparkDays { + result[idx[name]].Daily[day] = cnt + } + } + if err := srows.Err(); err != nil { + serverError(w, r, err) + return + } + } respond(w, http.StatusOK, result) } } +// handleStatsHeatmap counts incidents by UTC weekday (0=Sunday) and hour: +// always 7 rows of 24 cells, so a client never has to fill gaps. +func handleStatsHeatmap(db *sql.DB) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + where, args := statsFilter(r.URL.Query(), "triggered_at", callerTeamIDs(r.Context())) + rows, err := db.QueryContext(r.Context(), fmt.Sprintf(` + SELECT EXTRACT(DOW FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int, + EXTRACT(HOUR FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int, + COUNT(*) + FROM incidents + WHERE %s + GROUP BY 1, 2`, where), args.all()...) + if err != nil { + serverError(w, r, err) + return + } + defer rows.Close() + grid := make([][]int64, 7) + for d := range grid { + grid[d] = make([]int64, 24) + } + for rows.Next() { + var d, h int + var c int64 + if err := rows.Scan(&d, &h, &c); err != nil { + serverError(w, r, err) + return + } + grid[d][h] = c + } + if err := rows.Err(); err != nil { + serverError(w, r, err) + return + } + respond(w, http.StatusOK, map[string]any{"grid": grid}) + } +} + +// maxOccurrences caps the drilldown's table; the aggregates still cover all. +const maxOccurrences = 200 + +// handleStatsAlert is the per-alert drilldown. Every figure is computed over +// the incidents that alert opened in the window, so it agrees with the top list. +func handleStatsAlert(db *sql.DB) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // chi hands back the still-escaped segment when the path has %2F in it. + name, err := url.PathUnescape(chi.URLParam(r, "name")) + if err != nil { + respond(w, http.StatusBadRequest, errResp("invalid alert name")) + return + } + where, args := statsFilter(r.URL.Query(), "i.triggered_at", callerTeamIDs(r.Context())) + where = strings.ReplaceAll(where, "archived_at", "i.archived_at") + where = strings.ReplaceAll(where, "team_id =", "i.team_id =") + where += " AND split_part(i.signature, '|', 1) = " + args.add(name) + + rows, err := db.QueryContext(r.Context(), fmt.Sprintf(` + SELECT i.id, i.triggered_at, i.acknowledged_at, i.resolved_at, i.status, i.severity, + COALESCE(u.username, sa.name) + FROM incidents i + LEFT JOIN users u ON u.id = i.acknowledged_by + LEFT JOIN service_accounts sa ON sa.id = i.acknowledged_by_service_account_id + WHERE %s + ORDER BY i.triggered_at DESC`, where), args.all()...) + if err != nil { + serverError(w, r, err) + return + } + defer rows.Close() + + type occurrence struct { + IncidentID int64 `json:"incident_id"` + TriggeredAt int64 `json:"triggered_at"` + AckSeconds *int64 `json:"ack_seconds"` + ResolveSeconds *int64 `json:"resolve_seconds"` + Status string `json:"status"` + Severity *string `json:"severity,omitempty"` + AcknowledgedBy *string `json:"acknowledged_by,omitempty"` + } + all := []occurrence{} + for rows.Next() { + var o occurrence + var ack, res *int64 + if err := rows.Scan(&o.IncidentID, &o.TriggeredAt, &ack, &res, &o.Status, &o.Severity, &o.AcknowledgedBy); err != nil { + serverError(w, r, err) + return + } + if ack != nil { + v := *ack - o.TriggeredAt + o.AckSeconds = &v + } + if res != nil { + v := *res - o.TriggeredAt + o.ResolveSeconds = &v + } + all = append(all, o) + } + if err := rows.Err(); err != nil { + serverError(w, r, err) + return + } + if len(all) == 0 { + respond(w, http.StatusNotFound, errResp("no incidents for this alert")) + return + } + + byHour := make([]int64, 24) + byDay := make([]int64, 7) + var acks, fixes []int64 + var longest int64 + for _, o := range all { + t := time.Unix(o.TriggeredAt, 0).UTC() + byHour[t.Hour()]++ + byDay[int(t.Weekday())]++ + if o.AckSeconds != nil { + acks = append(acks, *o.AckSeconds) + } + if o.ResolveSeconds != nil { + fixes = append(fixes, *o.ResolveSeconds) + longest = max(longest, *o.ResolveSeconds) + } + } + // all is newest first, so the gap runs from the last row back to the first. + var meanGap *float64 + if n := len(all); n > 1 { + g := float64(all[0].TriggeredAt-all[n-1].TriggeredAt) / float64(n-1) + meanGap = &g + } + stat := func(xs []int64) (mean, median *float64) { + if len(xs) == 0 { + return nil, nil + } + s := append([]int64(nil), xs...) + sort.Slice(s, func(i, j int) bool { return s[i] < s[j] }) + var sum int64 + for _, x := range s { + sum += x + } + m, md := float64(sum)/float64(len(s)), float64(s[len(s)/2]) + return &m, &md + } + meanAck, medAck := stat(acks) + meanFix, medFix := stat(fixes) + + occ := all + if len(occ) > maxOccurrences { + occ = occ[:maxOccurrences] + } + resp := map[string]any{ + "name": name, + "count": len(all), + "first_seen": all[len(all)-1].TriggeredAt, + "last_seen": all[0].TriggeredAt, + "mean_gap_seconds": meanGap, + "mean_ack_seconds": meanAck, + "median_ack_seconds": medAck, + "mean_resolve_seconds": meanFix, + "median_resolve_seconds": medFix, + "longest_open_seconds": nil, + "by_hour": byHour, + "by_weekday": byDay, + "occurrences": occ, + } + if len(fixes) > 0 { + resp["longest_open_seconds"] = longest + } + respond(w, http.StatusOK, resp) + } +} + func handleStatsByHour(db *sql.DB) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context())) diff --git a/internal/api/stats_alert_test.go b/internal/api/stats_alert_test.go new file mode 100644 index 0000000..8b98089 --- /dev/null +++ b/internal/api/stats_alert_test.go @@ -0,0 +1,88 @@ +package api_test + +import ( + "net/http" + "testing" +) + +func TestStatsTop_CountsIncidentsWithSparkAndMeans(t *testing.T) { + s := newTS(t) + red := newTeam(t, s, "red") + postToIntegration(t, s, red.key, "fp-1", "DiskFull") + postToIntegration(t, s, red.key, "fp-2", "DiskFull") // same group key: one incident + postToIntegration(t, s, red.key, "fp-3", "Memory") + + top := list(t, red.call(http.MethodGet, "/api/stats/alerts/top", nil)) + if len(top) != 2 { + t.Fatalf("expected 2 names, got %d", len(top)) + } + for _, row := range top { + if row["count"].(float64) != 1 { + t.Errorf("%v: count %v, want 1 incident", row["name"], row["count"]) + } + daily := row["daily"].([]any) + if len(daily) != 30 { + t.Fatalf("daily has %d days, want 30", len(daily)) + } + if row["mean_resolve_seconds"] != nil { + t.Errorf("unresolved incident must have null mean, got %v", row["mean_resolve_seconds"]) + } + } +} + +func TestStatsHeatmap_ShapeAndScope(t *testing.T) { + s := newTS(t) + red := newTeam(t, s, "red") + blue := newTeam(t, s, "blue") + postToIntegration(t, s, red.key, "fp-r", "RedOnly") + postToIntegration(t, s, blue.key, "fp-b", "BlueOnly") + + var out struct { + Grid [][]int64 `json:"grid"` + } + decode(t, red.call(http.MethodGet, "/api/stats/alerts/heatmap", nil), &out) + if len(out.Grid) != 7 || len(out.Grid[0]) != 24 { + t.Fatalf("grid is %dx?, want 7x24", len(out.Grid)) + } + var sum int64 + for _, row := range out.Grid { + for _, c := range row { + sum += c + } + } + if sum != 1 { + t.Errorf("red's heatmap counted %d incidents, want 1", sum) + } +} + +func TestStatsAlert_DrilldownAndScope(t *testing.T) { + s := newTS(t) + red := newTeam(t, s, "red") + blue := newTeam(t, s, "blue") + postToIntegration(t, s, red.key, "fp-r", "DiskFull") + postToIntegration(t, s, blue.key, "fp-b", "BlueOnly") + + var d map[string]any + decode(t, red.call(http.MethodGet, "/api/stats/alerts/DiskFull", nil), &d) + if d["count"].(float64) != 1 { + t.Errorf("count = %v, want 1", d["count"]) + } + if len(d["by_hour"].([]any)) != 24 || len(d["by_weekday"].([]any)) != 7 { + t.Errorf("by_hour/by_weekday have wrong lengths") + } + if occ := d["occurrences"].([]any); len(occ) != 1 { + t.Errorf("occurrences = %d, want 1", len(occ)) + } + if d["mean_ack_seconds"] != nil || d["longest_open_seconds"] != nil { + t.Errorf("nothing acked or resolved yet, means must be null: %v", d) + } + + // Another team's alert, and a name nobody has, are both a plain 404. + for _, name := range []string{"BlueOnly", "Nope"} { + resp := red.call(http.MethodGet, "/api/stats/alerts/"+name, nil) + resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s: expected 404, got %d", name, resp.StatusCode) + } + } +} diff --git a/internal/web/static/app.css b/internal/web/static/app.css index 8bc7ccc..b30a128 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -1320,9 +1320,71 @@ button.rota-week:hover { background: var(--surface-2); color: var(--text); } .columns .col-bar { fill: var(--accent); } .columns .col:hover .col-bar { opacity: 0.75; } .columns .col-label { fill: var(--faint); font-size: 10px; font-family: var(--font); } +.chart-head { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: baseline; gap: 4px 8px; margin-bottom: 10px; } +.chart-head .chart-title { margin-bottom: 0; } +.chart-note { font-size: 12px; color: var(--muted); } +.stat-note { margin-top: 2px; font-size: 12px; color: var(--muted); } +.stats-head { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 12px; } +.stats-head .chips { padding-left: 0; padding-right: 0; flex: 1 1 100%; } +.stats-alert-name { font-family: var(--mono, ui-monospace, monospace); font-size: 20px; overflow-wrap: anywhere; } +.stats-split { display: grid; gap: 14px; } +.pattern-strip { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 20px; font-size: 14px; } +.pattern-strip .chart-title { margin-bottom: 0; } +.patterns { display: grid; gap: 14px; } +.pattern-title { font-size: 15px; font-weight: 600; } +.pattern-sub { margin-top: 2px; font-size: 13px; color: var(--muted); } + +/* Top alerts: each row is a link to the alert's drilldown. */ +.toplist { display: grid; overflow-x: auto; } +.top-row { + display: grid; align-items: center; gap: 12px; min-width: 640px; min-height: 44px; padding: 6px 4px; + grid-template-columns: minmax(140px, 2.4fr) minmax(130px, 1.4fr) 100px 100px 70px 18px; + border-top: 1px solid var(--border); color: inherit; text-decoration: none; font-size: 13px; +} +a.top-row:hover { background: var(--surface-2); } +.top-head { min-height: 0; border-top: 0; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); grid-template-columns: minmax(140px, 2.4fr) minmax(130px, 1.4fr) 100px 100px 70px 18px; } +.top-count { display: flex; align-items: center; gap: 8px; } +.top-count .hbar-track { flex: 1; height: 6px; } +.top-count .hbar-count { order: -1; min-width: 2ch; color: var(--text); font-weight: 600; } +.top-pct, .top-cell { color: var(--muted); font-variant-numeric: tabular-nums; } +.spark { display: flex; align-items: flex-end; gap: 1px; height: 24px; } +.spark-bar { width: 2px; border-radius: 1px; background: var(--border-strong); } +.spark-bar.on { background: var(--accent); } + +/* Heatmap: weekday × hour. */ +.heatmap { display: grid; gap: 3px; overflow-x: auto; min-width: 0; } +.heatmap > * { min-width: 520px; } +.heat-row { display: flex; align-items: center; gap: 8px; } +.heat-day { width: 30px; flex: none; font-size: 12px; color: var(--muted); } +.heat-cells { flex: 1; display: grid; grid-template-columns: repeat(24, minmax(0, 1fr)); gap: 3px; } +.heat-cell { height: 24px; border-radius: 4px; background: var(--surface-2); } +.heat-cell.lv1 { background: color-mix(in srgb, var(--accent) 30%, var(--surface-2)); } +.heat-cell.lv2 { background: color-mix(in srgb, var(--accent) 55%, var(--surface-2)); } +.heat-cell.lv3 { background: color-mix(in srgb, var(--accent) 80%, var(--surface-2)); } +.heat-cell.lv4 { background: var(--accent); } +.heat-hour { font-size: 11px; color: var(--muted); font-variant-numeric: tabular-nums; } + +/* Drilldown: timeline, resolve bars, occurrences. */ +.tl { position: relative; height: 56px; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; background: var(--surface-2); } +.tl-mark { position: absolute; height: 18px; min-width: 8px; border-radius: 5px; background: var(--accent); } +.tl-axis { display: flex; justify-content: space-between; margin-top: 6px; font-size: 11px; color: var(--muted); } +.fixbars { display: grid; gap: 8px; } +.fix-row { display: flex; align-items: center; gap: 10px; font-size: 12.5px; } +.fix-label { width: 104px; flex: none; color: var(--muted); font-variant-numeric: tabular-nums; } +.fix-track { flex: 1; min-width: 0; display: flex; height: 16px; border-radius: 4px; background: var(--surface-2); overflow: hidden; } +.fix-ack { background: var(--warn); } +.fix-work { background: var(--accent); } +.fix-total { width: 56px; flex: none; text-align: right; font-weight: 600; font-variant-numeric: tabular-nums; } +.occ-wrap { overflow-x: auto; } +.occ { width: 100%; min-width: 560px; border-collapse: collapse; font-size: 13px; font-variant-numeric: tabular-nums; } +.occ th { text-align: left; padding: 6px 8px; font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); } +.occ td { padding: 10px 8px; border-top: 1px solid var(--border); } @media (min-width: 900px) { .stat-tiles { grid-template-columns: repeat(3, 1fr); } } +@media (min-width: 900px) { + .stats-split { grid-template-columns: 3fr 1fr; align-items: stretch; } +} /* On-call, wide: the hero and the week on the left, your shifts on the right. .view-page > * caps its children at 760px, so the grid lifts that. */ diff --git a/internal/web/static/js/api.js b/internal/web/static/js/api.js index 6e1e6ee..41dc235 100644 --- a/internal/web/static/js/api.js +++ b/internal/web/static/js/api.js @@ -115,6 +115,8 @@ export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/note export const statsIncidents = (query) => call('GET', '/stats/incidents', { query }); export const statsTop = (query) => call('GET', '/stats/alerts/top', { query }); export const statsByHour = (query) => call('GET', '/stats/alerts/by-hour', { query }); +export const statsHeatmap = (query) => call('GET', '/stats/alerts/heatmap', { query }); +export const statsAlert = (name, query) => call('GET', `/stats/alerts/${encodeURIComponent(name)}`, { query }); export const statsByDay = (query) => call('GET', '/stats/alerts/by-day', { query }); // alerts diff --git a/internal/web/static/js/app.js b/internal/web/static/js/app.js index c4f0a87..62d5d7b 100644 --- a/internal/web/static/js/app.js +++ b/internal/web/static/js/app.js @@ -59,6 +59,8 @@ function parseRoute(pathname) { // /admin/teams/7 fall through to the queue. const g = pathname.match(/^\/admin\/teams\/(\d+)\/?$/); if (g) return { section: 'adminteam', team: Number(g[1]) }; + const sa = pathname.match(/^\/stats\/alert\/([^/]+)\/?$/); + if (sa) return { section: 'stats', alert: decodeURIComponent(sa[1]) }; const name = pathname.replace(/^\/|\/$/g, ''); // The Admin and Team tabs' sub-sections are routes of their own. Each view // owns the table of its own, since each also builds the strip that links to @@ -147,7 +149,7 @@ function render() { // the settings should not land you halfway down them. So does a changed // subject — one team to the next is two pages, not one scrolled page. else if (prev.section !== route.section || prev.tab !== route.tab - || prev.user !== route.user || prev.team !== route.team) window.scrollTo(0, 0); + || prev.user !== route.user || prev.team !== route.team || prev.alert !== route.alert) window.scrollTo(0, 0); updateTitle(); } diff --git a/internal/web/static/js/charts.js b/internal/web/static/js/charts.js new file mode 100644 index 0000000..9ca14cd --- /dev/null +++ b/internal/web/static/js/charts.js @@ -0,0 +1,117 @@ +// Shared by the stats overview and the per-alert drilldown: the time-range +// chips, the chart cards and the SVG column chart. + +import { h } from './ui.js'; + +const DAY_MS = 24 * 60 * 60 * 1000; + +// `days` counts back from today, inclusive; the server reads from/to as UTC +// dates, so these are too. +export const RANGES = [ + { id: 'today', label: 'Today', days: 1 }, + { id: '7d', label: '7d', days: 7 }, + { id: '30d', label: '30d', days: 30 }, + { id: '90d', label: '90d', days: 90 }, + { id: 'all', label: 'All', days: null }, +]; + +// The chosen range outlives navigating between the overview and a drilldown. +let range = '30d'; +export const getRange = () => range; +export const setRangeId = (id) => { range = id; }; + +const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10); + +export function rangeQuery() { + const r = RANGES.find((x) => x.id === range); + if (!r.days) return {}; + const now = Date.now(); + return { from: utcDate(now - (r.days - 1) * DAY_MS), to: utcDate(now) }; +} + +export function chips(onPick, ranges = RANGES) { + return h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Time range' }, + ranges.map((r) => h('button', { + class: 'chip', + type: 'button', + role: 'tab', + 'aria-selected': String(r.id === range), + onclick: () => onPick(r.id), + text: r.label, + }))); +} + +export function card(title, content, note) { + return h('section', { class: 'chart-card card card-pad' }, + h('div', { class: 'chart-head' }, + h('h3', { class: 'chart-title', text: title }), + note && h('span', { class: 'chart-note', text: note })), + content); +} + +export const pad2 = (n) => String(n).padStart(2, '0'); +export const WEEKDAYS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']; +// Monday first, as the mockup has it; server weekdays are 0=Sunday. +export const MONDAY_FIRST = [1, 2, 3, 4, 5, 6, 0]; + +const SVG_NS = 'http://www.w3.org/2000/svg'; + +function svg(tag, attrs = {}, text) { + const el = document.createElementNS(SVG_NS, tag); + for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v)); + if (text != null) el.textContent = text; + return el; +} + +// A column chart: one bar per item, the value in a tooltip, and a label under +// the items marked `tick`. +export function columns(items, label) { + const W = 480; + const H = 140; + const base = H - 18; + const step = W / items.length; + const max = Math.max(...items.map((x) => x.value), 1); + const root = svg('svg', { + class: 'columns', viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': label, + }); + root.appendChild(svg('line', { class: 'axis', x1: 0, x2: W, y1: base, y2: base })); + items.forEach((it, i) => { + const bh = it.value ? Math.max(2, (it.value / max) * (base - 6)) : 0; + const x = i * step + step * 0.15; + const g = svg('g', { class: 'col' }); + g.appendChild(svg('title', {}, `${it.label}: ${it.value}`)); + // A full-height transparent hit area, so a tiny bar is still hoverable. + g.appendChild(svg('rect', { class: 'col-hit', x: i * step, y: 0, width: step, height: base })); + if (bh) g.appendChild(svg('rect', { class: 'col-bar', x, y: base - bh, width: step * 0.7, height: bh, rx: 2 })); + root.appendChild(g); + if (it.tick) { + root.appendChild(svg('text', { class: 'col-label', x: i * step + step / 2, y: H - 4, 'text-anchor': 'middle' }, it.label)); + } + }); + return root; +} + +// A 7×24 weekday-by-hour grid from `grid[weekday][hour]` (0=Sunday); cell +// opacity steps with the count relative to the busiest cell. +export function heatmap(grid) { + const max = Math.max(...grid.flat(), 1); + const level = (c) => (c === 0 ? 0 : Math.min(4, Math.ceil((c / max) * 4))); + const rows = MONDAY_FIRST.map((w) => h('div', { class: 'heat-row' }, + h('span', { class: 'heat-day', text: WEEKDAYS[w] }), + h('div', { class: 'heat-cells' }, grid[w].map((c, hr) => h('div', { + class: `heat-cell lv${level(c)}`, + title: `${WEEKDAYS[w]} ${pad2(hr)}:00 UTC: ${c}`, + }))))); + const axis = h('div', { class: 'heat-row' }, + h('span', { class: 'heat-day' }), + h('div', { class: 'heat-cells' }, Array.from({ length: 24 }, (_, hr) => + h('span', { class: 'heat-hour', text: hr % 3 === 0 ? pad2(hr) : '' })))); + const peak = grid.flatMap((row, d) => row.map((c, hr) => ({ d, hr, c }))).reduce((a, b) => (b.c > a.c ? b : a)); + return h('div', { + class: 'heatmap', + role: 'img', + 'aria-label': peak.c + ? `Heatmap of incidents by weekday and hour in UTC. The busiest slot is ${WEEKDAYS[peak.d]} at ${pad2(peak.hr)}:00 with ${peak.c}.` + : 'Heatmap of incidents by weekday and hour in UTC. Nothing in this range.', + }, rows, axis); +} diff --git a/internal/web/static/js/stats.js b/internal/web/static/js/stats.js index 41ec91a..24e34c6 100644 --- a/internal/web/static/js/stats.js +++ b/internal/web/static/js/stats.js @@ -1,99 +1,87 @@ // Statistics: how many incidents, how fast they are answered, and when and // what the alerts are. The same figures the TUI's Stats tab shows, over a // range picked with the chips. The server scopes them to the caller's teams. +// /stats/alert/{name} is the drilldown behind a row of the top-alerts list. import * as api from './api.js'; -import { h, clear, emptyState, spinner } from './ui.js'; +import { h, clear, emptyState, spinner, icon } from './ui.js'; import { duration } from './format.js'; - -const DAY_MS = 24 * 60 * 60 * 1000; - -// `days` counts back from today, inclusive; the server reads from/to as UTC -// dates, so these are too. -const RANGES = [ - { id: 'today', label: 'Today', days: 1 }, - { id: '7d', label: '7d', days: 7 }, - { id: '30d', label: '30d', days: 30 }, - { id: '90d', label: '90d', days: 90 }, - { id: 'all', label: 'All', days: null }, -]; +import { chips, card, heatmap, rangeQuery, getRange, setRangeId, pad2 } from './charts.js'; +import * as drill from './statsalert.js'; const view = () => document.getElementById('view-stats'); -let range = '30d'; +// What the page shows: the overview (null) or one alert's drilldown. +let alertName = null; let data = null; let error = null; -const utcDate = (ms) => new Date(ms).toISOString().slice(0, 10); - -function query(r) { - if (!r.days) return {}; - const now = Date.now(); - return { from: utcDate(now - (r.days - 1) * DAY_MS), to: utcDate(now) }; -} - -export function show() { +export function show(route) { + const next = route?.alert ?? null; + if (next !== alertName) data = null; + alertName = next; + error = null; render(); refresh(); } export async function refresh() { - const requested = range; - const q = query(RANGES.find((x) => x.id === range)); + const requested = getRange(); + const name = alertName; + const q = rangeQuery(); try { - const [incidents, top, byHour, byDay] = await Promise.all([ - api.statsIncidents(q), - api.statsTop({ ...q, limit: 10 }), - api.statsByHour(q), - api.statsByDay(q), - ]); - if (requested !== range) return; - data = { incidents, top, byHour, byDay }; + if (name != null) { + try { + data = await drill.load(name); + } catch (err) { + if (err.status !== 404) throw err; + data = 'none'; + } + } else { + const [incidents, top, heat] = await Promise.all([ + api.statsIncidents(q), + api.statsTop({ ...q, limit: 10 }), + api.statsHeatmap(q), + ]); + data = { incidents, top, heat }; + } + if (requested !== getRange() || name !== alertName) return; error = null; } catch (err) { - if (requested !== range) return; + if (requested !== getRange() || name !== alertName) return; error = err.message; } render(); } function setRange(id) { - if (id === range) return; - range = id; + if (id === getRange()) return; + setRangeId(id); data = null; render(); refresh(); } function render() { - const chips = h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Time range' }, - RANGES.map((r) => h('button', { - class: 'chip', - type: 'button', - role: 'tab', - 'aria-selected': String(r.id === range), - onclick: () => setRange(r.id), - text: r.label, - }))); - + if (alertName != null) { + clear(view(), drill.body(alertName, data, error, setRange)); + return; + } let body; if (error && !data) body = h('div', { class: 'load-error', text: error }); else if (!data) body = spinner(); - else if (!data.incidents.total && !data.byHour.some((x) => x.count)) { + else if (!data.incidents.total) { body = emptyState('No data in this range', 'Nothing happened in this window.', 'chart'); } else { body = h('div', { class: 'stats' }, error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }), tiles(data.incidents), - data.top.length > 0 && card('Top alerts', topAlerts(data.top)), - card('Alerts by hour (UTC)', columns( - data.byHour.map((x) => ({ label: String(x.hour), value: x.count, tick: x.hour % 6 === 0 })), - 'Alerts by hour of day')), - card('Alerts by day', columns( - data.byDay.map((x) => ({ label: x.day_name.slice(0, 3), value: x.count, tick: true })), - 'Alerts by day of week'))); + data.top.length > 0 && card('Top alerts', topAlerts(data.top), 'Select an alert to see when and how often it fires'), + h('div', { class: 'stats-split' }, + card('When incidents open', heatmap(data.heat.grid), 'Weekday × hour (UTC)'), + patternsCard(data))); } - clear(view(), h('div', {}, chips, body)); + clear(view(), h('div', {}, chips(setRange), body)); } // A missing mean means nothing has been acknowledged or resolved yet. @@ -112,57 +100,65 @@ function tiles(s) { tile('Mean time to resolve', mean(s.mttr_seconds))); } -function card(title, content) { - return h('section', { class: 'chart-card card card-pad' }, - h('h3', { class: 'chart-title', text: title }), content); -} - -// Ranked names with a bar scaled to the busiest one. -function topAlerts(items) { - const max = Math.max(...items.map((x) => x.count), 1); - return h('ol', { class: 'hbars' }, items.map((x) => { - const fill = h('span', { class: 'hbar-fill' }); - fill.style.width = `${Math.max(2, (x.count / max) * 100)}%`; - return h('li', { class: 'hbar' }, - h('span', { class: 'hbar-name', title: x.name, text: x.name }), - h('span', { class: 'hbar-track' }, fill), - h('span', { class: 'hbar-count', text: String(x.count) })); +function spark(daily) { + const max = Math.max(...daily, 1); + return h('span', { class: 'spark', 'aria-hidden': 'true' }, daily.map((c) => { + const b = h('span', { class: c ? 'spark-bar on' : 'spark-bar' }); + b.style.height = `${c ? 6 + (c / max) * 16 : 2}px`; + return b; })); } -const SVG_NS = 'http://www.w3.org/2000/svg'; +const lastSeen = (unix) => { + const d = new Date(unix * 1000); + return `${d.toLocaleString('en', { month: 'short', timeZone: 'UTC' })} ${d.getUTCDate()}, ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}`; +}; -function svg(tag, attrs = {}, text) { - const el = document.createElementNS(SVG_NS, tag); - for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, String(v)); - if (text != null) el.textContent = text; - return el; +// Ranked names, each a link to its drilldown: a bar scaled to the busiest, the +// last 30 days as a sparkline, when it last fired and how long it takes to fix. +function topAlerts(items) { + const max = Math.max(...items.map((x) => x.count), 1); + const total = items.reduce((n, x) => n + x.count, 0); + return h('div', { class: 'toplist' }, + h('div', { class: 'top-row top-head', 'aria-hidden': 'true' }, + ['Alert', 'Count', 'Last 30 days', 'Last seen', 'Mean fix'].map((t) => h('span', { text: t }))), + items.map((x) => { + const fill = h('span', { class: 'hbar-fill' }); + fill.style.width = `${Math.max(2, (x.count / max) * 100)}%`; + return h('a', { class: 'top-row', href: drill.path(x.name) }, + h('span', { class: 'hbar-name', title: x.name, text: x.name }), + h('span', { class: 'top-count' }, + h('span', { class: 'hbar-count', text: String(x.count) }), + h('span', { class: 'hbar-track' }, fill), + h('span', { class: 'top-pct', text: `${Math.round((x.count / total) * 100)}%` })), + spark(x.daily), + h('span', { class: 'top-cell', text: lastSeen(x.last_seen) }), + h('span', { class: 'top-cell', text: mean(x.mean_resolve_seconds) }), + icon('chevronRight')); + })); } -// A column chart: one bar per item, the value in a tooltip, and a label under -// the items marked `tick`. -function columns(items, label) { - const W = 480; - const H = 140; - const base = H - 18; - const step = W / items.length; - const max = Math.max(...items.map((x) => x.value), 1); - const root = svg('svg', { - class: 'columns', viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': label, - }); - root.appendChild(svg('line', { class: 'axis', x1: 0, x2: W, y1: base, y2: base })); - items.forEach((it, i) => { - const bh = it.value ? Math.max(2, (it.value / max) * (base - 6)) : 0; - const x = i * step + step * 0.15; - const g = svg('g', { class: 'col' }); - g.appendChild(svg('title', {}, `${it.label}: ${it.value}`)); - // A full-height transparent hit area, so a tiny bar is still hoverable. - g.appendChild(svg('rect', { class: 'col-hit', x: i * step, y: 0, width: step, height: base })); - if (bh) g.appendChild(svg('rect', { class: 'col-bar', x, y: base - bh, width: step * 0.7, height: bh, rx: 2 })); - root.appendChild(g); - if (it.tick) { - root.appendChild(svg('text', { class: 'col-label', x: i * step + step / 2, y: H - 4, 'text-anchor': 'middle' }, it.label)); - } - }); - return root; +// Observations drawn from the figures already on the page. +function patternsCard(d) { + const flat = d.heat.grid; + const total = flat.flat().reduce((a, b) => a + b, 0); + const days = flat.map((row) => row.reduce((a, b) => a + b, 0)); + const hours = Array.from({ length: 24 }, (_, hr) => flat.reduce((a, row) => a + row[hr], 0)); + const topDay = days.indexOf(Math.max(...days)); + const topHour = hours.indexOf(Math.max(...hours)); + const topAlert = d.top[0]; + const full = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; + const items = []; + if (total > 1) { + items.push([`${full[topDay]}s are the noisiest day`, `${days[topDay]} of ${total} incidents opened on a ${full[topDay]}.`]); + items.push([`Peak hour is ${pad2(topHour)}:00 UTC`, `${hours[topHour]} incidents opened in that single hour.`]); + } + if (topAlert && total > 1 && topAlert.count / total >= 0.4) { + items.push(['One alert drives most of the volume', `${topAlert.name} is ${Math.round((topAlert.count / total) * 100)}% of everything that fired.`]); + } + const body = items.length + ? items.map(([t, s]) => h('div', { class: 'pattern' }, h('div', { class: 'pattern-title', text: t }), h('div', { class: 'pattern-sub', text: s }))) + : h('p', { class: 'chart-note', text: 'Not enough incidents yet to see a pattern.' }); + return card('Patterns', h('div', { class: 'patterns' }, body, + topAlert && h('a', { class: 'btn btn-ghost', href: drill.path(topAlert.name), text: `Open ${topAlert.name} →` }))); } diff --git a/internal/web/static/js/statsalert.js b/internal/web/static/js/statsalert.js new file mode 100644 index 0000000..2b44883 --- /dev/null +++ b/internal/web/static/js/statsalert.js @@ -0,0 +1,136 @@ +// The drilldown behind a row of the stats page's top-alerts list: when one +// alert fired, how fast it was picked up and fixed. An occurrence is an +// incident that alert opened (alert rows carry no history of their own). + +import * as api from './api.js'; +import { h, emptyState, spinner } from './ui.js'; +import { duration } from './format.js'; +import { chips, card, columns, rangeQuery, pad2, WEEKDAYS, MONDAY_FIRST } from './charts.js'; + +const sec = (s) => (s == null ? '—' : duration(s * 1000)); +const fmtTime = (unix) => { + const d = new Date(unix * 1000); + const mon = d.toLocaleString('en', { month: 'short', timeZone: 'UTC' }); + return `${mon} ${d.getUTCDate()}, ${pad2(d.getUTCHours())}:${pad2(d.getUTCMinutes())}`; +}; + +export const path = (name) => `/stats/alert/${encodeURIComponent(name)}`; + +export async function load(name) { + return api.statsAlert(name, rangeQuery()); +} + +function tiles(d) { + const tile = (label, value, note) => h('div', { class: 'stat-tile' }, + h('div', { class: 'stat-value', text: String(value) }), + h('div', { class: 'stat-label', text: label }), + note && h('div', { class: 'stat-note', text: note })); + return h('div', { class: 'stat-tiles' }, + tile('Occurrences', d.count), + tile('Last seen', fmtTime(d.last_seen), 'UTC'), + tile('Average gap', d.mean_gap_seconds == null ? '—' : sec(d.mean_gap_seconds), 'between occurrences'), + tile('Mean time to acknowledge', sec(d.mean_ack_seconds), d.median_ack_seconds == null ? '' : `median ${sec(d.median_ack_seconds)}`), + tile('Mean time to resolve', sec(d.mean_resolve_seconds), d.median_resolve_seconds == null ? '' : `median ${sec(d.median_resolve_seconds)}`), + tile('Longest open', sec(d.longest_open_seconds))); +} + +// Plain-language observations, derived from the same occurrences. +function patterns(d) { + const n = d.count; + const out = []; + const sunday = d.by_weekday[0]; + const top = d.by_weekday.indexOf(Math.max(...d.by_weekday)); + if (n > 1 && d.by_weekday[top] > n / 2) out.push(`${d.by_weekday[top]} of ${n} fired on a ${['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'][top]}`); + else if (sunday > 0 && n > 1) out.push(`${sunday} of ${n} fired on a Sunday`); + const peak = d.by_hour.indexOf(Math.max(...d.by_hour)); + if (n > 1 && d.by_hour[peak] > 1) out.push(`Peak hour is ${pad2(peak)}:00 UTC (${d.by_hour[peak]} of ${n})`); + const times = d.occurrences.map((o) => o.triggered_at).sort((a, b) => a - b); + const repeats = times.filter((t, i) => i > 0 && t - times[i - 1] < 86400).length; + if (repeats) out.push(`${repeats} of ${n} fired again within 24 hours of the previous one`); + return out; +} + +function patternStrip(d) { + const items = patterns(d); + if (!items.length) return null; + return h('div', { class: 'pattern-strip card card-pad' }, + h('span', { class: 'chart-title', text: 'Pattern' }), + items.map((t) => h('span', { text: t }))); +} + +// One block per occurrence along a time axis from the first to the last, as wide +// as the incident stayed open. Overlaps drop to a second lane. +function timeline(d) { + const occ = [...d.occurrences].sort((a, b) => a.triggered_at - b.triggered_at); + const start = occ[0].triggered_at; + const end = Math.max(...occ.map((o) => o.triggered_at + (o.resolve_seconds || 0)), start + 3600); + const span = end - start; + let prev = -100; + const marks = occ.map((o) => { + const left = ((o.triggered_at - start) / span) * 100; + const lane = left - prev < 1.5 ? 1 : 0; + prev = left; + const m = h('div', { + class: 'tl-mark', + title: `${fmtTime(o.triggered_at)} UTC · ${o.resolve_seconds == null ? 'still open' : `open for ${sec(o.resolve_seconds)}`}`, + }); + m.style.left = `${left}%`; + m.style.width = `${((o.resolve_seconds || 0) / span) * 100}%`; + m.style.top = `${8 + lane * 22}px`; + return m; + }); + return h('div', {}, + h('div', { class: 'tl', role: 'img', 'aria-label': `Timeline of ${occ.length} occurrences between ${fmtTime(start)} and ${fmtTime(end)} UTC.` }, marks), + h('div', { class: 'tl-axis' }, + h('span', { text: fmtTime(start) }), h('span', { text: fmtTime(end) }))); +} + +// Per occurrence: the wait for an acknowledgement, then the time being worked. +function fixBars(d) { + const rows = d.occurrences.filter((o) => o.resolve_seconds != null).slice(0, 12); + if (!rows.length) return h('p', { class: 'chart-note', text: 'Nothing resolved in this range.' }); + const max = Math.max(...rows.map((o) => o.resolve_seconds), 1); + return h('div', { class: 'fixbars' }, rows.map((o) => { + const ack = Math.min(o.ack_seconds ?? 0, o.resolve_seconds); + const a = h('span', { class: 'fix-ack', title: `Waiting for acknowledgement: ${sec(ack)}` }); + const w = h('span', { class: 'fix-work', title: `Being worked: ${sec(o.resolve_seconds - ack)}` }); + a.style.width = `${(ack / max) * 100}%`; + w.style.width = `${((o.resolve_seconds - ack) / max) * 100}%`; + return h('div', { class: 'fix-row' }, + h('span', { class: 'fix-label', text: fmtTime(o.triggered_at) }), + h('span', { class: 'fix-track' }, a, w), + h('span', { class: 'fix-total', text: sec(o.resolve_seconds) })); + })); +} + +function table(d) { + return h('div', { class: 'occ-wrap' }, + h('table', { class: 'occ' }, + h('thead', {}, h('tr', {}, ['Fired (UTC)', 'Time to ack', 'Duration', 'Acknowledged by', 'Status'].map((t) => h('th', { text: t })))), + h('tbody', {}, d.occurrences.map((o) => h('tr', {}, + h('td', {}, h('a', { href: `/incidents/${o.incident_id}`, text: fmtTime(o.triggered_at) }), + ` ${WEEKDAYS[new Date(o.triggered_at * 1000).getUTCDay()]}`), + h('td', { text: sec(o.ack_seconds) }), + h('td', { text: sec(o.resolve_seconds) }), + h('td', { text: o.acknowledged_by || '—' }), + h('td', { text: o.status })))))); +} + +export function body(name, d, error, onRange) { + const head = h('div', { class: 'stats-head' }, + h('a', { class: 'btn btn-ghost', href: '/stats', text: '← Stats' }), + h('h2', { class: 'stats-alert-name', text: name }), + chips(onRange)); + if (error && !d) return h('div', {}, head, h('div', { class: 'load-error', text: error })); + if (!d) return h('div', {}, head, spinner()); + if (d === 'none') return h('div', {}, head, emptyState('No occurrences in this range', 'This alert did not open an incident in this window.', 'chart')); + return h('div', {}, head, h('div', { class: 'stats' }, + error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }), + tiles(d), + patternStrip(d), + card('When it fired', timeline(d), 'Each block spans how long the incident stayed open'), + card('By hour (UTC)', columns(d.by_hour.map((v, hr) => ({ label: pad2(hr), value: v, tick: hr % 6 === 0 })), 'Occurrences by hour of day')), + card('By weekday', columns(MONDAY_FIRST.map((w) => ({ label: WEEKDAYS[w], value: d.by_weekday[w], tick: true })), 'Occurrences by weekday')), + card('Time to resolve', fixBars(d), 'Amber waited for an acknowledgement; blue was worked'), + card('All occurrences', table(d), `Newest first${d.count > d.occurrences.length ? `, latest ${d.occurrences.length} of ${d.count}` : ''}`))); +} diff --git a/internal/web/stats_test.go b/internal/web/stats_test.go index d743878..5d7f3eb 100644 --- a/internal/web/stats_test.go +++ b/internal/web/stats_test.go @@ -20,7 +20,9 @@ func TestStatsPageIsEmbedded(t *testing.T) { t.Errorf("index.html lacks %s", want) } } - if _, err := fs.Stat(sub, "js/stats.js"); err != nil { - t.Errorf("js/stats.js not embedded: %v", err) + for _, f := range []string{"js/stats.js", "js/statsalert.js", "js/charts.js"} { + if _, err := fs.Stat(sub, f); err != nil { + t.Errorf("%s not embedded: %v", f, err) + } } }