Niklas Ye
6235a2f737
The stats page could only say how many alerts fired. It now answers when
one alert fires, how often, and how fast it is picked up and fixed: each
row of the top-alerts list opens /stats/alert/{name}, with a timeline,
by-hour and by-weekday charts, time-to-resolve per occurrence and a table
of every occurrence linking to its incident. The overview gains a
weekday-by-hour heatmap and a few derived observations.
An occurrence is an incident, not an alert row. Alert rows are reused per
fingerprint and mutated in place, so they carry no history; incidents are
the only record of each time something fired and was answered. That means
/api/stats/alerts/top now counts incidents opened (filtered on
triggered_at) so the list and the drilldown agree. The response keeps name
and count and only adds fields, so existing clients keep working, but the
counts can differ from before.
No migration: the lookups use incidents_signature_idx. Label breakdown,
team and severity filters and period-over-period deltas from the design
draft are left for later.
Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN
476 lines
15 KiB
Go
476 lines
15 KiB
Go
package api
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
)
|
|
|
|
func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))
|
|
|
|
// COALESCE because SUM over zero rows is NULL, not 0, and a count of
|
|
// nothing is 0 — without it an empty window is a 500 rather than a
|
|
// legitimately empty report.
|
|
var total, firing, resolved int64
|
|
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
|
|
SELECT COUNT(*),
|
|
COALESCE(SUM(CASE WHEN status = 'firing' THEN 1 ELSE 0 END), 0),
|
|
COALESCE(SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END), 0)
|
|
FROM alerts WHERE %s`, where), args.all()...,
|
|
).Scan(&total, &firing, &resolved)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
respond(w, http.StatusOK, map[string]int64{
|
|
"total": total,
|
|
"firing": firing,
|
|
"resolved": resolved,
|
|
})
|
|
}
|
|
}
|
|
|
|
// alertNameSQL is the alert-name half of an incident's signature (see
|
|
// incidentSignature: "name|stable,labels"). An occurrence of an alert is an
|
|
// incident, because alert rows are reused per fingerprint and carry no history.
|
|
const alertNameSQL = "split_part(signature, '|', 1)"
|
|
|
|
const sparkDays = 30
|
|
|
|
// handleStatsTop ranks alert names by how many incidents they opened in the
|
|
// window, with what a list row needs besides the count: when it last fired, how
|
|
// long it takes to fix, and a per-day series for the last sparkDays days.
|
|
func handleStatsTop(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
q := r.URL.Query()
|
|
teams := callerTeamIDs(r.Context())
|
|
where, args := statsFilter(q, "triggered_at", teams)
|
|
|
|
limit := 10
|
|
if l := q.Get("limit"); l != "" {
|
|
var n int
|
|
fmt.Sscanf(l, "%d", &n)
|
|
if n > 0 && n <= 100 {
|
|
limit = n
|
|
}
|
|
}
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT %[1]s AS name, COUNT(*) AS cnt, MAX(triggered_at),
|
|
AVG(CASE WHEN resolved_at IS NOT NULL THEN resolved_at - triggered_at END)
|
|
FROM incidents
|
|
WHERE %[2]s
|
|
GROUP BY name
|
|
ORDER BY cnt DESC, name ASC
|
|
LIMIT %[3]s`, alertNameSQL, where, args.add(limit)), args.all()...)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
type entry struct {
|
|
Name string `json:"name"`
|
|
Count int64 `json:"count"`
|
|
LastSeen int64 `json:"last_seen"`
|
|
MeanResolve *float64 `json:"mean_resolve_seconds"`
|
|
Daily []int64 `json:"daily"`
|
|
}
|
|
result := []entry{}
|
|
names := []string{}
|
|
for rows.Next() {
|
|
var e entry
|
|
if err := rows.Scan(&e.Name, &e.Count, &e.LastSeen, &e.MeanResolve); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
e.Daily = make([]int64, sparkDays)
|
|
result = append(result, e)
|
|
names = append(names, e.Name)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
|
|
// The sparkline always covers the last sparkDays days ending at ?to (or
|
|
// today), whatever ?from says, so the rows stay comparable.
|
|
end := time.Now().UTC().Truncate(24 * time.Hour)
|
|
if to, err := time.Parse("2006-01-02", q.Get("to")); err == nil {
|
|
end = to.UTC()
|
|
}
|
|
first := end.AddDate(0, 0, -(sparkDays - 1))
|
|
sq := url.Values{}
|
|
sq.Set("from", first.Format("2006-01-02"))
|
|
sq.Set("to", end.Format("2006-01-02"))
|
|
swhere, sargs := statsFilter(sq, "triggered_at", teams)
|
|
if len(names) > 0 {
|
|
swhere += " AND " + alertNameSQL + " = ANY(" + sargs.add(names) + ")"
|
|
srows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT %s AS name, ((triggered_at - %s) / 86400)::int AS day, COUNT(*)
|
|
FROM incidents
|
|
WHERE %s
|
|
GROUP BY name, day`, alertNameSQL, sargs.add(first.Unix()), swhere), sargs.all()...)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
defer srows.Close()
|
|
idx := map[string]int{}
|
|
for i, e := range result {
|
|
idx[e.Name] = i
|
|
}
|
|
for srows.Next() {
|
|
var name string
|
|
var day int
|
|
var cnt int64
|
|
if err := srows.Scan(&name, &day, &cnt); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
if day >= 0 && day < sparkDays {
|
|
result[idx[name]].Daily[day] = cnt
|
|
}
|
|
}
|
|
if err := srows.Err(); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
}
|
|
respond(w, http.StatusOK, result)
|
|
}
|
|
}
|
|
|
|
// handleStatsHeatmap counts incidents by UTC weekday (0=Sunday) and hour:
|
|
// always 7 rows of 24 cells, so a client never has to fill gaps.
|
|
func handleStatsHeatmap(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query(), "triggered_at", callerTeamIDs(r.Context()))
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT EXTRACT(DOW FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
|
|
EXTRACT(HOUR FROM to_timestamp(triggered_at) AT TIME ZONE 'UTC')::int,
|
|
COUNT(*)
|
|
FROM incidents
|
|
WHERE %s
|
|
GROUP BY 1, 2`, where), args.all()...)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
grid := make([][]int64, 7)
|
|
for d := range grid {
|
|
grid[d] = make([]int64, 24)
|
|
}
|
|
for rows.Next() {
|
|
var d, h int
|
|
var c int64
|
|
if err := rows.Scan(&d, &h, &c); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
grid[d][h] = c
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
respond(w, http.StatusOK, map[string]any{"grid": grid})
|
|
}
|
|
}
|
|
|
|
// maxOccurrences caps the drilldown's table; the aggregates still cover all.
|
|
const maxOccurrences = 200
|
|
|
|
// handleStatsAlert is the per-alert drilldown. Every figure is computed over
|
|
// the incidents that alert opened in the window, so it agrees with the top list.
|
|
func handleStatsAlert(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
// chi hands back the still-escaped segment when the path has %2F in it.
|
|
name, err := url.PathUnescape(chi.URLParam(r, "name"))
|
|
if err != nil {
|
|
respond(w, http.StatusBadRequest, errResp("invalid alert name"))
|
|
return
|
|
}
|
|
where, args := statsFilter(r.URL.Query(), "i.triggered_at", callerTeamIDs(r.Context()))
|
|
where = strings.ReplaceAll(where, "archived_at", "i.archived_at")
|
|
where = strings.ReplaceAll(where, "team_id =", "i.team_id =")
|
|
where += " AND split_part(i.signature, '|', 1) = " + args.add(name)
|
|
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT i.id, i.triggered_at, i.acknowledged_at, i.resolved_at, i.status, i.severity,
|
|
COALESCE(u.username, sa.name)
|
|
FROM incidents i
|
|
LEFT JOIN users u ON u.id = i.acknowledged_by
|
|
LEFT JOIN service_accounts sa ON sa.id = i.acknowledged_by_service_account_id
|
|
WHERE %s
|
|
ORDER BY i.triggered_at DESC`, where), args.all()...)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
type occurrence struct {
|
|
IncidentID int64 `json:"incident_id"`
|
|
TriggeredAt int64 `json:"triggered_at"`
|
|
AckSeconds *int64 `json:"ack_seconds"`
|
|
ResolveSeconds *int64 `json:"resolve_seconds"`
|
|
Status string `json:"status"`
|
|
Severity *string `json:"severity,omitempty"`
|
|
AcknowledgedBy *string `json:"acknowledged_by,omitempty"`
|
|
}
|
|
all := []occurrence{}
|
|
for rows.Next() {
|
|
var o occurrence
|
|
var ack, res *int64
|
|
if err := rows.Scan(&o.IncidentID, &o.TriggeredAt, &ack, &res, &o.Status, &o.Severity, &o.AcknowledgedBy); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
if ack != nil {
|
|
v := *ack - o.TriggeredAt
|
|
o.AckSeconds = &v
|
|
}
|
|
if res != nil {
|
|
v := *res - o.TriggeredAt
|
|
o.ResolveSeconds = &v
|
|
}
|
|
all = append(all, o)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
if len(all) == 0 {
|
|
respond(w, http.StatusNotFound, errResp("no incidents for this alert"))
|
|
return
|
|
}
|
|
|
|
byHour := make([]int64, 24)
|
|
byDay := make([]int64, 7)
|
|
var acks, fixes []int64
|
|
var longest int64
|
|
for _, o := range all {
|
|
t := time.Unix(o.TriggeredAt, 0).UTC()
|
|
byHour[t.Hour()]++
|
|
byDay[int(t.Weekday())]++
|
|
if o.AckSeconds != nil {
|
|
acks = append(acks, *o.AckSeconds)
|
|
}
|
|
if o.ResolveSeconds != nil {
|
|
fixes = append(fixes, *o.ResolveSeconds)
|
|
longest = max(longest, *o.ResolveSeconds)
|
|
}
|
|
}
|
|
// all is newest first, so the gap runs from the last row back to the first.
|
|
var meanGap *float64
|
|
if n := len(all); n > 1 {
|
|
g := float64(all[0].TriggeredAt-all[n-1].TriggeredAt) / float64(n-1)
|
|
meanGap = &g
|
|
}
|
|
stat := func(xs []int64) (mean, median *float64) {
|
|
if len(xs) == 0 {
|
|
return nil, nil
|
|
}
|
|
s := append([]int64(nil), xs...)
|
|
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
|
var sum int64
|
|
for _, x := range s {
|
|
sum += x
|
|
}
|
|
m, md := float64(sum)/float64(len(s)), float64(s[len(s)/2])
|
|
return &m, &md
|
|
}
|
|
meanAck, medAck := stat(acks)
|
|
meanFix, medFix := stat(fixes)
|
|
|
|
occ := all
|
|
if len(occ) > maxOccurrences {
|
|
occ = occ[:maxOccurrences]
|
|
}
|
|
resp := map[string]any{
|
|
"name": name,
|
|
"count": len(all),
|
|
"first_seen": all[len(all)-1].TriggeredAt,
|
|
"last_seen": all[0].TriggeredAt,
|
|
"mean_gap_seconds": meanGap,
|
|
"mean_ack_seconds": meanAck,
|
|
"median_ack_seconds": medAck,
|
|
"mean_resolve_seconds": meanFix,
|
|
"median_resolve_seconds": medFix,
|
|
"longest_open_seconds": nil,
|
|
"by_hour": byHour,
|
|
"by_weekday": byDay,
|
|
"occurrences": occ,
|
|
}
|
|
if len(fixes) > 0 {
|
|
resp["longest_open_seconds"] = longest
|
|
}
|
|
respond(w, http.StatusOK, resp)
|
|
}
|
|
}
|
|
|
|
func handleStatsByHour(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))
|
|
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT EXTRACT(HOUR FROM to_timestamp(received_at) AT TIME ZONE 'UTC')::int AS hr,
|
|
COUNT(*) AS cnt
|
|
FROM alerts
|
|
WHERE %s
|
|
GROUP BY hr
|
|
ORDER BY hr ASC`, where), args.all()...)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
counts := make(map[int]int64, 24)
|
|
for rows.Next() {
|
|
var hr int
|
|
var cnt int64
|
|
if err := rows.Scan(&hr, &cnt); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
counts[hr] = cnt
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
|
|
type entry struct {
|
|
Hour int `json:"hour"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
result := make([]entry, 24)
|
|
for h := range result {
|
|
result[h] = entry{Hour: h, Count: counts[h]}
|
|
}
|
|
respond(w, http.StatusOK, result)
|
|
}
|
|
}
|
|
|
|
func handleStatsByDay(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query(), "received_at", callerTeamIDs(r.Context()))
|
|
|
|
// Postgres EXTRACT(DOW …) → 0=Sunday … 6=Saturday.
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT EXTRACT(DOW FROM to_timestamp(received_at) AT TIME ZONE 'UTC')::int AS dow,
|
|
COUNT(*) AS cnt
|
|
FROM alerts
|
|
WHERE %s
|
|
GROUP BY dow
|
|
ORDER BY dow ASC`, where), args.all()...)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
counts := make(map[int]int64, 7)
|
|
for rows.Next() {
|
|
var dow int
|
|
var cnt int64
|
|
if err := rows.Scan(&dow, &cnt); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
counts[dow] = cnt
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
|
|
dayNames := [7]string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
|
|
type entry struct {
|
|
Day int `json:"day"`
|
|
DayName string `json:"day_name"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
result := make([]entry, 7)
|
|
for d := range result {
|
|
result[d] = entry{Day: d, DayName: dayNames[d], Count: counts[d]}
|
|
}
|
|
respond(w, http.StatusOK, result)
|
|
}
|
|
}
|
|
|
|
// handleStatsIncidents reports the queue and the two numbers a rota actually
|
|
// cares about: how long it takes someone to pick work up, and how long it takes
|
|
// to finish. Neither was computable before incidents existed — alert rows are
|
|
// mutated in place and carry no acknowledgement or closure time.
|
|
func handleStatsIncidents(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query(), "triggered_at", callerTeamIDs(r.Context()))
|
|
|
|
// The counts are COALESCEd because SUM over zero rows is NULL, not 0.
|
|
// The averages are not: mtta and mttr stay null on purpose, since zero
|
|
// would read as "instant" rather than "nothing to measure yet".
|
|
var total, triggered, acknowledged, resolved int64
|
|
var mtta, mttr *float64
|
|
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
|
|
SELECT COUNT(*),
|
|
COALESCE(SUM(CASE WHEN status = 'triggered' THEN 1 ELSE 0 END), 0),
|
|
COALESCE(SUM(CASE WHEN status = 'acknowledged' THEN 1 ELSE 0 END), 0),
|
|
COALESCE(SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END), 0),
|
|
AVG(CASE WHEN acknowledged_at IS NOT NULL
|
|
THEN acknowledged_at - triggered_at END),
|
|
AVG(CASE WHEN resolved_at IS NOT NULL
|
|
THEN resolved_at - triggered_at END)
|
|
FROM incidents WHERE %s`, where), args.all()...,
|
|
).Scan(&total, &triggered, &acknowledged, &resolved, &mtta, &mttr)
|
|
if err != nil {
|
|
serverError(w, r, err)
|
|
return
|
|
}
|
|
|
|
respond(w, http.StatusOK, map[string]any{
|
|
"total": total,
|
|
"triggered": triggered,
|
|
"acknowledged": acknowledged,
|
|
"resolved": resolved,
|
|
// Null until something has actually been acknowledged or resolved —
|
|
// zero would read as "instant", which is a different claim.
|
|
"mtta_seconds": mtta,
|
|
"mttr_seconds": mttr,
|
|
})
|
|
}
|
|
}
|
|
|
|
// statsFilter builds a WHERE clause and args from optional ?from and ?to query
|
|
// params, filtering on timeCol. Archived rows are always excluded, matching the
|
|
// default list views.
|
|
//
|
|
// teamIDs scopes every figure to the caller's own teams: a report that counted
|
|
// other teams' incidents would leak their volume and their names through the
|
|
// top-alerts list, and would not be a number about the reader's work anyway.
|
|
func statsFilter(q url.Values, timeCol string, teamIDs []int64) (where string, args *sqlArgs) {
|
|
args = &sqlArgs{}
|
|
clauses := []string{"archived_at IS NULL", "team_id = ANY(" + args.add(teamIDs) + ")"}
|
|
if from := q.Get("from"); from != "" {
|
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
|
clauses = append(clauses, timeCol+" >= "+args.add(t.UTC().Unix()))
|
|
}
|
|
}
|
|
if to := q.Get("to"); to != "" {
|
|
if t, err := time.Parse("2006-01-02", to); err == nil {
|
|
clauses = append(clauses, timeCol+" < "+args.add(t.UTC().AddDate(0, 0, 1).Unix()))
|
|
}
|
|
}
|
|
return strings.Join(clauses, " AND "), args
|
|
}
|