Files
terdut-server/internal/api/stats.go
T
Niklas Ye 42e846f876
Release / build (amd64, darwin) (push) Failing after 12s
Release / build (arm64, darwin) (push) Failing after 11s
Release / build (arm64, linux) (push) Failing after 11s
Release / release (push) Has been skipped
Release / docker (push) Failing after 19s
Release / build (amd64, linux) (push) Failing after 12s
Release / chart (push) Failing after 9s
Expire stale firing alerts
A resolved webhook was the only path out of the firing state, so a
notification that was dropped, silenced, or lost to a restart pinned an
alert as firing forever — Prometheus showed it resolved while
terdut-server kept listing it. The archiver only ever touched resolved
alerts, and both the list and stats queries compared status with plain
equality, so a stale row was indistinguishable from a live one.

A sweeper pass now resolves firing alerts on either of two signals: the
ends_at watermark Alertmanager sets on outgoing firing notifications has
passed (plus a grace period for clock skew), or no webhook has refreshed
the alert within TERDUT_STALE_AFTER (default 6h, above Alertmanager's 4h
repeat_interval). Such alerts get resolution_source = 'expiry',
distinguishing them from a real 'alertmanager' resolve.

Two related webhook bugs fixed alongside:

  - The upsert had no ordering guard, so a retried firing notification
    arriving after the resolved one resurrected the alert. Payloads for
    an older alert instance are now discarded: a stale retry carries the
    same startsAt, a genuine re-fire a newer one.
  - archived_at was never cleared on re-fire, leaving a re-fired alert
    archived and invisible in the default list.

Stats now exclude archived alerts to match the default list view; this
lowers historical firing/resolved totals.

The chart exposes both sweeper durations via sweeper.staleAfter and
sweeper.archiveAfter.
2026-07-28 11:49:39 +02:00

180 lines
4.7 KiB
Go

package api
import (
"database/sql"
"fmt"
"net/http"
"net/url"
"strings"
"time"
)
func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
where, args := statsFilter(r.URL.Query())
var total, firing, resolved int64
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
SELECT COUNT(*),
SUM(CASE WHEN status = 'firing' THEN 1 ELSE 0 END),
SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END)
FROM alerts WHERE %s`, where), args...,
).Scan(&total, &firing, &resolved)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, map[string]int64{
"total": total,
"firing": firing,
"resolved": resolved,
})
}
}
func handleStatsTop(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
where, args := statsFilter(r.URL.Query())
limit := 10
if l := r.URL.Query().Get("limit"); l != "" {
var n int
fmt.Sscanf(l, "%d", &n)
if n > 0 && n <= 100 {
limit = n
}
}
args = append(args, limit)
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
SELECT name, COUNT(*) AS cnt
FROM alerts
WHERE %s
GROUP BY name
ORDER BY cnt DESC
LIMIT ?`, where), args...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
type entry struct {
Name string `json:"name"`
Count int64 `json:"count"`
}
result := []entry{}
for rows.Next() {
var e entry
if err := rows.Scan(&e.Name, &e.Count); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
result = append(result, e)
}
respond(w, http.StatusOK, result)
}
}
func handleStatsByHour(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
where, args := statsFilter(r.URL.Query())
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
SELECT CAST(strftime('%%H', datetime(received_at, 'unixepoch')) AS INTEGER) AS hr,
COUNT(*) AS cnt
FROM alerts
WHERE %s
GROUP BY hr
ORDER BY hr ASC`, where), args...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
counts := make(map[int]int64, 24)
for rows.Next() {
var hr int
var cnt int64
if err := rows.Scan(&hr, &cnt); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
counts[hr] = cnt
}
type entry struct {
Hour int `json:"hour"`
Count int64 `json:"count"`
}
result := make([]entry, 24)
for h := range result {
result[h] = entry{Hour: h, Count: counts[h]}
}
respond(w, http.StatusOK, result)
}
}
func handleStatsByDay(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
where, args := statsFilter(r.URL.Query())
// SQLite strftime('%w') → 0=Sunday … 6=Saturday
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
SELECT CAST(strftime('%%w', datetime(received_at, 'unixepoch')) AS INTEGER) AS dow,
COUNT(*) AS cnt
FROM alerts
WHERE %s
GROUP BY dow
ORDER BY dow ASC`, where), args...)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
defer rows.Close()
counts := make(map[int]int64, 7)
for rows.Next() {
var dow int
var cnt int64
if err := rows.Scan(&dow, &cnt); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
counts[dow] = cnt
}
dayNames := [7]string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
type entry struct {
Day int `json:"day"`
DayName string `json:"day_name"`
Count int64 `json:"count"`
}
result := make([]entry, 7)
for d := range result {
result[d] = entry{Day: d, DayName: dayNames[d], Count: counts[d]}
}
respond(w, http.StatusOK, result)
}
}
// statsFilter builds a WHERE clause and args from optional ?from and ?to query params.
// Archived alerts are always excluded, matching the default GET /api/alerts view.
func statsFilter(q url.Values) (where string, args []any) {
clauses := []string{"archived_at IS NULL"}
if from := q.Get("from"); from != "" {
if t, err := time.Parse("2006-01-02", from); err == nil {
clauses = append(clauses, "received_at >= ?")
args = append(args, t.UTC().Unix())
}
}
if to := q.Get("to"); to != "" {
if t, err := time.Parse("2006-01-02", to); err == nil {
clauses = append(clauses, "received_at < ?")
args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
}
}
return strings.Join(clauses, " AND "), args
}