42e846f876
Release / build (amd64, darwin) (push) Failing after 12s
Release / build (arm64, darwin) (push) Failing after 11s
Release / build (arm64, linux) (push) Failing after 11s
Release / release (push) Has been skipped
Release / docker (push) Failing after 19s
Release / build (amd64, linux) (push) Failing after 12s
Release / chart (push) Failing after 9s
A resolved webhook was the only path out of the firing state, so a
notification that was dropped, silenced, or lost to a restart pinned an
alert as firing forever — Prometheus showed it resolved while
terdut-server kept listing it. The archiver only ever touched resolved
alerts, and both the list and stats queries compared status with plain
equality, so a stale row was indistinguishable from a live one.
A sweeper pass now resolves firing alerts on either of two signals: the
ends_at watermark Alertmanager sets on outgoing firing notifications has
passed (plus a grace period for clock skew), or no webhook has refreshed
the alert within TERDUT_STALE_AFTER (default 6h, above Alertmanager's 4h
repeat_interval). Such alerts get resolution_source = 'expiry',
distinguishing them from a real 'alertmanager' resolve.
Two related webhook bugs fixed alongside:
- The upsert had no ordering guard, so a retried firing notification
arriving after the resolved one resurrected the alert. Payloads for
an older alert instance are now discarded: a stale retry carries the
same startsAt, a genuine re-fire a newer one.
- archived_at was never cleared on re-fire, leaving a re-fired alert
archived and invisible in the default list.
Stats now exclude archived alerts to match the default list view; this
lowers historical firing/resolved totals.
The chart exposes both sweeper durations via sweeper.staleAfter and
sweeper.archiveAfter.
180 lines
4.7 KiB
Go
180 lines
4.7 KiB
Go
package api
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query())
|
|
|
|
var total, firing, resolved int64
|
|
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
|
|
SELECT COUNT(*),
|
|
SUM(CASE WHEN status = 'firing' THEN 1 ELSE 0 END),
|
|
SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END)
|
|
FROM alerts WHERE %s`, where), args...,
|
|
).Scan(&total, &firing, &resolved)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
respond(w, http.StatusOK, map[string]int64{
|
|
"total": total,
|
|
"firing": firing,
|
|
"resolved": resolved,
|
|
})
|
|
}
|
|
}
|
|
|
|
func handleStatsTop(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query())
|
|
|
|
limit := 10
|
|
if l := r.URL.Query().Get("limit"); l != "" {
|
|
var n int
|
|
fmt.Sscanf(l, "%d", &n)
|
|
if n > 0 && n <= 100 {
|
|
limit = n
|
|
}
|
|
}
|
|
args = append(args, limit)
|
|
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT name, COUNT(*) AS cnt
|
|
FROM alerts
|
|
WHERE %s
|
|
GROUP BY name
|
|
ORDER BY cnt DESC
|
|
LIMIT ?`, where), args...)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
type entry struct {
|
|
Name string `json:"name"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
result := []entry{}
|
|
for rows.Next() {
|
|
var e entry
|
|
if err := rows.Scan(&e.Name, &e.Count); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
result = append(result, e)
|
|
}
|
|
respond(w, http.StatusOK, result)
|
|
}
|
|
}
|
|
|
|
func handleStatsByHour(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query())
|
|
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT CAST(strftime('%%H', datetime(received_at, 'unixepoch')) AS INTEGER) AS hr,
|
|
COUNT(*) AS cnt
|
|
FROM alerts
|
|
WHERE %s
|
|
GROUP BY hr
|
|
ORDER BY hr ASC`, where), args...)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
counts := make(map[int]int64, 24)
|
|
for rows.Next() {
|
|
var hr int
|
|
var cnt int64
|
|
if err := rows.Scan(&hr, &cnt); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
counts[hr] = cnt
|
|
}
|
|
|
|
type entry struct {
|
|
Hour int `json:"hour"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
result := make([]entry, 24)
|
|
for h := range result {
|
|
result[h] = entry{Hour: h, Count: counts[h]}
|
|
}
|
|
respond(w, http.StatusOK, result)
|
|
}
|
|
}
|
|
|
|
func handleStatsByDay(db *sql.DB) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
where, args := statsFilter(r.URL.Query())
|
|
|
|
// SQLite strftime('%w') → 0=Sunday … 6=Saturday
|
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
|
SELECT CAST(strftime('%%w', datetime(received_at, 'unixepoch')) AS INTEGER) AS dow,
|
|
COUNT(*) AS cnt
|
|
FROM alerts
|
|
WHERE %s
|
|
GROUP BY dow
|
|
ORDER BY dow ASC`, where), args...)
|
|
if err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
counts := make(map[int]int64, 7)
|
|
for rows.Next() {
|
|
var dow int
|
|
var cnt int64
|
|
if err := rows.Scan(&dow, &cnt); err != nil {
|
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
return
|
|
}
|
|
counts[dow] = cnt
|
|
}
|
|
|
|
dayNames := [7]string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
|
|
type entry struct {
|
|
Day int `json:"day"`
|
|
DayName string `json:"day_name"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
result := make([]entry, 7)
|
|
for d := range result {
|
|
result[d] = entry{Day: d, DayName: dayNames[d], Count: counts[d]}
|
|
}
|
|
respond(w, http.StatusOK, result)
|
|
}
|
|
}
|
|
|
|
// statsFilter builds a WHERE clause and args from optional ?from and ?to query params.
|
|
// Archived alerts are always excluded, matching the default GET /api/alerts view.
|
|
func statsFilter(q url.Values) (where string, args []any) {
|
|
clauses := []string{"archived_at IS NULL"}
|
|
if from := q.Get("from"); from != "" {
|
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
|
clauses = append(clauses, "received_at >= ?")
|
|
args = append(args, t.UTC().Unix())
|
|
}
|
|
}
|
|
if to := q.Get("to"); to != "" {
|
|
if t, err := time.Parse("2006-01-02", to); err == nil {
|
|
clauses = append(clauses, "received_at < ?")
|
|
args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
|
|
}
|
|
}
|
|
return strings.Join(clauses, " AND "), args
|
|
}
|