Add alert archiving
Release / build (amd64, darwin) (push) Failing after 2m46s
Release / build (amd64, linux) (push) Failing after 2m26s
Release / build (arm64, darwin) (push) Failing after 1m40s
Release / build (arm64, linux) (push) Failing after 10s
Release / release (push) Has been skipped
Release / chart (push) Failing after 11s
Release / docker (push) Failing after 19s
Release / build (amd64, darwin) (push) Failing after 2m46s
Release / build (amd64, linux) (push) Failing after 2m26s
Release / build (arm64, darwin) (push) Failing after 1m40s
Release / build (arm64, linux) (push) Failing after 10s
Release / release (push) Has been skipped
Release / chart (push) Failing after 11s
Release / docker (push) Failing after 19s
Alerts can be manually archived (POST /api/alerts/{id}/archive) or
unarchived (DELETE /api/alerts/{id}/archive). A background goroutine
auto-archives resolved alerts older than TERDUT_ARCHIVE_AFTER (default 7d).
GET /api/alerts hides archived alerts by default; ?archived=true shows them.
This commit is contained in:
@@ -41,6 +41,8 @@ func main() {
|
|||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
|
go api.StartArchiver(ctx, database, cfg.ArchiveAfter)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
|
log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
|
||||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
|
|||||||
+57
-2
@@ -20,7 +20,8 @@ const alertSelectFrom = `
|
|||||||
SELECT a.id, a.fingerprint, a.name, a.status,
|
SELECT a.id, a.fingerprint, a.name, a.status,
|
||||||
a.labels, a.annotations,
|
a.labels, a.annotations,
|
||||||
a.starts_at, a.ends_at, a.generator_url, a.received_at,
|
a.starts_at, a.ends_at, a.generator_url, a.received_at,
|
||||||
a.acknowledged_by, a.acknowledged_at, u.username
|
a.acknowledged_by, a.acknowledged_at, u.username,
|
||||||
|
a.archived_at
|
||||||
FROM alerts a
|
FROM alerts a
|
||||||
LEFT JOIN users u ON u.id = a.acknowledged_by`
|
LEFT JOIN users u ON u.id = a.acknowledged_by`
|
||||||
|
|
||||||
@@ -39,6 +40,12 @@ func handleListAlerts(db *sql.DB) http.HandlerFunc {
|
|||||||
where = append(where, "a.name = ?")
|
where = append(where, "a.name = ?")
|
||||||
args = append(args, name)
|
args = append(args, name)
|
||||||
}
|
}
|
||||||
|
if archived := q.Get("archived"); archived == "true" {
|
||||||
|
where = append(where, "a.archived_at IS NOT NULL")
|
||||||
|
} else {
|
||||||
|
where = append(where, "a.archived_at IS NULL")
|
||||||
|
}
|
||||||
|
|
||||||
if from := q.Get("from"); from != "" {
|
if from := q.Get("from"); from != "" {
|
||||||
if t, err := time.Parse("2006-01-02", from); err == nil {
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
||||||
where = append(where, "a.received_at >= ?")
|
where = append(where, "a.received_at >= ?")
|
||||||
@@ -169,7 +176,7 @@ func scanAlert(s scanner) (models.Alert, error) {
|
|||||||
var a models.Alert
|
var a models.Alert
|
||||||
var labelsJSON, annotationsJSON string
|
var labelsJSON, annotationsJSON string
|
||||||
var startsAtUnix, receivedAtUnix int64
|
var startsAtUnix, receivedAtUnix int64
|
||||||
var endsAtUnix, ackAtUnix *int64
|
var endsAtUnix, ackAtUnix, archivedAtUnix *int64
|
||||||
var ackByID *int64
|
var ackByID *int64
|
||||||
var ackByUser *string
|
var ackByUser *string
|
||||||
|
|
||||||
@@ -179,6 +186,7 @@ func scanAlert(s scanner) (models.Alert, error) {
|
|||||||
&startsAtUnix, &endsAtUnix,
|
&startsAtUnix, &endsAtUnix,
|
||||||
&a.GeneratorURL, &receivedAtUnix,
|
&a.GeneratorURL, &receivedAtUnix,
|
||||||
&ackByID, &ackAtUnix, &ackByUser,
|
&ackByID, &ackAtUnix, &ackByUser,
|
||||||
|
&archivedAtUnix,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return a, err
|
return a, err
|
||||||
}
|
}
|
||||||
@@ -197,5 +205,52 @@ func scanAlert(s scanner) (models.Alert, error) {
|
|||||||
a.AcknowledgedByUser = ackByUser
|
a.AcknowledgedByUser = ackByUser
|
||||||
a.AcknowledgedAt = &t
|
a.AcknowledgedAt = &t
|
||||||
}
|
}
|
||||||
|
if archivedAtUnix != nil {
|
||||||
|
t := time.Unix(*archivedAtUnix, 0).UTC()
|
||||||
|
a.ArchivedAt = &t
|
||||||
|
}
|
||||||
return a, nil
|
return a, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func handleArchive(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
res, err := db.ExecContext(r.Context(),
|
||||||
|
"UPDATE alerts SET archived_at = unixepoch() WHERE id = ?", id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
respond(w, http.StatusNotFound, errResp("alert not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a, _ := fetchAlert(r.Context(), db, id)
|
||||||
|
respond(w, http.StatusOK, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleUnarchive(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
res, err := db.ExecContext(r.Context(),
|
||||||
|
"UPDATE alerts SET archived_at = NULL WHERE id = ?", id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
respond(w, http.StatusNotFound, errResp("alert not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -357,6 +357,66 @@ func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Archive
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestArchive_RoundTrip(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{{
|
||||||
|
"status": "resolved", "fingerprint": "arch1",
|
||||||
|
"labels": map[string]string{"alertname": "Archivable"},
|
||||||
|
"annotations": map[string]string{},
|
||||||
|
"startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z",
|
||||||
|
"generatorURL": "",
|
||||||
|
}})
|
||||||
|
|
||||||
|
// 1. Alert appears in default list (not archived).
|
||||||
|
var alerts []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
|
if len(alerts) != 1 {
|
||||||
|
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
id := int(alerts[0]["id"].(float64))
|
||||||
|
|
||||||
|
// 2. Archive it.
|
||||||
|
resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("archive: expected 200, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var archived map[string]any
|
||||||
|
decode(t, resp, &archived)
|
||||||
|
if archived["archived_at"] == nil {
|
||||||
|
t.Error("expected archived_at to be set in response")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Default list excludes it.
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
|
if len(alerts) != 0 {
|
||||||
|
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. archived=true shows it.
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
|
||||||
|
if len(alerts) != 1 {
|
||||||
|
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Un-archive.
|
||||||
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
t.Fatalf("unarchive: expected 204, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
// 6. Back in default list.
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
|
if len(alerts) != 1 {
|
||||||
|
t.Errorf("expected unarchived alert to reappear, got %d results", len(alerts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
|
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
|
||||||
s := newTS(t)
|
s := newTS(t)
|
||||||
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
|
||||||
|
ticker := time.NewTicker(15 * time.Minute)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
run := func() {
|
||||||
|
cutoff := time.Now().Add(-archiveAfter).Unix()
|
||||||
|
res, err := db.ExecContext(ctx,
|
||||||
|
`UPDATE alerts SET archived_at = unixepoch()
|
||||||
|
WHERE status = 'resolved'
|
||||||
|
AND archived_at IS NULL
|
||||||
|
AND COALESCE(ends_at, received_at) < ?`, cutoff)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("archiver: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
|
log.Printf("archiver: archived %d resolved alert(s)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
run()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
run()
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,6 +35,8 @@ func NewRouter(db *sql.DB) http.Handler {
|
|||||||
r.Get("/api/alerts/{id}", handleGetAlert(db))
|
r.Get("/api/alerts/{id}", handleGetAlert(db))
|
||||||
r.Post("/api/alerts/{id}/acknowledge", handleAcknowledge(db))
|
r.Post("/api/alerts/{id}/acknowledge", handleAcknowledge(db))
|
||||||
r.Delete("/api/alerts/{id}/acknowledge", handleUnacknowledge(db))
|
r.Delete("/api/alerts/{id}/acknowledge", handleUnacknowledge(db))
|
||||||
|
r.Post("/api/alerts/{id}/archive", handleArchive(db))
|
||||||
|
r.Delete("/api/alerts/{id}/archive", handleUnarchive(db))
|
||||||
r.Get("/api/alerts/{id}/comments", handleListComments(db))
|
r.Get("/api/alerts/{id}/comments", handleListComments(db))
|
||||||
r.Post("/api/alerts/{id}/comments", handleCreateComment(db))
|
r.Post("/api/alerts/{id}/comments", handleCreateComment(db))
|
||||||
r.Delete("/api/alerts/{id}/comments/{commentID}", handleDeleteComment(db))
|
r.Delete("/api/alerts/{id}/comments/{commentID}", handleDeleteComment(db))
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import "os"
|
import (
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Addr string
|
Addr string
|
||||||
DBPath string
|
DBPath string
|
||||||
|
ArchiveAfter time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load() Config {
|
func Load() Config {
|
||||||
@@ -16,5 +20,11 @@ func Load() Config {
|
|||||||
if dbPath == "" {
|
if dbPath == "" {
|
||||||
dbPath = "terdut.db"
|
dbPath = "terdut.db"
|
||||||
}
|
}
|
||||||
return Config{Addr: addr, DBPath: dbPath}
|
archiveAfter := 7 * 24 * time.Hour
|
||||||
|
if s := os.Getenv("TERDUT_ARCHIVE_AFTER"); s != "" {
|
||||||
|
if d, err := time.ParseDuration(s); err == nil {
|
||||||
|
archiveAfter = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Config{Addr: addr, DBPath: dbPath, ArchiveAfter: archiveAfter}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE alerts ADD COLUMN archived_at INTEGER;
|
||||||
|
CREATE INDEX alerts_archived_at_idx ON alerts(archived_at);
|
||||||
@@ -18,4 +18,6 @@ type Alert struct {
|
|||||||
AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"`
|
AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"`
|
||||||
AcknowledgedByUser *string `json:"acknowledged_by,omitempty"`
|
AcknowledgedByUser *string `json:"acknowledged_by,omitempty"`
|
||||||
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
||||||
|
|
||||||
|
ArchivedAt *time.Time `json:"archived_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user