Compare commits

..

4 Commits

Author SHA1 Message Date
Niklas Ye debc4bf78c Add alert archiving
Release / build (amd64, darwin) (push) Failing after 2m46s
Release / build (amd64, linux) (push) Failing after 2m26s
Release / build (arm64, darwin) (push) Failing after 1m40s
Release / build (arm64, linux) (push) Failing after 10s
Release / release (push) Has been skipped
Release / chart (push) Failing after 11s
Release / docker (push) Failing after 19s
Alerts can be manually archived (POST /api/alerts/{id}/archive) or
unarchived (DELETE /api/alerts/{id}/archive). A background goroutine
auto-archives resolved alerts older than TERDUT_ARCHIVE_AFTER (default 7d).
GET /api/alerts hides archived alerts by default; ?archived=true shows them.
2026-05-22 13:22:45 +02:00
Niklas Ye 36468a68ed chart: add bootstrap job (v0.2.0)
Post-install/post-upgrade Job that calls /api/bootstrap on first deploy
and stores the admin API key in a Secret (<release>-admin-key by default).
Exits cleanly on subsequent upgrades when bootstrap is already complete.
Adds ServiceAccount, Role (secrets:create), and RoleBinding as hook resources.
2026-05-22 10:11:30 +02:00
Niklas Ye 885ba73d12 chart: guard version sed behind tag pattern check 2026-05-21 18:14:09 +02:00
Niklas Ye 1451682cdd chart: add skip_existing and workflow_dispatch trigger 2026-05-21 18:08:30 +02:00
14 changed files with 339 additions and 10 deletions
+5
View File
@@ -4,6 +4,7 @@ on:
push:
tags:
- 'v*'
workflow_dispatch:
jobs:
build:
@@ -94,12 +95,16 @@ jobs:
- name: Update chart versions
run: |
VERSION="${{ github.ref_name }}"
if [[ "$VERSION" =~ ^v[0-9] ]]; then
CHART_VERSION="${VERSION#v}"
sed -i "s/^version:.*/version: ${CHART_VERSION}/" charts/terdut-server/Chart.yaml
sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" charts/terdut-server/Chart.yaml
fi
- name: Run chart-releaser
uses: helm/chart-releaser-action@v1.6.0
with:
skip_existing: true
env:
CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+1 -1
View File
@@ -2,5 +2,5 @@ apiVersion: v2
name: terdut-server
description: A Helm chart for Terminal Duty — on-call alert management server
type: application
version: 0.1.0
version: 0.2.0
appVersion: "latest"
@@ -32,3 +32,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/name: {{ include "terdut-server.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{- define "terdut-server.bootstrapSecretName" -}}
{{- .Values.bootstrap.secretName | default (printf "%s-admin-key" (include "terdut-server.fullname" .)) }}
{{- end }}
@@ -0,0 +1,91 @@
{{- if .Values.bootstrap.enabled }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "terdut-server.fullname" . }}-bootstrap
namespace: {{ .Release.Namespace }}
labels:
{{- include "terdut-server.labels" . | nindent 4 }}
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-weight: "0"
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
spec:
backoffLimit: 3
template:
metadata:
labels:
{{- include "terdut-server.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: bootstrap
spec:
restartPolicy: OnFailure
serviceAccountName: {{ include "terdut-server.fullname" . }}-bootstrap
containers:
- name: bootstrap
image: alpine:3
command:
- /bin/sh
- -c
- |
apk add --no-cache curl > /dev/null 2>&1
SERVICE_URL="http://{{ include "terdut-server.fullname" . }}:{{ .Values.service.port }}"
SECRET_NAME="{{ include "terdut-server.bootstrapSecretName" . }}"
K8S_API="https://kubernetes.default.svc"
SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)"
echo "Waiting for terdut-server to be ready..."
RETRIES=60
while [ "$RETRIES" -gt 0 ]; do
curl -sf "$SERVICE_URL/healthz" > /dev/null 2>&1 && break
RETRIES=$((RETRIES - 1))
sleep 2
done
if [ "$RETRIES" -eq 0 ]; then
echo "Timed out waiting for server to be ready."
exit 1
fi
echo "Server is ready."
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$SERVICE_URL/api/bootstrap" \
-H "Content-Type: application/json" \
-d '{"username":"{{ .Values.bootstrap.username }}","email":"{{ .Values.bootstrap.email }}"}')
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -1)
if [ "$HTTP_CODE" = "403" ]; then
echo "Server already bootstrapped, nothing to do."
exit 0
fi
if [ "$HTTP_CODE" != "201" ]; then
echo "Bootstrap failed (HTTP $HTTP_CODE): $BODY"
exit 1
fi
API_KEY=$(echo "$BODY" | grep -o '"key":"[^"]*"' | cut -d'"' -f4)
if [ -z "$API_KEY" ]; then
echo "Failed to extract API key from response."
exit 1
fi
echo "Bootstrap succeeded. Storing API key in secret '$SECRET_NAME'."
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
-X POST "$K8S_API/api/v1/namespaces/$NAMESPACE/secrets" \
--cacert "$CA_CERT" \
-H "Authorization: Bearer $SA_TOKEN" \
-H "Content-Type: application/json" \
-d "$(printf '{"apiVersion":"v1","kind":"Secret","metadata":{"name":"%s"},"stringData":{"api-key":"%s"}}' "$SECRET_NAME" "$API_KEY")")
if [ "$HTTP_CODE" != "201" ]; then
echo "Failed to create secret (HTTP $HTTP_CODE)."
exit 1
fi
echo "Secret '$SECRET_NAME' created successfully."
{{- end }}
@@ -0,0 +1,50 @@
{{- if .Values.bootstrap.enabled }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "terdut-server.fullname" . }}-bootstrap
namespace: {{ .Release.Namespace }}
labels:
{{- include "terdut-server.labels" . | nindent 4 }}
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-weight: "-1"
helm.sh/hook-delete-policy: before-hook-creation
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "terdut-server.fullname" . }}-bootstrap
namespace: {{ .Release.Namespace }}
labels:
{{- include "terdut-server.labels" . | nindent 4 }}
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-weight: "-1"
helm.sh/hook-delete-policy: before-hook-creation
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "terdut-server.fullname" . }}-bootstrap
namespace: {{ .Release.Namespace }}
labels:
{{- include "terdut-server.labels" . | nindent 4 }}
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-weight: "-1"
helm.sh/hook-delete-policy: before-hook-creation
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ include "terdut-server.fullname" . }}-bootstrap
subjects:
- kind: ServiceAccount
name: {{ include "terdut-server.fullname" . }}-bootstrap
namespace: {{ .Release.Namespace }}
{{- end }}
+7
View File
@@ -14,3 +14,10 @@ storage:
service:
type: ClusterIP
port: 8080
bootstrap:
enabled: true
username: admin
email: admin@example.com
# secretName overrides the default of <fullname>-admin-key
secretName: ""
+2
View File
@@ -41,6 +41,8 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
go api.StartArchiver(ctx, database, cfg.ArchiveAfter)
go func() {
log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+57 -2
View File
@@ -20,7 +20,8 @@ const alertSelectFrom = `
SELECT a.id, a.fingerprint, a.name, a.status,
a.labels, a.annotations,
a.starts_at, a.ends_at, a.generator_url, a.received_at,
a.acknowledged_by, a.acknowledged_at, u.username
a.acknowledged_by, a.acknowledged_at, u.username,
a.archived_at
FROM alerts a
LEFT JOIN users u ON u.id = a.acknowledged_by`
@@ -39,6 +40,12 @@ func handleListAlerts(db *sql.DB) http.HandlerFunc {
where = append(where, "a.name = ?")
args = append(args, name)
}
if archived := q.Get("archived"); archived == "true" {
where = append(where, "a.archived_at IS NOT NULL")
} else {
where = append(where, "a.archived_at IS NULL")
}
if from := q.Get("from"); from != "" {
if t, err := time.Parse("2006-01-02", from); err == nil {
where = append(where, "a.received_at >= ?")
@@ -169,7 +176,7 @@ func scanAlert(s scanner) (models.Alert, error) {
var a models.Alert
var labelsJSON, annotationsJSON string
var startsAtUnix, receivedAtUnix int64
var endsAtUnix, ackAtUnix *int64
var endsAtUnix, ackAtUnix, archivedAtUnix *int64
var ackByID *int64
var ackByUser *string
@@ -179,6 +186,7 @@ func scanAlert(s scanner) (models.Alert, error) {
&startsAtUnix, &endsAtUnix,
&a.GeneratorURL, &receivedAtUnix,
&ackByID, &ackAtUnix, &ackByUser,
&archivedAtUnix,
); err != nil {
return a, err
}
@@ -197,5 +205,52 @@ func scanAlert(s scanner) (models.Alert, error) {
a.AcknowledgedByUser = ackByUser
a.AcknowledgedAt = &t
}
if archivedAtUnix != nil {
t := time.Unix(*archivedAtUnix, 0).UTC()
a.ArchivedAt = &t
}
return a, nil
}
func handleArchive(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
return
}
res, err := db.ExecContext(r.Context(),
"UPDATE alerts SET archived_at = unixepoch() WHERE id = ?", id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("alert not found"))
return
}
a, _ := fetchAlert(r.Context(), db, id)
respond(w, http.StatusOK, a)
}
}
func handleUnarchive(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
return
}
res, err := db.ExecContext(r.Context(),
"UPDATE alerts SET archived_at = NULL WHERE id = ?", id)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("alert not found"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}
+60
View File
@@ -357,6 +357,66 @@ func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
}
}
// ---------------------------------------------------------------------------
// Archive
// ---------------------------------------------------------------------------
func TestArchive_RoundTrip(t *testing.T) {
s := newTS(t)
postWebhook(t, s, []map[string]any{{
"status": "resolved", "fingerprint": "arch1",
"labels": map[string]string{"alertname": "Archivable"},
"annotations": map[string]string{},
"startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z",
"generatorURL": "",
}})
// 1. Alert appears in default list (not archived).
var alerts []map[string]any
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 {
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
}
id := int(alerts[0]["id"].(float64))
// 2. Archive it.
resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("archive: expected 200, got %d", resp.StatusCode)
}
var archived map[string]any
decode(t, resp, &archived)
if archived["archived_at"] == nil {
t.Error("expected archived_at to be set in response")
}
// 3. Default list excludes it.
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 0 {
t.Errorf("expected archived alert to be hidden, got %d results", len(alerts))
}
// 4. archived=true shows it.
decode(t, s.req(t, http.MethodGet, "/api/alerts?archived=true", nil), &alerts)
if len(alerts) != 1 {
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
}
// 5. Un-archive.
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("unarchive: expected 204, got %d", resp.StatusCode)
}
resp.Body.Close()
// 6. Back in default list.
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
if len(alerts) != 1 {
t.Errorf("expected unarchived alert to reappear, got %d results", len(alerts))
}
}
func TestStats_ByDayReturnsSevenSlots(t *testing.T) {
s := newTS(t)
resp := s.req(t, http.MethodGet, "/api/stats/alerts/by-day", nil)
+39
View File
@@ -0,0 +1,39 @@
package api
import (
"context"
"database/sql"
"log"
"time"
)
func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
ticker := time.NewTicker(15 * time.Minute)
defer ticker.Stop()
run := func() {
cutoff := time.Now().Add(-archiveAfter).Unix()
res, err := db.ExecContext(ctx,
`UPDATE alerts SET archived_at = unixepoch()
WHERE status = 'resolved'
AND archived_at IS NULL
AND COALESCE(ends_at, received_at) < ?`, cutoff)
if err != nil {
log.Printf("archiver: %v", err)
return
}
if n, _ := res.RowsAffected(); n > 0 {
log.Printf("archiver: archived %d resolved alert(s)", n)
}
}
run()
for {
select {
case <-ticker.C:
run()
case <-ctx.Done():
return
}
}
}
+2
View File
@@ -35,6 +35,8 @@ func NewRouter(db *sql.DB) http.Handler {
r.Get("/api/alerts/{id}", handleGetAlert(db))
r.Post("/api/alerts/{id}/acknowledge", handleAcknowledge(db))
r.Delete("/api/alerts/{id}/acknowledge", handleUnacknowledge(db))
r.Post("/api/alerts/{id}/archive", handleArchive(db))
r.Delete("/api/alerts/{id}/archive", handleUnarchive(db))
r.Get("/api/alerts/{id}/comments", handleListComments(db))
r.Post("/api/alerts/{id}/comments", handleCreateComment(db))
r.Delete("/api/alerts/{id}/comments/{commentID}", handleDeleteComment(db))
+12 -2
View File
@@ -1,10 +1,14 @@
package config
import "os"
import (
"os"
"time"
)
type Config struct {
Addr string
DBPath string
ArchiveAfter time.Duration
}
func Load() Config {
@@ -16,5 +20,11 @@ func Load() Config {
if dbPath == "" {
dbPath = "terdut.db"
}
return Config{Addr: addr, DBPath: dbPath}
archiveAfter := 7 * 24 * time.Hour
if s := os.Getenv("TERDUT_ARCHIVE_AFTER"); s != "" {
if d, err := time.ParseDuration(s); err == nil {
archiveAfter = d
}
}
return Config{Addr: addr, DBPath: dbPath, ArchiveAfter: archiveAfter}
}
@@ -0,0 +1,2 @@
ALTER TABLE alerts ADD COLUMN archived_at INTEGER;
CREATE INDEX alerts_archived_at_idx ON alerts(archived_at);
+2
View File
@@ -18,4 +18,6 @@ type Alert struct {
AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"`
AcknowledgedByUser *string `json:"acknowledged_by,omitempty"`
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
ArchivedAt *time.Time `json:"archived_at,omitempty"`
}