niklas/terdut-server
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 35 Wiki Activity

List dead man's switches with their status on Team -> Switches

Browse Source 
The page was a bare form: it did not say which switches existed or
whether they were alive. It now lists them, each with a Healthy, Dead or
Dormant badge, when its heartbeat was last heard and when it last opened
an incident (linked while that incident is open). A matcher that several
clusters satisfy is broken down per cluster, since a live cluster must
not hide a dead one. The form moved into a "New switch" sheet, and each
row has a Remove with a confirm.

That needed a switch to be a thing, so switches are rows now
(migration 009) with their own name, matcher, timeout and severity,
instead of one string with one team-wide timeout in deadman_configs.
Existing configuration is split into one row per matcher; a team whose
timeout was zero simply has none. The sweeper and the status endpoint
share one death rule (deadmanAlert.dead), so the page cannot disagree
with the pager. Incident group keys are unchanged, so incidents that
are open across the upgrade keep working.

The environment defaults (TERDUT_DEADMAN_*) are seeded into teams once
per install, recorded in settings, so a team that deletes its last
switch does not get it back on the next restart. Installs that already
had per-team rows are marked as seeded by the migration.

Removing a switch stops the watching but leaves an incident it already
opened open until someone resolves it.

API: GET/PUT /api/teams/{id}/deadman are replaced by
GET/POST /deadman/switches and DELETE /deadman/switches/{switchID}.
terdut-tui does not call them, so nothing to mirror there.
This commit is contained in:
Niklas Ye
2026-09-25 23:40:55 +02:00
parent 3ee8583f6f
commit f3918b863c
11 changed files with 868 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/teams.go
+92 -49
View File
@@ -539,17 +539,24 @@ func defaultTeamID(ctx context.Context, db *sql.DB) (int64, error) {
// A team's dead man's switches
// ---------------------------------------------------------------------------
// deadmanResponse is the wire shape of a team's switch configuration. The
// timeout is seconds rather than a duration string, because that is what the
// column holds and what arithmetic is done on; a client renders it.
type deadmanResponse struct {
TeamID int64 `json:"team_id"`
Matchers string `json:"matchers"`
// deadmanSwitchRequest is what creating a switch takes. The timeout is seconds,
// because that is what the column holds and what arithmetic is done on; a client
// renders it.
type deadmanSwitchRequest struct {
Name string `json:"name"`
Matcher string `json:"matcher"`
TimeoutSeconds int64 `json:"timeout_seconds"`
Severity string `json:"severity"`
}
func handleGetTeamDeadman(db *sql.DB) http.HandlerFunc {
// deadmanSeverities are the severities an incident can open at.
var deadmanSeverities = map[string]bool{"critical": true, "error": true, "warning": true, "info": true}
// handleListTeamDeadman lists a team's switches with what each one's heartbeats
// are doing. A team with none gets an empty list, which is a configuration and
// not an absence: answering 404 would make "off" indistinguishable from "this
// server does not do this".
func handleListTeamDeadman(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
teamID, ok := teamParam(w, r)
if !ok {
@@ -559,27 +566,26 @@ func handleGetTeamDeadman(db *sql.DB) http.HandlerFunc {
return
}
out := deadmanResponse{TeamID: teamID, Severity: "critical"}
err := db.QueryRowContext(r.Context(),
"SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1",
teamID).Scan(&out.Matchers, &out.TimeoutSeconds, &out.Severity)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
set, err := deadmanSetForTeam(r.Context(), db, teamID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
out, err := deadmanStatuses(r.Context(), db, teamID, set, time.Now())
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
// A team with no row watches nothing, which is a configuration and not
// an absence: answering 404 would make "off" indistinguishable from
// "this server does not do this".
respond(w, http.StatusOK, out)
}
}
// handleSetTeamDeadman replaces a team's switch configuration.
// handleCreateTeamDeadman adds one switch.
//
// Validated by parsing: a matcher string that survives ParseDeadmanConfig with
// nothing usable in it is rejected rather than stored, because a switch that
// silently watches nothing is the failure this feature exists to prevent.
func handleSetTeamDeadman(db *sql.DB) http.HandlerFunc {
// Validated by parsing: a matcher with no alertname is rejected rather than
// stored, because a switch that silently watches nothing is the failure this
// feature exists to prevent.
func handleCreateTeamDeadman(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
teamID, ok := teamParam(w, r)
if !ok {
@@ -589,50 +595,87 @@ func handleSetTeamDeadman(db *sql.DB) http.HandlerFunc {
return
}
var req struct {
Matchers string `json:"matchers"`
TimeoutSeconds int64 `json:"timeout_seconds"`
Severity string `json:"severity"`
}
var req deadmanSwitchRequest
if err := decodeJSON(r, &req); err != nil {
respond(w, http.StatusBadRequest, errResp("invalid request body"))
return
}
req.Matchers = strings.TrimSpace(req.Matchers)
req.Matcher = strings.TrimSpace(req.Matcher)
req.Name = strings.TrimSpace(req.Name)
if req.Severity == "" {
req.Severity = "critical"
}
if req.TimeoutSeconds < 0 {
respond(w, http.StatusBadRequest, errResp("timeout_seconds must not be negative"))
if !deadmanSeverities[req.Severity] {
respond(w, http.StatusBadRequest, errResp("severity must be critical, error, warning or info"))
return
}
if req.Matchers != "" {
parsed := parseDeadmanQuietly(req.Matchers, time.Duration(req.TimeoutSeconds)*time.Second, req.Severity)
if len(parsed.Matchers) == 0 {
respond(w, http.StatusBadRequest, errResp(
"no usable matchers: each must name an alertname, as in alertname=Watchdog,cluster=prod"))
return
}
if req.TimeoutSeconds <= 0 {
respond(w, http.StatusBadRequest, errResp("timeout_seconds must be positive"))
return
}
if strings.Contains(req.Matcher, ";") {
respond(w, http.StatusBadRequest, errResp("one matcher per switch: add another switch instead of separating with ;"))
return
}
m, err := parseDeadmanMatcher(req.Matcher)
if err != nil {
respond(w, http.StatusBadRequest, errResp(
"unusable matcher ("+err.Error()+"): each must name an alertname, as in alertname=Watchdog,cluster=prod"))
return
}
if req.Name == "" {
req.Name = m.config()
}
if len(req.Name) > 100 {
respond(w, http.StatusBadRequest, errResp("name is too long"))
return
}
if _, err := db.ExecContext(r.Context(), `
INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity, updated_at)
VALUES ($1, $2, $3, $4, `+nowEpoch+`)
ON CONFLICT (team_id) DO UPDATE SET
matchers = excluded.matchers,
timeout_seconds = excluded.timeout_seconds,
severity = excluded.severity,
updated_at = excluded.updated_at`,
teamID, req.Matchers, req.TimeoutSeconds, req.Severity); err != nil {
var id int64
if err := db.QueryRowContext(r.Context(), `
INSERT INTO deadman_switches (team_id, name, matcher, timeout_seconds, severity)
VALUES ($1, $2, $3, $4, $5) RETURNING id`,
teamID, req.Name, m.config(), req.TimeoutSeconds, req.Severity).Scan(&id); err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
respond(w, http.StatusOK, deadmanResponse{
TeamID: teamID,
Matchers: req.Matchers,
TimeoutSeconds: req.TimeoutSeconds,
Severity: req.Severity,
respond(w, http.StatusCreated, deadmanSwitchStatus{
ID: id, Name: req.Name, Matcher: m.config(),
TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
Status: switchDormant, Sources: []deadmanSource{},
})
}
}
// handleDeleteTeamDeadman removes a switch. An incident it already opened stays
// open until somebody resolves it: deleting the switch says "stop watching", not
// "the problem is gone".
func handleDeleteTeamDeadman(db *sql.DB) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
teamID, ok := teamParam(w, r)
if !ok {
return
}
if !requireTeamOwner(w, r, teamID) {
return
}
switchID, err := strconv.ParseInt(chi.URLParam(r, "switchID"), 10, 64)
if err != nil {
respond(w, http.StatusBadRequest, errResp("invalid switch id"))
return
}
res, err := db.ExecContext(r.Context(),
"DELETE FROM deadman_switches WHERE id = $1 AND team_id = $2", switchID, teamID)
if err != nil {
respond(w, http.StatusInternalServerError, errResp("internal error"))
return
}
if n, _ := res.RowsAffected(); n == 0 {
respond(w, http.StatusNotFound, errResp("switch not found"))
return
}
w.WriteHeader(http.StatusNoContent)
}
}