diff --git a/README.md b/README.md index faf5ddf..2b98acd 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ somewhere to exec. The sidecar, the PVC and the `backupSidecar` values are all g | `TERDUT_DB_DSN` | — | **Required.** Postgres connection string, e.g. `postgres://terdut:secret@localhost:5432/terdut?sslmode=require` | | `TERDUT_ARCHIVE_AFTER` | `168h` (7d) | How long a resolved alert or incident stays in the default list before being auto-archived | | `TERDUT_STALE_AFTER` | `6h` | How long a firing alert may go without a refreshing webhook before it is treated as resolved — **must exceed your Alertmanager `repeat_interval`** | -| `TERDUT_DEADMAN_MATCHERS` | `alertname=Watchdog` | Which alerts are [dead man's switches](#dead-mans-switch). `;` separates matchers, `,` the label conditions within one, `=` is exact equality. Every matcher must name an `alertname` | +| `TERDUT_DEADMAN_MATCHERS` | `alertname=Watchdog` | The **default** matchers a team starts with — switches are per team now, and this seeds teams that have no configuration of their own. `;` separates matchers, `,` the label conditions within one, `=` is exact equality. Every matcher must name an `alertname` | | `TERDUT_DEADMAN_TIMEOUT` | `15m` | How long a heartbeat may go unheard before its switch is declared dead — **must be shorter than the `repeat_interval` of the route carrying it**. `0` disables dead man's switch handling | | `TERDUT_DEADMAN_SEVERITY` | `critical` | Severity a dead man's switch incident opens at | | `TERDUT_NTFY_URL` | — | ntfy server to publish push notifications to. Empty disables notifications entirely | @@ -378,11 +378,23 @@ kube-prometheus-stack already ships the alert for this. `Watchdog` is nothing unless something downstream notices it stop. That is what `TERDUT_DEADMAN_MATCHERS` defaults to. +**Switches belong to a team**, which decides which of its own alerts are +heartbeats and how long a silence has to last. An owner sets them through +`PUT /api/teams/{teamID}/deadman`; a missed heartbeat opens an incident in the +team whose integration received it. + +The environment variables are the starting point, not the setting: at startup +every team **without** a configuration of its own is given one from them, and an +owner's later edit is never overwritten by a redeploy. A team created after +that starts watching nothing until its owner says otherwise — inheriting an +install-wide heartbeat would page a new team about a source it has never heard +of. + A matcher is a set of exact label conditions, one of which must be the -`alertname`: +`alertname`, in the same format the environment variable uses: ``` -TERDUT_DEADMAN_MATCHERS="alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat" +alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat ``` **The unit of monitoring is the fingerprint, not the alert name.** Two clusters @@ -523,6 +535,8 @@ incident. Move senders to a key and it goes away. | `GET` | `/api/teams/{teamID}/integrations` | member | List integrations. Never returns keys | | `POST` | `/api/teams/{teamID}/integrations` | **owner** | Mint an integration `{"name","kind"}` — key and URL shown once | | `DELETE` | `/api/teams/{teamID}/integrations/{integrationID}` | **owner** | Revoke an integration | +| `GET` | `/api/teams/{teamID}/deadman` | member | The team's [dead man's switch](#dead-mans-switch) configuration `{matchers, timeout_seconds, severity}` | +| `PUT` | `/api/teams/{teamID}/deadman` | **owner** | Replace it. `400` when no matcher names an `alertname`, because a switch that silently watches nothing is the failure this feature exists to prevent | ### Notifications @@ -764,6 +778,11 @@ What changes, and will need attention: fingerprint, the same Alertmanager groupKey, and put somebody on call on the same date. +**Dead man's switches moved too.** `TERDUT_DEADMAN_MATCHERS`, `_TIMEOUT` and +`_SEVERITY` are no longer the setting; they are the default each existing team +is seeded with at startup, after which an owner edits them per team through +`PUT /api/teams/{teamID}/deadman` and a redeploy never overwrites that. + Nothing else about an incident changes, and incidents never move between teams: an alert belongs to whichever team's key it arrived on. diff --git a/cmd/terdut/main.go b/cmd/terdut/main.go index b6f2fa4..947475b 100644 --- a/cmd/terdut/main.go +++ b/cmd/terdut/main.go @@ -36,9 +36,16 @@ func main() { RepeatEvery: cfg.NotifyRepeat, } + // Dead man's switches live per team now. The environment variables are the + // defaults a team starts from: every team without a configuration of its + // own gets one from them here, and an owner's later edit is never + // overwritten by a redeploy. deadman := api.ParseDeadmanConfig(cfg.DeadmanMatchers, cfg.DeadmanTimeout, cfg.DeadmanSeverity) + if err := api.SeedDeadmanConfigs(context.Background(), database, deadman); err != nil { + log.Fatalf("seed dead man's switch defaults: %v", err) + } - router := api.NewRouter(database, notify, deadman) + router := api.NewRouter(database, notify) srv := &http.Server{ Addr: cfg.Addr, @@ -51,7 +58,7 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, deadman, notify) + go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notify) go api.StartNotifier(ctx, database, notify) go func() { diff --git a/internal/api/alertmanager.go b/internal/api/alertmanager.go index f714970..ff669df 100644 --- a/internal/api/alertmanager.go +++ b/internal/api/alertmanager.go @@ -76,7 +76,7 @@ type ingested struct { // handleIntegrationWebhook receives alerts on a team's own integration key. // The key in the path is both the credential and the routing: it says who may // post, and which team the alerts belong to. -func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.HandlerFunc { +func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { teamID, err := teamIDForKey(r.Context(), db, chi.URLParam(r, "key")) if err != nil { @@ -90,7 +90,7 @@ func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanCo respond(w, http.StatusInternalServerError, errResp("internal error")) return } - receiveWebhook(w, r, db, notify, deadman, teamID) + receiveWebhook(w, r, db, notify, teamID) } } @@ -101,7 +101,7 @@ func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanCo // // It is deprecated and unauthenticated — anything that can reach the port can // open an incident. Move senders to an integration key and this goes away. -func handleLegacyWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.HandlerFunc { +func handleLegacyWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { teamID, err := defaultTeamID(r.Context(), db) if err != nil { @@ -111,11 +111,11 @@ func handleLegacyWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) } log.Printf("legacy webhook: unauthenticated payload routed to team %d; "+ "move this sender to an integration key", teamID) - receiveWebhook(w, r, db, notify, deadman, teamID) + receiveWebhook(w, r, db, notify, teamID) } } -func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify NotifyConfig, deadman DeadmanConfig, teamID int64) { +func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify NotifyConfig, teamID int64) { var payload amPayload if err := decodeJSON(r, &payload); err != nil { respond(w, http.StatusBadRequest, errResp("invalid payload")) @@ -125,7 +125,7 @@ func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify N // Alertmanager retries anything that is not 2xx, and a retry of a payload // we failed to store is more useful than an error it cannot act on — so // failures are logged, not surfaced. - if err := ingest(r.Context(), db, notify, deadman, teamID, payload); err != nil { + if err := ingest(r.Context(), db, notify, teamID, payload); err != nil { log.Printf("webhook ingest (team %d, group %q): %v", teamID, payload.GroupKey, err) } @@ -135,13 +135,21 @@ func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify N // ingest stores a payload's alerts and reconciles the incident for its group. // The whole payload is one transaction: an incident that opened but whose alerts // failed to link would be a work item nobody could act on. -func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, deadman DeadmanConfig, teamID int64, payload amPayload) error { +func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, teamID int64, payload amPayload) error { tx, err := db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() //nolint:errcheck + // Which arriving alerts are heartbeats is the team's own answer, read + // inside the transaction so an owner editing it mid-payload cannot split + // one webhook across two interpretations. + deadman, err := deadmanConfigForTeam(ctx, tx, teamID) + if err != nil { + return err + } + accepted, err := upsertAlerts(ctx, tx, deadman, teamID, payload.Alerts) if err != nil { return err diff --git a/internal/api/api_test.go b/internal/api/api_test.go index dfd9a0c..7ff6b62 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -9,6 +9,8 @@ import ( "io" "net/http" "net/http/httptest" + "sort" + "strings" "testing" "time" @@ -37,7 +39,7 @@ func newTS(t *testing.T, notify ...api.NotifyConfig) *ts { return newDeadmanTS(t, api.DeadmanConfig{}, cfg) } -// newDeadmanTS is newTS with dead man's switch handling configured. +// newDeadmanTS is newTS with the default team's dead man's switches configured. func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts { t.Helper() var cfg api.NotifyConfig @@ -46,7 +48,7 @@ func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyC } database := newTestDB(t) - srv := httptest.NewServer(api.NewRouter(database, cfg, deadman)) + srv := httptest.NewServer(api.NewRouter(database, cfg)) t.Cleanup(srv.Close) body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"}) @@ -62,7 +64,38 @@ func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyC json.NewDecoder(resp.Body).Decode(&result) key := result["api_key"].(map[string]any)["key"].(string) - return &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman} + s := &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman} + + // Dead man's switches belong to a team now, so a test that wants them + // configures the default team the way an owner would. + if deadman.Timeout > 0 { + setTeamDeadman(t, s, deadman) + } + return s +} + +// setTeamDeadman configures the default team's switches over the API, rendering +// the matchers back into the string form the endpoint takes. +func setTeamDeadman(t *testing.T, s *ts, cfg api.DeadmanConfig) { + t.Helper() + matchers := make([]string, 0, len(cfg.Matchers)) + for _, m := range cfg.Matchers { + parts := []string{"alertname=" + m.Name} + for k, v := range m.Labels { + parts = append(parts, k+"="+v) + } + sort.Strings(parts[1:]) + matchers = append(matchers, strings.Join(parts, ",")) + } + resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{ + "matchers": strings.Join(matchers, "; "), + "timeout_seconds": int64(cfg.Timeout.Seconds()), + "severity": cfg.Severity, + }) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("configure the team's dead man's switches: %d", resp.StatusCode) + } } // exec runs a statement against the test database. @@ -512,7 +545,7 @@ func TestArchive_AlertListFilter(t *testing.T) { } // 2. Let the sweeper archive it: ends_at is already well past archiveAfter. - api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.deadman, s.notify) + api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.notify) // 3. Default list excludes it. decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts) @@ -553,7 +586,7 @@ func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string func sweep(t *testing.T, s *ts, staleAfter time.Duration) { t.Helper() - api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.deadman, s.notify) + api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.notify) } // A firing alert Alertmanager stopped refreshing is resolved via the diff --git a/internal/api/archiver.go b/internal/api/archiver.go index b767b16..705b4b9 100644 --- a/internal/api/archiver.go +++ b/internal/api/archiver.go @@ -19,15 +19,15 @@ const ( // StartArchiver runs the alert sweeper until ctx is cancelled, starting with an // immediate pass so a restart reconciles state right away. -func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, deadman DeadmanConfig, notify NotifyConfig) { +func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, notify NotifyConfig) { ticker := time.NewTicker(sweepInterval) defer ticker.Stop() - Sweep(ctx, db, archiveAfter, staleAfter, deadman, notify) + Sweep(ctx, db, archiveAfter, staleAfter, notify) for { select { case <-ticker.C: - Sweep(ctx, db, archiveAfter, staleAfter, deadman, notify) + Sweep(ctx, db, archiveAfter, staleAfter, notify) case <-ctx.Done(): return } @@ -44,8 +44,8 @@ func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter tim // touch: a heartbeat answers to its own, much tighter, timeout, and the generic // staleness rules would otherwise resolve it as 'expiry' long before that. // Exported so tests can drive a pass without waiting on the ticker. -func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, deadman DeadmanConfig, notify NotifyConfig) { - heartbeats := sweepDeadman(ctx, db, deadman, notify) +func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, notify NotifyConfig) { + heartbeats := sweepDeadman(ctx, db, notify) expireStale(ctx, db, staleAfter, heartbeats) resolveSettledIncidents(ctx, db) archiveResolved(ctx, db, archiveAfter) diff --git a/internal/api/auth_test.go b/internal/api/auth_test.go index f2af701..5989bf4 100644 --- a/internal/api/auth_test.go +++ b/internal/api/auth_test.go @@ -248,7 +248,7 @@ func TestSession_ExpiredIsRejected(t *testing.T) { if code := status(t, b.do(t, http.MethodGet, "/api/me", nil)); code != http.StatusUnauthorized { t.Errorf("expired session: %d", code) } - api.Sweep(t.Context(), s.db, 0, 0, api.DeadmanConfig{}, api.NotifyConfig{}) + api.Sweep(t.Context(), s.db, 0, 0, api.NotifyConfig{}) var n int s.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&n) if n != 0 { @@ -301,7 +301,7 @@ func TestSetPassword_EndsOtherSessionsButNotThisOne(t *testing.T) { func TestBootstrap_WithPassword(t *testing.T) { database := newTestDB(t) - srv := httptest.NewServer(api.NewRouter(database, api.NotifyConfig{}, api.DeadmanConfig{})) + srv := httptest.NewServer(api.NewRouter(database, api.NotifyConfig{})) t.Cleanup(srv.Close) body := `{"username":"admin","email":"a@test.com","password":"` + adminPassword + `"}` diff --git a/internal/api/deadman.go b/internal/api/deadman.go index 8c1fa91..50b1d7a 100644 --- a/internal/api/deadman.go +++ b/internal/api/deadman.go @@ -189,35 +189,42 @@ func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerpr // It returns the ids of the alerts it owns, because the generic staleness // expiry must leave them alone — staleAfter and ends_at would otherwise resolve // a heartbeat long before its own, much tighter, timeout ever fired. -func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig) map[int64]bool { +// Each team is swept against its own configuration: its own matchers, its own +// timeout, its own severity. A team watching nothing is skipped entirely, which +// is most of them. +func sweepDeadman(ctx context.Context, db *sql.DB, notify NotifyConfig) map[int64]bool { owned := map[int64]bool{} - if !cfg.enabled() { - return owned - } - switches, err := deadmanAlerts(ctx, db, cfg) + configs, err := deadmanConfigs(ctx, db) if err != nil { - log.Printf("deadman: load switches: %v", err) + log.Printf("deadman: load configs: %v", err) return owned } now := time.Now() - cutoff := now.Add(-cfg.Timeout).Unix() - - for _, sw := range switches { - owned[sw.id] = true - - // An explicit resolved from Alertmanager is a stronger death signal than - // mere absence: the sender is telling us the heartbeat stopped, so there - // is nothing left to wait out. - if sw.resolved || sw.receivedAt < cutoff { - if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil { - log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err) - } + for teamID, cfg := range configs { + switches, err := deadmanAlerts(ctx, db, teamID, cfg) + if err != nil { + log.Printf("deadman: load switches for team %d: %v", teamID, err) continue } - if err := deadmanRecovered(ctx, db, sw); err != nil { - log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err) + cutoff := now.Add(-cfg.Timeout).Unix() + + for _, sw := range switches { + owned[sw.id] = true + + // An explicit resolved from Alertmanager is a stronger death signal + // than mere absence: the sender is telling us the heartbeat + // stopped, so there is nothing left to wait out. + if sw.resolved || sw.receivedAt < cutoff { + if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil { + log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err) + } + continue + } + if err := deadmanRecovered(ctx, db, sw); err != nil { + log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err) + } } } return owned @@ -228,7 +235,7 @@ func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify Not // happens in Go, which keeps one implementation of the rules. The rows are read // in full before the caller writes, so the writes do not run against an open // cursor over the same table. -func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadmanAlert, error) { +func deadmanAlerts(ctx context.Context, db *sql.DB, teamID int64, cfg DeadmanConfig) ([]deadmanAlert, error) { names := cfg.names() args := &sqlArgs{} nameList := make([]any, len(names)) @@ -239,7 +246,8 @@ func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadma rows, err := db.QueryContext(ctx, ` SELECT id, team_id, fingerprint, labels, status, received_at FROM alerts - WHERE name IN (`+args.addList(nameList)+`) + WHERE team_id = `+args.add(teamID)+` + AND name IN (`+args.addList(nameList)+`) AND archived_at IS NULL`, args.all()...) if err != nil { return nil, err @@ -382,3 +390,113 @@ func deadmanRecovered(ctx context.Context, db *sql.DB, sw deadmanAlert) error { log.Printf("deadman: %s is back, resolved incident %d", sw.matcher.String(), incidentID) return nil } + +// --------------------------------------------------------------------------- +// Per-team configuration +// --------------------------------------------------------------------------- + +// deadmanConfigForTeam reads one team's switches. A team with no row, or with +// nothing configured, gets a disabled config — which is the right answer rather +// than an error: most teams watch no heartbeat at all. +func deadmanConfigForTeam(ctx context.Context, q querier, teamID int64) (DeadmanConfig, error) { + var matchers, severity string + var timeout int64 + err := q.QueryRowContext(ctx, + "SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1", + teamID).Scan(&matchers, &timeout, &severity) + if err == sql.ErrNoRows { + return DeadmanConfig{}, nil + } + if err != nil { + return DeadmanConfig{}, err + } + return parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity), nil +} + +// deadmanConfigs reads every team's switches in one query, for the sweeper. +func deadmanConfigs(ctx context.Context, db *sql.DB) (map[int64]DeadmanConfig, error) { + rows, err := db.QueryContext(ctx, + "SELECT team_id, matchers, timeout_seconds, severity FROM deadman_configs") + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[int64]DeadmanConfig{} + for rows.Next() { + var teamID, timeout int64 + var matchers, severity string + if err := rows.Scan(&teamID, &matchers, &timeout, &severity); err != nil { + return nil, err + } + cfg := parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity) + if cfg.enabled() { + out[teamID] = cfg + } + } + return out, rows.Err() +} + +// SeedDeadmanConfigs gives every team without a row the server's environment +// configuration, so the install that upgrades into per-team switches keeps +// watching exactly what it was watching before. +// +// Idempotent, and never overwrites: once a team has a row it owns its own +// configuration, and a redeploy must not quietly put the environment's value +// back over an owner's edit. +// +// A team created after startup gets no row and therefore watches nothing until +// its owner says otherwise. That is deliberate: inheriting an install-wide +// heartbeat would page a new team about a source it has never heard of, and a +// switch nobody chose is the kind that gets muted rather than fixed. +func SeedDeadmanConfigs(ctx context.Context, db *sql.DB, cfg DeadmanConfig) error { + matchers := make([]string, 0, len(cfg.Matchers)) + for _, m := range cfg.Matchers { + parts := []string{"alertname=" + m.Name} + for k, v := range m.Labels { + parts = append(parts, k+"="+v) + } + sort.Strings(parts[1:]) + matchers = append(matchers, strings.Join(parts, ",")) + } + + _, err := db.ExecContext(ctx, ` + INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity) + SELECT id, $1, $2, $3 FROM teams + ON CONFLICT (team_id) DO NOTHING`, + strings.Join(matchers, "; "), int64(cfg.Timeout.Seconds()), cfg.Severity) + return err +} + +// parseDeadmanQuietly is ParseDeadmanConfig without the startup logging: a +// team's configuration is read on every sweep and every webhook, and logging it +// each time would bury everything else. +func parseDeadmanQuietly(matchers string, timeout time.Duration, severity string) DeadmanConfig { + cfg := DeadmanConfig{Timeout: timeout, Severity: severity} + for _, entry := range strings.Split(matchers, ";") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + m := DeadmanMatcher{Labels: map[string]string{}} + malformed := false + for _, cond := range strings.Split(entry, ",") { + k, v, ok := strings.Cut(cond, "=") + k, v = strings.TrimSpace(k), strings.TrimSpace(v) + if !ok || k == "" || v == "" { + malformed = true + break + } + if k == "alertname" { + m.Name = v + continue + } + m.Labels[k] = v + } + if malformed || m.Name == "" { + continue + } + cfg.Matchers = append(cfg.Matchers, m) + } + return cfg +} diff --git a/internal/api/deadman_test.go b/internal/api/deadman_test.go index d9cecfb..4b69aef 100644 --- a/internal/api/deadman_test.go +++ b/internal/api/deadman_test.go @@ -2,6 +2,7 @@ package api_test import ( "net/http" + "strings" "testing" "time" @@ -469,3 +470,123 @@ func TestDeadman_DisabledConfigIsInert(t *testing.T) { t.Errorf("expected the generic sweeper to own the alert, got %v", source) } } + +// --------------------------------------------------------------------------- +// Per-team configuration +// --------------------------------------------------------------------------- + +// Each team decides for itself what a heartbeat is. The same alert is a +// heartbeat in one team and an ordinary problem in another. +func TestDeadman_ConfigurationIsPerTeam(t *testing.T) { + s, _ := deadmanTS(t, deadmanCfg()) + watched := newTeam(t, s, "watched") + unwatched := newTeam(t, s, "unwatched") + + // Only the first team calls Watchdog a heartbeat. + resp := s.req(t, http.MethodPut, "/api/teams/"+id64(watched.id)+"/deadman", map[string]any{ + "matchers": "alertname=Watchdog", + "timeout_seconds": 3600, + "severity": "critical", + }) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("configure the watched team: %d", resp.StatusCode) + } + + postToIntegration(t, s, watched.key, "fp-watched", "Watchdog") + postToIntegration(t, s, unwatched.key, "fp-unwatched", "Watchdog") + + // A heartbeat opens nothing where it is one; an ordinary alert opens an + // incident where it is not. + if got := len(list(t, watched.call(http.MethodGet, "/api/incidents", nil))); got != 0 { + t.Errorf("the watched team's heartbeat opened %d incident(s), want 0", got) + } + if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 { + t.Errorf("the unwatched team's Watchdog opened %d incident(s), want 1", got) + } + + // Silence pages only the team that is watching. + s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2", + time.Now().Add(-2*time.Hour).Unix(), "fp-watched") + s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2", + time.Now().Add(-2*time.Hour).Unix(), "fp-unwatched") + sweep(t, s, noArchive) + + watchedIncidents := list(t, watched.call(http.MethodGet, "/api/incidents", nil)) + if len(watchedIncidents) != 1 { + t.Fatalf("silence opened %d incident(s) for the watching team, want 1", len(watchedIncidents)) + } + if title := watchedIncidents[0]["title"].(string); title != "No heartbeat from Watchdog" { + t.Errorf("unexpected incident title %q", title) + } + if teamID := int64(watchedIncidents[0]["team_id"].(float64)); teamID != watched.id { + t.Errorf("the incident opened in team %d, want %d", teamID, watched.id) + } + + // The unwatched team's alert went stale the ordinary way, so it has the one + // incident it always had — not a second, dead man's switch one. + if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 { + t.Errorf("the unwatched team ended with %d incident(s), want 1", got) + } +} + +// Configuration is an owner's to change and a member's to read, like the rest of +// a team's settings. +func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) { + s, _ := deadmanTS(t, deadmanCfg()) + team := newTeam(t, s, "red") + + // A plain member of that team. + var user struct { + ID int64 `json:"id"` + } + decode(t, s.req(t, http.MethodPost, "/api/users", + map[string]string{"username": "plain", "email": "plain@test.com"}), &user) + s.req(t, http.MethodPost, "/api/teams/"+id64(team.id)+"/members", + map[string]any{"user_id": user.ID, "role": "member"}).Body.Close() + var key struct { + Key string `json:"key"` + } + decode(t, s.req(t, http.MethodPost, "/api/users/"+id64(user.ID)+"/api-keys", + map[string]string{"name": "test"}), &key) + + req, _ := http.NewRequest(http.MethodPut, + s.URL+"/api/teams/"+id64(team.id)+"/deadman", + strings.NewReader(`{"matchers":"alertname=Watchdog","timeout_seconds":60}`)) + req.Header.Set("Authorization", "Bearer "+key.Key) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("put: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Errorf("a member editing the switches: expected 403, got %d", resp.StatusCode) + } + + read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman", nil) + read.Header.Set("Authorization", "Bearer "+key.Key) + got, err := http.DefaultClient.Do(read) + if err != nil { + t.Fatalf("get: %v", err) + } + got.Body.Close() + if got.StatusCode != http.StatusOK { + t.Errorf("a member reading the switches: expected 200, got %d", got.StatusCode) + } +} + +// A matcher with no alertname watches nothing, silently, which is the failure +// this feature exists to prevent — so it is refused at the door. +func TestDeadman_UnusableMatchersAreRejected(t *testing.T) { + s, _ := deadmanTS(t, deadmanCfg()) + + resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{ + "matchers": "cluster=prod", + "timeout_seconds": 900, + }) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("expected 400 for a matcher with no alertname, got %d", resp.StatusCode) + } +} diff --git a/internal/api/incidents_test.go b/internal/api/incidents_test.go index 745c0cd..a98b0b5 100644 --- a/internal/api/incidents_test.go +++ b/internal/api/incidents_test.go @@ -609,7 +609,7 @@ func TestSweeper_ArchivesResolvedIncidents(t *testing.T) { s.exec(t, "UPDATE incidents SET resolved_at = $1 WHERE id = 1", time.Now().Add(-30*24*time.Hour).Unix()) - api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.deadman, s.notify) + api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.notify) if inc := getIncident(t, s, 1); inc["archived_at"] == nil { t.Error("expected the sweeper to archive a long-resolved incident") diff --git a/internal/api/notify_test.go b/internal/api/notify_test.go index 3b4dc0b..2abc935 100644 --- a/internal/api/notify_test.go +++ b/internal/api/notify_test.go @@ -438,7 +438,7 @@ func TestNotify_SweepPurgesExpiredAckTokens(t *testing.T) { s.sweepNotify(t) s.exec(t, "UPDATE incident_ack_tokens SET expires_at = $1", time.Now().Add(-time.Minute).Unix()) - api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour, s.deadman, s.notify) + api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour, s.notify) var n int if err := s.db.QueryRow("SELECT COUNT(*) FROM incident_ack_tokens").Scan(&n); err != nil { diff --git a/internal/api/router.go b/internal/api/router.go index e42b79b..e448c81 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -9,11 +9,11 @@ import ( "github.com/go-chi/chi/v5/middleware" ) -// NewRouter builds the HTTP surface. notify and deadman are passed through to -// the webhook, the only handler that has to decide where a new incident's page -// goes and which arriving alerts are heartbeats rather than problems. A zero -// notify disables notifications; a zero deadman disables dead man's switches. -func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Handler { +// NewRouter builds the HTTP surface. notify is passed through to the webhook, +// the only handler that has to decide where a new incident's page goes; a zero +// notify disables notifications. Dead man's switches are per team and read from +// the database, so nothing about them is wired in here. +func NewRouter(db *sql.DB, notify NotifyConfig) http.Handler { r := chi.NewRouter() r.Use(middleware.Logger) r.Use(middleware.Recoverer) @@ -31,13 +31,13 @@ func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Hand // Alert ingestion. The key in the path says both that the sender may post // and which team the alerts belong to, which is why it needs no session. - r.Post("/api/integrations/{key}/alertmanager", handleIntegrationWebhook(db, notify, deadman)) + r.Post("/api/integrations/{key}/alertmanager", handleIntegrationWebhook(db, notify)) // DEPRECATED, and unauthenticated: anything that can reach the port can // open an incident here. Kept for one release so an upgrade does not stop // delivering while the Alertmanager config is edited; it routes everything // to the oldest team. Remove it once senders carry a key. - r.Post("/api/alertmanager/webhook", handleLegacyWebhook(db, notify, deadman)) + r.Post("/api/alertmanager/webhook", handleLegacyWebhook(db, notify)) // Signing in to the web UI. Login trades a password for a session cookie, // which AuthMiddleware accepts in place of an API key. @@ -101,6 +101,11 @@ func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Hand r.Post("/api/teams/{teamID}/members", handleAddTeamMember(db)) r.Delete("/api/teams/{teamID}/members/{userID}", handleRemoveTeamMember(db)) + // A team's own dead man's switches: which of its alerts are heartbeats, + // and how long a silence has to last before somebody is paged. + r.Get("/api/teams/{teamID}/deadman", handleGetTeamDeadman(db)) + r.Put("/api/teams/{teamID}/deadman", handleSetTeamDeadman(db)) + // Integrations: where a team's alerts come in, and the key that says so. r.Get("/api/teams/{teamID}/integrations", handleListIntegrations(db)) r.Post("/api/teams/{teamID}/integrations", handleCreateIntegration(db, notify.PublicURL)) diff --git a/internal/api/teams.go b/internal/api/teams.go index 24523e4..1e2bbac 100644 --- a/internal/api/teams.go +++ b/internal/api/teams.go @@ -471,3 +471,105 @@ func defaultTeamID(ctx context.Context, db *sql.DB) (int64, error) { err := db.QueryRowContext(ctx, "SELECT id FROM teams ORDER BY id LIMIT 1").Scan(&id) return id, err } + +// --------------------------------------------------------------------------- +// A team's dead man's switches +// --------------------------------------------------------------------------- + +// deadmanResponse is the wire shape of a team's switch configuration. The +// timeout is seconds rather than a duration string, because that is what the +// column holds and what arithmetic is done on; a client renders it. +type deadmanResponse struct { + TeamID int64 `json:"team_id"` + Matchers string `json:"matchers"` + TimeoutSeconds int64 `json:"timeout_seconds"` + Severity string `json:"severity"` +} + +func handleGetTeamDeadman(db *sql.DB) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + teamID, ok := teamParam(w, r) + if !ok { + return + } + if !requireTeamMember(w, r, teamID) { + return + } + + out := deadmanResponse{TeamID: teamID, Severity: "critical"} + err := db.QueryRowContext(r.Context(), + "SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1", + teamID).Scan(&out.Matchers, &out.TimeoutSeconds, &out.Severity) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + // A team with no row watches nothing, which is a configuration and not + // an absence: answering 404 would make "off" indistinguishable from + // "this server does not do this". + respond(w, http.StatusOK, out) + } +} + +// handleSetTeamDeadman replaces a team's switch configuration. +// +// Validated by parsing: a matcher string that survives ParseDeadmanConfig with +// nothing usable in it is rejected rather than stored, because a switch that +// silently watches nothing is the failure this feature exists to prevent. +func handleSetTeamDeadman(db *sql.DB) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + teamID, ok := teamParam(w, r) + if !ok { + return + } + if !requireTeamOwner(w, r, teamID) { + return + } + + var req struct { + Matchers string `json:"matchers"` + TimeoutSeconds int64 `json:"timeout_seconds"` + Severity string `json:"severity"` + } + if err := decodeJSON(r, &req); err != nil { + respond(w, http.StatusBadRequest, errResp("invalid request body")) + return + } + req.Matchers = strings.TrimSpace(req.Matchers) + if req.Severity == "" { + req.Severity = "critical" + } + if req.TimeoutSeconds < 0 { + respond(w, http.StatusBadRequest, errResp("timeout_seconds must not be negative")) + return + } + if req.Matchers != "" { + parsed := parseDeadmanQuietly(req.Matchers, time.Duration(req.TimeoutSeconds)*time.Second, req.Severity) + if len(parsed.Matchers) == 0 { + respond(w, http.StatusBadRequest, errResp( + "no usable matchers: each must name an alertname, as in alertname=Watchdog,cluster=prod")) + return + } + } + + if _, err := db.ExecContext(r.Context(), ` + INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity, updated_at) + VALUES ($1, $2, $3, $4, `+nowEpoch+`) + ON CONFLICT (team_id) DO UPDATE SET + matchers = excluded.matchers, + timeout_seconds = excluded.timeout_seconds, + severity = excluded.severity, + updated_at = excluded.updated_at`, + teamID, req.Matchers, req.TimeoutSeconds, req.Severity); err != nil { + respond(w, http.StatusInternalServerError, errResp("internal error")) + return + } + + respond(w, http.StatusOK, deadmanResponse{ + TeamID: teamID, + Matchers: req.Matchers, + TimeoutSeconds: req.TimeoutSeconds, + Severity: req.Severity, + }) + } +} diff --git a/internal/db/migrations/004_team_deadman.sql b/internal/db/migrations/004_team_deadman.sql new file mode 100644 index 0000000..3a066fd --- /dev/null +++ b/internal/db/migrations/004_team_deadman.sql @@ -0,0 +1,39 @@ +-- Dead man's switches become a team's own configuration. +-- +-- They were three environment variables — TERDUT_DEADMAN_MATCHERS, _TIMEOUT and +-- _SEVERITY — which made them one setting for the whole install. That was the +-- last piece of the alerting path a team could not control: a team could take +-- its own alerts on its own key and still not say which of them were +-- heartbeats, or how long a silence had to last before somebody was paged. +-- +-- One row per team rather than one row per switch. The unit of monitoring is +-- still the fingerprint, as it always was — two clusters sending the same +-- heartbeat alertname are two independent switches — and the matcher string +-- keeps the format the environment variable used, so a value can be moved from +-- one to the other unchanged. +-- +-- No rows are seeded here: a migration cannot read the environment. The server +-- inserts a row per team at startup from its own configuration, and the same +-- values therefore carry forward into the first team's row without anybody +-- retyping them. See seedDeadmanConfigs. +CREATE TABLE deadman_configs ( + team_id BIGINT PRIMARY KEY REFERENCES teams(id) ON DELETE CASCADE, + + -- ";" separates matchers, "," the label conditions within one, "=" is exact + -- equality: `alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat`. + -- Every matcher must name an alertname. Empty watches nothing. + matchers TEXT NOT NULL DEFAULT '', + + -- Seconds rather than a Go duration string: the column is compared and + -- arithmetic is done on it, and a value that has to be parsed before it can + -- be believed is a value that can be stored unparseable. Zero disables the + -- team's switches entirely. + timeout_seconds BIGINT NOT NULL DEFAULT 0, + + -- The severity these incidents open at. They have no member alerts to + -- derive one from, and a heartbeat's own severity label is meaningless — + -- Watchdog ships as "none". + severity TEXT NOT NULL DEFAULT 'critical', + + updated_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM now()))::bigint +); diff --git a/internal/web/static/app.css b/internal/web/static/app.css index bdc325f..17567ca 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -285,6 +285,16 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh min-width: 0; } .row-meta .labels { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; max-width: 100%; color: var(--faint); } +/* Which team's queue a row came from. Only rendered for somebody in more than + one team, so it never repeats the same word down the whole list. */ +/* Separates the status chips from the team chips in the queue's filter row. */ +.chip-sep { width: 1px; align-self: stretch; background: var(--border); margin: 0 2px; } + +.row-team { + padding: 1px 6px; border-radius: 4px; + background: var(--surface-2); border: 1px solid var(--border); + color: var(--muted); font-size: 12px; white-space: nowrap; +} .row.resolved .row-title { color: var(--muted); } .sev-critical { --sev: var(--crit); } diff --git a/internal/web/static/js/queue.js b/internal/web/static/js/queue.js index 3e82619..3fae5c0 100644 --- a/internal/web/static/js/queue.js +++ b/internal/web/static/js/queue.js @@ -26,12 +26,32 @@ const EMPTY = { }; let filter = loadFilter(); +let teamFilter = loadTeamFilter(); // '' for every team the viewer is in let items = null; // null while loading let error = null; let selected = null; let cursor = -1; // keyboard position in the list let built = false; +function loadTeamFilter() { + try { + return sessionStorage.getItem('terdut.queue.team') || ''; + } catch { + return ''; + } +} + +function setTeamFilter(id) { + teamFilter = id; + try { + sessionStorage.setItem('terdut.queue.team', id); + } catch { + /* storage unavailable */ + } + renderChips(); + refresh({ fresh: true }); +} + function loadFilter() { try { const f = sessionStorage.getItem('terdut.queue.filter'); @@ -64,7 +84,11 @@ export async function refresh({ fresh = false } = {}) { const requested = filter; try { // The open list is already fetched for the badges; no need to ask twice. - const result = filter === 'open' && !fresh ? state.open : await api.incidents(f.query); + // The cached open queue covers every team, so it can only be reused when + // no team filter is applied. + const query = teamFilter ? { ...f.query, team_id: teamFilter } : f.query; + const cached = filter === 'open' && !fresh && !teamFilter; + const result = cached ? state.open : await api.incidents(query); if (requested !== filter) return; items = result; error = null; @@ -88,7 +112,7 @@ function setFilter(id) { function renderChips() { const el = document.getElementById('queue-filters'); - clear(el, FILTERS.map((f) => + const chips = FILTERS.map((f) => h('button', { class: 'chip', type: 'button', @@ -97,7 +121,34 @@ function renderChips() { onclick: () => setFilter(f.id), text: f.label, }), - )); + ); + + // Somebody in one team has nothing to choose between, so the row of team + // chips appears only when there is more than one. The default is all of + // them: the combined queue is the point. + if (state.teams.length > 1) { + chips.push(h('span', { class: 'chip-sep' })); + chips.push(h('button', { + class: 'chip', + type: 'button', + role: 'tab', + 'aria-selected': String(teamFilter === ''), + onclick: () => setTeamFilter(''), + text: 'All teams', + })); + for (const team of state.teams) { + chips.push(h('button', { + class: 'chip', + type: 'button', + role: 'tab', + 'aria-selected': String(teamFilter === String(team.id)), + onclick: () => setTeamFilter(String(team.id)), + text: team.name, + })); + } + } + + clear(el, chips); } function renderList() { @@ -142,6 +193,13 @@ function row(inc, index) { // The server already puts the group labels in the title; show only the rest. const labels = labelSummary(Object.fromEntries( Object.entries(inc.group_labels || {}).filter(([k, v]) => !inc.title.includes(`${k}=${v}`)))); + // The team is shown only to somebody who is in more than one. For everybody + // else it is the same word on every row, which is noise rather than + // information. + const team = state.teams.length > 1 && inc.team_name + ? h('span', { class: 'row-team', text: inc.team_name }) + : null; + return h('a', { class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`, href: `/incidents/${inc.id}`, @@ -153,6 +211,7 @@ function row(inc, index) { h('div', { class: 'row-meta' }, status, assignee, + team, labels && h('span', { class: 'labels', text: labels }), ), );