README.md

@@ -161,7 +161,7 @@ somewhere to exec. The sidecar, the PVC and the `backupSidecar` values are all g
 | `TERDUT_DB_DSN` | — | **Required.** Postgres connection string, e.g. `postgres://terdut:secret@localhost:5432/terdut?sslmode=require` |
 | `TERDUT_ARCHIVE_AFTER` | `168h` (7d) | How long a resolved alert or incident stays in the default list before being auto-archived |
 | `TERDUT_STALE_AFTER` | `6h` | How long a firing alert may go without a refreshing webhook before it is treated as resolved — **must exceed your Alertmanager `repeat_interval`** |
-| `TERDUT_DEADMAN_MATCHERS` | `alertname=Watchdog` | Which alerts are [dead man's switches](#dead-mans-switch). `;` separates matchers, `,` the label conditions within one, `=` is exact equality. Every matcher must name an `alertname` |
+| `TERDUT_DEADMAN_MATCHERS` | `alertname=Watchdog` | The **default** matchers a team starts with — switches are per team now, and this seeds teams that have no configuration of their own. `;` separates matchers, `,` the label conditions within one, `=` is exact equality. Every matcher must name an `alertname` |
 | `TERDUT_DEADMAN_TIMEOUT` | `15m` | How long a heartbeat may go unheard before its switch is declared dead — **must be shorter than the `repeat_interval` of the route carrying it**. `0` disables dead man's switch handling |
 | `TERDUT_DEADMAN_SEVERITY` | `critical` | Severity a dead man's switch incident opens at |
 | `TERDUT_NTFY_URL` | — | ntfy server to publish push notifications to. Empty disables notifications entirely |
@@ -378,11 +378,23 @@ kube-prometheus-stack already ships the alert for this. `Watchdog` is
 nothing unless something downstream notices it stop. That is what
 `TERDUT_DEADMAN_MATCHERS` defaults to.
 
+**Switches belong to a team**, which decides which of its own alerts are
+heartbeats and how long a silence has to last. An owner sets them through
+`PUT /api/teams/{teamID}/deadman`; a missed heartbeat opens an incident in the
+team whose integration received it.
+
+The environment variables are the starting point, not the setting: at startup
+every team **without** a configuration of its own is given one from them, and an
+owner's later edit is never overwritten by a redeploy. A team created after
+that starts watching nothing until its owner says otherwise — inheriting an
+install-wide heartbeat would page a new team about a source it has never heard
+of.
+
 A matcher is a set of exact label conditions, one of which must be the
-`alertname`:
+`alertname`, in the same format the environment variable uses:
 
 ```
-TERDUT_DEADMAN_MATCHERS="alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat"
+alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat
 ```
 
 **The unit of monitoring is the fingerprint, not the alert name.** Two clusters
@@ -523,6 +535,8 @@ incident. Move senders to a key and it goes away.
 | `GET` | `/api/teams/{teamID}/integrations` | member | List integrations. Never returns keys |
 | `POST` | `/api/teams/{teamID}/integrations` | **owner** | Mint an integration `{"name","kind"}` — key and URL shown once |
 | `DELETE` | `/api/teams/{teamID}/integrations/{integrationID}` | **owner** | Revoke an integration |
+| `GET` | `/api/teams/{teamID}/deadman` | member | The team's [dead man's switch](#dead-mans-switch) configuration `{matchers, timeout_seconds, severity}` |
+| `PUT` | `/api/teams/{teamID}/deadman` | **owner** | Replace it. `400` when no matcher names an `alertname`, because a switch that silently watches nothing is the failure this feature exists to prevent |
 
 ### Notifications
 
@@ -764,6 +778,11 @@ What changes, and will need attention:
   fingerprint, the same Alertmanager groupKey, and put somebody on call on the
   same date.
 
+**Dead man's switches moved too.** `TERDUT_DEADMAN_MATCHERS`, `_TIMEOUT` and
+`_SEVERITY` are no longer the setting; they are the default each existing team
+is seeded with at startup, after which an owner edits them per team through
+`PUT /api/teams/{teamID}/deadman` and a redeploy never overwrites that.
+
 Nothing else about an incident changes, and incidents never move between teams:
 an alert belongs to whichever team's key it arrived on.
 

cmd/terdut/main.go

@@ -36,9 +36,16 @@ func main() {
 		RepeatEvery:   cfg.NotifyRepeat,
 	}
 
+	// Dead man's switches live per team now. The environment variables are the
+	// defaults a team starts from: every team without a configuration of its
+	// own gets one from them here, and an owner's later edit is never
+	// overwritten by a redeploy.
 	deadman := api.ParseDeadmanConfig(cfg.DeadmanMatchers, cfg.DeadmanTimeout, cfg.DeadmanSeverity)
+	if err := api.SeedDeadmanConfigs(context.Background(), database, deadman); err != nil {
+		log.Fatalf("seed dead man's switch defaults: %v", err)
+	}
 
-	router := api.NewRouter(database, notify, deadman)
+	router := api.NewRouter(database, notify)
 
 	srv := &http.Server{
 		Addr:         cfg.Addr,
@@ -51,7 +58,7 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, deadman, notify)
+	go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, notify)
 	go api.StartNotifier(ctx, database, notify)
 
 	go func() {

internal/api/alertmanager.go

@@ -76,7 +76,7 @@ type ingested struct {
 // handleIntegrationWebhook receives alerts on a team's own integration key.
 // The key in the path is both the credential and the routing: it says who may
 // post, and which team the alerts belong to.
-func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.HandlerFunc {
+func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		teamID, err := teamIDForKey(r.Context(), db, chi.URLParam(r, "key"))
 		if err != nil {
@@ -90,7 +90,7 @@ func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanCo
 			respond(w, http.StatusInternalServerError, errResp("internal error"))
 			return
 		}
-		receiveWebhook(w, r, db, notify, deadman, teamID)
+		receiveWebhook(w, r, db, notify, teamID)
 	}
 }
 
@@ -101,7 +101,7 @@ func handleIntegrationWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanCo
 //
 // It is deprecated and unauthenticated — anything that can reach the port can
 // open an incident. Move senders to an integration key and this goes away.
-func handleLegacyWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.HandlerFunc {
+func handleLegacyWebhook(db *sql.DB, notify NotifyConfig) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		teamID, err := defaultTeamID(r.Context(), db)
 		if err != nil {
@@ -111,11 +111,11 @@ func handleLegacyWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig)
 		}
 		log.Printf("legacy webhook: unauthenticated payload routed to team %d; "+
 			"move this sender to an integration key", teamID)
-		receiveWebhook(w, r, db, notify, deadman, teamID)
+		receiveWebhook(w, r, db, notify, teamID)
 	}
 }
 
-func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify NotifyConfig, deadman DeadmanConfig, teamID int64) {
+func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify NotifyConfig, teamID int64) {
 	var payload amPayload
 	if err := decodeJSON(r, &payload); err != nil {
 		respond(w, http.StatusBadRequest, errResp("invalid payload"))
@@ -125,7 +125,7 @@ func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify N
 	// Alertmanager retries anything that is not 2xx, and a retry of a payload
 	// we failed to store is more useful than an error it cannot act on — so
 	// failures are logged, not surfaced.
-	if err := ingest(r.Context(), db, notify, deadman, teamID, payload); err != nil {
+	if err := ingest(r.Context(), db, notify, teamID, payload); err != nil {
 		log.Printf("webhook ingest (team %d, group %q): %v", teamID, payload.GroupKey, err)
 	}
 
@@ -135,13 +135,21 @@ func receiveWebhook(w http.ResponseWriter, r *http.Request, db *sql.DB, notify N
 // ingest stores a payload's alerts and reconciles the incident for its group.
 // The whole payload is one transaction: an incident that opened but whose alerts
 // failed to link would be a work item nobody could act on.
-func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, deadman DeadmanConfig, teamID int64, payload amPayload) error {
+func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, teamID int64, payload amPayload) error {
 	tx, err := db.BeginTx(ctx, nil)
 	if err != nil {
 		return err
 	}
 	defer tx.Rollback() //nolint:errcheck
 
+	// Which arriving alerts are heartbeats is the team's own answer, read
+	// inside the transaction so an owner editing it mid-payload cannot split
+	// one webhook across two interpretations.
+	deadman, err := deadmanConfigForTeam(ctx, tx, teamID)
+	if err != nil {
+		return err
+	}
+
 	accepted, err := upsertAlerts(ctx, tx, deadman, teamID, payload.Alerts)
 	if err != nil {
 		return err

internal/api/api_test.go

@@ -9,6 +9,8 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -37,7 +39,7 @@ func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
 	return newDeadmanTS(t, api.DeadmanConfig{}, cfg)
 }
 
-// newDeadmanTS is newTS with dead man's switch handling configured.
+// newDeadmanTS is newTS with the default team's dead man's switches configured.
 func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts {
 	t.Helper()
 	var cfg api.NotifyConfig
@@ -46,7 +48,7 @@ func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyC
 	}
 
 	database := newTestDB(t)
-	srv := httptest.NewServer(api.NewRouter(database, cfg, deadman))
+	srv := httptest.NewServer(api.NewRouter(database, cfg))
 	t.Cleanup(srv.Close)
 
 	body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
@@ -62,7 +64,38 @@ func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyC
 	json.NewDecoder(resp.Body).Decode(&result)
 	key := result["api_key"].(map[string]any)["key"].(string)
 
-	return &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman}
+	s := &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman}
+
+	// Dead man's switches belong to a team now, so a test that wants them
+	// configures the default team the way an owner would.
+	if deadman.Timeout > 0 {
+		setTeamDeadman(t, s, deadman)
+	}
+	return s
+}
+
+// setTeamDeadman configures the default team's switches over the API, rendering
+// the matchers back into the string form the endpoint takes.
+func setTeamDeadman(t *testing.T, s *ts, cfg api.DeadmanConfig) {
+	t.Helper()
+	matchers := make([]string, 0, len(cfg.Matchers))
+	for _, m := range cfg.Matchers {
+		parts := []string{"alertname=" + m.Name}
+		for k, v := range m.Labels {
+			parts = append(parts, k+"="+v)
+		}
+		sort.Strings(parts[1:])
+		matchers = append(matchers, strings.Join(parts, ","))
+	}
+	resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{
+		"matchers":        strings.Join(matchers, "; "),
+		"timeout_seconds": int64(cfg.Timeout.Seconds()),
+		"severity":        cfg.Severity,
+	})
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("configure the team's dead man's switches: %d", resp.StatusCode)
+	}
 }
 
 // exec runs a statement against the test database.
@@ -512,7 +545,7 @@ func TestArchive_AlertListFilter(t *testing.T) {
 	}
 
 	// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
-	api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.deadman, s.notify)
+	api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.notify)
 
 	// 3. Default list excludes it.
 	decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
@@ -553,7 +586,7 @@ func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string
 
 func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
 	t.Helper()
-	api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.deadman, s.notify)
+	api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.notify)
 }
 
 // A firing alert Alertmanager stopped refreshing is resolved via the

internal/api/archiver.go

@@ -19,15 +19,15 @@ const (
 
 // StartArchiver runs the alert sweeper until ctx is cancelled, starting with an
 // immediate pass so a restart reconciles state right away.
-func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, deadman DeadmanConfig, notify NotifyConfig) {
+func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, notify NotifyConfig) {
 	ticker := time.NewTicker(sweepInterval)
 	defer ticker.Stop()
 
-	Sweep(ctx, db, archiveAfter, staleAfter, deadman, notify)
+	Sweep(ctx, db, archiveAfter, staleAfter, notify)
 	for {
 		select {
 		case <-ticker.C:
-			Sweep(ctx, db, archiveAfter, staleAfter, deadman, notify)
+			Sweep(ctx, db, archiveAfter, staleAfter, notify)
 		case <-ctx.Done():
 			return
 		}
@@ -44,8 +44,8 @@ func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter tim
 // touch: a heartbeat answers to its own, much tighter, timeout, and the generic
 // staleness rules would otherwise resolve it as 'expiry' long before that.
 // Exported so tests can drive a pass without waiting on the ticker.
-func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, deadman DeadmanConfig, notify NotifyConfig) {
+func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, notify NotifyConfig) {
-	heartbeats := sweepDeadman(ctx, db, deadman, notify)
+	heartbeats := sweepDeadman(ctx, db, notify)
 	expireStale(ctx, db, staleAfter, heartbeats)
 	resolveSettledIncidents(ctx, db)
 	archiveResolved(ctx, db, archiveAfter)

internal/api/auth_test.go

@@ -248,7 +248,7 @@ func TestSession_ExpiredIsRejected(t *testing.T) {
 	if code := status(t, b.do(t, http.MethodGet, "/api/me", nil)); code != http.StatusUnauthorized {
 		t.Errorf("expired session: %d", code)
 	}
-	api.Sweep(t.Context(), s.db, 0, 0, api.DeadmanConfig{}, api.NotifyConfig{})
+	api.Sweep(t.Context(), s.db, 0, 0, api.NotifyConfig{})
 	var n int
 	s.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&n)
 	if n != 0 {
@@ -301,7 +301,7 @@ func TestSetPassword_EndsOtherSessionsButNotThisOne(t *testing.T) {
 
 func TestBootstrap_WithPassword(t *testing.T) {
 	database := newTestDB(t)
-	srv := httptest.NewServer(api.NewRouter(database, api.NotifyConfig{}, api.DeadmanConfig{}))
+	srv := httptest.NewServer(api.NewRouter(database, api.NotifyConfig{}))
 	t.Cleanup(srv.Close)
 
 	body := `{"username":"admin","email":"a@test.com","password":"` + adminPassword + `"}`

internal/api/deadman.go

@@ -189,35 +189,42 @@ func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerpr
 // It returns the ids of the alerts it owns, because the generic staleness
 // expiry must leave them alone — staleAfter and ends_at would otherwise resolve
 // a heartbeat long before its own, much tighter, timeout ever fired.
-func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig) map[int64]bool {
+// Each team is swept against its own configuration: its own matchers, its own
+// timeout, its own severity. A team watching nothing is skipped entirely, which
+// is most of them.
+func sweepDeadman(ctx context.Context, db *sql.DB, notify NotifyConfig) map[int64]bool {
 	owned := map[int64]bool{}
-	if !cfg.enabled() {
-		return owned
-	}
 
-	switches, err := deadmanAlerts(ctx, db, cfg)
+	configs, err := deadmanConfigs(ctx, db)
 	if err != nil {
-		log.Printf("deadman: load switches: %v", err)
+		log.Printf("deadman: load configs: %v", err)
 		return owned
 	}
 
 	now := time.Now()
-	cutoff := now.Add(-cfg.Timeout).Unix()
+	for teamID, cfg := range configs {
-
+		switches, err := deadmanAlerts(ctx, db, teamID, cfg)
-	for _, sw := range switches {
+		if err != nil {
-		owned[sw.id] = true
+			log.Printf("deadman: load switches for team %d: %v", teamID, err)
-
-		// An explicit resolved from Alertmanager is a stronger death signal than
-		// mere absence: the sender is telling us the heartbeat stopped, so there
-		// is nothing left to wait out.
-		if sw.resolved || sw.receivedAt < cutoff {
-			if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil {
-				log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err)
-			}
 			continue
 		}
-		if err := deadmanRecovered(ctx, db, sw); err != nil {
+		cutoff := now.Add(-cfg.Timeout).Unix()
-			log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err)
+
+		for _, sw := range switches {
+			owned[sw.id] = true
+
+			// An explicit resolved from Alertmanager is a stronger death signal
+			// than mere absence: the sender is telling us the heartbeat
+			// stopped, so there is nothing left to wait out.
+			if sw.resolved || sw.receivedAt < cutoff {
+				if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil {
+					log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err)
+				}
+				continue
+			}
+			if err := deadmanRecovered(ctx, db, sw); err != nil {
+				log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err)
+			}
 		}
 	}
 	return owned
@@ -228,7 +235,7 @@ func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify Not
 // happens in Go, which keeps one implementation of the rules. The rows are read
 // in full before the caller writes, so the writes do not run against an open
 // cursor over the same table.
-func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadmanAlert, error) {
+func deadmanAlerts(ctx context.Context, db *sql.DB, teamID int64, cfg DeadmanConfig) ([]deadmanAlert, error) {
 	names := cfg.names()
 	args := &sqlArgs{}
 	nameList := make([]any, len(names))
@@ -239,7 +246,8 @@ func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadma
 	rows, err := db.QueryContext(ctx, `
 		SELECT id, team_id, fingerprint, labels, status, received_at
 		FROM   alerts
-		WHERE  name IN (`+args.addList(nameList)+`)
+		WHERE  team_id = `+args.add(teamID)+`
+		  AND  name IN (`+args.addList(nameList)+`)
 		  AND  archived_at IS NULL`, args.all()...)
 	if err != nil {
 		return nil, err
@@ -382,3 +390,113 @@ func deadmanRecovered(ctx context.Context, db *sql.DB, sw deadmanAlert) error {
 	log.Printf("deadman: %s is back, resolved incident %d", sw.matcher.String(), incidentID)
 	return nil
 }
+
+// ---------------------------------------------------------------------------
+// Per-team configuration
+// ---------------------------------------------------------------------------
+
+// deadmanConfigForTeam reads one team's switches. A team with no row, or with
+// nothing configured, gets a disabled config — which is the right answer rather
+// than an error: most teams watch no heartbeat at all.
+func deadmanConfigForTeam(ctx context.Context, q querier, teamID int64) (DeadmanConfig, error) {
+	var matchers, severity string
+	var timeout int64
+	err := q.QueryRowContext(ctx,
+		"SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1",
+		teamID).Scan(&matchers, &timeout, &severity)
+	if err == sql.ErrNoRows {
+		return DeadmanConfig{}, nil
+	}
+	if err != nil {
+		return DeadmanConfig{}, err
+	}
+	return parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity), nil
+}
+
+// deadmanConfigs reads every team's switches in one query, for the sweeper.
+func deadmanConfigs(ctx context.Context, db *sql.DB) (map[int64]DeadmanConfig, error) {
+	rows, err := db.QueryContext(ctx,
+		"SELECT team_id, matchers, timeout_seconds, severity FROM deadman_configs")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	out := map[int64]DeadmanConfig{}
+	for rows.Next() {
+		var teamID, timeout int64
+		var matchers, severity string
+		if err := rows.Scan(&teamID, &matchers, &timeout, &severity); err != nil {
+			return nil, err
+		}
+		cfg := parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity)
+		if cfg.enabled() {
+			out[teamID] = cfg
+		}
+	}
+	return out, rows.Err()
+}
+
+// SeedDeadmanConfigs gives every team without a row the server's environment
+// configuration, so the install that upgrades into per-team switches keeps
+// watching exactly what it was watching before.
+//
+// Idempotent, and never overwrites: once a team has a row it owns its own
+// configuration, and a redeploy must not quietly put the environment's value
+// back over an owner's edit.
+//
+// A team created after startup gets no row and therefore watches nothing until
+// its owner says otherwise. That is deliberate: inheriting an install-wide
+// heartbeat would page a new team about a source it has never heard of, and a
+// switch nobody chose is the kind that gets muted rather than fixed.
+func SeedDeadmanConfigs(ctx context.Context, db *sql.DB, cfg DeadmanConfig) error {
+	matchers := make([]string, 0, len(cfg.Matchers))
+	for _, m := range cfg.Matchers {
+		parts := []string{"alertname=" + m.Name}
+		for k, v := range m.Labels {
+			parts = append(parts, k+"="+v)
+		}
+		sort.Strings(parts[1:])
+		matchers = append(matchers, strings.Join(parts, ","))
+	}
+
+	_, err := db.ExecContext(ctx, `
+		INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity)
+		SELECT id, $1, $2, $3 FROM teams
+		ON CONFLICT (team_id) DO NOTHING`,
+		strings.Join(matchers, "; "), int64(cfg.Timeout.Seconds()), cfg.Severity)
+	return err
+}
+
+// parseDeadmanQuietly is ParseDeadmanConfig without the startup logging: a
+// team's configuration is read on every sweep and every webhook, and logging it
+// each time would bury everything else.
+func parseDeadmanQuietly(matchers string, timeout time.Duration, severity string) DeadmanConfig {
+	cfg := DeadmanConfig{Timeout: timeout, Severity: severity}
+	for _, entry := range strings.Split(matchers, ";") {
+		entry = strings.TrimSpace(entry)
+		if entry == "" {
+			continue
+		}
+		m := DeadmanMatcher{Labels: map[string]string{}}
+		malformed := false
+		for _, cond := range strings.Split(entry, ",") {
+			k, v, ok := strings.Cut(cond, "=")
+			k, v = strings.TrimSpace(k), strings.TrimSpace(v)
+			if !ok || k == "" || v == "" {
+				malformed = true
+				break
+			}
+			if k == "alertname" {
+				m.Name = v
+				continue
+			}
+			m.Labels[k] = v
+		}
+		if malformed || m.Name == "" {
+			continue
+		}
+		cfg.Matchers = append(cfg.Matchers, m)
+	}
+	return cfg
+}

internal/api/deadman_test.go

@@ -2,6 +2,7 @@ package api_test
 
 import (
 	"net/http"
+	"strings"
 	"testing"
 	"time"
 
@@ -469,3 +470,123 @@ func TestDeadman_DisabledConfigIsInert(t *testing.T) {
 		t.Errorf("expected the generic sweeper to own the alert, got %v", source)
 	}
 }
+
+// ---------------------------------------------------------------------------
+// Per-team configuration
+// ---------------------------------------------------------------------------
+
+// Each team decides for itself what a heartbeat is. The same alert is a
+// heartbeat in one team and an ordinary problem in another.
+func TestDeadman_ConfigurationIsPerTeam(t *testing.T) {
+	s, _ := deadmanTS(t, deadmanCfg())
+	watched := newTeam(t, s, "watched")
+	unwatched := newTeam(t, s, "unwatched")
+
+	// Only the first team calls Watchdog a heartbeat.
+	resp := s.req(t, http.MethodPut, "/api/teams/"+id64(watched.id)+"/deadman", map[string]any{
+		"matchers":        "alertname=Watchdog",
+		"timeout_seconds": 3600,
+		"severity":        "critical",
+	})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("configure the watched team: %d", resp.StatusCode)
+	}
+
+	postToIntegration(t, s, watched.key, "fp-watched", "Watchdog")
+	postToIntegration(t, s, unwatched.key, "fp-unwatched", "Watchdog")
+
+	// A heartbeat opens nothing where it is one; an ordinary alert opens an
+	// incident where it is not.
+	if got := len(list(t, watched.call(http.MethodGet, "/api/incidents", nil))); got != 0 {
+		t.Errorf("the watched team's heartbeat opened %d incident(s), want 0", got)
+	}
+	if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 {
+		t.Errorf("the unwatched team's Watchdog opened %d incident(s), want 1", got)
+	}
+
+	// Silence pages only the team that is watching.
+	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
+		time.Now().Add(-2*time.Hour).Unix(), "fp-watched")
+	s.exec(t, "UPDATE alerts SET received_at = $1 WHERE fingerprint = $2",
+		time.Now().Add(-2*time.Hour).Unix(), "fp-unwatched")
+	sweep(t, s, noArchive)
+
+	watchedIncidents := list(t, watched.call(http.MethodGet, "/api/incidents", nil))
+	if len(watchedIncidents) != 1 {
+		t.Fatalf("silence opened %d incident(s) for the watching team, want 1", len(watchedIncidents))
+	}
+	if title := watchedIncidents[0]["title"].(string); title != "No heartbeat from Watchdog" {
+		t.Errorf("unexpected incident title %q", title)
+	}
+	if teamID := int64(watchedIncidents[0]["team_id"].(float64)); teamID != watched.id {
+		t.Errorf("the incident opened in team %d, want %d", teamID, watched.id)
+	}
+
+	// The unwatched team's alert went stale the ordinary way, so it has the one
+	// incident it always had — not a second, dead man's switch one.
+	if got := len(list(t, unwatched.call(http.MethodGet, "/api/incidents", nil))); got != 1 {
+		t.Errorf("the unwatched team ended with %d incident(s), want 1", got)
+	}
+}
+
+// Configuration is an owner's to change and a member's to read, like the rest of
+// a team's settings.
+func TestDeadman_ConfigurationIsOwnerOnly(t *testing.T) {
+	s, _ := deadmanTS(t, deadmanCfg())
+	team := newTeam(t, s, "red")
+
+	// A plain member of that team.
+	var user struct {
+		ID int64 `json:"id"`
+	}
+	decode(t, s.req(t, http.MethodPost, "/api/users",
+		map[string]string{"username": "plain", "email": "plain@test.com"}), &user)
+	s.req(t, http.MethodPost, "/api/teams/"+id64(team.id)+"/members",
+		map[string]any{"user_id": user.ID, "role": "member"}).Body.Close()
+	var key struct {
+		Key string `json:"key"`
+	}
+	decode(t, s.req(t, http.MethodPost, "/api/users/"+id64(user.ID)+"/api-keys",
+		map[string]string{"name": "test"}), &key)
+
+	req, _ := http.NewRequest(http.MethodPut,
+		s.URL+"/api/teams/"+id64(team.id)+"/deadman",
+		strings.NewReader(`{"matchers":"alertname=Watchdog","timeout_seconds":60}`))
+	req.Header.Set("Authorization", "Bearer "+key.Key)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("put: %v", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusForbidden {
+		t.Errorf("a member editing the switches: expected 403, got %d", resp.StatusCode)
+	}
+
+	read, _ := http.NewRequest(http.MethodGet, s.URL+"/api/teams/"+id64(team.id)+"/deadman", nil)
+	read.Header.Set("Authorization", "Bearer "+key.Key)
+	got, err := http.DefaultClient.Do(read)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	got.Body.Close()
+	if got.StatusCode != http.StatusOK {
+		t.Errorf("a member reading the switches: expected 200, got %d", got.StatusCode)
+	}
+}
+
+// A matcher with no alertname watches nothing, silently, which is the failure
+// this feature exists to prevent — so it is refused at the door.
+func TestDeadman_UnusableMatchersAreRejected(t *testing.T) {
+	s, _ := deadmanTS(t, deadmanCfg())
+
+	resp := s.req(t, http.MethodPut, "/api/teams/"+defaultTeam+"/deadman", map[string]any{
+		"matchers":        "cluster=prod",
+		"timeout_seconds": 900,
+	})
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Errorf("expected 400 for a matcher with no alertname, got %d", resp.StatusCode)
+	}
+}

internal/api/incidents_test.go

@@ -609,7 +609,7 @@ func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
 
 	s.exec(t, "UPDATE incidents SET resolved_at = $1 WHERE id = 1",
 		time.Now().Add(-30*24*time.Hour).Unix())
-	api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.deadman, s.notify)
+	api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.notify)
 
 	if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
 		t.Error("expected the sweeper to archive a long-resolved incident")

internal/api/notify_test.go

@@ -438,7 +438,7 @@ func TestNotify_SweepPurgesExpiredAckTokens(t *testing.T) {
 	s.sweepNotify(t)
 	s.exec(t, "UPDATE incident_ack_tokens SET expires_at = $1", time.Now().Add(-time.Minute).Unix())
 
-	api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour, s.deadman, s.notify)
+	api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour, s.notify)
 
 	var n int
 	if err := s.db.QueryRow("SELECT COUNT(*) FROM incident_ack_tokens").Scan(&n); err != nil {

internal/api/router.go

@@ -9,11 +9,11 @@ import (
 	"github.com/go-chi/chi/v5/middleware"
 )
 
-// NewRouter builds the HTTP surface. notify and deadman are passed through to
+// NewRouter builds the HTTP surface. notify is passed through to the webhook,
-// the webhook, the only handler that has to decide where a new incident's page
+// the only handler that has to decide where a new incident's page goes; a zero
-// goes and which arriving alerts are heartbeats rather than problems. A zero
+// notify disables notifications. Dead man's switches are per team and read from
-// notify disables notifications; a zero deadman disables dead man's switches.
+// the database, so nothing about them is wired in here.
-func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Handler {
+func NewRouter(db *sql.DB, notify NotifyConfig) http.Handler {
 	r := chi.NewRouter()
 	r.Use(middleware.Logger)
 	r.Use(middleware.Recoverer)
@@ -31,13 +31,13 @@ func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Hand
 
 	// Alert ingestion. The key in the path says both that the sender may post
 	// and which team the alerts belong to, which is why it needs no session.
-	r.Post("/api/integrations/{key}/alertmanager", handleIntegrationWebhook(db, notify, deadman))
+	r.Post("/api/integrations/{key}/alertmanager", handleIntegrationWebhook(db, notify))
 
 	// DEPRECATED, and unauthenticated: anything that can reach the port can
 	// open an incident here. Kept for one release so an upgrade does not stop
 	// delivering while the Alertmanager config is edited; it routes everything
 	// to the oldest team. Remove it once senders carry a key.
-	r.Post("/api/alertmanager/webhook", handleLegacyWebhook(db, notify, deadman))
+	r.Post("/api/alertmanager/webhook", handleLegacyWebhook(db, notify))
 
 	// Signing in to the web UI. Login trades a password for a session cookie,
 	// which AuthMiddleware accepts in place of an API key.
@@ -101,6 +101,11 @@ func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Hand
 		r.Post("/api/teams/{teamID}/members", handleAddTeamMember(db))
 		r.Delete("/api/teams/{teamID}/members/{userID}", handleRemoveTeamMember(db))
 
+		// A team's own dead man's switches: which of its alerts are heartbeats,
+		// and how long a silence has to last before somebody is paged.
+		r.Get("/api/teams/{teamID}/deadman", handleGetTeamDeadman(db))
+		r.Put("/api/teams/{teamID}/deadman", handleSetTeamDeadman(db))
+
 		// Integrations: where a team's alerts come in, and the key that says so.
 		r.Get("/api/teams/{teamID}/integrations", handleListIntegrations(db))
 		r.Post("/api/teams/{teamID}/integrations", handleCreateIntegration(db, notify.PublicURL))

internal/api/teams.go

@@ -471,3 +471,105 @@ func defaultTeamID(ctx context.Context, db *sql.DB) (int64, error) {
 	err := db.QueryRowContext(ctx, "SELECT id FROM teams ORDER BY id LIMIT 1").Scan(&id)
 	return id, err
 }
+
+// ---------------------------------------------------------------------------
+// A team's dead man's switches
+// ---------------------------------------------------------------------------
+
+// deadmanResponse is the wire shape of a team's switch configuration. The
+// timeout is seconds rather than a duration string, because that is what the
+// column holds and what arithmetic is done on; a client renders it.
+type deadmanResponse struct {
+	TeamID         int64  `json:"team_id"`
+	Matchers       string `json:"matchers"`
+	TimeoutSeconds int64  `json:"timeout_seconds"`
+	Severity       string `json:"severity"`
+}
+
+func handleGetTeamDeadman(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		teamID, ok := teamParam(w, r)
+		if !ok {
+			return
+		}
+		if !requireTeamMember(w, r, teamID) {
+			return
+		}
+
+		out := deadmanResponse{TeamID: teamID, Severity: "critical"}
+		err := db.QueryRowContext(r.Context(),
+			"SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1",
+			teamID).Scan(&out.Matchers, &out.TimeoutSeconds, &out.Severity)
+		if err != nil && !errors.Is(err, sql.ErrNoRows) {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+		// A team with no row watches nothing, which is a configuration and not
+		// an absence: answering 404 would make "off" indistinguishable from
+		// "this server does not do this".
+		respond(w, http.StatusOK, out)
+	}
+}
+
+// handleSetTeamDeadman replaces a team's switch configuration.
+//
+// Validated by parsing: a matcher string that survives ParseDeadmanConfig with
+// nothing usable in it is rejected rather than stored, because a switch that
+// silently watches nothing is the failure this feature exists to prevent.
+func handleSetTeamDeadman(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		teamID, ok := teamParam(w, r)
+		if !ok {
+			return
+		}
+		if !requireTeamOwner(w, r, teamID) {
+			return
+		}
+
+		var req struct {
+			Matchers       string `json:"matchers"`
+			TimeoutSeconds int64  `json:"timeout_seconds"`
+			Severity       string `json:"severity"`
+		}
+		if err := decodeJSON(r, &req); err != nil {
+			respond(w, http.StatusBadRequest, errResp("invalid request body"))
+			return
+		}
+		req.Matchers = strings.TrimSpace(req.Matchers)
+		if req.Severity == "" {
+			req.Severity = "critical"
+		}
+		if req.TimeoutSeconds < 0 {
+			respond(w, http.StatusBadRequest, errResp("timeout_seconds must not be negative"))
+			return
+		}
+		if req.Matchers != "" {
+			parsed := parseDeadmanQuietly(req.Matchers, time.Duration(req.TimeoutSeconds)*time.Second, req.Severity)
+			if len(parsed.Matchers) == 0 {
+				respond(w, http.StatusBadRequest, errResp(
+					"no usable matchers: each must name an alertname, as in alertname=Watchdog,cluster=prod"))
+				return
+			}
+		}
+
+		if _, err := db.ExecContext(r.Context(), `
+			INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity, updated_at)
+			VALUES ($1, $2, $3, $4, `+nowEpoch+`)
+			ON CONFLICT (team_id) DO UPDATE SET
+				matchers        = excluded.matchers,
+				timeout_seconds = excluded.timeout_seconds,
+				severity        = excluded.severity,
+				updated_at      = excluded.updated_at`,
+			teamID, req.Matchers, req.TimeoutSeconds, req.Severity); err != nil {
+			respond(w, http.StatusInternalServerError, errResp("internal error"))
+			return
+		}
+
+		respond(w, http.StatusOK, deadmanResponse{
+			TeamID:         teamID,
+			Matchers:       req.Matchers,
+			TimeoutSeconds: req.TimeoutSeconds,
+			Severity:       req.Severity,
+		})
+	}
+}

internal/db/migrations/004_team_deadman.sql

@@ -0,0 +1,39 @@
+-- Dead man's switches become a team's own configuration.
+--
+-- They were three environment variables — TERDUT_DEADMAN_MATCHERS, _TIMEOUT and
+-- _SEVERITY — which made them one setting for the whole install. That was the
+-- last piece of the alerting path a team could not control: a team could take
+-- its own alerts on its own key and still not say which of them were
+-- heartbeats, or how long a silence had to last before somebody was paged.
+--
+-- One row per team rather than one row per switch. The unit of monitoring is
+-- still the fingerprint, as it always was — two clusters sending the same
+-- heartbeat alertname are two independent switches — and the matcher string
+-- keeps the format the environment variable used, so a value can be moved from
+-- one to the other unchanged.
+--
+-- No rows are seeded here: a migration cannot read the environment. The server
+-- inserts a row per team at startup from its own configuration, and the same
+-- values therefore carry forward into the first team's row without anybody
+-- retyping them. See seedDeadmanConfigs.
+CREATE TABLE deadman_configs (
+    team_id BIGINT PRIMARY KEY REFERENCES teams(id) ON DELETE CASCADE,
+
+    -- ";" separates matchers, "," the label conditions within one, "=" is exact
+    -- equality: `alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat`.
+    -- Every matcher must name an alertname. Empty watches nothing.
+    matchers TEXT NOT NULL DEFAULT '',
+
+    -- Seconds rather than a Go duration string: the column is compared and
+    -- arithmetic is done on it, and a value that has to be parsed before it can
+    -- be believed is a value that can be stored unparseable. Zero disables the
+    -- team's switches entirely.
+    timeout_seconds BIGINT NOT NULL DEFAULT 0,
+
+    -- The severity these incidents open at. They have no member alerts to
+    -- derive one from, and a heartbeat's own severity label is meaningless —
+    -- Watchdog ships as "none".
+    severity TEXT NOT NULL DEFAULT 'critical',
+
+    updated_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM now()))::bigint
+);

internal/web/static/app.css

@@ -285,6 +285,16 @@ input:focus, textarea:focus { outline: none; border-color: var(--accent); box-sh
   min-width: 0;
 }
 .row-meta .labels { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; max-width: 100%; color: var(--faint); }
+/* Which team's queue a row came from. Only rendered for somebody in more than
+   one team, so it never repeats the same word down the whole list. */
+/* Separates the status chips from the team chips in the queue's filter row. */
+.chip-sep { width: 1px; align-self: stretch; background: var(--border); margin: 0 2px; }
+
+.row-team {
+  padding: 1px 6px; border-radius: 4px;
+  background: var(--surface-2); border: 1px solid var(--border);
+  color: var(--muted); font-size: 12px; white-space: nowrap;
+}
 .row.resolved .row-title { color: var(--muted); }
 
 .sev-critical { --sev: var(--crit); }

internal/web/static/js/queue.js

@@ -26,12 +26,32 @@ const EMPTY = {
 };
 
 let filter = loadFilter();
+let teamFilter = loadTeamFilter(); // '' for every team the viewer is in
 let items = null; // null while loading
 let error = null;
 let selected = null;
 let cursor = -1; // keyboard position in the list
 let built = false;
 
+function loadTeamFilter() {
+  try {
+    return sessionStorage.getItem('terdut.queue.team') || '';
+  } catch {
+    return '';
+  }
+}
+
+function setTeamFilter(id) {
+  teamFilter = id;
+  try {
+    sessionStorage.setItem('terdut.queue.team', id);
+  } catch {
+    /* storage unavailable */
+  }
+  renderChips();
+  refresh({ fresh: true });
+}
+
 function loadFilter() {
   try {
     const f = sessionStorage.getItem('terdut.queue.filter');
@@ -64,7 +84,11 @@ export async function refresh({ fresh = false } = {}) {
   const requested = filter;
   try {
     // The open list is already fetched for the badges; no need to ask twice.
-    const result = filter === 'open' && !fresh ? state.open : await api.incidents(f.query);
+    // The cached open queue covers every team, so it can only be reused when
+    // no team filter is applied.
+    const query = teamFilter ? { ...f.query, team_id: teamFilter } : f.query;
+    const cached = filter === 'open' && !fresh && !teamFilter;
+    const result = cached ? state.open : await api.incidents(query);
     if (requested !== filter) return;
     items = result;
     error = null;
@@ -88,7 +112,7 @@ function setFilter(id) {
 
 function renderChips() {
   const el = document.getElementById('queue-filters');
-  clear(el, FILTERS.map((f) =>
+  const chips = FILTERS.map((f) =>
     h('button', {
       class: 'chip',
       type: 'button',
@@ -97,7 +121,34 @@ function renderChips() {
       onclick: () => setFilter(f.id),
       text: f.label,
     }),
-  ));
+  );
+
+  // Somebody in one team has nothing to choose between, so the row of team
+  // chips appears only when there is more than one. The default is all of
+  // them: the combined queue is the point.
+  if (state.teams.length > 1) {
+    chips.push(h('span', { class: 'chip-sep' }));
+    chips.push(h('button', {
+      class: 'chip',
+      type: 'button',
+      role: 'tab',
+      'aria-selected': String(teamFilter === ''),
+      onclick: () => setTeamFilter(''),
+      text: 'All teams',
+    }));
+    for (const team of state.teams) {
+      chips.push(h('button', {
+        class: 'chip',
+        type: 'button',
+        role: 'tab',
+        'aria-selected': String(teamFilter === String(team.id)),
+        onclick: () => setTeamFilter(String(team.id)),
+        text: team.name,
+      }));
+    }
+  }
+
+  clear(el, chips);
 }
 
 function renderList() {
@@ -142,6 +193,13 @@ function row(inc, index) {
   // The server already puts the group labels in the title; show only the rest.
   const labels = labelSummary(Object.fromEntries(
     Object.entries(inc.group_labels || {}).filter(([k, v]) => !inc.title.includes(`${k}=${v}`))));
+  // The team is shown only to somebody who is in more than one. For everybody
+  // else it is the same word on every row, which is noise rather than
+  // information.
+  const team = state.teams.length > 1 && inc.team_name
+    ? h('span', { class: 'row-team', text: inc.team_name })
+    : null;
+
   return h('a', {
     class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`,
     href: `/incidents/${inc.id}`,
@@ -153,6 +211,7 @@ function row(inc, index) {
     h('div', { class: 'row-meta' },
       status,
       assignee,
+      team,
       labels && h('span', { class: 'labels', text: labels }),
     ),
   );