Give each team its own dead man's switches, and the UI a team to show

The rest of #4. Two halves that belong together because they are the
same sentence from opposite ends: a team decides which of its alerts are
heartbeats, and the UI has to be able to say which team it is talking
about.

Switches were three environment variables, which made them one setting
for the whole install. That was the last piece of the alerting path a
team could not control: it could take its own alerts on its own key and
still not say which of them were heartbeats, or how long a silence had
to last. They are a row per team now, edited by an owner through
PUT /api/teams/{teamID}/deadman, and the sweeper runs each team against
its own matchers, timeout and severity.

The environment variables become the starting point rather than the
setting. Every team without a configuration is seeded from them at
startup, so an upgrade keeps watching exactly what it was watching, and
SeedDeadmanConfigs never overwrites -- a redeploy must not put the
environment's value back over an owner's edit. A team created later
watches nothing until somebody says otherwise: inheriting an
install-wide heartbeat would page a new team about a source it has never
heard of, and a switch nobody chose is the kind that gets muted rather
than fixed.

A matcher string with no alertname in it is refused at the door instead
of stored. Storing it would produce a switch that watches nothing
silently, which is the exact failure the feature exists to prevent.

NewRouter and Sweep lose their DeadmanConfig parameter -- there is no
longer one answer to hand them. The type stays, because parsing a
matcher string is still parsing a matcher string.

The UI side: rows in the queue carry a team badge, the filter row gains
a team chip per team, and "on call now" shows one card per team. All
three appear only when the viewer is in more than one team -- otherwise
they are the same word repeated down a list, which is noise rather than
information, and the single-team install reads exactly as it did before
teams existed.

Verified against a live two-team server as well as in tests: the
combined queue labelled by team, the team_id filter, a heartbeat that is
a heartbeat in one team and an ordinary alert in another, and a new
team's switches starting empty while the upgraded team keeps the
environment's.

Claude-Session: https://claude.ai/code/session_01RHPj4ggeFdEjKKfm4SHbD7

internal/api/deadman.go

@@ -189,35 +189,42 @@ func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerpr
 // It returns the ids of the alerts it owns, because the generic staleness
 // expiry must leave them alone — staleAfter and ends_at would otherwise resolve
 // a heartbeat long before its own, much tighter, timeout ever fired.
-func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig) map[int64]bool {
+// Each team is swept against its own configuration: its own matchers, its own
+// timeout, its own severity. A team watching nothing is skipped entirely, which
+// is most of them.
+func sweepDeadman(ctx context.Context, db *sql.DB, notify NotifyConfig) map[int64]bool {
 	owned := map[int64]bool{}
-	if !cfg.enabled() {
-		return owned
-	}
 
-	switches, err := deadmanAlerts(ctx, db, cfg)
+	configs, err := deadmanConfigs(ctx, db)
 	if err != nil {
-		log.Printf("deadman: load switches: %v", err)
+		log.Printf("deadman: load configs: %v", err)
 		return owned
 	}
 
 	now := time.Now()
-	cutoff := now.Add(-cfg.Timeout).Unix()
-
-	for _, sw := range switches {
-		owned[sw.id] = true
-
-		// An explicit resolved from Alertmanager is a stronger death signal than
-		// mere absence: the sender is telling us the heartbeat stopped, so there
-		// is nothing left to wait out.
-		if sw.resolved || sw.receivedAt < cutoff {
-			if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil {
-				log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err)
-			}
+	for teamID, cfg := range configs {
+		switches, err := deadmanAlerts(ctx, db, teamID, cfg)
+		if err != nil {
+			log.Printf("deadman: load switches for team %d: %v", teamID, err)
 			continue
 		}
-		if err := deadmanRecovered(ctx, db, sw); err != nil {
-			log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err)
+		cutoff := now.Add(-cfg.Timeout).Unix()
+
+		for _, sw := range switches {
+			owned[sw.id] = true
+
+			// An explicit resolved from Alertmanager is a stronger death signal
+			// than mere absence: the sender is telling us the heartbeat
+			// stopped, so there is nothing left to wait out.
+			if sw.resolved || sw.receivedAt < cutoff {
+				if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil {
+					log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err)
+				}
+				continue
+			}
+			if err := deadmanRecovered(ctx, db, sw); err != nil {
+				log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err)
+			}
 		}
 	}
 	return owned
@@ -228,7 +235,7 @@ func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify Not
 // happens in Go, which keeps one implementation of the rules. The rows are read
 // in full before the caller writes, so the writes do not run against an open
 // cursor over the same table.
-func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadmanAlert, error) {
+func deadmanAlerts(ctx context.Context, db *sql.DB, teamID int64, cfg DeadmanConfig) ([]deadmanAlert, error) {
 	names := cfg.names()
 	args := &sqlArgs{}
 	nameList := make([]any, len(names))
@@ -239,7 +246,8 @@ func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadma
 	rows, err := db.QueryContext(ctx, `
 		SELECT id, team_id, fingerprint, labels, status, received_at
 		FROM   alerts
-		WHERE  name IN (`+args.addList(nameList)+`)
+		WHERE  team_id = `+args.add(teamID)+`
+		  AND  name IN (`+args.addList(nameList)+`)
 		  AND  archived_at IS NULL`, args.all()...)
 	if err != nil {
 		return nil, err
@@ -382,3 +390,113 @@ func deadmanRecovered(ctx context.Context, db *sql.DB, sw deadmanAlert) error {
 	log.Printf("deadman: %s is back, resolved incident %d", sw.matcher.String(), incidentID)
 	return nil
 }
+
+// ---------------------------------------------------------------------------
+// Per-team configuration
+// ---------------------------------------------------------------------------
+
+// deadmanConfigForTeam reads one team's switches. A team with no row, or with
+// nothing configured, gets a disabled config — which is the right answer rather
+// than an error: most teams watch no heartbeat at all.
+func deadmanConfigForTeam(ctx context.Context, q querier, teamID int64) (DeadmanConfig, error) {
+	var matchers, severity string
+	var timeout int64
+	err := q.QueryRowContext(ctx,
+		"SELECT matchers, timeout_seconds, severity FROM deadman_configs WHERE team_id = $1",
+		teamID).Scan(&matchers, &timeout, &severity)
+	if err == sql.ErrNoRows {
+		return DeadmanConfig{}, nil
+	}
+	if err != nil {
+		return DeadmanConfig{}, err
+	}
+	return parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity), nil
+}
+
+// deadmanConfigs reads every team's switches in one query, for the sweeper.
+func deadmanConfigs(ctx context.Context, db *sql.DB) (map[int64]DeadmanConfig, error) {
+	rows, err := db.QueryContext(ctx,
+		"SELECT team_id, matchers, timeout_seconds, severity FROM deadman_configs")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	out := map[int64]DeadmanConfig{}
+	for rows.Next() {
+		var teamID, timeout int64
+		var matchers, severity string
+		if err := rows.Scan(&teamID, &matchers, &timeout, &severity); err != nil {
+			return nil, err
+		}
+		cfg := parseDeadmanQuietly(matchers, time.Duration(timeout)*time.Second, severity)
+		if cfg.enabled() {
+			out[teamID] = cfg
+		}
+	}
+	return out, rows.Err()
+}
+
+// SeedDeadmanConfigs gives every team without a row the server's environment
+// configuration, so the install that upgrades into per-team switches keeps
+// watching exactly what it was watching before.
+//
+// Idempotent, and never overwrites: once a team has a row it owns its own
+// configuration, and a redeploy must not quietly put the environment's value
+// back over an owner's edit.
+//
+// A team created after startup gets no row and therefore watches nothing until
+// its owner says otherwise. That is deliberate: inheriting an install-wide
+// heartbeat would page a new team about a source it has never heard of, and a
+// switch nobody chose is the kind that gets muted rather than fixed.
+func SeedDeadmanConfigs(ctx context.Context, db *sql.DB, cfg DeadmanConfig) error {
+	matchers := make([]string, 0, len(cfg.Matchers))
+	for _, m := range cfg.Matchers {
+		parts := []string{"alertname=" + m.Name}
+		for k, v := range m.Labels {
+			parts = append(parts, k+"="+v)
+		}
+		sort.Strings(parts[1:])
+		matchers = append(matchers, strings.Join(parts, ","))
+	}
+
+	_, err := db.ExecContext(ctx, `
+		INSERT INTO deadman_configs (team_id, matchers, timeout_seconds, severity)
+		SELECT id, $1, $2, $3 FROM teams
+		ON CONFLICT (team_id) DO NOTHING`,
+		strings.Join(matchers, "; "), int64(cfg.Timeout.Seconds()), cfg.Severity)
+	return err
+}
+
+// parseDeadmanQuietly is ParseDeadmanConfig without the startup logging: a
+// team's configuration is read on every sweep and every webhook, and logging it
+// each time would bury everything else.
+func parseDeadmanQuietly(matchers string, timeout time.Duration, severity string) DeadmanConfig {
+	cfg := DeadmanConfig{Timeout: timeout, Severity: severity}
+	for _, entry := range strings.Split(matchers, ";") {
+		entry = strings.TrimSpace(entry)
+		if entry == "" {
+			continue
+		}
+		m := DeadmanMatcher{Labels: map[string]string{}}
+		malformed := false
+		for _, cond := range strings.Split(entry, ",") {
+			k, v, ok := strings.Cut(cond, "=")
+			k, v = strings.TrimSpace(k), strings.TrimSpace(v)
+			if !ok || k == "" || v == "" {
+				malformed = true
+				break
+			}
+			if k == "alertname" {
+				m.Name = v
+				continue
+			}
+			m.Labels[k] = v
+		}
+		if malformed || m.Name == "" {
+			continue
+		}
+		cfg.Matchers = append(cfg.Matchers, m)
+	}
+	return cfg
+}