Stage 3: TerdutEscalationRule + TerdutDeadmanSwitch

Both child CRDs resolve their own teamRef -> TerdutTeam.status via the new
shared resolveTeamAndClient helper (childref.go), never chaining up to
TerdutServer (DESIGN.md §5) -- TerdutTeam.status.serverEndpoint, added in
this same stage, is what makes that literally true.

TerdutEscalationRule: one PUT /api/teams/{id}/escalation per reconcile
(an upsert server-side, confirmed against source), resolving each "user"
target's username to a user_id via GET /api/users first and reporting
Ready: False, reason: UnknownUser if it doesn't resolve. No DELETE exists
for this resource, so its delete path PUTs an empty policy as the closest
available undo.

TerdutDeadmanSwitch: real create/update-in-place/delete, using
terdut-server v0.33.0's PUT (added specifically for this operator). No
unique-name constraint server-side, so idempotent-create here is
GET-list-and-match-by-name rather than adopt-on-409.

Extends tdclient with User/GetUserByUsername, the escalation request types
+ SetEscalation, and DeadmanSwitch + its CRUD methods. Also folds
ConditionTeamReady into the single shared ConditionReady constant, since
both were literally "Ready" and Stage 3 would otherwise have needed a
third same-valued constant.

internal/controller/terdutserver_controller_test.go's fakeTerdutServer
grows GET /api/users, PUT .../escalation, and the full dead man's switch
collection/item routes, replacing the old parseTeamPath/handleTeamByID
pair with a more general parseTeamSubPath/handleTeamSubPath dispatcher
that still covers every existing Stage 1/2 route unchanged.

make fmt lint test build all clean; envtest coverage for
internal/controller: 50.5% -> 71.7%.

internal/controller/childref.go

@@ -0,0 +1,54 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
+	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
+)
+
+// childError carries a condition reason/message, the same role teamError
+// and databaseError play for their own controllers: an expected,
+// requeue-and-retry outcome, not a reconcile failure.
+type childError struct {
+	reason  string
+	message string
+}
+
+func (e *childError) Error() string { return e.message }
+
+// resolveTeamAndClient implements DESIGN.md §5's "every child resolves its
+// own teamRef -> TerdutTeam.status, never chains up to TerdutServer"
+// rule -- shared by TerdutEscalationRule and TerdutDeadmanSwitch, which
+// both need exactly this and nothing else to call terdut-server's API.
+func resolveTeamAndClient(
+	ctx context.Context, c client.Client, operatorNamespace, namespace string,
+	ref terdutv1alpha1.TerdutTeamRef, newClient func(string) *tdclient.Client,
+) (*terdutv1alpha1.TerdutTeam, *tdclient.Client, *childError) {
+	var team terdutv1alpha1.TerdutTeam
+	if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, &team); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, nil, &childError{
+				reason:  terdutv1alpha1.ReasonTeamRefNotFound,
+				message: fmt.Sprintf("TerdutTeam %q not found in namespace %q", ref.Name, namespace),
+			}
+		}
+		return nil, nil, &childError{reason: terdutv1alpha1.ReasonTeamRefNotFound, message: err.Error()}
+	}
+	if team.Status.CredentialsSecretRef == nil || team.Status.TeamID == 0 {
+		return nil, nil, &childError{
+			reason:  terdutv1alpha1.ReasonWaitingForTeam,
+			message: fmt.Sprintf("TerdutTeam %q is not Ready yet", ref.Name),
+		}
+	}
+
+	teamKey, err := readOperatorSecret(ctx, c, operatorNamespace, team.Status.CredentialsSecretRef)
+	if err != nil {
+		return nil, nil, &childError{reason: terdutv1alpha1.ReasonWaitingForTeam, message: err.Error()}
+	}
+	return &team, newClient(team.Status.ServerEndpoint).WithToken(teamKey), nil
+}

internal/controller/terdutdeadmanswitch_controller.go

@@ -0,0 +1,199 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/recorder"
+
+	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
+	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
+)
+
+const deadmanFinalizerName = "terdut.ryuvia.com/terdutdeadmanswitch"
+
+// TerdutDeadmanSwitchReconciler reconciles a TerdutDeadmanSwitch object.
+type TerdutDeadmanSwitchReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+
+	OperatorNamespace string
+	Recorder          recorder.EventRecorder
+	NewClient         func(endpoint string) *tdclient.Client
+}
+
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutdeadmanswitches/finalizers,verbs=update
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
+
+func (r *TerdutDeadmanSwitchReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := logf.FromContext(ctx)
+
+	var sw terdutv1alpha1.TerdutDeadmanSwitch
+	if err := r.Get(ctx, req.NamespacedName, &sw); err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	newClient := r.NewClient
+	if newClient == nil {
+		newClient = tdclient.New
+	}
+
+	if !sw.DeletionTimestamp.IsZero() {
+		return r.reconcileDeadmanDelete(ctx, &sw, newClient)
+	}
+
+	if !controllerutil.ContainsFinalizer(&sw, deadmanFinalizerName) {
+		controllerutil.AddFinalizer(&sw, deadmanFinalizerName)
+		if err := r.Update(ctx, &sw); err != nil {
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{}, nil
+	}
+
+	team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, sw.Namespace, sw.Spec.TeamRef, newClient)
+	if resolveErr != nil {
+		return r.setDeadmanNotReady(ctx, &sw, resolveErr.reason, resolveErr.message, waitInterval)
+	}
+
+	timeout, err := time.ParseDuration(sw.Spec.Timeout)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("spec.timeout %q: %w", sw.Spec.Timeout, err)
+	}
+	severity := sw.Spec.Severity
+	if severity == "" {
+		severity = "critical"
+	}
+
+	if sw.Status.SwitchID == 0 {
+		if err := r.createOrAdoptDeadmanSwitch(ctx, &sw, tc, team.Status.TeamID, timeout, severity); err != nil {
+			return ctrl.Result{}, err
+		}
+	} else if err := tc.UpdateDeadmanSwitch(ctx, team.Status.TeamID, sw.Status.SwitchID, sw.Spec.Name, sw.Spec.Matcher, int64(timeout.Seconds()), severity); err != nil {
+		return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/deadman/switches/%d: %w", team.Status.TeamID, sw.Status.SwitchID, err)
+	}
+
+	meta.SetStatusCondition(&sw.Status.Conditions, metav1.Condition{
+		Type:    terdutv1alpha1.ConditionReady,
+		Status:  metav1.ConditionTrue,
+		Reason:  terdutv1alpha1.ReasonChildAdopted,
+		Message: fmt.Sprintf("switch %d applied on team %d", sw.Status.SwitchID, team.Status.TeamID),
+	})
+	sw.Status.ObservedGeneration = sw.Generation
+	if err := r.Status().Update(ctx, &sw); err != nil {
+		return ctrl.Result{}, err
+	}
+	if r.Recorder != nil {
+		r.Recorder.Eventf(&sw, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
+			"dead man's switch applied")
+	}
+	log.Info("TerdutDeadmanSwitch applied", "name", sw.Name, "switchID", sw.Status.SwitchID)
+
+	return ctrl.Result{RequeueAfter: resyncInterval}, nil
+}
+
+// createOrAdoptDeadmanSwitch implements this resource's own idempotent-
+// create shape (DESIGN.md §4.4, §5): there's no unique-name constraint
+// server-side to 409 on, so this lists first and matches by name (the
+// server's own derived name, when spec.name is empty) rather than adopting
+// after a conflict the API would never actually raise.
+func (r *TerdutDeadmanSwitchReconciler) createOrAdoptDeadmanSwitch(
+	ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, tc *tdclient.Client,
+	teamID int64, timeout time.Duration, severity string,
+) error {
+	existing, err := tc.ListDeadmanSwitches(ctx, teamID)
+	if err != nil {
+		return fmt.Errorf("GET /api/teams/%d/deadman/switches: %w", teamID, err)
+	}
+	if sw.Spec.Name != "" {
+		for _, s := range existing {
+			if s.Name == sw.Spec.Name {
+				sw.Status.SwitchID = s.ID
+				return nil
+			}
+		}
+	}
+
+	created, err := tc.CreateDeadmanSwitch(ctx, teamID, sw.Spec.Name, sw.Spec.Matcher, int64(timeout.Seconds()), severity)
+	if err != nil {
+		return fmt.Errorf("POST /api/teams/%d/deadman/switches: %w", teamID, err)
+	}
+	sw.Status.SwitchID = created.ID
+	return nil
+}
+
+func (r *TerdutDeadmanSwitchReconciler) setDeadmanNotReady(
+	ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, reason, message string, d time.Duration,
+) (ctrl.Result, error) {
+	meta.SetStatusCondition(&sw.Status.Conditions, metav1.Condition{
+		Type:    terdutv1alpha1.ConditionReady,
+		Status:  metav1.ConditionFalse,
+		Reason:  reason,
+		Message: message,
+	})
+	sw.Status.ObservedGeneration = sw.Generation
+	if err := r.Status().Update(ctx, sw); err != nil {
+		return ctrl.Result{}, err
+	}
+	if r.Recorder != nil {
+		r.Recorder.Eventf(sw, nil, corev1.EventTypeWarning, reason, reason, message)
+	}
+	return ctrl.Result{RequeueAfter: d}, nil
+}
+
+// reconcileDeadmanDelete calls the real DELETE this resource actually has
+// (unlike TerdutEscalationRule) if the team is still resolvable and a
+// switch was ever created, then removes the finalizer unconditionally.
+func (r *TerdutDeadmanSwitchReconciler) reconcileDeadmanDelete(
+	ctx context.Context, sw *terdutv1alpha1.TerdutDeadmanSwitch, newClient func(string) *tdclient.Client,
+) (ctrl.Result, error) {
+	if !controllerutil.ContainsFinalizer(sw, deadmanFinalizerName) {
+		return ctrl.Result{}, nil
+	}
+
+	if sw.Status.SwitchID != 0 {
+		if team, tc, resolveErr := resolveTeamAndClient(
+			ctx, r.Client, r.OperatorNamespace, sw.Namespace, sw.Spec.TeamRef, newClient,
+		); resolveErr == nil {
+			if err := tc.DeleteDeadmanSwitch(ctx, team.Status.TeamID, sw.Status.SwitchID); err != nil {
+				if r.Recorder != nil {
+					r.Recorder.Eventf(sw, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
+				}
+				return ctrl.Result{}, err
+			}
+		}
+	}
+
+	controllerutil.RemoveFinalizer(sw, deadmanFinalizerName)
+	return ctrl.Result{}, r.Update(ctx, sw)
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *TerdutDeadmanSwitchReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	if r.NewClient == nil {
+		r.NewClient = tdclient.New
+	}
+	if r.Recorder == nil {
+		r.Recorder = mgr.GetEventRecorder("terdutdeadmanswitch-controller")
+	}
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&terdutv1alpha1.TerdutDeadmanSwitch{}).
+		Named("terdutdeadmanswitch").
+		Complete(r)
+}

internal/controller/terdutdeadmanswitch_controller_test.go

@@ -0,0 +1,213 @@
+package controller
+
+import (
+	"context"
+	"net/http/httptest"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
+	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
+)
+
+var _ = Describe("TerdutDeadmanSwitch Controller", func() {
+	const operatorNamespace = "default"
+
+	var (
+		reconciler *TerdutDeadmanSwitchReconciler
+		fake       *fakeTerdutServer
+		fakeSrv    *httptest.Server
+		srv        *terdutv1alpha1.TerdutServer
+		team       *terdutv1alpha1.TerdutTeam
+		swName     string
+		swKey      types.NamespacedName
+	)
+
+	BeforeEach(func(ctx SpecContext) {
+		fake, fakeSrv = newFakeTerdutServer()
+		DeferCleanup(fakeSrv.Close)
+
+		srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("dmserver"), fakeSrv.URL)
+		team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("dmteam"), srv, fakeSrv.URL)
+
+		reconciler = &TerdutDeadmanSwitchReconciler{
+			Client:            k8sClient,
+			Scheme:            k8sClient.Scheme(),
+			OperatorNamespace: operatorNamespace,
+			NewClient:         func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
+		}
+		swName = uniqueName("switch")
+		swKey = types.NamespacedName{Name: swName, Namespace: operatorNamespace}
+	})
+
+	AfterEach(func(ctx SpecContext) {
+		sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
+		if err := k8sClient.Get(ctx, swKey, sw); err == nil {
+			sw.Finalizers = nil
+			_ = k8sClient.Update(ctx, sw)
+			_ = k8sClient.Delete(ctx, sw)
+		}
+
+		teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
+		if err := k8sClient.Get(ctx, teamKey, team); err == nil {
+			team.Finalizers = nil
+			_ = k8sClient.Update(ctx, team)
+			_ = k8sClient.Delete(ctx, team)
+		}
+
+		srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
+		if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
+			srv.Finalizers = nil
+			_ = k8sClient.Update(ctx, srv)
+			_ = k8sClient.Delete(ctx, srv)
+		}
+	})
+
+	createSwitch := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, name, matcher, timeout string) {
+		sw := &terdutv1alpha1.TerdutDeadmanSwitch{
+			ObjectMeta: metav1.ObjectMeta{Name: swName, Namespace: operatorNamespace},
+			Spec: terdutv1alpha1.TerdutDeadmanSwitchSpec{
+				TeamRef: teamRef,
+				Name:    name,
+				Matcher: matcher,
+				Timeout: timeout,
+			},
+		}
+		Expect(k8sClient.Create(ctx, sw)).To(Succeed())
+	}
+
+	reconcileOnce := func(ctx context.Context) {
+		_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: swKey})
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	readyCondition := func(ctx context.Context) metav1.Condition {
+		sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
+		Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
+		c := meta.FindStatusCondition(sw.Status.Conditions, terdutv1alpha1.ConditionReady)
+		Expect(c).NotTo(BeNil())
+		return *c
+	}
+
+	sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
+		return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
+	}
+
+	Describe("the happy path", func() {
+		It("creates the switch server-side", func(ctx SpecContext) {
+			createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx) // create
+
+			cond := readyCondition(ctx)
+			Expect(cond.Status).To(Equal(metav1.ConditionTrue))
+			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
+
+			sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
+			Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
+			Expect(sw.Status.SwitchID).NotTo(BeZero())
+
+			created, ok := fake.switches[team.Status.TeamID][sw.Status.SwitchID]
+			Expect(ok).To(BeTrue())
+			Expect(created.Matcher).To(Equal("alertname=Watchdog"))
+			Expect(created.TimeoutSeconds).To(Equal(int64(900)))
+			Expect(created.Severity).To(Equal("critical")) // kubebuilder default
+		})
+	})
+
+	Describe("list-and-match-by-name adoption", func() {
+		It("adopts an already-created switch instead of creating a duplicate", func(ctx SpecContext) {
+			// Simulates a prior, interrupted reconcile that got as far as
+			// POSTing the switch -- no 409 signal exists for this resource
+			// (DESIGN.md §4.4), so the recovery path is GET-list-and-match,
+			// not adopt-on-409.
+			fake.nextSwitchID = 1
+			fake.switches[team.Status.TeamID] = map[int64]tdclient.DeadmanSwitch{
+				1: {ID: 1, Name: "heartbeat", Matcher: "alertname=Watchdog", TimeoutSeconds: 900, Severity: "critical"},
+			}
+
+			createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+
+			sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
+			Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
+			Expect(sw.Status.SwitchID).To(Equal(int64(1)))
+			Expect(fake.switches[team.Status.TeamID]).To(HaveLen(1), "should not have created a second switch")
+		})
+	})
+
+	Describe("update-in-place on spec drift", func() {
+		It("PUTs the new spec rather than creating a second switch", func(ctx SpecContext) {
+			createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
+
+			sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
+			Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
+			switchID := sw.Status.SwitchID
+
+			sw.Spec.Timeout = "30m"
+			Expect(k8sClient.Update(ctx, sw)).To(Succeed())
+			reconcileOnce(ctx)
+
+			Expect(fake.switches[team.Status.TeamID]).To(HaveLen(1), "update-in-place, not a second switch")
+			Expect(fake.switches[team.Status.TeamID][switchID].TimeoutSeconds).To(Equal(int64(1800)))
+		})
+	})
+
+	Describe("waiting on the referenced TerdutTeam", func() {
+		It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
+			createSwitch(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, "heartbeat", "alertname=Watchdog", "15m")
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+
+			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
+		})
+
+		It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
+			unreadyName := uniqueName("dmteam-unready")
+			unready := &terdutv1alpha1.TerdutTeam{
+				ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
+				Spec: terdutv1alpha1.TerdutTeamSpec{
+					ServerRef:   terdutv1alpha1.TerdutServerRef{Name: srv.Name},
+					DisplayName: "unready",
+				},
+			}
+			Expect(k8sClient.Create(ctx, unready)).To(Succeed())
+			DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
+
+			createSwitch(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, "heartbeat", "alertname=Watchdog", "15m")
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+
+			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
+		})
+	})
+
+	Describe("deletion", func() {
+		It("deletes the switch server-side (the real DELETE this resource has) and removes the finalizer", func(ctx SpecContext) {
+			createSwitch(ctx, sameTeamRef(), "heartbeat", "alertname=Watchdog", "15m")
+			reconcileOnce(ctx)
+			reconcileOnce(ctx)
+			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
+
+			sw := &terdutv1alpha1.TerdutDeadmanSwitch{}
+			Expect(k8sClient.Get(ctx, swKey, sw)).To(Succeed())
+			switchID := sw.Status.SwitchID
+
+			Expect(k8sClient.Delete(ctx, sw)).To(Succeed())
+			reconcileOnce(ctx) // runs the finalizer
+
+			Expect(fake.switchDelete[switchID]).To(BeTrue())
+			err := k8sClient.Get(ctx, swKey, sw)
+			Expect(err).To(HaveOccurred(), "the TerdutDeadmanSwitch itself should be gone once the finalizer clears")
+		})
+	})
+})

internal/controller/terdutescalationrule_controller.go

@@ -0,0 +1,217 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/recorder"
+
+	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
+	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
+)
+
+// escalationFinalizerName exists for the one undo PUT /api/teams/{teamID}/escalation
+// supports, on delete: an empty policy (DESIGN.md §5's general finalizer
+// rule expects a server-side counterpart to be undone, but this resource's
+// API is GET/PUT-only, with no DELETE at all -- a zeroed PUT is the closest
+// equivalent terdut-server itself recognizes as "no policy"
+// (escalationPolicy.configured(), confirmed against source: "a policy row
+// with no levels is the same as no policy").
+const escalationFinalizerName = "terdut.ryuvia.com/terdutescalationrule"
+
+// TerdutEscalationRuleReconciler reconciles a TerdutEscalationRule object.
+type TerdutEscalationRuleReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+
+	OperatorNamespace string
+	Recorder          recorder.EventRecorder
+	NewClient         func(endpoint string) *tdclient.Client
+}
+
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutescalationrules/finalizers,verbs=update
+// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
+
+func (r *TerdutEscalationRuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := logf.FromContext(ctx)
+
+	var rule terdutv1alpha1.TerdutEscalationRule
+	if err := r.Get(ctx, req.NamespacedName, &rule); err != nil {
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	newClient := r.NewClient
+	if newClient == nil {
+		newClient = tdclient.New
+	}
+
+	if !rule.DeletionTimestamp.IsZero() {
+		return r.reconcileEscalationDelete(ctx, &rule, newClient)
+	}
+
+	if !controllerutil.ContainsFinalizer(&rule, escalationFinalizerName) {
+		controllerutil.AddFinalizer(&rule, escalationFinalizerName)
+		if err := r.Update(ctx, &rule); err != nil {
+			return ctrl.Result{}, err
+		}
+		return ctrl.Result{}, nil
+	}
+
+	team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, rule.Namespace, rule.Spec.TeamRef, newClient)
+	if resolveErr != nil {
+		return r.setEscalationNotReady(ctx, &rule, resolveErr.reason, resolveErr.message, waitInterval)
+	}
+
+	body, unknownUser, err := buildEscalationRequest(ctx, tc, rule.Spec)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	if unknownUser != "" {
+		return r.setEscalationNotReady(ctx, &rule, terdutv1alpha1.ReasonUnknownUser,
+			fmt.Sprintf("username %q does not resolve to any user", unknownUser), waitInterval)
+	}
+
+	if err := tc.SetEscalation(ctx, team.Status.TeamID, body); err != nil {
+		return ctrl.Result{}, fmt.Errorf("PUT /api/teams/%d/escalation: %w", team.Status.TeamID, err)
+	}
+
+	meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{
+		Type:    terdutv1alpha1.ConditionReady, // "Ready" -- same name, shared across every CRD (DESIGN.md §7)
+		Status:  metav1.ConditionTrue,
+		Reason:  terdutv1alpha1.ReasonChildAdopted,
+		Message: fmt.Sprintf("escalation policy applied to team %d", team.Status.TeamID),
+	})
+	rule.Status.ObservedGeneration = rule.Generation
+	if err := r.Status().Update(ctx, &rule); err != nil {
+		return ctrl.Result{}, err
+	}
+	if r.Recorder != nil {
+		r.Recorder.Eventf(&rule, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
+			"escalation policy applied")
+	}
+	log.Info("TerdutEscalationRule applied", "name", rule.Name, "teamID", team.Status.TeamID)
+
+	return ctrl.Result{RequeueAfter: resyncInterval}, nil
+}
+
+// buildEscalationRequest resolves every "user" target's username to a
+// user_id (DESIGN.md §4.3) and translates spec into the wire shape
+// SetEscalation sends. Returns the first unresolvable username, if any,
+// distinct from a plain error: that's an expected, reportable condition
+// (ReasonUnknownUser), not a reconcile failure.
+func buildEscalationRequest(
+	ctx context.Context, tc *tdclient.Client, spec terdutv1alpha1.TerdutEscalationRuleSpec,
+) (tdclient.SetEscalationRequest, string, error) {
+	levels := make([]tdclient.EscalationLevelRequest, len(spec.Levels))
+	for i, lvl := range spec.Levels {
+		timeout, err := time.ParseDuration(lvl.Timeout)
+		if err != nil {
+			return tdclient.SetEscalationRequest{}, "", fmt.Errorf("spec.levels[%d].timeout %q: %w", i, lvl.Timeout, err)
+		}
+
+		targets := make([]tdclient.EscalationTargetRequest, len(lvl.Targets))
+		for j, t := range lvl.Targets {
+			if t.Kind == terdutv1alpha1.EscalationTargetOncall {
+				targets[j] = tdclient.EscalationTargetRequest{Kind: string(terdutv1alpha1.EscalationTargetOncall)}
+				continue
+			}
+			user, err := tc.GetUserByUsername(ctx, t.Username)
+			if err != nil {
+				return tdclient.SetEscalationRequest{}, "", fmt.Errorf("GET /api/users (resolving %q): %w", t.Username, err)
+			}
+			if user == nil {
+				return tdclient.SetEscalationRequest{}, t.Username, nil
+			}
+			targets[j] = tdclient.EscalationTargetRequest{Kind: string(terdutv1alpha1.EscalationTargetUser), UserID: &user.ID}
+		}
+
+		levels[i] = tdclient.EscalationLevelRequest{
+			Position:       int64(i + 1),
+			TimeoutSeconds: int64(timeout.Seconds()),
+			Targets:        targets,
+		}
+	}
+
+	return tdclient.SetEscalationRequest{
+		RepeatCount:   spec.RepeatCount,
+		FallbackTopic: spec.FallbackTopic,
+		Levels:        levels,
+	}, "", nil
+}
+
+func (r *TerdutEscalationRuleReconciler) setEscalationNotReady(
+	ctx context.Context, rule *terdutv1alpha1.TerdutEscalationRule, reason, message string, d time.Duration,
+) (ctrl.Result, error) {
+	meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{
+		Type:    terdutv1alpha1.ConditionReady,
+		Status:  metav1.ConditionFalse,
+		Reason:  reason,
+		Message: message,
+	})
+	rule.Status.ObservedGeneration = rule.Generation
+	if err := r.Status().Update(ctx, rule); err != nil {
+		return ctrl.Result{}, err
+	}
+	if r.Recorder != nil {
+		r.Recorder.Eventf(rule, nil, corev1.EventTypeWarning, reason, reason, message)
+	}
+	return ctrl.Result{RequeueAfter: d}, nil
+}
+
+// reconcileEscalationDelete PUTs an empty policy (this resource's only
+// available "undo", per this file's own const comment) if the team is
+// still resolvable, then removes the finalizer unconditionally -- same
+// "the parent's probably going away too" reasoning TerdutTeam's own delete
+// path uses for a TerdutServer that's gone.
+func (r *TerdutEscalationRuleReconciler) reconcileEscalationDelete(
+	ctx context.Context, rule *terdutv1alpha1.TerdutEscalationRule, newClient func(string) *tdclient.Client,
+) (ctrl.Result, error) {
+	if !controllerutil.ContainsFinalizer(rule, escalationFinalizerName) {
+		return ctrl.Result{}, nil
+	}
+
+	if team, tc, resolveErr := resolveTeamAndClient(
+		ctx, r.Client, r.OperatorNamespace, rule.Namespace, rule.Spec.TeamRef, newClient,
+	); resolveErr == nil {
+		if err := tc.SetEscalation(ctx, team.Status.TeamID, tdclient.SetEscalationRequest{Levels: []tdclient.EscalationLevelRequest{}}); err != nil {
+			if r.Recorder != nil {
+				r.Recorder.Eventf(rule, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
+			}
+			return ctrl.Result{}, err
+		}
+	}
+
+	controllerutil.RemoveFinalizer(rule, escalationFinalizerName)
+	return ctrl.Result{}, r.Update(ctx, rule)
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *TerdutEscalationRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	if r.NewClient == nil {
+		r.NewClient = tdclient.New
+	}
+	if r.Recorder == nil {
+		r.Recorder = mgr.GetEventRecorder("terdutescalationrule-controller")
+	}
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&terdutv1alpha1.TerdutEscalationRule{}).
+		Named("terdutescalationrule").
+		Complete(r)
+}

internal/controller/terdutescalationrule_controller_test.go

@@ -0,0 +1,206 @@
+package controller
+
+import (
+	"context"
+	"net/http/httptest"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
+	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
+)
+
+var _ = Describe("TerdutEscalationRule Controller", func() {
+	const operatorNamespace = "default"
+
+	var (
+		reconciler *TerdutEscalationRuleReconciler
+		fake       *fakeTerdutServer
+		fakeSrv    *httptest.Server
+		srv        *terdutv1alpha1.TerdutServer
+		team       *terdutv1alpha1.TerdutTeam
+		ruleName   string
+		ruleKey    types.NamespacedName
+	)
+
+	BeforeEach(func(ctx SpecContext) {
+		fake, fakeSrv = newFakeTerdutServer()
+		DeferCleanup(fakeSrv.Close)
+
+		srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("erserver"), fakeSrv.URL)
+		team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("erteam"), srv, fakeSrv.URL)
+
+		reconciler = &TerdutEscalationRuleReconciler{
+			Client:            k8sClient,
+			Scheme:            k8sClient.Scheme(),
+			OperatorNamespace: operatorNamespace,
+			NewClient:         func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
+		}
+		ruleName = uniqueName("escalation")
+		ruleKey = types.NamespacedName{Name: ruleName, Namespace: operatorNamespace}
+	})
+
+	AfterEach(func(ctx SpecContext) {
+		rule := &terdutv1alpha1.TerdutEscalationRule{}
+		if err := k8sClient.Get(ctx, ruleKey, rule); err == nil {
+			rule.Finalizers = nil
+			_ = k8sClient.Update(ctx, rule)
+			_ = k8sClient.Delete(ctx, rule)
+		}
+
+		// team/srv carry their own finalizers from readyTerdutTeam/
+		// bootstrapReadyTerdutServer -- clear them directly the same way
+		// terdutteam_controller_test.go's own AfterEach does, rather than
+		// relying on either reconciler to ever run again here.
+		teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
+		if err := k8sClient.Get(ctx, teamKey, team); err == nil {
+			team.Finalizers = nil
+			_ = k8sClient.Update(ctx, team)
+			_ = k8sClient.Delete(ctx, team)
+		}
+
+		srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
+		if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
+			srv.Finalizers = nil
+			_ = k8sClient.Update(ctx, srv)
+			_ = k8sClient.Delete(ctx, srv)
+		}
+	})
+
+	createRule := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, levels []terdutv1alpha1.EscalationLevel) {
+		rule := &terdutv1alpha1.TerdutEscalationRule{
+			ObjectMeta: metav1.ObjectMeta{Name: ruleName, Namespace: operatorNamespace},
+			Spec: terdutv1alpha1.TerdutEscalationRuleSpec{
+				TeamRef: teamRef,
+				Levels:  levels,
+			},
+		}
+		Expect(k8sClient.Create(ctx, rule)).To(Succeed())
+	}
+
+	reconcileOnce := func(ctx context.Context) {
+		_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: ruleKey})
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	readyCondition := func(ctx context.Context) metav1.Condition {
+		rule := &terdutv1alpha1.TerdutEscalationRule{}
+		Expect(k8sClient.Get(ctx, ruleKey, rule)).To(Succeed())
+		c := meta.FindStatusCondition(rule.Status.Conditions, terdutv1alpha1.ConditionReady)
+		Expect(c).NotTo(BeNil())
+		return *c
+	}
+
+	sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
+		return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
+	}
+
+	Describe("the happy path", func() {
+		It("resolves usernames and applies the escalation policy", func(ctx SpecContext) {
+			fake.seedUser("alice")
+			createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
+				{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
+					{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"},
+				}},
+				{Timeout: "10m", Targets: []terdutv1alpha1.EscalationTarget{
+					{Kind: terdutv1alpha1.EscalationTargetOncall},
+				}},
+			})
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx) // resolve + apply
+
+			cond := readyCondition(ctx)
+			Expect(cond.Status).To(Equal(metav1.ConditionTrue))
+			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
+
+			applied, ok := fake.escalation[team.Status.TeamID]
+			Expect(ok).To(BeTrue())
+			Expect(applied.Levels).To(HaveLen(2))
+			Expect(applied.Levels[0].Targets[0].Kind).To(Equal("user"))
+			Expect(applied.Levels[0].Targets[0].UserID).NotTo(BeNil())
+			Expect(*applied.Levels[0].Targets[0].UserID).To(Equal(fake.users["alice"]))
+			Expect(applied.Levels[1].Targets[0].Kind).To(Equal("oncall"))
+			Expect(applied.Levels[1].Targets[0].UserID).To(BeNil())
+		})
+	})
+
+	Describe("an unresolvable username", func() {
+		It("reports UnknownUser and never calls PUT /api/teams/{id}/escalation", func(ctx SpecContext) {
+			createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
+				{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
+					{Kind: terdutv1alpha1.EscalationTargetUser, Username: "ghost"},
+				}},
+			})
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+
+			cond := readyCondition(ctx)
+			Expect(cond.Status).To(Equal(metav1.ConditionFalse))
+			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonUnknownUser))
+			_, applied := fake.escalation[team.Status.TeamID]
+			Expect(applied).To(BeFalse())
+		})
+	})
+
+	Describe("waiting on the referenced TerdutTeam", func() {
+		It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
+			createRule(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, []terdutv1alpha1.EscalationLevel{
+				{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
+			})
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+
+			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
+		})
+
+		It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
+			unreadyName := uniqueName("erteam-unready")
+			unready := &terdutv1alpha1.TerdutTeam{
+				ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
+				Spec: terdutv1alpha1.TerdutTeamSpec{
+					ServerRef:   terdutv1alpha1.TerdutServerRef{Name: srv.Name},
+					DisplayName: "unready",
+				},
+			}
+			Expect(k8sClient.Create(ctx, unready)).To(Succeed())
+			DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
+
+			createRule(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, []terdutv1alpha1.EscalationLevel{
+				{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
+			})
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx)
+
+			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
+		})
+	})
+
+	Describe("deletion", func() {
+		It("PUTs an empty policy (this resource's only available undo) and removes the finalizer", func(ctx SpecContext) {
+			fake.seedUser("alice")
+			createRule(ctx, sameTeamRef(), []terdutv1alpha1.EscalationLevel{
+				{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{
+					{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"},
+				}},
+			})
+			reconcileOnce(ctx)
+			reconcileOnce(ctx)
+			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
+			Expect(fake.escalation[team.Status.TeamID].Levels).To(HaveLen(1))
+
+			rule := &terdutv1alpha1.TerdutEscalationRule{}
+			Expect(k8sClient.Get(ctx, ruleKey, rule)).To(Succeed())
+			Expect(k8sClient.Delete(ctx, rule)).To(Succeed())
+			reconcileOnce(ctx) // runs the finalizer
+
+			Expect(fake.escalation[team.Status.TeamID].Levels).To(BeEmpty())
+			err := k8sClient.Get(ctx, ruleKey, rule)
+			Expect(err).To(HaveOccurred(), "the TerdutEscalationRule itself should be gone once the finalizer clears")
+		})
+	})
+})

internal/controller/terdutserver_controller_test.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
+	"strings"
 	"sync"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -51,20 +53,54 @@ type fakeTerdutServer struct {
 	teamNames  map[int64]string // id -> current name (renames update this)
 	teamOIDC   map[int64][2]string
 	teamDelete map[int64]bool // id -> true once DELETEd, for 404-on-redelete
+
+	// users backs GET /api/users for TerdutEscalationRule's username
+	// resolution (DESIGN.md §4.3) -- a fixed, pre-seeded directory, since
+	// nothing in this controller's own flow ever creates a user.
+	users map[string]int64 // username -> id
+
+	// escalation backs PUT /api/teams/{id}/escalation -- an upsert
+	// server-side (confirmed against source), so this is just "the last
+	// body PUT for this team", keyed by teamID, with no separate create
+	// step to model.
+	escalation map[int64]tdclient.SetEscalationRequest
+
+	// switches/nextSwitchID/switchDelete back the dead man's switch
+	// endpoints -- no unique-name constraint server-side (DESIGN.md §4.4),
+	// so switches is keyed by id, not name, same as the real API's own
+	// GET-list-and-match-by-name idempotent-create shape requires.
+	nextSwitchID int64
+	switches     map[int64]map[int64]tdclient.DeadmanSwitch // teamID -> switchID -> switch
+	switchDelete map[int64]bool                             // switchID -> true once DELETEd, for 404-on-redelete
 }
 
 func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) {
 	f := &fakeTerdutServer{
-		accounts:   map[string]int64{},
-		keyMints:   map[int64]int{},
-		teams:      map[string]int64{},
-		teamNames:  map[int64]string{},
-		teamOIDC:   map[int64][2]string{},
-		teamDelete: map[int64]bool{},
+		accounts:     map[string]int64{},
+		keyMints:     map[int64]int{},
+		teams:        map[string]int64{},
+		teamNames:    map[int64]string{},
+		teamOIDC:     map[int64][2]string{},
+		teamDelete:   map[int64]bool{},
+		users:        map[string]int64{},
+		escalation:   map[int64]tdclient.SetEscalationRequest{},
+		switches:     map[int64]map[int64]tdclient.DeadmanSwitch{},
+		switchDelete: map[int64]bool{},
 	}
 	return f, httptest.NewServer(f)
 }
 
+// seedUser registers a username the fake GET /api/users will return --
+// called from test setup, before the controller under test ever runs.
+// Callers read the assigned id back from f.users themselves, so this has
+// nothing left to return.
+func (f *fakeTerdutServer) seedUser(username string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.nextID++
+	f.users[username] = f.nextID
+}
+
 func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -137,6 +173,16 @@ func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		writeJSON(w, http.StatusOK, []tdclient.Team{{ID: id, Name: f.teamNames[id]}})
 
+	case r.URL.Path == "/api/users" && r.Method == http.MethodGet:
+		// No query filter -- GetUserByUsername fetches the whole list and
+		// matches client-side (confirmed against source: no server-side
+		// filter either), so the fake does the same.
+		users := make([]tdclient.User, 0, len(f.users))
+		for name, id := range f.users {
+			users = append(users, tdclient.User{ID: id, Username: name})
+		}
+		writeJSON(w, http.StatusOK, users)
+
 	default:
 		if id, name, ok := parseKeysPath(r.URL.Path); ok && r.Method == http.MethodPost {
 			f.keyMints[id]++
@@ -146,34 +192,21 @@ func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			})
 			return
 		}
-		if id, ok := parseTeamPath(r.URL.Path); ok {
-			f.handleTeamByID(w, r, id)
+		if id, rest, ok := parseTeamSubPath(r.URL.Path); ok {
+			f.handleTeamSubPath(w, r, id, rest)
 			return
 		}
 		w.WriteHeader(http.StatusNotFound)
 	}
 }
 
-// handleTeamByID answers PUT /api/teams/{id}, PUT /api/teams/{id}/oidc-groups
-// and DELETE /api/teams/{id}.
-func (f *fakeTerdutServer) handleTeamByID(w http.ResponseWriter, r *http.Request, id int64) {
-	oidcSuffix := fmt.Sprintf("/api/teams/%d/oidc-groups", id)
-
+// handleTeamSubPath answers everything under /api/teams/{id}: PUT (rename),
+// DELETE, PUT .../oidc-groups, PUT .../escalation, and the dead man's
+// switch collection/item endpoints. rest is whatever parseTeamSubPath found
+// after "/api/teams/{id}" -- "" for the bare resource.
+func (f *fakeTerdutServer) handleTeamSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
 	switch {
-	case r.URL.Path == oidcSuffix && r.Method == http.MethodPut:
-		var req struct {
-			MemberGroup string `json:"member_group"`
-			OwnerGroup  string `json:"owner_group"`
-		}
-		_ = json.NewDecoder(r.Body).Decode(&req)
-		if _, exists := f.teamNames[id]; !exists {
-			w.WriteHeader(http.StatusNotFound)
-			return
-		}
-		f.teamOIDC[id] = [2]string{req.MemberGroup, req.OwnerGroup}
-		w.WriteHeader(http.StatusNoContent)
-
-	case r.URL.Path == fmt.Sprintf("/api/teams/%d", id) && r.Method == http.MethodPut:
+	case rest == "" && r.Method == http.MethodPut:
 		var req struct {
 			Name string `json:"name"`
 		}
@@ -188,7 +221,7 @@ func (f *fakeTerdutServer) handleTeamByID(w http.ResponseWriter, r *http.Request
 		f.teams[req.Name] = id
 		w.WriteHeader(http.StatusNoContent)
 
-	case r.URL.Path == fmt.Sprintf("/api/teams/%d", id) && r.Method == http.MethodDelete:
+	case rest == "" && r.Method == http.MethodDelete:
 		name, exists := f.teamNames[id]
 		if !exists {
 			w.WriteHeader(http.StatusNotFound)
@@ -199,23 +232,132 @@ func (f *fakeTerdutServer) handleTeamByID(w http.ResponseWriter, r *http.Request
 		f.teamDelete[id] = true
 		w.WriteHeader(http.StatusNoContent)
 
+	case rest == "/oidc-groups" && r.Method == http.MethodPut:
+		var req struct {
+			MemberGroup string `json:"member_group"`
+			OwnerGroup  string `json:"owner_group"`
+		}
+		_ = json.NewDecoder(r.Body).Decode(&req)
+		if _, exists := f.teamNames[id]; !exists {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		f.teamOIDC[id] = [2]string{req.MemberGroup, req.OwnerGroup}
+		w.WriteHeader(http.StatusNoContent)
+
+	case rest == "/escalation" && r.Method == http.MethodPut:
+		var req tdclient.SetEscalationRequest
+		_ = json.NewDecoder(r.Body).Decode(&req)
+		f.escalation[id] = req
+		w.WriteHeader(http.StatusNoContent)
+
+	case rest == "/deadman/switches" && r.Method == http.MethodGet:
+		existing := f.switches[id]
+		out := make([]tdclient.DeadmanSwitch, 0, len(existing))
+		for _, s := range existing {
+			out = append(out, s)
+		}
+		writeJSON(w, http.StatusOK, out)
+
+	case rest == "/deadman/switches" && r.Method == http.MethodPost:
+		var req deadmanSwitchFakeRequest
+		_ = json.NewDecoder(r.Body).Decode(&req)
+		f.nextSwitchID++
+		switchID := f.nextSwitchID
+		name := req.Name
+		if name == "" {
+			name = "derived-" + req.Matcher
+		}
+		sw := tdclient.DeadmanSwitch{
+			ID: switchID, Name: name, Matcher: req.Matcher,
+			TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
+		}
+		if f.switches[id] == nil {
+			f.switches[id] = map[int64]tdclient.DeadmanSwitch{}
+		}
+		f.switches[id][switchID] = sw
+		writeJSON(w, http.StatusCreated, sw)
+
+	case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodPut:
+		switchID, ok := parseTrailingID(rest, "/deadman/switches/")
+		if !ok {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		if _, exists := f.switches[id][switchID]; !exists {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		var req deadmanSwitchFakeRequest
+		_ = json.NewDecoder(r.Body).Decode(&req)
+		name := req.Name
+		if name == "" {
+			name = f.switches[id][switchID].Name
+		}
+		f.switches[id][switchID] = tdclient.DeadmanSwitch{
+			ID: switchID, Name: name, Matcher: req.Matcher,
+			TimeoutSeconds: req.TimeoutSeconds, Severity: req.Severity,
+		}
+		w.WriteHeader(http.StatusNoContent)
+
+	case strings.HasPrefix(rest, "/deadman/switches/") && r.Method == http.MethodDelete:
+		switchID, ok := parseTrailingID(rest, "/deadman/switches/")
+		if !ok {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		if _, exists := f.switches[id][switchID]; !exists {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		delete(f.switches[id], switchID)
+		f.switchDelete[switchID] = true
+		w.WriteHeader(http.StatusNoContent)
+
 	default:
 		w.WriteHeader(http.StatusNotFound)
 	}
 }
 
-// parseTeamPath extracts the numeric id from "/api/teams/{id}" or
-// "/api/teams/{id}/oidc-groups" -- anything with more or fewer segments
-// doesn't match (handleTeamByID's own switch sorts out which of the two).
-func parseTeamPath(path string) (id int64, ok bool) {
-	var parsedID int64
-	if n, err := fmt.Sscanf(path, "/api/teams/%d/oidc-groups", &parsedID); err == nil && n == 1 {
-		return parsedID, true
+// deadmanSwitchFakeRequest mirrors tdclient's own (unexported)
+// deadmanSwitchRequest -- the fake needs its own copy to decode the same
+// wire shape without reaching across package boundaries for an internal type.
+type deadmanSwitchFakeRequest struct {
+	Name           string `json:"name,omitempty"`
+	Matcher        string `json:"matcher"`
+	TimeoutSeconds int64  `json:"timeout_seconds"`
+	Severity       string `json:"severity"`
+}
+
+// parseTeamSubPath splits "/api/teams/{id}" from anything after it --
+// "" for an exact match, "/oidc-groups", "/escalation", "/deadman/switches"
+// or "/deadman/switches/{switchID}" otherwise. Doesn't itself validate the
+// suffix; handleTeamSubPath's own switch does that.
+func parseTeamSubPath(path string) (id int64, rest string, ok bool) {
+	const prefix = "/api/teams/"
+	if !strings.HasPrefix(path, prefix) {
+		return 0, "", false
 	}
-	if n, err := fmt.Sscanf(path, "/api/teams/%d", &parsedID); err == nil && n == 1 {
-		return parsedID, true
+	trimmed := path[len(prefix):]
+	parts := strings.SplitN(trimmed, "/", 2)
+	parsedID, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil {
+		return 0, "", false
 	}
-	return 0, false
+	if len(parts) == 1 {
+		return parsedID, "", true
+	}
+	return parsedID, "/" + parts[1], true
+}
+
+// parseTrailingID parses the numeric id after prefix within rest, e.g.
+// parseTrailingID("/deadman/switches/7", "/deadman/switches/") -> 7, true.
+func parseTrailingID(rest, prefix string) (id int64, ok bool) {
+	parsedID, err := strconv.ParseInt(strings.TrimPrefix(rest, prefix), 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	return parsedID, true
 }
 
 func parseKeysPath(path string) (id int64, mintName string, ok bool) {

internal/controller/terdutteam_controller.go

@@ -95,7 +95,9 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	instanceClient := newClient(serviceURL(srv)).WithToken(instanceKey)
+	endpoint := serviceURL(srv)
+	team.Status.ServerEndpoint = endpoint
+	instanceClient := newClient(endpoint).WithToken(instanceKey)
 
 	if team.Status.TeamID == 0 {
 		if err := r.createOrAdoptTeam(ctx, &team, instanceClient); err != nil {
@@ -113,7 +115,7 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	teamClient := newClient(serviceURL(srv)).WithToken(teamKey)
+	teamClient := newClient(endpoint).WithToken(teamKey)
 
 	// Owner-gated on terdut-server, so this always runs with the
 	// team-scoped credential just minted above, never the instance-scoped
@@ -131,7 +133,7 @@ func (r *TerdutTeamReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	meta.SetStatusCondition(&team.Status.Conditions, metav1.Condition{
-		Type:    terdutv1alpha1.ConditionTeamReady,
+		Type:    terdutv1alpha1.ConditionReady,
 		Status:  metav1.ConditionTrue,
 		Reason:  terdutv1alpha1.ReasonTeamAdopted,
 		Message: fmt.Sprintf("team %d ready, credentials in Secret %q", team.Status.TeamID, team.Status.CredentialsSecretRef.Name),
@@ -208,7 +210,7 @@ func (r *TerdutTeamReconciler) setTeamNotReady(
 	ctx context.Context, team *terdutv1alpha1.TerdutTeam, reason, message string, d time.Duration,
 ) (ctrl.Result, error) {
 	meta.SetStatusCondition(&team.Status.Conditions, metav1.Condition{
-		Type:    terdutv1alpha1.ConditionTeamReady,
+		Type:    terdutv1alpha1.ConditionReady,
 		Status:  metav1.ConditionFalse,
 		Reason:  reason,
 		Message: message,

internal/controller/terdutteam_controller_test.go

@@ -86,7 +86,7 @@ var _ = Describe("TerdutTeam Controller", func() {
 	readyCondition := func(ctx context.Context) metav1.Condition {
 		team := &terdutv1alpha1.TerdutTeam{}
 		Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
-		c := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionTeamReady)
+		c := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
 		Expect(c).NotTo(BeNil())
 		return *c
 	}
@@ -120,7 +120,7 @@ var _ = Describe("TerdutTeam Controller", func() {
 
 	Describe("waiting on the referenced TerdutServer", func() {
 		It("reports ServerRefNotFound when the TerdutServer doesn't exist", func(ctx SpecContext) {
-			createTeam(ctx, "orphan", terdutv1alpha1.TerdutServerRef{Name: "does-not-exist"})
+			createTeam(ctx, "orphan", terdutv1alpha1.TerdutServerRef{Name: testRefNotFoundName})
 			reconcileOnce(ctx) // finalizer
 			reconcileOnce(ctx)
 
@@ -175,7 +175,7 @@ var _ = Describe("TerdutTeam Controller", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			Expect(k8sClient.Get(ctx, crossKey, team)).To(Succeed())
-			cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionTeamReady)
+			cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
 			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonRefNotPermitted))
 		})
 
@@ -201,7 +201,7 @@ var _ = Describe("TerdutTeam Controller", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			Expect(k8sClient.Get(ctx, crossKey, team)).To(Succeed())
-			cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionTeamReady)
+			cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
 			// Past the gate: Adopted (the fake server has no reason to
 			// reject this), definitely not RefNotPermitted.
 			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamAdopted))

internal/controller/testhelpers_test.go

@@ -23,6 +23,13 @@ const (
 	testImageRepo = "example.invalid/terdut-server"
 	testImageTag  = "test"
 	testDSN       = "postgres://terdut@test-postgres:5432/terdut?sslmode=require"
+
+	// testRefNotFoundName is a name no TerdutServer/TerdutTeam ever gets
+	// created with -- shared by every "...RefNotFound" test across
+	// terdutteam_controller_test.go, terdutescalationrule_controller_test.go
+	// and terdutdeadmanswitch_controller_test.go, so goconst doesn't flag
+	// three independent copies of the same literal.
+	testRefNotFoundName = "does-not-exist"
 )
 
 // bootstrapReadyTerdutServer creates a TerdutServer with a bring-your-own
@@ -70,6 +77,41 @@ func bootstrapReadyTerdutServer(ctx context.Context, namespace, name, fakeURL st
 	return srv
 }
 
+// readyTerdutTeam creates a TerdutTeam under srv (an already-Ready
+// TerdutServer, e.g. from bootstrapReadyTerdutServer) and drives it to
+// Ready against fakeURL -- shared by TerdutEscalationRule's and
+// TerdutDeadmanSwitch's own tests, which both just need a resolvable
+// teamRef (DESIGN.md §5), not TerdutTeam's own behavior.
+func readyTerdutTeam(ctx context.Context, namespace, name string, srv *terdutv1alpha1.TerdutServer, fakeURL string) *terdutv1alpha1.TerdutTeam {
+	GinkgoHelper()
+
+	reconciler := &TerdutTeamReconciler{
+		Client:            k8sClient,
+		Scheme:            k8sClient.Scheme(),
+		OperatorNamespace: namespace,
+		NewClient:         func(string) *tdclient.Client { return tdclient.New(fakeURL) },
+	}
+	objKey := types.NamespacedName{Name: name, Namespace: namespace}
+
+	team := &terdutv1alpha1.TerdutTeam{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec: terdutv1alpha1.TerdutTeamSpec{
+			ServerRef:   terdutv1alpha1.TerdutServerRef{Name: srv.Name},
+			DisplayName: name,
+		},
+	}
+	Expect(k8sClient.Create(ctx, team)).To(Succeed())
+
+	_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // finalizer
+	Expect(err).NotTo(HaveOccurred())
+	_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey}) // create+mint+apply
+	Expect(err).NotTo(HaveOccurred())
+
+	Expect(k8sClient.Get(ctx, objKey, team)).To(Succeed())
+	Expect(team.Status.CredentialsSecretRef).NotTo(BeNil(), "test setup: TerdutTeam %s/%s did not reach Ready", namespace, name)
+	return team
+}
+
 // uniqueNameCounter backs uniqueName. GinkgoRandomSeed()/GinkgoParallelProcess()
 // are constants for the whole suite run, not per-spec -- an earlier version
 // of this helper used them and collided across every spec that called it

internal/tdclient/client.go

@@ -357,3 +357,142 @@ func (c *Client) SetTeamOIDCGroups(ctx context.Context, teamID int64, memberGrou
 	}
 	return c.do(req, nil)
 }
+
+// User mirrors terdut-server's models.User, minus fields this client never
+// reads.
+type User struct {
+	ID       int64  `json:"id"`
+	Username string `json:"username"`
+}
+
+// GetUserByUsername calls GET /api/users and finds the one matching exactly
+// -- confirmed open to any authenticated caller, not gated by team
+// membership or admin (internal/api/router.go's own comment: "readable by
+// anyone signed in"), so the team-scoped credential a TerdutEscalationRule's
+// controller already holds is enough. There is no server-side filter, so
+// this always fetches the whole list; terdut-server's own query has no
+// pagination either (confirmed against source), so this matches what the
+// server itself considers an acceptable cost. Returns nil, nil on no match.
+func (c *Client) GetUserByUsername(ctx context.Context, username string) (*User, error) {
+	req, err := c.newRequest(ctx, http.MethodGet, "/api/users", nil)
+	if err != nil {
+		return nil, err
+	}
+	var users []User
+	if err := c.do(req, &users); err != nil {
+		return nil, err
+	}
+	for _, u := range users {
+		if u.Username == username {
+			return &u, nil
+		}
+	}
+	return nil, nil
+}
+
+// EscalationTargetRequest/EscalationLevelRequest/SetEscalationRequest mirror
+// terdut-server's escalationTargetJSON/escalationLevelJSON/escalationJSON
+// (internal/api/escalation.go) -- the PUT body, not the richer GET response
+// (escalationView), which this client never needs to decode since the
+// controller always computes its own desired state fresh from spec.
+type EscalationTargetRequest struct {
+	Kind   string `json:"kind"`
+	UserID *int64 `json:"user_id,omitempty"`
+}
+
+type EscalationLevelRequest struct {
+	Position       int64                     `json:"position"`
+	TimeoutSeconds int64                     `json:"timeout_seconds"`
+	Targets        []EscalationTargetRequest `json:"targets"`
+}
+
+type SetEscalationRequest struct {
+	RepeatCount   int64                    `json:"repeat_count"`
+	FallbackTopic string                   `json:"fallback_topic"`
+	Levels        []EscalationLevelRequest `json:"levels"`
+}
+
+// SetEscalation calls PUT /api/teams/{teamID}/escalation -- an upsert
+// server-side (confirmed against source: `INSERT ... ON CONFLICT (team_id)
+// DO UPDATE`), so there is no separate create step for this resource at
+// all, unlike Team or the dead man's switch.
+func (c *Client) SetEscalation(ctx context.Context, teamID int64, body SetEscalationRequest) error {
+	req, err := c.newRequest(ctx, http.MethodPut, fmt.Sprintf("/api/teams/%d/escalation", teamID), body)
+	if err != nil {
+		return err
+	}
+	return c.do(req, nil)
+}
+
+// DeadmanSwitch mirrors terdut-server's deadmanSwitchStatus
+// (internal/api/deadman.go), minus fields this client never reads.
+type DeadmanSwitch struct {
+	ID             int64  `json:"id"`
+	Name           string `json:"name"`
+	Matcher        string `json:"matcher"`
+	TimeoutSeconds int64  `json:"timeout_seconds"`
+	Severity       string `json:"severity"`
+}
+
+// ListDeadmanSwitches calls GET /api/teams/{teamID}/deadman/switches. There
+// is no unique-name constraint on this resource server-side (confirmed
+// against source), so this is the idempotent-create lookup for it --
+// GET-list-and-match-by-name, not adopt-on-409.
+func (c *Client) ListDeadmanSwitches(ctx context.Context, teamID int64) ([]DeadmanSwitch, error) {
+	req, err := c.newRequest(ctx, http.MethodGet, fmt.Sprintf("/api/teams/%d/deadman/switches", teamID), nil)
+	if err != nil {
+		return nil, err
+	}
+	var switches []DeadmanSwitch
+	if err := c.do(req, &switches); err != nil {
+		return nil, err
+	}
+	return switches, nil
+}
+
+// deadmanSwitchRequest mirrors terdut-server's own deadmanSwitchRequest
+// (internal/api/teams.go) -- the same body shape for both create and
+// update.
+type deadmanSwitchRequest struct {
+	Name           string `json:"name,omitempty"`
+	Matcher        string `json:"matcher"`
+	TimeoutSeconds int64  `json:"timeout_seconds"`
+	Severity       string `json:"severity"`
+}
+
+// CreateDeadmanSwitch calls POST /api/teams/{teamID}/deadman/switches.
+func (c *Client) CreateDeadmanSwitch(ctx context.Context, teamID int64, name, matcher string, timeoutSeconds int64, severity string) (*DeadmanSwitch, error) {
+	req, err := c.newRequest(ctx, http.MethodPost, fmt.Sprintf("/api/teams/%d/deadman/switches", teamID),
+		deadmanSwitchRequest{Name: name, Matcher: matcher, TimeoutSeconds: timeoutSeconds, Severity: severity})
+	if err != nil {
+		return nil, err
+	}
+	var sw DeadmanSwitch
+	if err := c.do(req, &sw); err != nil {
+		return nil, err
+	}
+	return &sw, nil
+}
+
+// UpdateDeadmanSwitch calls PUT /api/teams/{teamID}/deadman/switches/{switchID}
+// -- real update-in-place, added in terdut-server v0.33.0 specifically for
+// this controller (that handler's own doc comment names terdut-operator).
+func (c *Client) UpdateDeadmanSwitch(ctx context.Context, teamID, switchID int64, name, matcher string, timeoutSeconds int64, severity string) error {
+	req, err := c.newRequest(ctx, http.MethodPut,
+		fmt.Sprintf("/api/teams/%d/deadman/switches/%d", teamID, switchID),
+		deadmanSwitchRequest{Name: name, Matcher: matcher, TimeoutSeconds: timeoutSeconds, Severity: severity})
+	if err != nil {
+		return err
+	}
+	return c.do(req, nil)
+}
+
+// DeleteDeadmanSwitch calls DELETE /api/teams/{teamID}/deadman/switches/{switchID}.
+func (c *Client) DeleteDeadmanSwitch(ctx context.Context, teamID, switchID int64) error {
+	req, err := c.newRequest(ctx, http.MethodDelete,
+		fmt.Sprintf("/api/teams/%d/deadman/switches/%d", teamID, switchID), nil)
+	if err != nil {
+		return err
+	}
+	return c.do(req, nil)
+}