terdut-operator/internal/controller/terdutteam_controller_test.go

package controller

import (
	"context"
	"net/http/httptest"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"

	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)

// platformName is the display name most specs give their team.
const platformName = "platform"

// allNamespaces is allowedTeams.namespaces.from admitting every namespace.
const allNamespaces = "All"

var _ = Describe("TerdutTeam Controller", func() {
	const operatorNamespace = "default"

	var (
		reconciler *TerdutTeamReconciler
		fake       *fakeTerdutServer
		fakeSrv    *httptest.Server
		srv        *terdutv1alpha1.TerdutServer
		teamName   string
		teamKey    types.NamespacedName
	)

	BeforeEach(func(ctx SpecContext) {
		fake, fakeSrv = newFakeTerdutServer()
		DeferCleanup(fakeSrv.Close)

		srv = readyTerdutServer(ctx, uniqueName("ttserver"))

		reconciler = &TerdutTeamReconciler{
			Client:    k8sClient,
			Scheme:    k8sClient.Scheme(),
			NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
		}
		teamName = uniqueName("team")
		teamKey = types.NamespacedName{Name: teamName, Namespace: operatorNamespace}
	})

	AfterEach(func(ctx SpecContext) {
		team := &terdutv1alpha1.TerdutTeam{}
		if err := k8sClient.Get(ctx, teamKey, team); err == nil {
			team.Finalizers = nil
			_ = k8sClient.Update(ctx, team)
			_ = k8sClient.Delete(ctx, team)
		}

		srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
		if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
			srv.Finalizers = nil
			_ = k8sClient.Update(ctx, srv)
			_ = k8sClient.Delete(ctx, srv)
		}
		_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
			Name: srv.Name + "-operator-key", Namespace: operatorNamespace,
		}})
	})

	createTeam := func(ctx context.Context, displayName string, serverRef terdutv1alpha1.TerdutServerRef) {
		team := &terdutv1alpha1.TerdutTeam{
			ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: operatorNamespace},
			Spec:       terdutv1alpha1.TerdutTeamSpec{ServerRef: serverRef, DisplayName: displayName},
		}
		Expect(k8sClient.Create(ctx, team)).To(Succeed())
	}

	reconcileOnce := func(ctx context.Context) {
		_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: teamKey})
		Expect(err).NotTo(HaveOccurred())
	}

	readyCondition := func(ctx context.Context) metav1.Condition {
		team := &terdutv1alpha1.TerdutTeam{}
		Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
		c := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
		Expect(c).NotTo(BeNil())
		return *c
	}

	sameNSRef := func() terdutv1alpha1.TerdutServerRef {
		return terdutv1alpha1.TerdutServerRef{Name: srv.Name}
	}

	getTeam := func(ctx context.Context) *terdutv1alpha1.TerdutTeam {
		team := &terdutv1alpha1.TerdutTeam{}
		Expect(k8sClient.Get(ctx, teamKey, team)).To(Succeed())
		return team
	}

	// reconcileToReady drives a freshly created team through the finalizer pass
	// and one apply pass.
	reconcileToReady := func(ctx context.Context) {
		reconcileOnce(ctx) // finalizer
		reconcileOnce(ctx) // apply
	}

	Describe("the happy path", func() {
		It("creates the team under its CR identity with the operator key, and applies oidc groups", func(ctx SpecContext) {
			// The fake only answers a request carrying the server's operator key.
			var keySecret corev1.Secret
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name + "-operator-key", Namespace: operatorNamespace}, &keySecret)).To(Succeed())
			fake.expectKey = string(keySecret.Data[operatorKeyDataKey])

			team := &terdutv1alpha1.TerdutTeam{
				ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: operatorNamespace},
				Spec: terdutv1alpha1.TerdutTeamSpec{
					ServerRef: sameNSRef(), DisplayName: platformName,
					OIDC: terdutv1alpha1.TerdutTeamOIDC{MemberGroup: "members", OwnerGroup: "owners"},
				},
			}
			Expect(k8sClient.Create(ctx, team)).To(Succeed())
			reconcileToReady(ctx)

			cond := readyCondition(ctx)
			Expect(cond.Status).To(Equal(metav1.ConditionTrue))
			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamAdopted))

			team = getTeam(ctx)
			Expect(team.Status.TeamID).To(Equal(fake.teams[platformName]))
			Expect(fake.teamExt).To(HaveKeyWithValue(operatorNamespace+"/"+teamName, team.Status.TeamID))
			Expect(fake.teamOIDC[team.Status.TeamID]).To(Equal([2]string{"members", "owners"}))
		})
	})

	Describe("waiting on the referenced TerdutServer", func() {
		It("reports ServerRefNotFound when the TerdutServer doesn't exist", func(ctx SpecContext) {
			createTeam(ctx, "orphan", terdutv1alpha1.TerdutServerRef{Name: testRefNotFoundName})
			reconcileToReady(ctx)

			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonServerRefNotFound))
		})

		It("reports WaitingForServer when the TerdutServer exists but isn't Ready yet", func(ctx SpecContext) {
			unreadyName := uniqueName("ttserver-unready")
			unready := &terdutv1alpha1.TerdutServer{
				ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
				Spec: terdutv1alpha1.TerdutServerSpec{
					Image:      terdutv1alpha1.ImageSpec{Repository: testImageRepo, Tag: testImageTag},
					Networking: terdutv1alpha1.NetworkingSpec{Hostname: "unready.example.invalid", ServicePort: 8080},
					Database:   terdutv1alpha1.DatabaseSpec{DSN: testDSN},
				},
			}
			Expect(k8sClient.Create(ctx, unready)).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })

			createTeam(ctx, "waiting", terdutv1alpha1.TerdutServerRef{Name: unreadyName})
			reconcileToReady(ctx)

			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForServer))
		})
	})

	Describe("cross-namespace serverRef", func() {
		var otherNS string

		BeforeEach(func(ctx SpecContext) {
			otherNS = uniqueName("ns")
			Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherNS}})).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherNS}}) })
		})

		It("denies by default (From: None)", func(ctx SpecContext) {
			team := &terdutv1alpha1.TerdutTeam{
				ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: otherNS},
				Spec: terdutv1alpha1.TerdutTeamSpec{
					ServerRef:   terdutv1alpha1.TerdutServerRef{Name: srv.Name, Namespace: operatorNamespace},
					DisplayName: "cross-ns-denied",
				},
			}
			Expect(k8sClient.Create(ctx, team)).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, team) })

			crossKey := types.NamespacedName{Name: teamName, Namespace: otherNS}
			_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey}) // finalizer
			Expect(err).NotTo(HaveOccurred())
			_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey})
			Expect(err).NotTo(HaveOccurred())

			Expect(k8sClient.Get(ctx, crossKey, team)).To(Succeed())
			cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonRefNotPermitted))
		})

		It("permits when the TerdutServer's allowedTeams.namespaces.from is All", func(ctx SpecContext) {
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}, srv)).To(Succeed())
			srv.Spec.AllowedTeams.Namespaces.From = allNamespaces
			Expect(k8sClient.Update(ctx, srv)).To(Succeed())

			team := &terdutv1alpha1.TerdutTeam{
				ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: otherNS},
				Spec: terdutv1alpha1.TerdutTeamSpec{
					ServerRef:   terdutv1alpha1.TerdutServerRef{Name: srv.Name, Namespace: operatorNamespace},
					DisplayName: "cross-ns-allowed",
				},
			}
			Expect(k8sClient.Create(ctx, team)).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, team) })

			crossKey := types.NamespacedName{Name: teamName, Namespace: otherNS}
			_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey}) // finalizer
			Expect(err).NotTo(HaveOccurred())
			_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey})
			Expect(err).NotTo(HaveOccurred())

			Expect(k8sClient.Get(ctx, crossKey, team)).To(Succeed())
			cond := meta.FindStatusCondition(team.Status.Conditions, terdutv1alpha1.ConditionReady)
			// Past the gate: Adopted (the fake server has no reason to
			// reject this), definitely not RefNotPermitted.
			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamAdopted))
		})
	})

	Describe("team identity", func() {
		It("refuses a name that belongs to a team this CR did not create", func(ctx SpecContext) {
			fake.nextTeamID = 1
			fake.teams[platformName] = 1
			fake.teamNames[1] = platformName

			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)

			cond := readyCondition(ctx)
			Expect(cond.Status).To(Equal(metav1.ConditionFalse))
			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonTeamNameTaken))
			Expect(getTeam(ctx).Status.TeamID).To(BeZero())
			Expect(fake.teamOIDC).To(BeEmpty(), "nothing is configured on a team that is not ours")
		})

		It("keeps the same team when spec.displayName changes", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			id := getTeam(ctx).Status.TeamID

			team := getTeam(ctx)
			team.Spec.DisplayName = "platform-renamed"
			Expect(k8sClient.Update(ctx, team)).To(Succeed())
			reconcileOnce(ctx)

			Expect(getTeam(ctx).Status.TeamID).To(Equal(id))
			Expect(fake.teamNames[id]).To(Equal("platform-renamed"))
			Expect(fake.nextTeamID).To(Equal(id), "renamed in place, not recreated")
		})

		It("reports TeamNameTaken when a rename collides with another team", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			fake.nextTeamID++
			fake.teams["other"] = fake.nextTeamID
			fake.teamNames[fake.nextTeamID] = "other"

			team := getTeam(ctx)
			team.Spec.DisplayName = "other"
			Expect(k8sClient.Update(ctx, team)).To(Succeed())
			reconcileOnce(ctx)

			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamNameTaken))
		})

		It("finds its team again when the status was lost", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			id := getTeam(ctx).Status.TeamID

			team := getTeam(ctx)
			team.Status.TeamID = 0
			Expect(k8sClient.Status().Update(ctx, team)).To(Succeed())
			reconcileOnce(ctx)

			Expect(getTeam(ctx).Status.TeamID).To(Equal(id))
			Expect(fake.nextTeamID).To(Equal(id), "no second team was created")
		})

		It("recreates a team deleted behind its back", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			id := getTeam(ctx).Status.TeamID
			delete(fake.teams, platformName)
			delete(fake.teamNames, id)
			delete(fake.teamExt, operatorNamespace+"/"+teamName)

			reconcileOnce(ctx)

			Expect(getTeam(ctx).Status.TeamID).NotTo(Equal(id))
			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
		})
	})

	Describe("spec.escalation", func() {
		ladder := func() *terdutv1alpha1.EscalationSpec {
			return &terdutv1alpha1.EscalationSpec{
				RepeatCount: 2, FallbackTopic: "oncall",
				Levels: []terdutv1alpha1.EscalationLevel{
					{Timeout: "5m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetOncall}}},
					{Timeout: "15m", Targets: []terdutv1alpha1.EscalationTarget{{Kind: terdutv1alpha1.EscalationTargetUser, Username: "alice"}}},
				},
			}
		}
		createWithLadder := func(ctx context.Context, e *terdutv1alpha1.EscalationSpec) {
			Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutTeam{
				ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: operatorNamespace},
				Spec:       terdutv1alpha1.TerdutTeamSpec{ServerRef: sameNSRef(), DisplayName: platformName, Escalation: e},
			})).To(Succeed())
		}

		It("PUTs the ladder with usernames for the server to resolve", func(ctx SpecContext) {
			createWithLadder(ctx, ladder())
			reconcileToReady(ctx)

			got := fake.escalation[getTeam(ctx).Status.TeamID]
			Expect(got.RepeatCount).To(Equal(int64(2)))
			Expect(got.FallbackTopic).To(Equal("oncall"))
			Expect(got.Levels).To(HaveLen(2))
			Expect(got.Levels[0].TimeoutSeconds).To(Equal(int64(300)))
			Expect(got.Levels[1].Targets[0]).To(Equal(tdclient.EscalationTargetRequest{Kind: "user", Username: "alice"}))
		})

		It("waits with UnknownUser while the server does not know a username", func(ctx SpecContext) {
			fake.unknownUsers["alice"] = true
			createWithLadder(ctx, ladder())
			reconcileToReady(ctx)

			cond := readyCondition(ctx)
			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonUnknownUser))
			Expect(cond.Message).To(ContainSubstring("alice"))

			delete(fake.unknownUsers, "alice")
			reconcileOnce(ctx)
			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
		})

		It("clears the ladder when spec.escalation is removed", func(ctx SpecContext) {
			createWithLadder(ctx, ladder())
			reconcileToReady(ctx)

			team := getTeam(ctx)
			team.Spec.Escalation = nil
			Expect(k8sClient.Update(ctx, team)).To(Succeed())
			reconcileOnce(ctx)

			Expect(fake.escalation[team.Status.TeamID].Levels).To(BeEmpty())
		})

		It("reports InvalidSpec for a duration that does not parse", func(ctx SpecContext) {
			bad := ladder()
			bad.Levels[0].Timeout = "5 minutes"
			// The CRD pattern rejects it at admission when the API server enforces it;
			// envtest does, so write it past the pattern by creating a valid one first.
			bad.Levels[0].Timeout = "5m"
			createWithLadder(ctx, bad)
			reconcileToReady(ctx)
			team := getTeam(ctx)
			team.Spec.Escalation.Levels[0].Timeout = "0s"
			Expect(k8sClient.Update(ctx, team)).To(Succeed())
			reconcileOnce(ctx)

			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonInvalidSpec))
		})
	})

	Describe("spec.deadmanSwitches", func() {
		sw := func(name, matcher, timeout string) terdutv1alpha1.DeadmanSwitchSpec {
			return terdutv1alpha1.DeadmanSwitchSpec{Name: name, Matcher: matcher, Timeout: timeout, Severity: "critical"}
		}
		setSwitches := func(ctx context.Context, switches ...terdutv1alpha1.DeadmanSwitchSpec) {
			team := getTeam(ctx)
			team.Spec.DeadmanSwitches = switches
			Expect(k8sClient.Update(ctx, team)).To(Succeed())
			reconcileOnce(ctx)
		}
		names := func(teamID int64) []string {
			out := make([]string, 0, len(fake.switches[teamID]))
			for _, s := range fake.switches[teamID] {
				out = append(out, s.Name)
			}
			return out
		}

		It("creates, updates and prunes switches by name", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			id := getTeam(ctx).Status.TeamID

			setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "15m"), sw("edge", "alertname=Edge", "5m"))
			Expect(names(id)).To(ConsistOf("watchdog", "edge"))

			// Same name, new timeout: updated in place, same id.
			var watchdogID int64
			for _, s := range fake.switches[id] {
				if s.Name == "watchdog" {
					watchdogID = s.ID
				}
			}
			setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "30m"))
			Expect(names(id)).To(ConsistOf("watchdog"))
			Expect(fake.switches[id][watchdogID].TimeoutSeconds).To(Equal(int64(1800)))
			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
		})

		It("removes a switch somebody added on the server", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			id := getTeam(ctx).Status.TeamID
			fake.nextSwitchID++
			fake.switches[id] = map[int64]tdclient.DeadmanSwitch{
				fake.nextSwitchID: {ID: fake.nextSwitchID, Name: "stray", Matcher: "alertname=X", TimeoutSeconds: 60, Severity: "critical"},
			}

			reconcileOnce(ctx)

			Expect(fake.switches[id]).To(BeEmpty())
		})

		It("recreates a switch deleted on the server", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			id := getTeam(ctx).Status.TeamID
			setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "15m"))
			fake.switches[id] = map[int64]tdclient.DeadmanSwitch{}

			reconcileOnce(ctx)

			Expect(names(id)).To(ConsistOf("watchdog"))
		})

		It("reports InvalidSpec for a timeout that does not parse", func(ctx SpecContext) {
			createTeam(ctx, platformName, sameNSRef())
			reconcileToReady(ctx)
			setSwitches(ctx, sw("watchdog", "alertname=Watchdog", "0s"))

			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonInvalidSpec))
		})
	})

	Describe("deletion", func() {
		It("deletes the team on the server and clears the finalizer", func(ctx SpecContext) {
			createTeam(ctx, "to-delete", sameNSRef())
			reconcileToReady(ctx)
			teamID := getTeam(ctx).Status.TeamID

			Expect(k8sClient.Delete(ctx, getTeam(ctx))).To(Succeed())
			reconcileOnce(ctx) // runs the finalizer

			Expect(fake.teamDelete[teamID]).To(BeTrue())
			Expect(k8sClient.Get(ctx, teamKey, &terdutv1alpha1.TerdutTeam{})).NotTo(Succeed(),
				"the TerdutTeam should be gone once the finalizer clears")
		})

		It("still deletes the team after the server's allowedTeams consent is revoked", func(ctx SpecContext) {
			otherNS := uniqueName("ns")
			Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherNS}})).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherNS}}) })

			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}, srv)).To(Succeed())
			srv.Spec.AllowedTeams.Namespaces.From = allNamespaces
			Expect(k8sClient.Update(ctx, srv)).To(Succeed())

			crossKey := types.NamespacedName{Name: teamName, Namespace: otherNS}
			Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutTeam{
				ObjectMeta: metav1.ObjectMeta{Name: teamName, Namespace: otherNS},
				Spec: terdutv1alpha1.TerdutTeamSpec{
					ServerRef:   terdutv1alpha1.TerdutServerRef{Name: srv.Name, Namespace: operatorNamespace},
					DisplayName: "revoked",
				},
			})).To(Succeed())
			for range 2 {
				_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey})
				Expect(err).NotTo(HaveOccurred())
			}
			cross := &terdutv1alpha1.TerdutTeam{}
			Expect(k8sClient.Get(ctx, crossKey, cross)).To(Succeed())
			teamID := cross.Status.TeamID
			Expect(teamID).NotTo(BeZero())

			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}, srv)).To(Succeed())
			srv.Spec.AllowedTeams.Namespaces.From = "None"
			Expect(k8sClient.Update(ctx, srv)).To(Succeed())

			Expect(k8sClient.Delete(ctx, cross)).To(Succeed())
			_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: crossKey})
			Expect(err).NotTo(HaveOccurred())

			Expect(fake.teamDelete[teamID]).To(BeTrue(), "revoking consent must not orphan the team")
		})

		It("lets the CR go when its TerdutServer is already gone", func(ctx SpecContext) {
			createTeam(ctx, "to-delete", sameNSRef())
			reconcileToReady(ctx)

			serverKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
			Expect(k8sClient.Get(ctx, serverKey, srv)).To(Succeed())
			srv.Finalizers = nil
			Expect(k8sClient.Update(ctx, srv)).To(Succeed())
			Expect(k8sClient.Delete(ctx, srv)).To(Succeed())

			Expect(k8sClient.Delete(ctx, getTeam(ctx))).To(Succeed())
			reconcileOnce(ctx)

			Expect(k8sClient.Get(ctx, teamKey, &terdutv1alpha1.TerdutTeam{})).NotTo(Succeed())
		})
	})
})