terdut-operator/internal/controller/terdutserver_controller_test.go

package controller

import (
	"context"
	"fmt"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	policyv1 "k8s.io/api/policy/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"

	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
)

var _ = Describe("TerdutServer Controller", func() {
	const operatorNamespace = "default"

	var (
		reconciler *TerdutServerReconciler
		name       string
		objKey     types.NamespacedName
	)

	BeforeEach(func() {
		reconciler = &TerdutServerReconciler{
			Client: k8sClient,
			Scheme: k8sClient.Scheme(),
		}
		name = fmt.Sprintf("test-server-%d-%d", GinkgoRandomSeed(), GinkgoParallelProcess())
		objKey = types.NamespacedName{Name: name, Namespace: operatorNamespace}
	})

	AfterEach(func(ctx SpecContext) {
		srv := &terdutv1alpha1.TerdutServer{}
		if err := k8sClient.Get(ctx, objKey, srv); err == nil {
			srv.Finalizers = nil
			_ = k8sClient.Update(ctx, srv)
			_ = k8sClient.Delete(ctx, srv)
		}
		_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-operator-key", Namespace: operatorNamespace}})
	})

	dsnSpec := func() terdutv1alpha1.TerdutServerSpec {
		return terdutv1alpha1.TerdutServerSpec{
			Image:      terdutv1alpha1.ImageSpec{Repository: testImageRepo, Tag: testImageTag},
			Networking: terdutv1alpha1.NetworkingSpec{Hostname: "terdut.example.invalid", ServicePort: 8080},
			Database:   terdutv1alpha1.DatabaseSpec{DSN: testDSN},
		}
	}

	createServer := func(ctx context.Context, spec terdutv1alpha1.TerdutServerSpec) {
		srv := &terdutv1alpha1.TerdutServer{
			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: operatorNamespace},
			Spec:       spec,
		}
		Expect(k8sClient.Create(ctx, srv)).To(Succeed())
	}

	reconcileOnce := func(ctx context.Context) ctrl.Result {
		res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objKey})
		Expect(err).NotTo(HaveOccurred())
		return res
	}

	markDeploymentReady := func(ctx context.Context) {
		var deploy appsv1.Deployment
		Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
		deploy.Status.ReadyReplicas = 1
		deploy.Status.Replicas = 1
		Expect(k8sClient.Status().Update(ctx, &deploy)).To(Succeed())
	}

	readyCondition := func(ctx context.Context) metav1.Condition {
		srv := &terdutv1alpha1.TerdutServer{}
		Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
		c := meta.FindStatusCondition(srv.Status.Conditions, terdutv1alpha1.ConditionReady)
		Expect(c).NotTo(BeNil(), "Ready condition should always be set after a reconcile")
		return *c
	}

	envOf := func(ctx context.Context) map[string]corev1.EnvVar {
		var deploy appsv1.Deployment
		Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
		out := map[string]corev1.EnvVar{}
		for _, e := range deploy.Spec.Template.Spec.Containers[0].Env {
			out[e.Name] = e
		}
		return out
	}

	Describe("bring-your-own DSN path", func() {
		It("creates Deployment, Service and operator key, then reaches Ready once a replica is up", func(ctx SpecContext) {
			createServer(ctx, dsnSpec())

			reconcileOnce(ctx)
			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForDeployment))

			var deploy appsv1.Deployment
			Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
			Expect(deploy.Spec.Template.Spec.Containers[0].Image).To(Equal("example.invalid/terdut-server:test"))
			env := envOf(ctx)
			Expect(env["TERDUT_DB_DSN"].Value).To(Equal(testDSN))
			Expect(env["TERDUT_OPERATOR_MODE"].Value).To(Equal("true"))
			Expect(env).NotTo(HaveKey("TERDUT_DEADMAN_MATCHERS"), "dead man's switches are per team now")

			var svc corev1.Service
			Expect(k8sClient.Get(ctx, objKey, &svc)).To(Succeed())
			Expect(svc.Spec.Ports[0].Port).To(Equal(int32(8080)))

			markDeploymentReady(ctx)
			reconcileOnce(ctx)

			srv := &terdutv1alpha1.TerdutServer{}
			Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
			ready := meta.FindStatusCondition(srv.Status.Conditions, terdutv1alpha1.ConditionReady)
			Expect(ready.Status).To(Equal(metav1.ConditionTrue))
			Expect(ready.Reason).To(Equal(terdutv1alpha1.ReasonAdopted))
			Expect(srv.Status.CredentialsSecretRef).To(Equal(&terdutv1alpha1.SecretKeyRef{
				Name: name + "-operator-key", Key: operatorKeyDataKey,
			}))
		})
	})

	Describe("the operator key", func() {
		keySecret := func(ctx context.Context) corev1.Secret {
			var s corev1.Secret
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name + "-operator-key", Namespace: operatorNamespace}, &s)).To(Succeed())
			return s
		}

		It("is generated once, owned by the TerdutServer, and wired into the pod", func(ctx SpecContext) {
			createServer(ctx, dsnSpec())
			reconcileOnce(ctx)

			s := keySecret(ctx)
			value := string(s.Data[operatorKeyDataKey])
			Expect(len(value)).To(BeNumerically(">=", 32), "terdut-server refuses a shorter TERDUT_OPERATOR_KEY")
			Expect(value).To(HavePrefix(operatorKeyPrefix))
			Expect(s.OwnerReferences).To(ContainElement(HaveField("Name", name)))

			env := envOf(ctx)
			Expect(env["TERDUT_OPERATOR_KEY"].ValueFrom).NotTo(BeNil())
			Expect(env["TERDUT_OPERATOR_KEY"].ValueFrom.SecretKeyRef.Name).To(Equal(name + "-operator-key"))
			Expect(env["TERDUT_OPERATOR_KEY"].Value).To(BeEmpty(), "the key itself must never appear in the pod spec")

			var deploy appsv1.Deployment
			Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
			Expect(deploy.Spec.Template.Annotations).To(HaveKeyWithValue(operatorKeyHashAnnotation, operatorKeyHash(value)))

			// A later reconcile keeps the same key: the running server was seeded with it.
			reconcileOnce(ctx)
			Expect(string(keySecret(ctx).Data[operatorKeyDataKey])).To(Equal(value))
		})

		It("rolls the Deployment when the Secret is replaced", func(ctx SpecContext) {
			createServer(ctx, dsnSpec())
			reconcileOnce(ctx)
			first := string(keySecret(ctx).Data[operatorKeyDataKey])

			s := keySecret(ctx)
			Expect(k8sClient.Delete(ctx, &s)).To(Succeed())
			reconcileOnce(ctx)

			second := string(keySecret(ctx).Data[operatorKeyDataKey])
			Expect(second).NotTo(Equal(first))
			var deploy appsv1.Deployment
			Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
			Expect(deploy.Spec.Template.Annotations).To(HaveKeyWithValue(operatorKeyHashAnnotation, operatorKeyHash(second)))
		})
	})

	Describe("spec.oidc claims", func() {
		It("passes the claim names and trustEmail through", func(ctx SpecContext) {
			spec := dsnSpec()
			spec.OIDC = terdutv1alpha1.OIDCSpec{
				Enabled: true, Issuer: "https://sso.example.invalid", ClientID: "terdut",
				UsernameClaim: "upn", EmailClaim: "mail", GroupsClaim: "roles", TrustEmail: true,
				SessionMaxAge: "12h",
			}
			createServer(ctx, spec)
			reconcileOnce(ctx)

			env := envOf(ctx)
			Expect(env["TERDUT_OIDC_USERNAME_CLAIM"].Value).To(Equal("upn"))
			Expect(env["TERDUT_OIDC_EMAIL_CLAIM"].Value).To(Equal("mail"))
			Expect(env["TERDUT_OIDC_GROUPS_CLAIM"].Value).To(Equal("roles"))
			Expect(env["TERDUT_OIDC_TRUST_EMAIL"].Value).To(Equal("true"))
		})
	})

	Describe("the Zalando postgresClusterRef path", func() {
		zalandoSpec := func(clusterName string) terdutv1alpha1.TerdutServerSpec {
			s := dsnSpec()
			s.Database = terdutv1alpha1.DatabaseSpec{
				PostgresClusterRef: &terdutv1alpha1.PostgresClusterRef{Name: clusterName},
			}
			return s
		}

		It("waits with reason PostgresClusterNotFound when the postgresql CR doesn't exist yet", func(ctx SpecContext) {
			createServer(ctx, zalandoSpec("missing-cluster"))
			reconcileOnce(ctx)

			Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonPostgresClusterNotFound))
		})

		It("resolves a real DSN and reaches Ready once the CR and its generated Secret both exist", func(ctx SpecContext) {
			clusterName := "pg-" + name
			cluster := &unstructured.Unstructured{}
			cluster.SetGroupVersionKind(postgresqlGVK)
			cluster.SetName(clusterName)
			cluster.SetNamespace(operatorNamespace)
			Expect(unstructured.SetNestedField(cluster.Object, map[string]any{}, "spec")).To(Succeed())
			Expect(k8sClient.Create(ctx, cluster)).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, cluster) })

			credsSecretName := fmt.Sprintf("terdut.%s.credentials.postgresql.acid.zalan.do", clusterName)
			zalandoSecret := &corev1.Secret{
				ObjectMeta: metav1.ObjectMeta{Name: credsSecretName, Namespace: operatorNamespace},
				Data:       map[string][]byte{"password": []byte("whatever")},
			}
			Expect(k8sClient.Create(ctx, zalandoSecret)).To(Succeed())
			DeferCleanup(func() { _ = k8sClient.Delete(ctx, zalandoSecret) })

			createServer(ctx, zalandoSpec(clusterName))
			reconcileOnce(ctx) // Deployment/Service created with resolved DB env

			var deploy appsv1.Deployment
			Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
			var dsn string
			for _, e := range deploy.Spec.Template.Spec.Containers[0].Env {
				if e.Name == "TERDUT_DB_DSN" {
					dsn = e.Value
				}
			}
			Expect(dsn).To(Equal(fmt.Sprintf("postgres://terdut@%s.%s.svc:5432/terdut?sslmode=require", clusterName, operatorNamespace)))

			markDeploymentReady(ctx)
			reconcileOnce(ctx)

			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
		})
	})

	Describe("spec.pod", func() {
		It("wires pod-level customization onto the right spot on the Deployment", func(ctx SpecContext) {
			spec := dsnSpec()
			qty := resource.MustParse("250m")
			spec.Pod = terdutv1alpha1.PodSpec{
				Resources:          corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: qty}},
				Tolerations:        []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpEqual, Value: "terdut", Effect: corev1.TaintEffectNoSchedule}},
				ExtraEnv:           []corev1.EnvVar{{Name: "EXTRA_FLAG", Value: "on"}},
				ServiceAccountName: "terdut-server-custom",
				ExtraVolumes:       []corev1.Volume{{Name: "extra-ca", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}},
				ExtraVolumeMounts:  []corev1.VolumeMount{{Name: "extra-ca", MountPath: "/etc/extra-ca"}},
			}
			createServer(ctx, spec)
			reconcileOnce(ctx)

			var deploy appsv1.Deployment
			Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())

			podSpec := deploy.Spec.Template.Spec
			Expect(podSpec.Tolerations).To(ConsistOf(spec.Pod.Tolerations))
			Expect(podSpec.ServiceAccountName).To(Equal("terdut-server-custom"))
			Expect(podSpec.Volumes).To(ConsistOf(spec.Pod.ExtraVolumes))

			main := podSpec.Containers[0]
			Expect(main.Name).To(Equal("terdut-server"))
			Expect(main.Resources).To(Equal(spec.Pod.Resources))
			Expect(main.VolumeMounts).To(ConsistOf(spec.Pod.ExtraVolumeMounts))
			Expect(main.Env).To(ContainElement(corev1.EnvVar{Name: "EXTRA_FLAG", Value: "on"}))

			initContainer := podSpec.InitContainers[0]
			Expect(initContainer.Name).To(Equal("wait-for-postgres"))
			Expect(initContainer.VolumeMounts).To(BeEmpty(), "extraVolumeMounts must not leak onto wait-for-postgres")
		})
	})

	Describe("spec.pod.disruptionBudget", func() {
		It("creates an owned PodDisruptionBudget when set, and deletes it once cleared", func(ctx SpecContext) {
			spec := dsnSpec()
			minAvail := intstr.FromInt32(1)
			spec.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail}
			createServer(ctx, spec)
			reconcileOnce(ctx)

			var pdb policyv1.PodDisruptionBudget
			Expect(k8sClient.Get(ctx, objKey, &pdb)).To(Succeed())
			Expect(pdb.Spec.Selector.MatchLabels).To(Equal(labelsFor(&terdutv1alpha1.TerdutServer{ObjectMeta: metav1.ObjectMeta{Name: name}})))
			Expect(pdb.Spec.MinAvailable).To(Equal(&minAvail))
			Expect(pdb.OwnerReferences).To(ContainElement(HaveField("Name", name)))

			srv := &terdutv1alpha1.TerdutServer{}
			Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
			srv.Spec.Pod.DisruptionBudget = nil
			Expect(k8sClient.Update(ctx, srv)).To(Succeed())
			reconcileOnce(ctx)

			err := k8sClient.Get(ctx, objKey, &policyv1.PodDisruptionBudget{})
			Expect(apierrors.IsNotFound(err)).To(BeTrue(), "PodDisruptionBudget should be deleted once spec.pod.disruptionBudget is cleared")
		})

		It("rejects both minAvailable and maxUnavailable set together, and neither set", func(ctx SpecContext) {
			bothSet := dsnSpec()
			minAvail, maxUnavail := intstr.FromInt32(1), intstr.FromInt32(1)
			bothSet.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail, MaxUnavailable: &maxUnavail}
			Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutServer{
				ObjectMeta: metav1.ObjectMeta{Name: name + "-both", Namespace: operatorNamespace},
				Spec:       bothSet,
			})).To(HaveOccurred())

			neitherSet := dsnSpec()
			neitherSet.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{}
			Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutServer{
				ObjectMeta: metav1.ObjectMeta{Name: name + "-neither", Namespace: operatorNamespace},
				Spec:       neitherSet,
			})).To(HaveOccurred())
		})
	})

	When("the TerdutServer object no longer exists", func() {
		It("returns no error (deleted between enqueue and reconcile)", func(ctx SpecContext) {
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: "never-created", Namespace: operatorNamespace},
			})
			Expect(err).NotTo(HaveOccurred())
		})
	})
})