Let TerdutServer customize its pod, and never manage its own ingress

spec.pod (api/v1alpha1/terdutserver_types.go): annotations, nodeSelector,
tolerations, affinity, topologySpreadConstraints, resources, pod and
container securityContext, serviceAccountName, extraEnv/extraEnvFrom,
extraVolumes/extraVolumeMounts, imagePullSecrets, and an optional
disruptionBudget. All direct corev1 passthrough -- no wrapper types buy
anything for any of these, matching how CloudNativePG and the Zalando
postgres-operator both expose the same knobs, and matching this repo's
own SweeperSpec precedent ("wrap only when a round-trip through a
different type buys something"). affinity is pure user-supplied
passthrough, not a toggle-plus-generated-default the way a multi-replica
cluster operator's pod anti-affinity usually is: this operator never
auto-generates one, since spec.replicas above 1 isn't a supported
topology (the sweeper/notifier singleton constraint). Considered and
declined for this round: priorityClassName, pod labels beyond
annotations, and a HorizontalPodAutoscaler -- the last of those would
directly contradict the singleton constraint above.

disruptionBudget is the one field here that isn't a plain PodTemplateSpec
knob: when set, the controller now reconciles a PodDisruptionBudget
selecting the TerdutServer's own pods (new terdutserver_pdb.go); clearing
it deletes any it previously created. New RBAC marker on
poddisruptionbudgets to match.

Driven by a public-release pass: looking past this project's own use case
at what a mature, general-purpose operator CRD exposes here (researched
against Zalando postgres-operator and CloudNativePG specifically), not
just the fields this install happened to need.

Separately, and found while answering a question about exposing
TerdutServer through Istio instead of Gateway API: spec.networking's own
doc comment quietly promised a Gateway API HTTPRoute this operator would
build eventually ("a near-term follow-up, not deferred"). That promise is
wrong for a public release -- an operator managing someone's ingress
mechanism for them is a worse default than not touching it at all, and a
surprise HTTPRoute appearing once that follow-up eventually landed would
have been exactly backwards for an Istio (or plain-Ingress, or
intentionally-unexposed) install. Made the non-goal explicit and
permanent instead (DESIGN.md §1), removed the dead `gatewayListener`
field it was the only consumer of (zero runtime call sites anywhere --
setting it already had no effect, so this is a schema cleanup, not a
behavior change), and corrected ROADMAP.md's framing. hostname/servicePort
stay: both are live (TERDUT_PUBLIC_URL, container/Service port), this
operator just never acts on hostname for exposure. Added
examples/networking (Gateway API HTTPRoute, Istio VirtualService) showing
how to expose the plain ClusterIP Service the operator already creates --
outside the operator itself, as illustrations, not as something
examples/demo applies automatically.

No new terdut-server version requirement: both changes are CRD/controller-
only, nothing about the API this operator's bootstrap flow depends on
changed.
This commit is contained in:
Niklas Ye
2026-10-02 18:50:38 +02:00
parent d9315322fc
commit 6a699d4341
20 changed files with 8631 additions and 96 deletions
@@ -8,6 +8,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
policyv1 "k8s.io/api/policy/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -97,6 +98,7 @@ type TerdutServerReconciler struct {
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=acid.zalan.do,resources=postgresqls,verbs=get;list;watch
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
@@ -137,6 +139,9 @@ func (r *TerdutServerReconciler) Reconcile(ctx context.Context, req ctrl.Request
if err := r.reconcileService(ctx, &srv); err != nil {
return ctrl.Result{}, err
}
if err := r.reconcilePodDisruptionBudget(ctx, &srv); err != nil {
return ctrl.Result{}, err
}
meta.SetStatusCondition(&srv.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionDatabaseReady,
@@ -246,6 +251,7 @@ func (r *TerdutServerReconciler) SetupWithManager(mgr ctrl.Manager) error {
For(&terdutv1alpha1.TerdutServer{}).
Owns(&appsv1.Deployment{}).
Owns(&corev1.Service{}).
Owns(&policyv1.PodDisruptionBudget{}).
Named("terdutserver").
Complete(r)
}
@@ -14,10 +14,14 @@ import (
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
policyv1 "k8s.io/api/policy/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -705,6 +709,95 @@ var _ = Describe("TerdutServer Controller", func() {
})
})
Describe("spec.pod", func() {
It("wires pod-level customization onto the right spot on the Deployment", func(ctx SpecContext) {
fake, fakeSrv := newFakeTerdutServer()
_ = fake
DeferCleanup(fakeSrv.Close)
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
spec := dsnSpec()
qty := resource.MustParse("250m")
spec.Pod = terdutv1alpha1.PodSpec{
Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: qty}},
Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpEqual, Value: "terdut", Effect: corev1.TaintEffectNoSchedule}},
ExtraEnv: []corev1.EnvVar{{Name: "EXTRA_FLAG", Value: "on"}},
ServiceAccountName: "terdut-server-custom",
ExtraVolumes: []corev1.Volume{{Name: "extra-ca", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}},
ExtraVolumeMounts: []corev1.VolumeMount{{Name: "extra-ca", MountPath: "/etc/extra-ca"}},
}
createServer(ctx, spec)
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx) // Deployment/Service/PDB
var deploy appsv1.Deployment
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
podSpec := deploy.Spec.Template.Spec
Expect(podSpec.Tolerations).To(ConsistOf(spec.Pod.Tolerations))
Expect(podSpec.ServiceAccountName).To(Equal("terdut-server-custom"))
Expect(podSpec.Volumes).To(ConsistOf(spec.Pod.ExtraVolumes))
main := podSpec.Containers[0]
Expect(main.Name).To(Equal("terdut-server"))
Expect(main.Resources).To(Equal(spec.Pod.Resources))
Expect(main.VolumeMounts).To(ConsistOf(spec.Pod.ExtraVolumeMounts))
Expect(main.Env).To(ContainElement(corev1.EnvVar{Name: "EXTRA_FLAG", Value: "on"}))
initContainer := podSpec.InitContainers[0]
Expect(initContainer.Name).To(Equal("wait-for-postgres"))
Expect(initContainer.VolumeMounts).To(BeEmpty(), "extraVolumeMounts must not leak onto wait-for-postgres")
})
})
Describe("spec.pod.disruptionBudget", func() {
It("creates an owned PodDisruptionBudget when set, and deletes it once cleared", func(ctx SpecContext) {
fake, fakeSrv := newFakeTerdutServer()
_ = fake
DeferCleanup(fakeSrv.Close)
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
spec := dsnSpec()
minAvail := intstr.FromInt32(1)
spec.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail}
createServer(ctx, spec)
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx) // Deployment/Service/PDB
var pdb policyv1.PodDisruptionBudget
Expect(k8sClient.Get(ctx, objKey, &pdb)).To(Succeed())
Expect(pdb.Spec.Selector.MatchLabels).To(Equal(labelsFor(&terdutv1alpha1.TerdutServer{ObjectMeta: metav1.ObjectMeta{Name: name}})))
Expect(pdb.Spec.MinAvailable).To(Equal(&minAvail))
Expect(pdb.OwnerReferences).To(ContainElement(HaveField("Name", name)))
srv := &terdutv1alpha1.TerdutServer{}
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
srv.Spec.Pod.DisruptionBudget = nil
Expect(k8sClient.Update(ctx, srv)).To(Succeed())
reconcileOnce(ctx)
err := k8sClient.Get(ctx, objKey, &policyv1.PodDisruptionBudget{})
Expect(apierrors.IsNotFound(err)).To(BeTrue(), "PodDisruptionBudget should be deleted once spec.pod.disruptionBudget is cleared")
})
It("rejects both minAvailable and maxUnavailable set together, and neither set", func(ctx SpecContext) {
bothSet := dsnSpec()
minAvail, maxUnavail := intstr.FromInt32(1), intstr.FromInt32(1)
bothSet.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail, MaxUnavailable: &maxUnavail}
Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutServer{
ObjectMeta: metav1.ObjectMeta{Name: name + "-both", Namespace: operatorNamespace},
Spec: bothSet,
})).To(HaveOccurred())
neitherSet := dsnSpec()
neitherSet.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{}
Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutServer{
ObjectMeta: metav1.ObjectMeta{Name: name + "-neither", Namespace: operatorNamespace},
Spec: neitherSet,
})).To(HaveOccurred())
})
})
Describe("deletion", func() {
It("removes the credentials and checkpoint Secrets and the finalizer", func(ctx SpecContext) {
fake, fakeSrv := newFakeTerdutServer()
+30 -8
View File
@@ -48,11 +48,26 @@ func (r *TerdutServerReconciler) reconcileDeployment(
// overlapping during a rollout would both page for the same
// incident (matches the chart's own deployment.yaml comment).
deploy.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}
pod := srv.Spec.Pod
deploy.Spec.Template = corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels},
// pod.Annotations is assigned directly, not merged -- nothing
// else sets pod-template annotations today. If a future change
// needs the controller to set one of its own (e.g. a
// Prometheus-scrape annotation), this needs to become a real
// map merge with a stated precedence rather than silently
// clobbering one side.
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: pod.Annotations},
Spec: corev1.PodSpec{
EnableServiceLinks: new(false),
InitContainers: []corev1.Container{waitForPostgresContainer(dbEnv)},
EnableServiceLinks: new(false),
NodeSelector: pod.NodeSelector,
Tolerations: pod.Tolerations,
Affinity: pod.Affinity,
TopologySpreadConstraints: pod.TopologySpreadConstraints,
SecurityContext: pod.SecurityContext,
ServiceAccountName: pod.ServiceAccountName,
ImagePullSecrets: pod.ImagePullSecrets,
InitContainers: []corev1.Container{waitForPostgresContainer(dbEnv)},
Volumes: pod.ExtraVolumes,
Containers: []corev1.Container{{
Name: "terdut-server",
Image: fmt.Sprintf("%s:%s", srv.Spec.Image.Repository, srv.Spec.Image.Tag),
@@ -61,9 +76,13 @@ func (r *TerdutServerReconciler) reconcileDeployment(
ContainerPort: servicePort(srv),
Protocol: corev1.ProtocolTCP,
}},
Env: buildEnv(srv, dbEnv),
LivenessProbe: healthzProbe(),
ReadinessProbe: healthzProbe(),
Env: buildEnv(srv, dbEnv),
EnvFrom: pod.ExtraEnvFrom,
VolumeMounts: pod.ExtraVolumeMounts,
Resources: pod.Resources,
SecurityContext: pod.ContainerSecurityContext,
LivenessProbe: healthzProbe(),
ReadinessProbe: healthzProbe(),
}},
},
}
@@ -151,7 +170,10 @@ func healthzProbe() *corev1.Probe {
// field-for-field (confirmed against that source, not reconstructed from
// DESIGN.md's illustrative YAML alone) — dbEnv (TERDUT_DB_DSN, optionally
// PGPASSWORD) comes from resolveDatabaseEnv, since which of §8's two paths
// produced it doesn't matter past this point.
// produced it doesn't matter past this point. spec.pod.extraEnv is appended
// last, after every fixed var -- this is the one place that owns "what env
// this container gets," so the escape hatch lives here rather than being
// appended separately in reconcileDeployment.
func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.EnvVar {
env := []corev1.EnvVar{{Name: "TERDUT_ADDR", Value: fmt.Sprintf(":%d", servicePort(srv))}}
env = append(env, dbEnv...)
@@ -230,7 +252,7 @@ func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.
}
}
return env
return append(env, srv.Spec.Pod.ExtraEnv...)
}
func secretEnvSource(ref *terdutv1alpha1.SecretKeyRef) *corev1.EnvVarSource {
+38
View File
@@ -0,0 +1,38 @@
package controller
import (
"context"
policyv1 "k8s.io/api/policy/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
)
// reconcilePodDisruptionBudget creates/updates the PodDisruptionBudget
// spec.pod.disruptionBudget asks for, or deletes a previously-created one
// when the field has been cleared -- the one conditionally-created child
// object in this controller (Deployment/Service are unconditional). Owned
// by srv, same plain-OwnerReference shape as Deployment/Service (DESIGN.md
// §7): same namespace, GC handles it, no finalizer needed.
func (r *TerdutServerReconciler) reconcilePodDisruptionBudget(ctx context.Context, srv *terdutv1alpha1.TerdutServer) error {
pdb := &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: srv.Name, Namespace: srv.Namespace}}
spec := srv.Spec.Pod.DisruptionBudget
if spec == nil {
if err := r.Delete(ctx, pdb); err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, pdb, func() error {
pdb.Spec.Selector = &metav1.LabelSelector{MatchLabels: labelsFor(srv)}
pdb.Spec.MinAvailable = spec.MinAvailable
pdb.Spec.MaxUnavailable = spec.MaxUnavailable
return controllerutil.SetControllerReference(srv, pdb, r.Scheme)
})
return err
}