Let TerdutServer customize its pod, and never manage its own ingress
spec.pod (api/v1alpha1/terdutserver_types.go): annotations, nodeSelector,
tolerations, affinity, topologySpreadConstraints, resources, pod and
container securityContext, serviceAccountName, extraEnv/extraEnvFrom,
extraVolumes/extraVolumeMounts, imagePullSecrets, and an optional
disruptionBudget. All direct corev1 passthrough -- no wrapper types buy
anything for any of these, matching how CloudNativePG and the Zalando
postgres-operator both expose the same knobs, and matching this repo's
own SweeperSpec precedent ("wrap only when a round-trip through a
different type buys something"). affinity is pure user-supplied
passthrough, not a toggle-plus-generated-default the way a multi-replica
cluster operator's pod anti-affinity usually is: this operator never
auto-generates one, since spec.replicas above 1 isn't a supported
topology (the sweeper/notifier singleton constraint). Considered and
declined for this round: priorityClassName, pod labels beyond
annotations, and a HorizontalPodAutoscaler -- the last of those would
directly contradict the singleton constraint above.
disruptionBudget is the one field here that isn't a plain PodTemplateSpec
knob: when set, the controller now reconciles a PodDisruptionBudget
selecting the TerdutServer's own pods (new terdutserver_pdb.go); clearing
it deletes any it previously created. New RBAC marker on
poddisruptionbudgets to match.
Driven by a public-release pass: looking past this project's own use case
at what a mature, general-purpose operator CRD exposes here (researched
against Zalando postgres-operator and CloudNativePG specifically), not
just the fields this install happened to need.
Separately, and found while answering a question about exposing
TerdutServer through Istio instead of Gateway API: spec.networking's own
doc comment quietly promised a Gateway API HTTPRoute this operator would
build eventually ("a near-term follow-up, not deferred"). That promise is
wrong for a public release -- an operator managing someone's ingress
mechanism for them is a worse default than not touching it at all, and a
surprise HTTPRoute appearing once that follow-up eventually landed would
have been exactly backwards for an Istio (or plain-Ingress, or
intentionally-unexposed) install. Made the non-goal explicit and
permanent instead (DESIGN.md §1), removed the dead `gatewayListener`
field it was the only consumer of (zero runtime call sites anywhere --
setting it already had no effect, so this is a schema cleanup, not a
behavior change), and corrected ROADMAP.md's framing. hostname/servicePort
stay: both are live (TERDUT_PUBLIC_URL, container/Service port), this
operator just never acts on hostname for exposure. Added
examples/networking (Gateway API HTTPRoute, Istio VirtualService) showing
how to expose the plain ClusterIP Service the operator already creates --
outside the operator itself, as illustrations, not as something
examples/demo applies automatically.
No new terdut-server version requirement: both changes are CRD/controller-
only, nothing about the API this operator's bootstrap flow depends on
changed.
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
policyv1 "k8s.io/api/policy/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -97,6 +98,7 @@ type TerdutServerReconciler struct {
|
||||
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
|
||||
// +kubebuilder:rbac:groups=acid.zalan.do,resources=postgresqls,verbs=get;list;watch
|
||||
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
|
||||
|
||||
@@ -137,6 +139,9 @@ func (r *TerdutServerReconciler) Reconcile(ctx context.Context, req ctrl.Request
|
||||
if err := r.reconcileService(ctx, &srv); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if err := r.reconcilePodDisruptionBudget(ctx, &srv); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
meta.SetStatusCondition(&srv.Status.Conditions, metav1.Condition{
|
||||
Type: terdutv1alpha1.ConditionDatabaseReady,
|
||||
@@ -246,6 +251,7 @@ func (r *TerdutServerReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
For(&terdutv1alpha1.TerdutServer{}).
|
||||
Owns(&appsv1.Deployment{}).
|
||||
Owns(&corev1.Service{}).
|
||||
Owns(&policyv1.PodDisruptionBudget{}).
|
||||
Named("terdutserver").
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
@@ -14,10 +14,14 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
policyv1 "k8s.io/api/policy/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
@@ -705,6 +709,95 @@ var _ = Describe("TerdutServer Controller", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("spec.pod", func() {
|
||||
It("wires pod-level customization onto the right spot on the Deployment", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
_ = fake
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
spec := dsnSpec()
|
||||
qty := resource.MustParse("250m")
|
||||
spec.Pod = terdutv1alpha1.PodSpec{
|
||||
Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: qty}},
|
||||
Tolerations: []corev1.Toleration{{Key: "dedicated", Operator: corev1.TolerationOpEqual, Value: "terdut", Effect: corev1.TaintEffectNoSchedule}},
|
||||
ExtraEnv: []corev1.EnvVar{{Name: "EXTRA_FLAG", Value: "on"}},
|
||||
ServiceAccountName: "terdut-server-custom",
|
||||
ExtraVolumes: []corev1.Volume{{Name: "extra-ca", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}},
|
||||
ExtraVolumeMounts: []corev1.VolumeMount{{Name: "extra-ca", MountPath: "/etc/extra-ca"}},
|
||||
}
|
||||
createServer(ctx, spec)
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service/PDB
|
||||
|
||||
var deploy appsv1.Deployment
|
||||
Expect(k8sClient.Get(ctx, objKey, &deploy)).To(Succeed())
|
||||
|
||||
podSpec := deploy.Spec.Template.Spec
|
||||
Expect(podSpec.Tolerations).To(ConsistOf(spec.Pod.Tolerations))
|
||||
Expect(podSpec.ServiceAccountName).To(Equal("terdut-server-custom"))
|
||||
Expect(podSpec.Volumes).To(ConsistOf(spec.Pod.ExtraVolumes))
|
||||
|
||||
main := podSpec.Containers[0]
|
||||
Expect(main.Name).To(Equal("terdut-server"))
|
||||
Expect(main.Resources).To(Equal(spec.Pod.Resources))
|
||||
Expect(main.VolumeMounts).To(ConsistOf(spec.Pod.ExtraVolumeMounts))
|
||||
Expect(main.Env).To(ContainElement(corev1.EnvVar{Name: "EXTRA_FLAG", Value: "on"}))
|
||||
|
||||
initContainer := podSpec.InitContainers[0]
|
||||
Expect(initContainer.Name).To(Equal("wait-for-postgres"))
|
||||
Expect(initContainer.VolumeMounts).To(BeEmpty(), "extraVolumeMounts must not leak onto wait-for-postgres")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("spec.pod.disruptionBudget", func() {
|
||||
It("creates an owned PodDisruptionBudget when set, and deletes it once cleared", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
_ = fake
|
||||
DeferCleanup(fakeSrv.Close)
|
||||
reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
|
||||
|
||||
spec := dsnSpec()
|
||||
minAvail := intstr.FromInt32(1)
|
||||
spec.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail}
|
||||
createServer(ctx, spec)
|
||||
reconcileOnce(ctx) // finalizer
|
||||
reconcileOnce(ctx) // Deployment/Service/PDB
|
||||
|
||||
var pdb policyv1.PodDisruptionBudget
|
||||
Expect(k8sClient.Get(ctx, objKey, &pdb)).To(Succeed())
|
||||
Expect(pdb.Spec.Selector.MatchLabels).To(Equal(labelsFor(&terdutv1alpha1.TerdutServer{ObjectMeta: metav1.ObjectMeta{Name: name}})))
|
||||
Expect(pdb.Spec.MinAvailable).To(Equal(&minAvail))
|
||||
Expect(pdb.OwnerReferences).To(ContainElement(HaveField("Name", name)))
|
||||
|
||||
srv := &terdutv1alpha1.TerdutServer{}
|
||||
Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
|
||||
srv.Spec.Pod.DisruptionBudget = nil
|
||||
Expect(k8sClient.Update(ctx, srv)).To(Succeed())
|
||||
reconcileOnce(ctx)
|
||||
|
||||
err := k8sClient.Get(ctx, objKey, &policyv1.PodDisruptionBudget{})
|
||||
Expect(apierrors.IsNotFound(err)).To(BeTrue(), "PodDisruptionBudget should be deleted once spec.pod.disruptionBudget is cleared")
|
||||
})
|
||||
|
||||
It("rejects both minAvailable and maxUnavailable set together, and neither set", func(ctx SpecContext) {
|
||||
bothSet := dsnSpec()
|
||||
minAvail, maxUnavail := intstr.FromInt32(1), intstr.FromInt32(1)
|
||||
bothSet.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{MinAvailable: &minAvail, MaxUnavailable: &maxUnavail}
|
||||
Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutServer{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name + "-both", Namespace: operatorNamespace},
|
||||
Spec: bothSet,
|
||||
})).To(HaveOccurred())
|
||||
|
||||
neitherSet := dsnSpec()
|
||||
neitherSet.Pod.DisruptionBudget = &terdutv1alpha1.PodDisruptionBudgetSpec{}
|
||||
Expect(k8sClient.Create(ctx, &terdutv1alpha1.TerdutServer{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name + "-neither", Namespace: operatorNamespace},
|
||||
Spec: neitherSet,
|
||||
})).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("deletion", func() {
|
||||
It("removes the credentials and checkpoint Secrets and the finalizer", func(ctx SpecContext) {
|
||||
fake, fakeSrv := newFakeTerdutServer()
|
||||
|
||||
@@ -48,11 +48,26 @@ func (r *TerdutServerReconciler) reconcileDeployment(
|
||||
// overlapping during a rollout would both page for the same
|
||||
// incident (matches the chart's own deployment.yaml comment).
|
||||
deploy.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}
|
||||
pod := srv.Spec.Pod
|
||||
deploy.Spec.Template = corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels},
|
||||
// pod.Annotations is assigned directly, not merged -- nothing
|
||||
// else sets pod-template annotations today. If a future change
|
||||
// needs the controller to set one of its own (e.g. a
|
||||
// Prometheus-scrape annotation), this needs to become a real
|
||||
// map merge with a stated precedence rather than silently
|
||||
// clobbering one side.
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: labels, Annotations: pod.Annotations},
|
||||
Spec: corev1.PodSpec{
|
||||
EnableServiceLinks: new(false),
|
||||
InitContainers: []corev1.Container{waitForPostgresContainer(dbEnv)},
|
||||
EnableServiceLinks: new(false),
|
||||
NodeSelector: pod.NodeSelector,
|
||||
Tolerations: pod.Tolerations,
|
||||
Affinity: pod.Affinity,
|
||||
TopologySpreadConstraints: pod.TopologySpreadConstraints,
|
||||
SecurityContext: pod.SecurityContext,
|
||||
ServiceAccountName: pod.ServiceAccountName,
|
||||
ImagePullSecrets: pod.ImagePullSecrets,
|
||||
InitContainers: []corev1.Container{waitForPostgresContainer(dbEnv)},
|
||||
Volumes: pod.ExtraVolumes,
|
||||
Containers: []corev1.Container{{
|
||||
Name: "terdut-server",
|
||||
Image: fmt.Sprintf("%s:%s", srv.Spec.Image.Repository, srv.Spec.Image.Tag),
|
||||
@@ -61,9 +76,13 @@ func (r *TerdutServerReconciler) reconcileDeployment(
|
||||
ContainerPort: servicePort(srv),
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
}},
|
||||
Env: buildEnv(srv, dbEnv),
|
||||
LivenessProbe: healthzProbe(),
|
||||
ReadinessProbe: healthzProbe(),
|
||||
Env: buildEnv(srv, dbEnv),
|
||||
EnvFrom: pod.ExtraEnvFrom,
|
||||
VolumeMounts: pod.ExtraVolumeMounts,
|
||||
Resources: pod.Resources,
|
||||
SecurityContext: pod.ContainerSecurityContext,
|
||||
LivenessProbe: healthzProbe(),
|
||||
ReadinessProbe: healthzProbe(),
|
||||
}},
|
||||
},
|
||||
}
|
||||
@@ -151,7 +170,10 @@ func healthzProbe() *corev1.Probe {
|
||||
// field-for-field (confirmed against that source, not reconstructed from
|
||||
// DESIGN.md's illustrative YAML alone) — dbEnv (TERDUT_DB_DSN, optionally
|
||||
// PGPASSWORD) comes from resolveDatabaseEnv, since which of §8's two paths
|
||||
// produced it doesn't matter past this point.
|
||||
// produced it doesn't matter past this point. spec.pod.extraEnv is appended
|
||||
// last, after every fixed var -- this is the one place that owns "what env
|
||||
// this container gets," so the escape hatch lives here rather than being
|
||||
// appended separately in reconcileDeployment.
|
||||
func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.EnvVar {
|
||||
env := []corev1.EnvVar{{Name: "TERDUT_ADDR", Value: fmt.Sprintf(":%d", servicePort(srv))}}
|
||||
env = append(env, dbEnv...)
|
||||
@@ -230,7 +252,7 @@ func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.
|
||||
}
|
||||
}
|
||||
|
||||
return env
|
||||
return append(env, srv.Spec.Pod.ExtraEnv...)
|
||||
}
|
||||
|
||||
func secretEnvSource(ref *terdutv1alpha1.SecretKeyRef) *corev1.EnvVarSource {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
policyv1 "k8s.io/api/policy/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
||||
)
|
||||
|
||||
// reconcilePodDisruptionBudget creates/updates the PodDisruptionBudget
|
||||
// spec.pod.disruptionBudget asks for, or deletes a previously-created one
|
||||
// when the field has been cleared -- the one conditionally-created child
|
||||
// object in this controller (Deployment/Service are unconditional). Owned
|
||||
// by srv, same plain-OwnerReference shape as Deployment/Service (DESIGN.md
|
||||
// §7): same namespace, GC handles it, no finalizer needed.
|
||||
func (r *TerdutServerReconciler) reconcilePodDisruptionBudget(ctx context.Context, srv *terdutv1alpha1.TerdutServer) error {
|
||||
pdb := &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: srv.Name, Namespace: srv.Namespace}}
|
||||
|
||||
spec := srv.Spec.Pod.DisruptionBudget
|
||||
if spec == nil {
|
||||
if err := r.Delete(ctx, pdb); err != nil && !apierrors.IsNotFound(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, pdb, func() error {
|
||||
pdb.Spec.Selector = &metav1.LabelSelector{MatchLabels: labelsFor(srv)}
|
||||
pdb.Spec.MinAvailable = spec.MinAvailable
|
||||
pdb.Spec.MaxUnavailable = spec.MaxUnavailable
|
||||
return controllerutil.SetControllerReference(srv, pdb, r.Scheme)
|
||||
})
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user