Niklas Ye
ae97d28444
This operator's own Deployment template crash-looped a few times against a from-scratch postgres-operator cluster still doing initdb and Patroni leader election -- exactly the gap examples/demo's own README just documented for it. terdut-server's ping-retry budget on startup (internal/db/db.go in that repo) is sized for a much shorter, different race (NetworkPolicy propagation, a few seconds), not genuine first-time cluster creation, so it exhausted and the process exited before ever binding its HTTP port -- a startupProbe cannot fix that, since the crash happens before there is anything to probe. Same root cause and same fix as charts/terdut-server's own deployment.yaml template as of that repo's v0.33.2. waitForPostgresContainer reuses dbEnv unchanged: both of resolveDatabaseEnv's two paths (DSN, postgresClusterRef) put TERDUT_DB_DSN first, so it's already exactly what pg_isready needs, and pg_isready needs no credentials, so dbEnv's optional PGPASSWORD riding along too is harmless rather than load-bearing. Covered by the existing envtest suite (asserts on Containers[0], the main container, unaffected by adding InitContainers) -- `make test` passes unchanged, 71.7% coverage on internal/controller. Updates examples/demo's own README, which no longer needs to warn about this.
251 lines
9.7 KiB
Go
251 lines
9.7 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
|
|
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
|
|
)
|
|
|
|
// labelsFor returns the selector labels for srv's Deployment/Service —
|
|
// fixed once a Deployment exists (its selector is immutable), so this must
|
|
// never depend on anything in srv.Spec that could change later.
|
|
func labelsFor(srv *terdutv1alpha1.TerdutServer) map[string]string {
|
|
return map[string]string{
|
|
"app.kubernetes.io/name": "terdut-server",
|
|
"app.kubernetes.io/instance": srv.Name,
|
|
}
|
|
}
|
|
|
|
// reconcileDeployment creates or updates the Deployment running
|
|
// terdut-server, owned by srv (DESIGN.md §7: same-namespace generated
|
|
// objects carry a plain OwnerReference, no finalizer). Returns the live
|
|
// object (not just the desired one) so the caller can check
|
|
// status.readyReplicas.
|
|
func (r *TerdutServerReconciler) reconcileDeployment(
|
|
ctx context.Context, srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar,
|
|
) (*appsv1.Deployment, error) {
|
|
deploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: srv.Name, Namespace: srv.Namespace}}
|
|
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, deploy, func() error {
|
|
replicas := srv.Spec.Replicas
|
|
if replicas == 0 {
|
|
replicas = 1
|
|
}
|
|
labels := labelsFor(srv)
|
|
|
|
deploy.Spec.Replicas = &replicas
|
|
deploy.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels}
|
|
// Recreate, not RollingUpdate: the sweeper and the notifier are
|
|
// unsynchronised singletons inside terdut-server, and two replicas
|
|
// overlapping during a rollout would both page for the same
|
|
// incident (matches the chart's own deployment.yaml comment).
|
|
deploy.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType}
|
|
deploy.Spec.Template = corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{Labels: labels},
|
|
Spec: corev1.PodSpec{
|
|
EnableServiceLinks: new(false),
|
|
InitContainers: []corev1.Container{waitForPostgresContainer(dbEnv)},
|
|
Containers: []corev1.Container{{
|
|
Name: "terdut-server",
|
|
Image: fmt.Sprintf("%s:%s", srv.Spec.Image.Repository, srv.Spec.Image.Tag),
|
|
Ports: []corev1.ContainerPort{{
|
|
Name: "http",
|
|
ContainerPort: servicePort(srv),
|
|
Protocol: corev1.ProtocolTCP,
|
|
}},
|
|
Env: buildEnv(srv, dbEnv),
|
|
LivenessProbe: healthzProbe(),
|
|
ReadinessProbe: healthzProbe(),
|
|
}},
|
|
},
|
|
}
|
|
return controllerutil.SetControllerReference(srv, deploy, r.Scheme)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// CreateOrUpdate's own object is desired-state-only on a fresh create
|
|
// (no .status yet); re-fetch so the ready-replica check the caller does
|
|
// next sees the real, live object.
|
|
if err := r.Get(ctx, client.ObjectKeyFromObject(deploy), deploy); err != nil {
|
|
return nil, err
|
|
}
|
|
return deploy, nil
|
|
}
|
|
|
|
// reconcileService creates or updates the ClusterIP Service in front of the
|
|
// Deployment, owned by srv.
|
|
func (r *TerdutServerReconciler) reconcileService(ctx context.Context, srv *terdutv1alpha1.TerdutServer) error {
|
|
svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: srv.Name, Namespace: srv.Namespace}}
|
|
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error {
|
|
svc.Spec.Selector = labelsFor(srv)
|
|
svc.Spec.Ports = []corev1.ServicePort{{
|
|
Name: "http",
|
|
Port: servicePort(srv),
|
|
TargetPort: intstr.FromString("http"),
|
|
Protocol: corev1.ProtocolTCP,
|
|
}}
|
|
return controllerutil.SetControllerReference(srv, svc, r.Scheme)
|
|
})
|
|
return err
|
|
}
|
|
|
|
func servicePort(srv *terdutv1alpha1.TerdutServer) int32 {
|
|
if srv.Spec.Networking.ServicePort == 0 {
|
|
return 8080
|
|
}
|
|
return srv.Spec.Networking.ServicePort
|
|
}
|
|
|
|
// waitForPostgresContainer blocks the main container from starting until
|
|
// Postgres accepts connections, matching charts/terdut-server's own
|
|
// deployment.yaml template as of v0.33.2 (that repo's CLAUDE.md/release
|
|
// notes) -- that chart grew this the moment this exact Deployment, created
|
|
// by this controller, crash-looped a few times against a from-scratch
|
|
// postgres-operator cluster still doing initdb and Patroni leader election:
|
|
// terdut-server's own ping-retry budget on startup (internal/db/db.go) is
|
|
// sized for a much shorter, different race (NetworkPolicy propagation, a
|
|
// few seconds), not for genuine first-time cluster creation, so it
|
|
// exhausted and the process exited before ever binding its HTTP port -- a
|
|
// startupProbe cannot help there, since the crash happens before there is
|
|
// anything to probe.
|
|
//
|
|
// Reuses dbEnv unchanged: both of resolveDatabaseEnv's paths put
|
|
// TERDUT_DB_DSN first (terdutserver_database.go), so it's already exactly
|
|
// what pg_isready needs, and pg_isready needs no credentials -- it reports
|
|
// PQPING_OK on anything that amounts to a Postgres backend answering,
|
|
// including an auth challenge -- so including dbEnv's optional PGPASSWORD
|
|
// here too is harmless rather than load-bearing.
|
|
func waitForPostgresContainer(dbEnv []corev1.EnvVar) corev1.Container {
|
|
return corev1.Container{
|
|
Name: "wait-for-postgres",
|
|
Image: "postgres:17-alpine",
|
|
Env: dbEnv,
|
|
Command: []string{"sh", "-c",
|
|
`until pg_isready -d "$TERDUT_DB_DSN"; do echo "wait-for-postgres: not ready yet, retrying in 2s"; sleep 2; done`,
|
|
},
|
|
}
|
|
}
|
|
|
|
func healthzProbe() *corev1.Probe {
|
|
return &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/healthz",
|
|
Port: intstr.FromString("http"),
|
|
},
|
|
},
|
|
InitialDelaySeconds: 5,
|
|
}
|
|
}
|
|
|
|
// buildEnv mirrors charts/terdut-server's own deployment.yaml env block
|
|
// field-for-field (confirmed against that source, not reconstructed from
|
|
// DESIGN.md's illustrative YAML alone) — dbEnv (TERDUT_DB_DSN, optionally
|
|
// PGPASSWORD) comes from resolveDatabaseEnv, since which of §8's two paths
|
|
// produced it doesn't matter past this point.
|
|
func buildEnv(srv *terdutv1alpha1.TerdutServer, dbEnv []corev1.EnvVar) []corev1.EnvVar {
|
|
env := []corev1.EnvVar{{Name: "TERDUT_ADDR", Value: fmt.Sprintf(":%d", servicePort(srv))}}
|
|
env = append(env, dbEnv...)
|
|
|
|
sweeper := srv.Spec.Sweeper
|
|
env = append(env,
|
|
corev1.EnvVar{Name: "TERDUT_STALE_AFTER", Value: sweeper.StaleAfter},
|
|
corev1.EnvVar{Name: "TERDUT_ARCHIVE_AFTER", Value: sweeper.ArchiveAfter},
|
|
)
|
|
|
|
deadman := srv.Spec.Deadman
|
|
env = append(env,
|
|
corev1.EnvVar{Name: "TERDUT_DEADMAN_MATCHERS", Value: deadman.Matchers},
|
|
corev1.EnvVar{Name: "TERDUT_DEADMAN_TIMEOUT", Value: deadman.Timeout},
|
|
corev1.EnvVar{Name: "TERDUT_DEADMAN_SEVERITY", Value: deadman.Severity},
|
|
)
|
|
|
|
notify := srv.Spec.Notify
|
|
if notify.NtfyURL != "" {
|
|
env = append(env,
|
|
corev1.EnvVar{Name: "TERDUT_NTFY_URL", Value: notify.NtfyURL},
|
|
corev1.EnvVar{Name: "TERDUT_NTFY_FALLBACK_TOPIC", Value: notify.FallbackTopic},
|
|
corev1.EnvVar{Name: "TERDUT_NOTIFY_REPEAT", Value: notify.RepeatEvery},
|
|
)
|
|
if notify.TokenSecretRef != nil {
|
|
env = append(env, corev1.EnvVar{
|
|
Name: "TERDUT_NTFY_TOKEN",
|
|
ValueFrom: secretEnvSource(notify.TokenSecretRef),
|
|
})
|
|
}
|
|
}
|
|
|
|
publicURL := fmt.Sprintf("https://%s", srv.Spec.Networking.Hostname)
|
|
env = append(env,
|
|
corev1.EnvVar{Name: "TERDUT_PUBLIC_URL", Value: publicURL},
|
|
corev1.EnvVar{Name: "TERDUT_PASSWORD_LOGIN", Value: boolString(srv.Spec.PasswordLogin)},
|
|
// Always on, unlike the chart's own default-off operatorMode: every
|
|
// write this operator's own TerdutTeam/EscalationRule/etc.
|
|
// controllers make goes through a service account already, and the
|
|
// whole reason to run this operator is gitops-managed config, not a
|
|
// human editing an operator-managed install's teams/policies by
|
|
// hand (chart's values.yaml comment: "a statement that something
|
|
// like terdut-operator... owns this install's configuration from
|
|
// here on" -- which is unconditionally true for anything this
|
|
// operator creates).
|
|
corev1.EnvVar{Name: "TERDUT_OPERATOR_MODE", Value: "true"},
|
|
)
|
|
|
|
if oidc := srv.Spec.OIDC; oidc.Enabled {
|
|
env = append(env,
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_ISSUER", Value: oidc.Issuer},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_CLIENT_ID", Value: oidc.ClientID},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_NAME", Value: oidc.Name},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_SCOPES", Value: oidc.Scopes},
|
|
// terdut-server's own defaults for the claims/trust-email knobs
|
|
// the chart exposes but DESIGN.md's spec doesn't (§4.1's doc
|
|
// comment on OIDCSpec) -- not configurable here, not an
|
|
// oversight.
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_USERNAME_CLAIM", Value: "preferred_username"},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_EMAIL_CLAIM", Value: "email"},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_GROUPS_CLAIM", Value: "groups"},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_TRUST_EMAIL", Value: "false"},
|
|
corev1.EnvVar{Name: "TERDUT_OIDC_SESSION_MAX_AGE", Value: oidc.SessionMaxAge},
|
|
)
|
|
if oidc.ClientSecretRef != nil {
|
|
env = append(env, corev1.EnvVar{
|
|
Name: "TERDUT_OIDC_CLIENT_SECRET",
|
|
ValueFrom: secretEnvSource(oidc.ClientSecretRef),
|
|
})
|
|
}
|
|
if len(oidc.AllowedGroups) > 0 {
|
|
env = append(env, corev1.EnvVar{Name: "TERDUT_OIDC_ALLOWED_GROUPS", Value: strings.Join(oidc.AllowedGroups, ",")})
|
|
}
|
|
if oidc.AdminGroup != "" {
|
|
env = append(env, corev1.EnvVar{Name: "TERDUT_OIDC_ADMIN_GROUP", Value: oidc.AdminGroup})
|
|
}
|
|
}
|
|
|
|
return env
|
|
}
|
|
|
|
func secretEnvSource(ref *terdutv1alpha1.SecretKeyRef) *corev1.EnvVarSource {
|
|
return &corev1.EnvVarSource{
|
|
SecretKeyRef: &corev1.SecretKeySelector{
|
|
LocalObjectReference: corev1.LocalObjectReference{Name: ref.Name},
|
|
Key: ref.Key,
|
|
},
|
|
}
|
|
}
|
|
|
|
func boolString(b bool) string {
|
|
if b {
|
|
return "true"
|
|
}
|
|
return "false"
|
|
}
|