Stage 4: TerdutAlertSource
CI / test (push) Successful in 1m34s

Covers webhook Secret generation/ownership (DESIGN.md §4.5, §7), the
WebhookSecretLost fail-closed condition, and the kind-change
delete-and-recreate rotation path.

Idempotent-create here is deliberately neither adopt-on-409
(Team/service-account) nor list-and-match-by-name (TerdutDeadmanSwitch):
terdut-server shows the webhook key exactly once, at creation, and never
again, so no server-side lookup could ever recover it after a crash.
Instead the generated webhook Secret itself -- written immediately after
the POST, before status is ever touched -- is this CR's only durable
record that a create already succeeded; found with status.integrationID
still unset on a later reconcile, it's read back directly rather than
POSTing a second, orphaned integration. Found missing with
status.integrationID *set* instead, that's the already-designed
WebhookSecretLost case: fail closed, not self-healed, since the key is
genuinely gone and recreating it would rotate a live webhook URL with no
spec change to explain why.

Renaming (PATCH) never touches the key, so it's applied unconditionally
every reconcile, same as the escalation policy's whole-policy PUT. A
spec.kind change is the one case with no in-place update verb at all:
DELETE the old integration, delete the stale webhook Secret, then run the
same create path fresh -- fires a Warning event since this breaks whatever
still sends to the old URL.

Also: fakeTerdutServer grows POST/PATCH/DELETE .../integrations routes
behind a new handleIntegrationSubPath, split out of handleTeamSubPath to
stay under gocyclo's threshold; three goconst-flagged test literals
("does-not-exist", "unready") and one unparam-flagged test helper
parameter (bootstrapReadyTerdutServer's always-"default" namespace) get
shared/removed now that a fourth same-shaped caller made the repetition
concrete enough for the linter to flag.

DESIGN.md §13 gains one honest gap found while grounding this stage, not
introduced by it: no child CRD specially detects a mid-life teamRef
change; all three always resolve spec.teamRef fresh and trust the
already-stored server-side id remains valid there.

make fmt lint test build all clean; internal/controller envtest coverage
holds at 71.6%.
This commit is contained in:
Niklas Ye
2026-10-01 14:13:19 +02:00
parent b0d50e305a
commit 048f4448c4
23 changed files with 1395 additions and 9 deletions
@@ -0,0 +1,313 @@
package controller
import (
"context"
"fmt"
"strconv"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/recorder"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
const alertSourceFinalizerName = "terdut.ryuvia.com/terdutalertsource"
// Data keys inside the generated webhook Secret (DESIGN.md §4.5). url/key
// are what a sender actually needs; integrationID exists purely so this
// Secret -- once written -- is also this CR's own record that a create
// already succeeded (see reconcileCreate's own comment for why that
// matters more here than for any other child kind).
const (
webhookSecretURLKey = "url"
webhookSecretKeyDataKey = "key"
webhookSecretIntegrationIDKey = "integrationID"
)
// TerdutAlertSourceReconciler reconciles a TerdutAlertSource object.
type TerdutAlertSourceReconciler struct {
client.Client
Scheme *runtime.Scheme
OperatorNamespace string
Recorder recorder.EventRecorder
NewClient func(endpoint string) *tdclient.Client
}
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutalertsources/finalizers,verbs=update
// +kubebuilder:rbac:groups=terdut.ryuvia.com,resources=terdutteams,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
func (r *TerdutAlertSourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := logf.FromContext(ctx)
var as terdutv1alpha1.TerdutAlertSource
if err := r.Get(ctx, req.NamespacedName, &as); err != nil {
if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
newClient := r.NewClient
if newClient == nil {
newClient = tdclient.New
}
if !as.DeletionTimestamp.IsZero() {
return r.reconcileDelete(ctx, &as, newClient)
}
if !controllerutil.ContainsFinalizer(&as, alertSourceFinalizerName) {
controllerutil.AddFinalizer(&as, alertSourceFinalizerName)
if err := r.Update(ctx, &as); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
team, tc, resolveErr := resolveTeamAndClient(ctx, r.Client, r.OperatorNamespace, as.Namespace, as.Spec.TeamRef, newClient)
if resolveErr != nil {
return r.setNotReady(ctx, &as, resolveErr.reason, resolveErr.message, waitInterval)
}
teamID := team.Status.TeamID
if as.Status.IntegrationID == 0 {
if err := r.reconcileCreate(ctx, &as, tc, teamID); err != nil {
return ctrl.Result{}, err
}
} else {
secretName := webhookSecretName(&as)
var secret corev1.Secret
if err := r.Get(ctx, client.ObjectKey{Namespace: as.Namespace, Name: secretName}, &secret); err != nil {
if !apierrors.IsNotFound(err) {
return ctrl.Result{}, err
}
// Fail-closed, not self-healed (DESIGN.md §5): the key is
// genuinely gone and can never be re-read from terdut-server,
// so minting a replacement here would silently rotate a live
// webhook URL with no spec change to explain why. Deleting
// and recreating this CR is the supported recovery.
return r.setNotReady(ctx, &as, terdutv1alpha1.ReasonWebhookSecretLost,
fmt.Sprintf("webhook Secret %s/%s is gone; delete and recreate this TerdutAlertSource to rotate a new one", as.Namespace, secretName),
waitInterval)
}
if as.Spec.Kind != as.Status.LastAppliedKind {
if err := r.rotateKind(ctx, &as, tc, teamID); err != nil {
return ctrl.Result{}, err
}
} else if err := tc.RenameIntegration(ctx, teamID, as.Status.IntegrationID, as.Spec.Name); err != nil {
return ctrl.Result{}, fmt.Errorf("PATCH /api/teams/%d/integrations/%d: %w", teamID, as.Status.IntegrationID, err)
}
}
meta.SetStatusCondition(&as.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionTrue,
Reason: terdutv1alpha1.ReasonChildAdopted,
Message: fmt.Sprintf("integration %d applied on team %d", as.Status.IntegrationID, teamID),
})
as.Status.ObservedGeneration = as.Generation
if err := r.Status().Update(ctx, &as); err != nil {
return ctrl.Result{}, err
}
if r.Recorder != nil {
r.Recorder.Eventf(&as, nil, corev1.EventTypeNormal, terdutv1alpha1.ReasonChildAdopted, terdutv1alpha1.ReasonChildAdopted,
"alert source applied")
}
log.Info("TerdutAlertSource applied", "name", as.Name, "integrationID", as.Status.IntegrationID)
return ctrl.Result{RequeueAfter: resyncInterval}, nil
}
// webhookSecretName is deterministic from this object's own, immutable
// metadata.name -- not spec.name, which can be renamed freely without the
// Secret needing to follow (DESIGN.md §4.5: renaming never rotates the key).
func webhookSecretName(as *terdutv1alpha1.TerdutAlertSource) string {
return as.Name + "-terdut-webhook"
}
// reconcileCreate implements this resource's own idempotent-create shape --
// deliberately not list-and-match (TerdutDeadmanSwitch) or adopt-on-409
// (Team/service-account): terdut-server shows the webhook key exactly once,
// at creation, and never again (DESIGN.md §4.5), so there is no server-side
// lookup that could ever recover it. Instead, the webhook Secret itself --
// written immediately after a successful POST, before status is ever
// touched -- is this CR's only durable record that a create already
// succeeded. A crash between the POST and the Secret write is the one
// unrecoverable case: on the next reconcile the Secret still won't exist,
// so this mints a brand new integration rather than risk adopting an
// orphaned, keyless row by name -- same accepted tradeoff as the "orphaned
// first key some interrupted attempt minted and never used" footnote
// DESIGN.md §6 already documents for service-account keys.
func (r *TerdutAlertSourceReconciler) reconcileCreate(
ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, tc *tdclient.Client, teamID int64,
) error {
secretName := webhookSecretName(as)
var secret corev1.Secret
err := r.Get(ctx, client.ObjectKey{Namespace: as.Namespace, Name: secretName}, &secret)
switch {
case err == nil:
// Crash recovery: a previous reconcile got as far as writing the
// Secret but died before writing status -- read the id back out
// of it rather than calling the server again.
id, parseErr := strconv.ParseInt(string(secret.Data[webhookSecretIntegrationIDKey]), 10, 64)
if parseErr != nil {
return fmt.Errorf("webhook Secret %s/%s has no valid %s: %w", as.Namespace, secretName, webhookSecretIntegrationIDKey, parseErr)
}
as.Status.IntegrationID = id
as.Status.LastAppliedKind = as.Spec.Kind
as.Status.WebhookURLSecretRef = &terdutv1alpha1.LocalSecretRef{Name: secretName}
return nil
case !apierrors.IsNotFound(err):
return err
}
created, err := tc.CreateIntegration(ctx, teamID, as.Spec.Name, as.Spec.Kind)
if err != nil {
return fmt.Errorf("POST /api/teams/%d/integrations: %w", teamID, err)
}
if err := r.writeWebhookSecret(ctx, as, secretName, created); err != nil {
return err
}
as.Status.IntegrationID = created.ID
as.Status.LastAppliedKind = as.Spec.Kind
as.Status.WebhookURLSecretRef = &terdutv1alpha1.LocalSecretRef{Name: secretName}
return nil
}
// rotateKind deletes the currently-live integration and creates a fresh one
// under the new kind (DESIGN.md §5's reconciliation table: kind is the one
// spec field with no in-place update verb at all), firing a Warning event
// since this rotates the webhook key and breaks whatever still sends to the
// old URL.
func (r *TerdutAlertSourceReconciler) rotateKind(
ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, tc *tdclient.Client, teamID int64,
) error {
oldKind, oldID := as.Status.LastAppliedKind, as.Status.IntegrationID
if err := tc.DeleteIntegration(ctx, teamID, oldID); err != nil {
return fmt.Errorf("DELETE /api/teams/%d/integrations/%d (rotating kind %q -> %q): %w",
teamID, oldID, oldKind, as.Spec.Kind, err)
}
// reconcileCreate's own crash-recovery path trusts this Secret's mere
// existence as "a create already succeeded" -- it must be gone before
// calling it here, or rotation would misread the about-to-be-stale
// old id right back out of it instead of minting a replacement.
secretName := webhookSecretName(as)
if err := r.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: as.Namespace}}); err != nil && !apierrors.IsNotFound(err) {
return fmt.Errorf("deleting stale webhook Secret %s/%s: %w", as.Namespace, secretName, err)
}
as.Status.IntegrationID = 0
as.Status.WebhookURLSecretRef = nil
if err := r.reconcileCreate(ctx, as, tc, teamID); err != nil {
return err
}
if r.Recorder != nil {
r.Recorder.Eventf(as, nil, corev1.EventTypeWarning, terdutv1alpha1.ReasonKindRotated, terdutv1alpha1.ReasonKindRotated,
"spec.kind changed from %q to %q: deleted integration %d and minted a new one (%d) -- the webhook URL/key changed, update whatever was sending to the old one",
oldKind, as.Spec.Kind, oldID, as.Status.IntegrationID)
}
return nil
}
// writeWebhookSecret creates or replaces the generated Secret holding
// integ's key material, owned by as (plain OwnerReference, same namespace,
// per DESIGN.md §7 -- no finalizer needed for this one, unlike the
// cross-namespace credential Secrets elsewhere in this operator).
func (r *TerdutAlertSourceReconciler) writeWebhookSecret(
ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, name string, integ *tdclient.Integration,
) error {
secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: as.Namespace}}
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error {
secret.Data = map[string][]byte{
webhookSecretURLKey: []byte(integ.URL),
webhookSecretKeyDataKey: []byte(integ.Key),
webhookSecretIntegrationIDKey: []byte(strconv.FormatInt(integ.ID, 10)),
}
return controllerutil.SetControllerReference(as, secret, r.Scheme)
})
return err
}
func (r *TerdutAlertSourceReconciler) setNotReady(
ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, reason, message string, d time.Duration,
) (ctrl.Result, error) {
meta.SetStatusCondition(&as.Status.Conditions, metav1.Condition{
Type: terdutv1alpha1.ConditionReady,
Status: metav1.ConditionFalse,
Reason: reason,
Message: message,
})
as.Status.ObservedGeneration = as.Generation
if err := r.Status().Update(ctx, as); err != nil {
return ctrl.Result{}, err
}
if r.Recorder != nil {
r.Recorder.Eventf(as, nil, corev1.EventTypeWarning, reason, reason, message)
}
return ctrl.Result{RequeueAfter: d}, nil
}
// reconcileDelete calls the real DELETE this resource has (unlike
// TerdutEscalationRule) if the team is still resolvable and an integration
// was ever created, then removes the finalizer unconditionally. The
// webhook Secret needs no explicit cleanup here -- it's same-namespace and
// OwnerReference-GC'd (DESIGN.md §7), not this finalizer's job.
func (r *TerdutAlertSourceReconciler) reconcileDelete(
ctx context.Context, as *terdutv1alpha1.TerdutAlertSource, newClient func(string) *tdclient.Client,
) (ctrl.Result, error) {
if !controllerutil.ContainsFinalizer(as, alertSourceFinalizerName) {
return ctrl.Result{}, nil
}
if as.Status.IntegrationID != 0 {
if team, tc, resolveErr := resolveTeamAndClient(
ctx, r.Client, r.OperatorNamespace, as.Namespace, as.Spec.TeamRef, newClient,
); resolveErr == nil {
if err := tc.DeleteIntegration(ctx, team.Status.TeamID, as.Status.IntegrationID); err != nil {
if r.Recorder != nil {
r.Recorder.Eventf(as, nil, corev1.EventTypeWarning, "DeleteFailed", "DeleteFailed", err.Error())
}
return ctrl.Result{}, err
}
}
}
controllerutil.RemoveFinalizer(as, alertSourceFinalizerName)
return ctrl.Result{}, r.Update(ctx, as)
}
// SetupWithManager sets up the controller with the Manager.
func (r *TerdutAlertSourceReconciler) SetupWithManager(mgr ctrl.Manager) error {
if r.NewClient == nil {
r.NewClient = tdclient.New
}
if r.Recorder == nil {
r.Recorder = mgr.GetEventRecorder("terdutalertsource-controller")
}
return ctrl.NewControllerManagedBy(mgr).
For(&terdutv1alpha1.TerdutAlertSource{}).
// Watched, not just the CR (DESIGN.md §5): the webhook Secret's
// loss has to be noticed on its own, independent of any spec
// change to this CR, for ReasonWebhookSecretLost to ever fire.
Owns(&corev1.Secret{}).
Named("terdutalertsource").
Complete(r)
}
@@ -0,0 +1,297 @@
package controller
import (
"context"
"net/http/httptest"
"strconv"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)
var _ = Describe("TerdutAlertSource Controller", func() {
const operatorNamespace = "default"
var (
reconciler *TerdutAlertSourceReconciler
fake *fakeTerdutServer
fakeSrv *httptest.Server
srv *terdutv1alpha1.TerdutServer
team *terdutv1alpha1.TerdutTeam
srcName string
srcKey types.NamespacedName
)
BeforeEach(func(ctx SpecContext) {
fake, fakeSrv = newFakeTerdutServer()
DeferCleanup(fakeSrv.Close)
srv = bootstrapReadyTerdutServer(ctx, uniqueName("asserver"), fakeSrv.URL)
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("asteam"), srv, fakeSrv.URL)
reconciler = &TerdutAlertSourceReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
OperatorNamespace: operatorNamespace,
NewClient: func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) },
}
srcName = uniqueName("alertsource")
srcKey = types.NamespacedName{Name: srcName, Namespace: operatorNamespace}
})
AfterEach(func(ctx SpecContext) {
as := &terdutv1alpha1.TerdutAlertSource{}
if err := k8sClient.Get(ctx, srcKey, as); err == nil {
as.Finalizers = nil
_ = k8sClient.Update(ctx, as)
_ = k8sClient.Delete(ctx, as)
}
_ = k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
Name: srcName + "-terdut-webhook", Namespace: operatorNamespace,
}})
teamKey := types.NamespacedName{Name: team.Name, Namespace: operatorNamespace}
if err := k8sClient.Get(ctx, teamKey, team); err == nil {
team.Finalizers = nil
_ = k8sClient.Update(ctx, team)
_ = k8sClient.Delete(ctx, team)
}
srvKey := types.NamespacedName{Name: srv.Name, Namespace: operatorNamespace}
if err := k8sClient.Get(ctx, srvKey, srv); err == nil {
srv.Finalizers = nil
_ = k8sClient.Update(ctx, srv)
_ = k8sClient.Delete(ctx, srv)
}
})
createSource := func(ctx context.Context, teamRef terdutv1alpha1.TerdutTeamRef, kind, name string) {
as := &terdutv1alpha1.TerdutAlertSource{
ObjectMeta: metav1.ObjectMeta{Name: srcName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutAlertSourceSpec{TeamRef: teamRef, Kind: kind, Name: name},
}
Expect(k8sClient.Create(ctx, as)).To(Succeed())
}
reconcileOnce := func(ctx context.Context) {
_, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: srcKey})
Expect(err).NotTo(HaveOccurred())
}
readyCondition := func(ctx context.Context) metav1.Condition {
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
c := meta.FindStatusCondition(as.Status.Conditions, terdutv1alpha1.ConditionReady)
Expect(c).NotTo(BeNil())
return *c
}
sameTeamRef := func() terdutv1alpha1.TerdutTeamRef {
return terdutv1alpha1.TerdutTeamRef{Name: team.Name}
}
webhookSecret := func(ctx context.Context) corev1.Secret {
var secret corev1.Secret
Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: srcName + "-terdut-webhook", Namespace: operatorNamespace,
}, &secret)).To(Succeed())
return secret
}
Describe("the happy path", func() {
It("creates the integration and writes the webhook Secret", func(ctx SpecContext) {
createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx) // create
cond := readyCondition(ctx)
Expect(cond.Status).To(Equal(metav1.ConditionTrue))
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonChildAdopted))
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
Expect(as.Status.IntegrationID).NotTo(BeZero())
Expect(as.Status.LastAppliedKind).To(Equal("alertmanager"))
Expect(as.Status.WebhookURLSecretRef).NotTo(BeNil())
Expect(as.Status.WebhookURLSecretRef.Name).To(Equal(srcName + "-terdut-webhook"))
secret := webhookSecret(ctx)
Expect(secret.Data[webhookSecretKeyDataKey]).NotTo(BeEmpty())
Expect(secret.Data[webhookSecretURLKey]).NotTo(BeEmpty())
Expect(secret.OwnerReferences).To(HaveLen(1), "same-namespace generated Secret should be owner-referenced, not finalizer-cleaned")
created, ok := fake.integrations[team.Status.TeamID][as.Status.IntegrationID]
Expect(ok).To(BeTrue())
Expect(created.Name).To(Equal("prod-alertmanager"))
Expect(created.Kind).To(Equal("alertmanager"))
})
})
Describe("renaming", func() {
It("PATCHes the new name without rotating the key", func(ctx SpecContext) {
createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager")
reconcileOnce(ctx)
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
integrationID := as.Status.IntegrationID
secretBefore := webhookSecret(ctx)
as.Spec.Name = "prod-alertmanager-renamed"
Expect(k8sClient.Update(ctx, as)).To(Succeed())
reconcileOnce(ctx)
Expect(fake.integrations[team.Status.TeamID][integrationID].Name).To(Equal("prod-alertmanager-renamed"))
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
Expect(as.Status.IntegrationID).To(Equal(integrationID), "renaming must not rotate the integration id")
secretAfter := webhookSecret(ctx)
Expect(secretAfter.Data[webhookSecretKeyDataKey]).To(Equal(secretBefore.Data[webhookSecretKeyDataKey]), "renaming must not rotate the webhook key")
})
})
Describe("a kind change", func() {
It("deletes the old integration, mints a new one, and rotates the webhook Secret", func(ctx SpecContext) {
createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager")
reconcileOnce(ctx)
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
oldIntegrationID := as.Status.IntegrationID
secretBefore := webhookSecret(ctx)
// terdut-server only supports one kind today (DESIGN.md §4.5),
// so there's no second real value to rotate to -- this fake
// server doesn't validate kind at all, so re-POSTing under the
// same literal string still exercises the full
// delete-then-create rotation path and produces a fresh id
// and key, which is everything this test needs to verify.
as.Spec.Kind = "alertmanager"
as.Status.LastAppliedKind = "something-else"
Expect(k8sClient.Status().Update(ctx, as)).To(Succeed())
reconcileOnce(ctx)
Expect(fake.integrationDelete[oldIntegrationID]).To(BeTrue())
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
Expect(as.Status.IntegrationID).NotTo(Equal(oldIntegrationID))
Expect(as.Status.LastAppliedKind).To(Equal("alertmanager"))
secretAfter := webhookSecret(ctx)
Expect(secretAfter.Data[webhookSecretKeyDataKey]).NotTo(Equal(secretBefore.Data[webhookSecretKeyDataKey]), "a kind rotation must mint a new webhook key")
})
})
Describe("crash recovery between POST and status write", func() {
It("adopts the webhook Secret's own record instead of minting a second integration", func(ctx SpecContext) {
createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager")
reconcileOnce(ctx) // finalizer only -- status.integrationID stays 0
// Simulate a previous reconcile that got as far as writing the
// Secret (the only durable record this resource can have, per
// its own controller comment) but crashed before status.
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: srcName + "-terdut-webhook", Namespace: operatorNamespace},
Data: map[string][]byte{
webhookSecretURLKey: []byte("https://terdut.example.invalid/api/integrations/orphaned-key/alertmanager"),
webhookSecretKeyDataKey: []byte("orphaned-key"),
webhookSecretIntegrationIDKey: []byte(strconv.FormatInt(999, 10)),
},
}
Expect(k8sClient.Create(ctx, secret)).To(Succeed())
reconcileOnce(ctx)
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
Expect(as.Status.IntegrationID).To(Equal(int64(999)))
Expect(fake.integrations[team.Status.TeamID]).To(BeEmpty(), "should never have called POST at all")
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
})
})
Describe("webhook Secret loss", func() {
It("reports WebhookSecretLost and does not mint a replacement", func(ctx SpecContext) {
createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager")
reconcileOnce(ctx)
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
integrationID := as.Status.IntegrationID
Expect(k8sClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
Name: srcName + "-terdut-webhook", Namespace: operatorNamespace,
}})).To(Succeed())
reconcileOnce(ctx)
cond := readyCondition(ctx)
Expect(cond.Status).To(Equal(metav1.ConditionFalse))
Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonWebhookSecretLost))
Expect(fake.integrationDelete[integrationID]).To(BeFalse(), "fail-closed: must not touch the still-live integration server-side either")
})
})
Describe("waiting on the referenced TerdutTeam", func() {
It("reports TeamRefNotFound when the TerdutTeam doesn't exist", func(ctx SpecContext) {
createSource(ctx, terdutv1alpha1.TerdutTeamRef{Name: testRefNotFoundName}, "alertmanager", "prod-alertmanager")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonTeamRefNotFound))
})
It("reports WaitingForTeam when the TerdutTeam exists but isn't Ready yet", func(ctx SpecContext) {
unreadyName := uniqueName("asteam-unready")
unready := &terdutv1alpha1.TerdutTeam{
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutTeamSpec{
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
DisplayName: testUnreadyDisplayName,
},
}
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
DeferCleanup(func() { _ = k8sClient.Delete(ctx, unready) })
createSource(ctx, terdutv1alpha1.TerdutTeamRef{Name: unreadyName}, "alertmanager", "prod-alertmanager")
reconcileOnce(ctx) // finalizer
reconcileOnce(ctx)
Expect(readyCondition(ctx).Reason).To(Equal(terdutv1alpha1.ReasonWaitingForTeam))
})
})
Describe("deletion", func() {
It("deletes the integration server-side and removes the finalizer", func(ctx SpecContext) {
createSource(ctx, sameTeamRef(), "alertmanager", "prod-alertmanager")
reconcileOnce(ctx)
reconcileOnce(ctx)
Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
as := &terdutv1alpha1.TerdutAlertSource{}
Expect(k8sClient.Get(ctx, srcKey, as)).To(Succeed())
integrationID := as.Status.IntegrationID
Expect(k8sClient.Delete(ctx, as)).To(Succeed())
reconcileOnce(ctx) // runs the finalizer
Expect(fake.integrationDelete[integrationID]).To(BeTrue())
err := k8sClient.Get(ctx, srcKey, as)
Expect(err).To(HaveOccurred(), "the TerdutAlertSource itself should be gone once the finalizer clears")
})
})
})
@@ -32,7 +32,7 @@ var _ = Describe("TerdutDeadmanSwitch Controller", func() {
fake, fakeSrv = newFakeTerdutServer()
DeferCleanup(fakeSrv.Close)
srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("dmserver"), fakeSrv.URL)
srv = bootstrapReadyTerdutServer(ctx, uniqueName("dmserver"), fakeSrv.URL)
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("dmteam"), srv, fakeSrv.URL)
reconciler = &TerdutDeadmanSwitchReconciler{
@@ -177,7 +177,7 @@ var _ = Describe("TerdutDeadmanSwitch Controller", func() {
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutTeamSpec{
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
DisplayName: "unready",
DisplayName: testUnreadyDisplayName,
},
}
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
@@ -32,7 +32,7 @@ var _ = Describe("TerdutEscalationRule Controller", func() {
fake, fakeSrv = newFakeTerdutServer()
DeferCleanup(fakeSrv.Close)
srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("erserver"), fakeSrv.URL)
srv = bootstrapReadyTerdutServer(ctx, uniqueName("erserver"), fakeSrv.URL)
team = readyTerdutTeam(ctx, operatorNamespace, uniqueName("erteam"), srv, fakeSrv.URL)
reconciler = &TerdutEscalationRuleReconciler{
@@ -164,7 +164,7 @@ var _ = Describe("TerdutEscalationRule Controller", func() {
ObjectMeta: metav1.ObjectMeta{Name: unreadyName, Namespace: operatorNamespace},
Spec: terdutv1alpha1.TerdutTeamSpec{
ServerRef: terdutv1alpha1.TerdutServerRef{Name: srv.Name},
DisplayName: "unready",
DisplayName: testUnreadyDisplayName,
},
}
Expect(k8sClient.Create(ctx, unready)).To(Succeed())
@@ -72,6 +72,13 @@ type fakeTerdutServer struct {
nextSwitchID int64
switches map[int64]map[int64]tdclient.DeadmanSwitch // teamID -> switchID -> switch
switchDelete map[int64]bool // switchID -> true once DELETEd, for 404-on-redelete
// integrations/nextIntegrationID/integrationDelete back the alert
// source endpoints -- no unique-name constraint server-side either
// (DESIGN.md §4.5), same shape as switches, keyed by id.
nextIntegrationID int64
integrations map[int64]map[int64]tdclient.Integration // teamID -> integrationID -> integration
integrationDelete map[int64]bool // integrationID -> true once DELETEd, for 404-on-redelete
}
func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) {
@@ -86,6 +93,9 @@ func newFakeTerdutServer() (*fakeTerdutServer, *httptest.Server) {
escalation: map[int64]tdclient.SetEscalationRequest{},
switches: map[int64]map[int64]tdclient.DeadmanSwitch{},
switchDelete: map[int64]bool{},
integrations: map[int64]map[int64]tdclient.Integration{},
integrationDelete: map[int64]bool{},
}
return f, httptest.NewServer(f)
}
@@ -314,6 +324,76 @@ func (f *fakeTerdutServer) handleTeamSubPath(w http.ResponseWriter, r *http.Requ
f.switchDelete[switchID] = true
w.WriteHeader(http.StatusNoContent)
case rest == "/integrations" || strings.HasPrefix(rest, "/integrations/"):
f.handleIntegrationSubPath(w, r, id, rest)
default:
w.WriteHeader(http.StatusNotFound)
}
}
// handleIntegrationSubPath answers POST /api/teams/{id}/integrations,
// PATCH .../integrations/{integrationID} and DELETE .../integrations/{integrationID}
// -- split out of handleTeamSubPath so that switch's own cyclomatic
// complexity stays under golangci-lint's gocyclo threshold.
func (f *fakeTerdutServer) handleIntegrationSubPath(w http.ResponseWriter, r *http.Request, id int64, rest string) {
switch {
case rest == "/integrations" && r.Method == http.MethodPost:
var req struct {
Name string `json:"name"`
Kind string `json:"kind"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
f.nextIntegrationID++
integID := f.nextIntegrationID
integ := tdclient.Integration{
ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name,
// Key/URL are only ever in *this* response -- never again,
// matching terdut-server's own one-time-show semantics
// (DESIGN.md §4.5) -- so what's stored for later GET/PATCH
// calls in this fake deliberately omits them too.
Key: fmt.Sprintf("webhook-key-%d", integID),
URL: fmt.Sprintf("https://terdut.example.invalid/api/integrations/webhook-key-%d/%s", integID, req.Kind),
}
if f.integrations[id] == nil {
f.integrations[id] = map[int64]tdclient.Integration{}
}
f.integrations[id][integID] = tdclient.Integration{ID: integID, TeamID: id, Kind: req.Kind, Name: req.Name}
writeJSON(w, http.StatusCreated, integ)
case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodPatch:
integID, ok := parseTrailingID(rest, "/integrations/")
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
existing, exists := f.integrations[id][integID]
if !exists {
w.WriteHeader(http.StatusNotFound)
return
}
var req struct {
Name string `json:"name"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
existing.Name = req.Name
f.integrations[id][integID] = existing
w.WriteHeader(http.StatusNoContent)
case strings.HasPrefix(rest, "/integrations/") && r.Method == http.MethodDelete:
integID, ok := parseTrailingID(rest, "/integrations/")
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
if _, exists := f.integrations[id][integID]; !exists {
w.WriteHeader(http.StatusNotFound)
return
}
delete(f.integrations[id], integID)
f.integrationDelete[integID] = true
w.WriteHeader(http.StatusNoContent)
default:
w.WriteHeader(http.StatusNotFound)
}
@@ -33,7 +33,7 @@ var _ = Describe("TerdutTeam Controller", func() {
fake, fakeSrv = newFakeTerdutServer()
DeferCleanup(fakeSrv.Close)
srv = bootstrapReadyTerdutServer(ctx, operatorNamespace, uniqueName("ttserver"), fakeSrv.URL)
srv = bootstrapReadyTerdutServer(ctx, uniqueName("ttserver"), fakeSrv.URL)
reconciler = &TerdutTeamReconciler{
Client: k8sClient,
+20 -4
View File
@@ -26,10 +26,25 @@ const (
// testRefNotFoundName is a name no TerdutServer/TerdutTeam ever gets
// created with -- shared by every "...RefNotFound" test across
// terdutteam_controller_test.go, terdutescalationrule_controller_test.go
// and terdutdeadmanswitch_controller_test.go, so goconst doesn't flag
// three independent copies of the same literal.
// terdutteam_controller_test.go, terdutescalationrule_controller_test.go,
// terdutdeadmanswitch_controller_test.go and
// terdutalertsource_controller_test.go, so goconst doesn't flag four
// independent copies of the same literal.
testRefNotFoundName = "does-not-exist"
// testUnreadyDisplayName is the TerdutTeam.spec.displayName every
// "...exists but isn't Ready yet" test across the same four files uses
// for its deliberately-never-reconciled fixture team, for the same
// goconst reason as testRefNotFoundName above.
testUnreadyDisplayName = "unready"
// testOperatorNamespace is every test file's own namespace for both
// the operator's generated/credential objects and the CRs under test
// -- always "default" in this suite, so bootstrapReadyTerdutServer
// below takes no namespace parameter of its own (golangci-lint's
// unparam flagged it once a fourth same-valued caller made that
// obvious): there's never a second value to pass.
testOperatorNamespace = "default"
)
// bootstrapReadyTerdutServer creates a TerdutServer with a bring-your-own
@@ -37,8 +52,9 @@ const (
// terdutserver_controller_test.go's own happy-path test exercises directly
// -- shared here so TerdutTeam's tests (which need a real, Ready
// TerdutServer to resolve against) don't duplicate it.
func bootstrapReadyTerdutServer(ctx context.Context, namespace, name, fakeURL string) *terdutv1alpha1.TerdutServer {
func bootstrapReadyTerdutServer(ctx context.Context, name, fakeURL string) *terdutv1alpha1.TerdutServer {
GinkgoHelper()
namespace := testOperatorNamespace
reconciler := &TerdutServerReconciler{
Client: k8sClient,
+70
View File
@@ -23,6 +23,10 @@ import (
// fieldName is the JSON key every create/rename request body below shares.
const fieldName = "name"
// fieldKind is the JSON key an integration's create request body shares
// with the terdutv1alpha1.TerdutAlertSourceSpec field of the same name.
const fieldKind = "kind"
type Client struct {
baseURL string
httpClient *http.Client
@@ -496,3 +500,69 @@ func (c *Client) DeleteDeadmanSwitch(ctx context.Context, teamID, switchID int64
}
return c.do(req, nil)
}
// Integration mirrors terdut-server's models.Integration, minus
// CreatedAt/LastUsedAt, which this client never reads. Key/URL are only
// ever populated by CreateIntegration's own response -- the one moment
// either value exists outside terdut-server's own database (DESIGN.md
// §4.5: shown once, never re-readable, same handling as the bootstrap
// admin key).
type Integration struct {
ID int64 `json:"id"`
TeamID int64 `json:"team_id"`
Kind string `json:"kind"`
Name string `json:"name"`
Key string `json:"key,omitempty"`
URL string `json:"url,omitempty"`
}
// CreateIntegration calls POST /api/teams/{teamID}/integrations --
// owner-gated (requireTeamOwner), so c must hold this team's own
// team-scoped credential. No conflict handling exists server-side at all
// for this resource (no unique constraint on name, confirmed against
// source) -- deliberately not treated as this resource's idempotent-create
// recovery path; see the controller's own reasoning for why a crash
// between this call succeeding and the webhook Secret being written can't
// be recovered by listing and adopting a same-named row the way
// TerdutDeadmanSwitch does.
func (c *Client) CreateIntegration(ctx context.Context, teamID int64, name, kind string) (*Integration, error) {
req, err := c.newRequest(ctx, http.MethodPost, fmt.Sprintf("/api/teams/%d/integrations", teamID),
map[string]string{fieldName: name, fieldKind: kind})
if err != nil {
return nil, err
}
var integ Integration
if err := c.do(req, &integ); err != nil {
return nil, err
}
return &integ, nil
}
// RenameIntegration calls PATCH /api/teams/{teamID}/integrations/{integrationID}
// -- owner-gated, same credential requirement as CreateIntegration. Never
// touches the key (terdut-server's own handler comment: "the key is
// untouched, so nothing posting with it notices"), so this is safe to call
// every reconcile unconditionally rather than only on detected drift --
// the same "cheap, so just always sync it" reasoning TerdutEscalationRule's
// whole-policy PUT uses.
func (c *Client) RenameIntegration(ctx context.Context, teamID, integrationID int64, name string) error {
req, err := c.newRequest(ctx, http.MethodPatch,
fmt.Sprintf("/api/teams/%d/integrations/%d", teamID, integrationID),
map[string]string{fieldName: name})
if err != nil {
return err
}
return c.do(req, nil)
}
// DeleteIntegration calls DELETE /api/teams/{teamID}/integrations/{integrationID}
// -- owner-gated, same credential requirement as CreateIntegration. Used
// both by the finalizer and by the kind-change rotation path (DESIGN.md §5).
func (c *Client) DeleteIntegration(ctx context.Context, teamID, integrationID int64) error {
req, err := c.newRequest(ctx, http.MethodDelete,
fmt.Sprintf("/api/teams/%d/integrations/%d", teamID, integrationID), nil)
if err != nil {
return err
}
return c.do(req, nil)
}