Keep the instance credential across a TerdutServer delete, and adopt it on recreate

Deleting a TerdutServer removed the credential Secrets but never touched the database, so a recreated one found a server that was already bootstrapped and no key for it: /api/bootstrap answered 403 and the operator stopped at BootstrapStateLost, whose message and DESIGN.md both said "delete and recreate". That is how the terdut-demo install on the cluster got stuck on 2026-10-03: Helm's cleanupOnFail deleted its TerdutServer after a failed upgrade, the recreate found the bootstrapped database, and it sat at Ready: False for five days until the database was reset by hand. Recreating cannot fix it, because the finalizer clears Secrets and the database is not its to reset, so "a fresh create starts clean" was only ever true when the database went with it. spec.credentials.deletionPolicy is Retain by default: the finalizer keeps the instance credential Secret (Delete removes it, as before). The bootstrap checkpoint is always removed. Before calling /api/bootstrap, reconcile now looks for the retained Secret and asks the server for the operator's own service account with its token. Accepted: adopt it and skip bootstrap. Rejected with 401/403: the Secret outlived a database reset, so ignore it and bootstrap like a first install, which replaces it. Any other error retries. terdut-server's own tests already call that endpoint with an instance-scoped key, so the permission is not new. BootstrapStateLost is still the answer when the server is bootstrapped and no credential it accepts survives, but its message now names the Secret to restore and says that recreating does not clear the database. DESIGN.md §6 says the same, and the chart passes the setting through as terdutServer.credentials.deletionPolicy. A retained Secret of a TerdutServer that is gone for good is an orphan to delete by hand. It is inert: nothing adopts it unless the server accepts the token. Checked on the kind demo with a locally built image against the real terdut-server v0.43.0: deleting the TerdutServer kept the Secret, recreating it reached Ready with the same credential (identical hash) and both TerdutTeams came back Ready with their original ids. The controller specs cover adoption, a rejected token after a reset, the bootstrapped-and-rejected failure, and both deletion policies. Co-authored-by: Claude <noreply@anthropic.com>
2026-10-08 21:07:38 +02:00
parent 6572f63157
commit 62664c93ff
10 changed files with 361 additions and 50 deletions
@@ -54,9 +54,13 @@ type fakeTerdutServer struct {
 	mu           sync.Mutex
 	bootstrapped bool
 	bootstrap403 bool // force every /api/bootstrap call to 403, even the first
-	nextID       int64
-	accounts     map[string]int64 // name -> id
-	keyMints     map[int64]int    // id -> number of keys minted so far
+	// rejectedTokens: bearer tokens the fake answers 401 on GET
+	// /api/service-accounts, the way a server that never issued the key
+	// (a database reset since) would.
+	rejectedTokens map[string]bool
+	nextID         int64
+	accounts       map[string]int64 // name -> id
+	keyMints       map[int64]int    // id -> number of keys minted so far

 	nextTeamID int64
 	teams      map[string]int64 // name -> id
@@ -170,6 +174,10 @@ func (f *fakeTerdutServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		})

 	case r.URL.Path == "/api/service-accounts" && r.Method == http.MethodGet:
+		if f.rejectedTokens[strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")] {
+			writeJSON(w, http.StatusUnauthorized, map[string]string{errJSONKey: "invalid or expired API key"})
+			return
+		}
 		name := r.URL.Query().Get("name")
 		id, exists := f.accounts[name]
 		if !exists {
@@ -875,13 +883,14 @@ var _ = Describe("TerdutServer Controller", func() {
 	})

 	Describe("deletion", func() {
-		It("removes the credentials and checkpoint Secrets and the finalizer", func(ctx SpecContext) {
-			fake, fakeSrv := newFakeTerdutServer()
-			_ = fake
+		// runToReady brings a server to Ready against a fresh fake and
+		// returns the name of the instance credential Secret it minted.
+		runToReady := func(ctx SpecContext, spec terdutv1alpha1.TerdutServerSpec) string {
+			_, fakeSrv := newFakeTerdutServer()
 			DeferCleanup(fakeSrv.Close)
 			reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }

-			createServer(ctx, dsnSpec())
+			createServer(ctx, spec)
 			reconcileOnce(ctx)
 			reconcileOnce(ctx)
 			markDeploymentReady(ctx)
@@ -889,17 +898,114 @@ var _ = Describe("TerdutServer Controller", func() {

 			srv := &terdutv1alpha1.TerdutServer{}
 			Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
-			credsName := srv.Status.CredentialsSecretRef.Name
-
+			return srv.Status.CredentialsSecretRef.Name
+		}
+		deleteAndFinalize := func(ctx SpecContext) {
+			srv := &terdutv1alpha1.TerdutServer{}
+			Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
 			Expect(k8sClient.Delete(ctx, srv)).To(Succeed())
 			reconcileOnce(ctx) // runs the finalizer
+			Expect(k8sClient.Get(ctx, objKey, srv)).NotTo(Succeed(),
+				"the TerdutServer itself should be gone once the finalizer clears")
+		}
+		secretExists := func(ctx SpecContext, secretName string) bool {
+			var s corev1.Secret
+			err := k8sClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: operatorNamespace}, &s)
+			return err == nil
+		}

-			err := k8sClient.Get(ctx, objKey, srv)
-			Expect(err).To(HaveOccurred(), "the TerdutServer itself should be gone once the finalizer clears")
+		It("keeps the instance credential by default and removes the checkpoint", func(ctx SpecContext) {
+			credsName := runToReady(ctx, dsnSpec())
+			// A checkpoint left behind, as if the best-effort delete had failed.
+			Expect(k8sClient.Create(ctx, &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: checkpointSecretNameFor(name), Namespace: operatorNamespace},
+				Data:       map[string][]byte{credentialsSecretDataKey: []byte("admin-key-raw")},
+			})).To(Succeed())

-			var leftover corev1.Secret
-			err = k8sClient.Get(ctx, types.NamespacedName{Name: credsName, Namespace: operatorNamespace}, &leftover)
-			Expect(err).To(HaveOccurred(), "the credentials Secret should have been cleaned up by the finalizer")
+			deleteAndFinalize(ctx)
+
+			Expect(secretExists(ctx, credsName)).To(BeTrue(),
+				"the credential is kept so a recreated TerdutServer can adopt it")
+			Expect(secretExists(ctx, checkpointSecretNameFor(name))).To(BeFalse(),
+				"the checkpoint is a short-lived admin key and is always removed")
+		})
+
+		It("removes the credentials and checkpoint Secrets and the finalizer when the policy is Delete", func(ctx SpecContext) {
+			spec := dsnSpec()
+			spec.Credentials.DeletionPolicy = terdutv1alpha1.CredentialsDelete
+			credsName := runToReady(ctx, spec)
+
+			deleteAndFinalize(ctx)
+
+			Expect(secretExists(ctx, credsName)).To(BeFalse(),
+				"the credentials Secret should have been cleaned up by the finalizer")
+		})
+	})
+
+	Describe("recreating a TerdutServer against an already-bootstrapped server", func() {
+		// retainedSecret stands in for what the previous TerdutServer left.
+		retainedSecret := func(ctx SpecContext, token string) {
+			Expect(k8sClient.Create(ctx, &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: credentialsSecretNameFor(name), Namespace: operatorNamespace},
+				Data:       map[string][]byte{credentialsSecretDataKey: []byte(token)},
+			})).To(Succeed())
+		}
+		bringUp := func(ctx SpecContext, fakeSrv *httptest.Server) {
+			DeferCleanup(fakeSrv.Close)
+			reconciler.NewClient = func(string) *tdclient.Client { return tdclient.New(fakeSrv.URL) }
+			createServer(ctx, dsnSpec())
+			reconcileOnce(ctx) // finalizer
+			reconcileOnce(ctx) // Deployment/Service
+			markDeploymentReady(ctx)
+			reconcileOnce(ctx)
+		}
+		credsToken := func(ctx SpecContext) string {
+			var s corev1.Secret
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: credentialsSecretNameFor(name), Namespace: operatorNamespace}, &s)).To(Succeed())
+			return string(s.Data[credentialsSecretDataKey])
+		}
+
+		It("adopts the retained credential when the server still accepts it, without bootstrapping", func(ctx SpecContext) {
+			fake, fakeSrv := newFakeTerdutServer()
+			fake.bootstrapped = true // every /api/bootstrap call 403s
+			fake.accounts[serviceAccountName] = 7
+			retainedSecret(ctx, "retained-key")
+
+			bringUp(ctx, fakeSrv)
+
+			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
+			srv := &terdutv1alpha1.TerdutServer{}
+			Expect(k8sClient.Get(ctx, objKey, srv)).To(Succeed())
+			Expect(srv.Status.CredentialsSecretRef).NotTo(BeNil())
+			Expect(srv.Status.CredentialsSecretRef.Name).To(Equal(credentialsSecretNameFor(name)))
+			Expect(credsToken(ctx)).To(Equal("retained-key"), "the retained key is reused, not replaced")
+		})
+
+		It("ignores a retained credential the server rejects and bootstraps afresh after a database reset", func(ctx SpecContext) {
+			fake, fakeSrv := newFakeTerdutServer() // a fresh database: bootstrap succeeds
+			fake.rejectedTokens = map[string]bool{"stale-key": true}
+			retainedSecret(ctx, "stale-key")
+
+			bringUp(ctx, fakeSrv)
+
+			Expect(readyCondition(ctx).Status).To(Equal(metav1.ConditionTrue))
+			Expect(credsToken(ctx)).NotTo(Equal("stale-key"), "the stale credential is replaced by a freshly minted one")
+		})
+
+		It("fails closed, naming the Secret to restore, when the server is bootstrapped and the retained credential is rejected", func(ctx SpecContext) {
+			fake, fakeSrv := newFakeTerdutServer()
+			fake.bootstrapped = true
+			fake.rejectedTokens = map[string]bool{"stale-key": true}
+			retainedSecret(ctx, "stale-key")
+
+			bringUp(ctx, fakeSrv)
+
+			cond := readyCondition(ctx)
+			Expect(cond.Status).To(Equal(metav1.ConditionFalse))
+			Expect(cond.Reason).To(Equal(terdutv1alpha1.ReasonBootstrapStateLost))
+			Expect(cond.Message).To(ContainSubstring(credentialsSecretNameFor(name)),
+				"the message names the Secret that would restore it")
+			Expect(cond.Message).To(ContainSubstring("does not clear the database"))
 		})
 	})