terdut-operator/internal/controller/terdutserver_bootstrap.go

package controller

import (
	"context"
	"errors"
	"fmt"
	"net/http"

	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"

	terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1"
	"git.ryuvia.com/niklas/terdut-operator/internal/tdclient"
)

// bootstrapStateLostError is DESIGN.md §6's one genuinely pathological
// case: the server's database is already bootstrapped, and no credential for
// it survives here -- a checkpointed admin key was used and then lost before
// the lasting credential could be persisted, or the instance credential
// Secret was deleted (spec.credentials.deletionPolicy: Delete, or by hand).
// Distinct from a plain error so Reconcile can route it to a Ready: False
// condition rather than treating it as a transient reconcile failure: no
// retry can fix it.
type bootstrapStateLostError struct {
	detail     string
	secretName string // the instance credential Secret that would have fixed it
}

func (e *bootstrapStateLostError) Error() string {
	return fmt.Sprintf(
		"server reports already bootstrapped, but there is no credential for it here: %s. "+
			"The operator cannot mint one on its own, and deleting and recreating this TerdutServer does "+
			"not clear the database. Restore Secret %q in the operator's namespace if you have a copy, "+
			"or reset the server's database and recreate this TerdutServer", e.detail, e.secretName)
}

// reconcileBootstrap implements DESIGN.md §6 point 1's self-registration
// flow, checkpointed against the two real crash windows in it rather than
// leaving them as theoretical gaps. Only called once
// srv.Status.CredentialsSecretRef is nil and the Deployment has a ready
// replica.
func (r *TerdutServerReconciler) reconcileBootstrap(ctx context.Context, srv *terdutv1alpha1.TerdutServer) error {
	// A TerdutServer recreated against a database that is already
	// bootstrapped: its instance credential may still be here (the default
	// deletion policy keeps it), and then there is nothing to bootstrap.
	adopted, err := r.adoptRetainedCredentials(ctx, srv)
	if err != nil {
		return err
	}
	if adopted {
		return nil
	}

	adminKey, err := r.getOrCreateCheckpointedAdminKey(ctx, srv)
	if err != nil {
		return err
	}

	bc := r.NewClient(serviceURL(srv)).WithToken(adminKey)
	instanceKey, err := r.getOrMintInstanceServiceAccountKey(ctx, bc)
	if err != nil {
		return err
	}

	credsName := credentialsSecretName(srv)
	if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, credsName, instanceKey); err != nil {
		return err
	}
	srv.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: credsName, Key: credentialsSecretDataKey}

	// Best-effort: the checkpoint has done its job. Leaving it behind on a
	// delete failure here isn't a correctness problem (the next reconcile
	// finds status.CredentialsSecretRef already set and never looks at the
	// checkpoint again) — it would just be an unused Secret sitting around,
	// cleaned up for real by the finalizer on delete.
	checkpoint := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: checkpointSecretName(srv), Namespace: r.OperatorNamespace}}
	_ = r.Delete(ctx, checkpoint)

	return nil
}

// adoptRetainedCredentials looks for the instance credential Secret an
// earlier TerdutServer of the same name and namespace left behind
// (spec.credentials.deletionPolicy: Retain), and adopts it if the server
// still accepts its token. Reports whether it did.
//
// The token is tried before it is trusted: a Secret that outlived a database
// reset holds a key the server has never heard of, and adopting that would
// make every later call 401. A rejected token (401/403) is not an error here;
// it just means there is nothing to adopt, and bootstrap proceeds as it would
// for a first install -- which succeeds against a freshly reset database and
// replaces the Secret. Anything else (the server unreachable, a 5xx) is
// returned for a retry rather than guessed at.
func (r *TerdutServerReconciler) adoptRetainedCredentials(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (bool, error) {
	name := credentialsSecretName(srv)
	var sec corev1.Secret
	if err := r.Get(ctx, client.ObjectKey{Namespace: r.OperatorNamespace, Name: name}, &sec); err != nil {
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		return false, err
	}
	token := string(sec.Data[credentialsSecretDataKey])
	if token == "" {
		return false, nil
	}

	// The operator's own service account is the one thing this credential
	// is for, so asking the server for it by name checks the token and that
	// it belongs to that account in the same call.
	sa, err := r.NewClient(serviceURL(srv)).WithToken(token).GetServiceAccountByName(ctx, serviceAccountName)
	if err != nil {
		if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok &&
			(statusErr.Code == http.StatusUnauthorized || statusErr.Code == http.StatusForbidden) {
			return false, nil
		}
		return false, fmt.Errorf("checking the retained credential in Secret %q: %w", name, err)
	}
	if sa == nil {
		return false, nil
	}

	srv.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: name, Key: credentialsSecretDataKey}
	return true, nil
}

// getOrCreateCheckpointedAdminKey returns a usable admin key: from the
// checkpoint Secret if an earlier, interrupted attempt already got one, or
// freshly from /api/bootstrap, immediately checkpointed before it's used
// for anything else.
func (r *TerdutServerReconciler) getOrCreateCheckpointedAdminKey(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (string, error) {
	checkpointName := checkpointSecretName(srv)

	var checkpoint corev1.Secret
	err := r.Get(ctx, client.ObjectKey{Namespace: r.OperatorNamespace, Name: checkpointName}, &checkpoint)
	switch {
	case err == nil:
		return string(checkpoint.Data[credentialsSecretDataKey]), nil
	case !apierrors.IsNotFound(err):
		return "", err
	}

	bc := r.NewClient(serviceURL(srv))
	result, err := bc.Bootstrap(ctx, bootstrapUsername, bootstrapEmail)
	if err != nil {
		if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok && statusErr.Code == http.StatusForbidden {
			// §1: this operator is the only thing that ever bootstraps a
			// server it created, so a 403 here (no checkpoint, no
			// status.credentialsSecretRef) means a prior reconcile already
			// won this exact race and its checkpoint was lost afterward --
			// the one case §6 doesn't try to paper over.
			return "", &bootstrapStateLostError{
				detail:     "/api/bootstrap returned 403",
				secretName: credentialsSecretName(srv),
			}
		}
		return "", fmt.Errorf("POST /api/bootstrap: %w", err)
	}

	if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, checkpointName, result.APIKey.Key); err != nil {
		return "", fmt.Errorf("checkpointing admin key: %w", err)
	}
	return result.APIKey.Key, nil
}

// getOrMintInstanceServiceAccountKey mints the operator's own instance-
// scoped service account, or, if an earlier interrupted attempt already
// created it (409), adopts it and mints a fresh key rather than treating
// the conflict as an error (DESIGN.md §6 point 1, §5's general
// adopt-on-conflict rule).
func (r *TerdutServerReconciler) getOrMintInstanceServiceAccountKey(ctx context.Context, bc *tdclient.Client) (string, error) {
	result, err := bc.CreateInstanceServiceAccount(ctx, serviceAccountName)
	if err == nil {
		return result.Key.Key, nil
	}

	statusErr, ok := errors.AsType[*tdclient.StatusError](err)
	if !ok || statusErr.Code != http.StatusConflict {
		return "", fmt.Errorf("POST /api/service-accounts: %w", err)
	}

	sa, err := bc.GetServiceAccountByName(ctx, serviceAccountName)
	if err != nil {
		return "", fmt.Errorf("GET /api/service-accounts?name=%s (adopting after 409): %w", serviceAccountName, err)
	}
	if sa == nil {
		return "", fmt.Errorf("POST /api/service-accounts 409'd for %q but GET found nothing", serviceAccountName)
	}
	key, err := bc.CreateServiceAccountKey(ctx, sa.ID, "initial")
	if err != nil {
		return "", fmt.Errorf("POST /api/service-accounts/%d/keys (adopting after 409): %w", sa.ID, err)
	}
	return key.Key, nil
}