package controller import ( "context" "errors" "fmt" "net/http" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" terdutv1alpha1 "git.ryuvia.com/niklas/terdut-operator/api/v1alpha1" "git.ryuvia.com/niklas/terdut-operator/internal/tdclient" ) // bootstrapStateLostError is DESIGN.md §6's one genuinely pathological // case: the server's database is already bootstrapped, and no credential for // it survives here -- a checkpointed admin key was used and then lost before // the lasting credential could be persisted, or the instance credential // Secret was deleted (spec.credentials.deletionPolicy: Delete, or by hand). // Distinct from a plain error so Reconcile can route it to a Ready: False // condition rather than treating it as a transient reconcile failure: no // retry can fix it. type bootstrapStateLostError struct { detail string secretName string // the instance credential Secret that would have fixed it } func (e *bootstrapStateLostError) Error() string { return fmt.Sprintf( "server reports already bootstrapped, but there is no credential for it here: %s. "+ "The operator cannot mint one on its own, and deleting and recreating this TerdutServer does "+ "not clear the database. Restore Secret %q in the operator's namespace if you have a copy, "+ "or reset the server's database and recreate this TerdutServer", e.detail, e.secretName) } // reconcileBootstrap implements DESIGN.md §6 point 1's self-registration // flow, checkpointed against the two real crash windows in it rather than // leaving them as theoretical gaps. Only called once // srv.Status.CredentialsSecretRef is nil and the Deployment has a ready // replica. func (r *TerdutServerReconciler) reconcileBootstrap(ctx context.Context, srv *terdutv1alpha1.TerdutServer) error { // A TerdutServer recreated against a database that is already // bootstrapped: its instance credential may still be here (the default // deletion policy keeps it), and then there is nothing to bootstrap. adopted, err := r.adoptRetainedCredentials(ctx, srv) if err != nil { return err } if adopted { return nil } adminKey, err := r.getOrCreateCheckpointedAdminKey(ctx, srv) if err != nil { return err } bc := r.NewClient(serviceURL(srv)).WithToken(adminKey) instanceKey, err := r.getOrMintInstanceServiceAccountKey(ctx, bc) if err != nil { return err } credsName := credentialsSecretName(srv) if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, credsName, instanceKey); err != nil { return err } srv.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: credsName, Key: credentialsSecretDataKey} // Best-effort: the checkpoint has done its job. Leaving it behind on a // delete failure here isn't a correctness problem (the next reconcile // finds status.CredentialsSecretRef already set and never looks at the // checkpoint again) — it would just be an unused Secret sitting around, // cleaned up for real by the finalizer on delete. checkpoint := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: checkpointSecretName(srv), Namespace: r.OperatorNamespace}} _ = r.Delete(ctx, checkpoint) return nil } // adoptRetainedCredentials looks for the instance credential Secret an // earlier TerdutServer of the same name and namespace left behind // (spec.credentials.deletionPolicy: Retain), and adopts it if the server // still accepts its token. Reports whether it did. // // The token is tried before it is trusted: a Secret that outlived a database // reset holds a key the server has never heard of, and adopting that would // make every later call 401. A rejected token (401/403) is not an error here; // it just means there is nothing to adopt, and bootstrap proceeds as it would // for a first install -- which succeeds against a freshly reset database and // replaces the Secret. Anything else (the server unreachable, a 5xx) is // returned for a retry rather than guessed at. func (r *TerdutServerReconciler) adoptRetainedCredentials(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (bool, error) { name := credentialsSecretName(srv) var sec corev1.Secret if err := r.Get(ctx, client.ObjectKey{Namespace: r.OperatorNamespace, Name: name}, &sec); err != nil { if apierrors.IsNotFound(err) { return false, nil } return false, err } token := string(sec.Data[credentialsSecretDataKey]) if token == "" { return false, nil } // The operator's own service account is the one thing this credential // is for, so asking the server for it by name checks the token and that // it belongs to that account in the same call. sa, err := r.NewClient(serviceURL(srv)).WithToken(token).GetServiceAccountByName(ctx, serviceAccountName) if err != nil { if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok && (statusErr.Code == http.StatusUnauthorized || statusErr.Code == http.StatusForbidden) { return false, nil } return false, fmt.Errorf("checking the retained credential in Secret %q: %w", name, err) } if sa == nil { return false, nil } srv.Status.CredentialsSecretRef = &terdutv1alpha1.SecretKeyRef{Name: name, Key: credentialsSecretDataKey} return true, nil } // getOrCreateCheckpointedAdminKey returns a usable admin key: from the // checkpoint Secret if an earlier, interrupted attempt already got one, or // freshly from /api/bootstrap, immediately checkpointed before it's used // for anything else. func (r *TerdutServerReconciler) getOrCreateCheckpointedAdminKey(ctx context.Context, srv *terdutv1alpha1.TerdutServer) (string, error) { checkpointName := checkpointSecretName(srv) var checkpoint corev1.Secret err := r.Get(ctx, client.ObjectKey{Namespace: r.OperatorNamespace, Name: checkpointName}, &checkpoint) switch { case err == nil: return string(checkpoint.Data[credentialsSecretDataKey]), nil case !apierrors.IsNotFound(err): return "", err } bc := r.NewClient(serviceURL(srv)) result, err := bc.Bootstrap(ctx, bootstrapUsername, bootstrapEmail) if err != nil { if statusErr, ok := errors.AsType[*tdclient.StatusError](err); ok && statusErr.Code == http.StatusForbidden { // §1: this operator is the only thing that ever bootstraps a // server it created, so a 403 here (no checkpoint, no // status.credentialsSecretRef) means a prior reconcile already // won this exact race and its checkpoint was lost afterward -- // the one case §6 doesn't try to paper over. return "", &bootstrapStateLostError{ detail: "/api/bootstrap returned 403", secretName: credentialsSecretName(srv), } } return "", fmt.Errorf("POST /api/bootstrap: %w", err) } if err := writeOperatorSecret(ctx, r.Client, r.OperatorNamespace, checkpointName, result.APIKey.Key); err != nil { return "", fmt.Errorf("checkpointing admin key: %w", err) } return result.APIKey.Key, nil } // getOrMintInstanceServiceAccountKey mints the operator's own instance- // scoped service account, or, if an earlier interrupted attempt already // created it (409), adopts it and mints a fresh key rather than treating // the conflict as an error (DESIGN.md §6 point 1, §5's general // adopt-on-conflict rule). func (r *TerdutServerReconciler) getOrMintInstanceServiceAccountKey(ctx context.Context, bc *tdclient.Client) (string, error) { result, err := bc.CreateInstanceServiceAccount(ctx, serviceAccountName) if err == nil { return result.Key.Key, nil } statusErr, ok := errors.AsType[*tdclient.StatusError](err) if !ok || statusErr.Code != http.StatusConflict { return "", fmt.Errorf("POST /api/service-accounts: %w", err) } sa, err := bc.GetServiceAccountByName(ctx, serviceAccountName) if err != nil { return "", fmt.Errorf("GET /api/service-accounts?name=%s (adopting after 409): %w", serviceAccountName, err) } if sa == nil { return "", fmt.Errorf("POST /api/service-accounts 409'd for %q but GET found nothing", serviceAccountName) } key, err := bc.CreateServiceAccountKey(ctx, sa.ID, "initial") if err != nil { return "", fmt.Errorf("POST /api/service-accounts/%d/keys (adopting after 409): %w", sa.ID, err) } return key.Key, nil }