Niklas Ye
c28a9b7b49
The server exits when the database does not answer within about ten seconds, so while the database is down the pod crash-loops with a restart backoff that grows to five minutes, and /healthz only answers after it has connected. The hook gave up after a fixed two minutes per attempt, and Flux's default five-minute timeout came first anyway. A failed hook fails the release and nothing runs it again until the next upgrade, so a database that was merely slow left the install without an administrator. Each attempt now waits bootstrap.waitSeconds (default 600), logging progress, and the Job gets an activeDeadlineSeconds derived from it so a database that never comes up still ends the hook. No change to the binary, API or schema. Claude-Session: https://claude.ai/code/session_016mBLURvJoMuUEr9cB2RpUN
177 lines
8.8 KiB
YAML
177 lines
8.8 KiB
YAML
{{- if .Values.bootstrap.enabled }}
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: {{ include "terdut-server.fullname" . }}-bootstrap
|
|
namespace: {{ .Release.Namespace }}
|
|
labels:
|
|
{{- include "terdut-server.labels" . | nindent 4 }}
|
|
annotations:
|
|
helm.sh/hook: post-install,post-upgrade
|
|
helm.sh/hook-weight: "0"
|
|
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
|
|
spec:
|
|
backoffLimit: 3
|
|
# Bounds the whole Job, retries included, so a database that never comes up ends the
|
|
# hook instead of leaving it pending. Each attempt waits up to bootstrap.waitSeconds
|
|
# for the server, and the allowance covers every attempt plus the pod backoff between.
|
|
activeDeadlineSeconds: {{ add (mul (int .Values.bootstrap.waitSeconds) 4) 300 }}
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "terdut-server.selectorLabels" . | nindent 8 }}
|
|
app.kubernetes.io/component: bootstrap
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
serviceAccountName: {{ include "terdut-server.fullname" . }}-bootstrap
|
|
containers:
|
|
- name: bootstrap
|
|
# alpine/curl, not alpine:3 + `apk add curl`. Installing the binary at run time
|
|
# writes it into the container's writable upper layer, which is exactly the
|
|
# signature Falco's `Drop and execute new binary in container` (MITRE TA0003)
|
|
# exists to catch -- this hook emitted two Critical events on every single
|
|
# upgrade. See Ryuvia/charts#100. It also made `helm upgrade` depend on the
|
|
# Alpine CDN answering, since this runs as a post-upgrade hook and a failed
|
|
# hook fails the release.
|
|
#
|
|
# Still a full Alpine underneath, so sh, cat, sleep, grep, cut, head and tail
|
|
# are all present (verified in-cluster 2026-09-04). The image declares
|
|
# ENTRYPOINT ["/entrypoint.sh"], which `command:` below overrides -- do not
|
|
# change `command:` to `args:`.
|
|
image: alpine/curl:8.21.0@sha256:a1c44bab54d88e18ea9a6a4ecefab7f2d230b968567b78960fcaff8d51b7f067
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
SERVICE_URL="http://{{ include "terdut-server.fullname" . }}:{{ .Values.service.port }}"
|
|
SECRET_NAME="{{ include "terdut-server.bootstrapSecretName" . }}"
|
|
USERNAME="{{ .Values.bootstrap.username }}"
|
|
EMAIL="{{ .Values.bootstrap.email }}"
|
|
K8S_API="https://kubernetes.default.svc"
|
|
SA_TOKEN="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
|
|
CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
NAMESPACE="$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)"
|
|
SECRETS="$K8S_API/api/v1/namespaces/$NAMESPACE/secrets"
|
|
|
|
# kapi METHOD URL [BODY]: sets CODE and BODY from the Kubernetes API.
|
|
kapi() {
|
|
_ctype="application/json"
|
|
[ "$1" = "PATCH" ] && _ctype="application/merge-patch+json"
|
|
if [ -n "$3" ]; then
|
|
_resp=$(printf '%s' "$3" | curl -s -w "\n%{http_code}" -X "$1" "$2" \
|
|
--cacert "$CA_CERT" -H "Authorization: Bearer $SA_TOKEN" \
|
|
-H "Content-Type: $_ctype" -d @-)
|
|
else
|
|
_resp=$(curl -s -w "\n%{http_code}" -X "$1" "$2" \
|
|
--cacert "$CA_CERT" -H "Authorization: Bearer $SA_TOKEN")
|
|
fi
|
|
CODE=$(echo "$_resp" | tail -1)
|
|
BODY=$(echo "$_resp" | sed '$d')
|
|
}
|
|
|
|
# The server exits when the database does not answer within ~10 seconds, so
|
|
# while the database is down the pod crash-loops, with a restart backoff that
|
|
# grows to five minutes, and /healthz only answers once it has connected. Wait
|
|
# for that for as long as bootstrap.waitSeconds allows, not a fixed two minutes:
|
|
# a failed hook fails the release and nothing runs it again until the next
|
|
# upgrade, so a database that was merely slow would leave no administrator.
|
|
WAIT={{ int .Values.bootstrap.waitSeconds }}
|
|
echo "Waiting up to ${WAIT}s for terdut-server to be ready (it needs the database)..."
|
|
WAITED=0
|
|
while ! curl -sf "$SERVICE_URL/healthz" > /dev/null 2>&1; do
|
|
if [ "$WAITED" -ge "$WAIT" ]; then
|
|
echo "Timed out after ${WAITED}s waiting for the server to be ready."
|
|
echo "Check the database and the terdut-server pod's logs, then run helm upgrade again."
|
|
exit 1
|
|
fi
|
|
[ $((WAITED % 30)) -eq 0 ] && [ "$WAITED" -gt 0 ] && echo "Still waiting (${WAITED}s)..."
|
|
sleep 2
|
|
WAITED=$((WAITED + 2))
|
|
done
|
|
echo "Server is ready after ${WAITED}s."
|
|
|
|
# The Secret is the source of truth for the administrator's password: reuse
|
|
# the one it holds, so recreating the database brings the same account back.
|
|
PASSWORD=""
|
|
EXISTS=0
|
|
kapi GET "$SECRETS/$SECRET_NAME"
|
|
if [ "$CODE" = "200" ]; then
|
|
EXISTS=1
|
|
PASSWORD=$(echo "$BODY" | grep -o '"password": *"[^"]*"' | head -1 | cut -d'"' -f4 | base64 -d)
|
|
elif [ "$CODE" != "404" ]; then
|
|
echo "Failed to read secret '$SECRET_NAME' (HTTP $CODE)."
|
|
exit 1
|
|
fi
|
|
|
|
# Written BEFORE the server is asked to create the account, so a crash in
|
|
# between cannot leave an administrator whose password nobody has.
|
|
GENERATED=0
|
|
if [ -z "$PASSWORD" ]; then
|
|
PASSWORD=$(head -c 256 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | head -c 32)
|
|
if [ "${#PASSWORD}" -lt 32 ]; then
|
|
echo "Failed to generate a password."
|
|
exit 1
|
|
fi
|
|
GENERATED=1
|
|
DATA=$(printf '"username":"%s","password":"%s"' "$USERNAME" "$PASSWORD")
|
|
if [ "$EXISTS" = "1" ]; then
|
|
kapi PATCH "$SECRETS/$SECRET_NAME" "{\"stringData\":{$DATA}}"
|
|
else
|
|
kapi POST "$SECRETS" "{\"apiVersion\":\"v1\",\"kind\":\"Secret\",\"metadata\":{\"name\":\"$SECRET_NAME\"},\"stringData\":{$DATA}}"
|
|
fi
|
|
case "$CODE" in 200|201) ;; *)
|
|
echo "Failed to store the password in secret '$SECRET_NAME' (HTTP $CODE)."
|
|
exit 1 ;;
|
|
esac
|
|
fi
|
|
|
|
RESPONSE=$(printf '{"username":"%s","email":"%s","password":"%s"}' "$USERNAME" "$EMAIL" "$PASSWORD" \
|
|
| curl -s -w "\n%{http_code}" -X POST "$SERVICE_URL/api/bootstrap" \
|
|
-H "Content-Type: application/json" -d @-)
|
|
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
|
|
RESP_BODY=$(echo "$RESPONSE" | head -1)
|
|
|
|
if [ "$HTTP_CODE" = "403" ]; then
|
|
# Somebody else made the first account. The Secret is only worth keeping
|
|
# if its password signs in.
|
|
LOGIN=$(printf '{"username":"%s","password":"%s"}' "$USERNAME" "$PASSWORD" \
|
|
| curl -s -o /dev/null -w "%{http_code}" -X POST "$SERVICE_URL/api/login" \
|
|
-H "Content-Type: application/json" -d @-)
|
|
if [ "$LOGIN" = "200" ]; then
|
|
echo "Server already bootstrapped; the password in '$SECRET_NAME' signs in."
|
|
exit 0
|
|
fi
|
|
echo "Server already bootstrapped by something else; '$USERNAME' does not sign in with the password in '$SECRET_NAME'."
|
|
if [ "$GENERATED" = "1" ]; then
|
|
if [ "$EXISTS" = "1" ]; then
|
|
kapi PATCH "$SECRETS/$SECRET_NAME" '{"data":{"username":null,"password":null}}'
|
|
else
|
|
kapi DELETE "$SECRETS/$SECRET_NAME"
|
|
fi
|
|
echo "Removed the password this run generated."
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$HTTP_CODE" != "201" ]; then
|
|
echo "Bootstrap failed (HTTP $HTTP_CODE): $RESP_BODY"
|
|
exit 1
|
|
fi
|
|
|
|
API_KEY=$(echo "$RESP_BODY" | grep -o '"key":"[^"]*"' | cut -d'"' -f4)
|
|
if [ -z "$API_KEY" ]; then
|
|
echo "Failed to extract API key from response."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Bootstrap succeeded. Storing the API key in secret '$SECRET_NAME'."
|
|
kapi PATCH "$SECRETS/$SECRET_NAME" "{\"stringData\":{\"api-key\":\"$API_KEY\"}}"
|
|
if [ "$CODE" != "200" ]; then
|
|
echo "Failed to store the API key (HTTP $CODE)."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Secret '$SECRET_NAME' holds username, password and api-key."
|
|
{{- end }}
|