Niklas Ye
b9ee5a8722
Pulls the demo's images on the host and loads them into the kind node, since the node may not trust the TLS chain to git.ryuvia.com. Tags come from the chart's appVersion, 01-server.yaml and 00-postgres.yaml, so a bumped pin is picked up. A full run does the same first unless PRELOAD_IMAGES=false; PULL_ALWAYS=true re-pulls tags the host has. Images are saved for the host platform and loaded as an archive: `kind load docker-image` imports with --all-platforms, which fails for a multi-arch image under docker's containerd image store. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
402 lines
15 KiB
Bash
Executable File
402 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Stands up the complete terdut demo (terdut-operator + every CRD kind this
|
|
# repo ships + a working local login + a few synthetic incidents) on a kind
|
|
# cluster, fully automated. Password login only -- this demo kit's own
|
|
# 01-server.yaml carries no oidc: block at all, so there is nothing to
|
|
# disable; OIDC is simply absent.
|
|
#
|
|
# Usage:
|
|
# ./run-demo.sh deploy the whole demo (idempotent: safe to
|
|
# re-run against a cluster that already has it)
|
|
# ./run-demo.sh --load-images
|
|
# only (re)load the demo's images into the
|
|
# existing kind cluster -- after bumping a pin,
|
|
# or when a pod is stuck in ImagePullBackOff
|
|
# ./run-demo.sh --teardown delete the demo namespace (and, if
|
|
# TEARDOWN_CLUSTER=true, the kind cluster too)
|
|
# ./run-demo.sh --help
|
|
#
|
|
# Images are pulled on the host and loaded into the kind node, because the
|
|
# node itself cannot always pull from git.ryuvia.com (x509 errors from a
|
|
# corporate TLS proxy whose CA the host trusts but the node does not). The
|
|
# tags come from the same files the demo deploys: the operator chart's
|
|
# appVersion, 01-server.yaml and 00-postgres.yaml.
|
|
#
|
|
# All of the defaults below are overridable as environment variables.
|
|
set -euo pipefail
|
|
|
|
CLUSTER_NAME="${CLUSTER_NAME:-terdut-demo}"
|
|
NAMESPACE="${NAMESPACE:-terdut-operator-demo}"
|
|
OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-terdut-operator-system}"
|
|
RELEASE_NAME="${RELEASE_NAME:-terdut-operator}"
|
|
|
|
ALICE_USERNAME="${ALICE_USERNAME:-alice}"
|
|
ALICE_EMAIL="${ALICE_EMAIL:-alice@terdut-demo.local}"
|
|
DEMO_PASSWORD="${DEMO_PASSWORD:-terdut-demo-1234}"
|
|
|
|
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
|
LOCAL_PORT="${LOCAL_PORT:-8080}"
|
|
|
|
HELM_TIMEOUT="${HELM_TIMEOUT:-180s}"
|
|
WAIT_TIMEOUT="${WAIT_TIMEOUT:-180s}"
|
|
|
|
TEARDOWN_CLUSTER="${TEARDOWN_CLUSTER:-false}"
|
|
|
|
# Pull on the host and `kind load` every demo image before installing.
|
|
PRELOAD_IMAGES="${PRELOAD_IMAGES:-true}"
|
|
# Re-pull images the host already has (otherwise only missing ones are pulled).
|
|
PULL_ALWAYS="${PULL_ALWAYS:-false}"
|
|
CONTAINER_TOOL="${CONTAINER_TOOL:-docker}"
|
|
|
|
PF_PIDFILE="${PF_PIDFILE:-/tmp/terdut-demo-port-forward.pid}"
|
|
PF_LOGFILE="${PF_LOGFILE:-/tmp/terdut-demo-port-forward.log}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CHART_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)/charts/terdut-operator"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log() { printf '[run-demo] %s\n' "$*" >&2; }
|
|
die() { printf '[run-demo] FAILED: %s\n' "$*" >&2; exit 1; }
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $0 [--load-images|--teardown|--help]
|
|
|
|
Deploys (or tears down) the complete terdut demo on a kind cluster.
|
|
|
|
--load-images only pull the demo's images on the host and load them into
|
|
the kind cluster (created if missing); no install, no apply.
|
|
PULL_ALWAYS=true re-pulls images the host already has.
|
|
--teardown delete the demo namespace (TEARDOWN_CLUSTER=true: the cluster too)
|
|
|
|
A normal run preloads the images too; PRELOAD_IMAGES=false skips that.
|
|
|
|
See the top of this file for every overridable environment variable.
|
|
EOF
|
|
}
|
|
|
|
# Only kills a port-forward THIS invocation started, so a successful run
|
|
# never has its background job reaped by its own exit trap.
|
|
STARTED_PF_PID=""
|
|
cleanup_on_failure() {
|
|
local rc=$?
|
|
if [ "$rc" -ne 0 ] && [ -n "$STARTED_PF_PID" ]; then
|
|
log "run failed -- stopping the port-forward it started (pid $STARTED_PF_PID)"
|
|
kill "$STARTED_PF_PID" 2>/dev/null || true
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
trap cleanup_on_failure EXIT
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# steps
|
|
# ---------------------------------------------------------------------------
|
|
|
|
preflight() {
|
|
local missing=()
|
|
for bin in kubectl kind helm jq curl; do
|
|
command -v "$bin" >/dev/null 2>&1 || missing+=("$bin")
|
|
done
|
|
if [ "${#missing[@]}" -gt 0 ]; then
|
|
die "missing required tools on PATH: ${missing[*]}"
|
|
fi
|
|
}
|
|
|
|
ensure_kind_cluster() {
|
|
case "$(kind get clusters 2>/dev/null)" in
|
|
*"$CLUSTER_NAME"*)
|
|
log "kind cluster '$CLUSTER_NAME' already exists, skipping creation" ;;
|
|
*)
|
|
log "creating kind cluster '$CLUSTER_NAME'"
|
|
kind create cluster --name "$CLUSTER_NAME" ;;
|
|
esac
|
|
kubectl config use-context "kind-${CLUSTER_NAME}" >/dev/null
|
|
}
|
|
|
|
# manager.image.repository from the chart's values.yaml.
|
|
chart_value_image_repo() {
|
|
awk '/^manager:/{f=1;next} f&&/^[^ #]/{exit} f&&$1=="repository:"{print $2;exit}' "$1"
|
|
}
|
|
|
|
# Prints every image the demo runs, one per line.
|
|
demo_images() {
|
|
local op_repo op_tag srv_repo srv_tag
|
|
op_repo="$(chart_value_image_repo "$CHART_DIR/values.yaml")"
|
|
op_tag="$(awk -F'"' '/^appVersion:/{print $2}' "$CHART_DIR/Chart.yaml")"
|
|
srv_repo="$(awk '$1=="repository:"{print $2;exit}' "$SCRIPT_DIR/01-server.yaml")"
|
|
srv_tag="$(awk '$1=="tag:"{print $2;exit}' "$SCRIPT_DIR/01-server.yaml")"
|
|
[ -n "$op_repo" ] && [ -n "$op_tag" ] || die "could not read the operator image from $CHART_DIR"
|
|
[ -n "$srv_repo" ] && [ -n "$srv_tag" ] || die "could not read the server image from 01-server.yaml"
|
|
printf '%s\n' "${op_repo}:${op_tag}" "${srv_repo}:${srv_tag}"
|
|
# postgres:17-alpine: the demo database, and the operator's wait-for-postgres init container.
|
|
awk '$1=="image:"{print $2}' "$SCRIPT_DIR/00-postgres.yaml" | sort -u
|
|
}
|
|
|
|
load_images() {
|
|
command -v "$CONTAINER_TOOL" >/dev/null 2>&1 \
|
|
|| die "$CONTAINER_TOOL not on PATH (needed to pull images for kind; set CONTAINER_TOOL)"
|
|
# Not `kind load docker-image`: it imports with --all-platforms, which fails
|
|
# ("content digest ... not found") for a multi-arch image when the host's
|
|
# containerd image store only holds its own platform. Saving just that
|
|
# platform to an archive and loading the archive avoids it.
|
|
local platform="" save_args=() img archive
|
|
if [ "$CONTAINER_TOOL" = "docker" ]; then
|
|
platform="$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}' 2>/dev/null || true)"
|
|
fi
|
|
if [ -n "$platform" ]; then
|
|
save_args=(--platform "$platform")
|
|
fi
|
|
archive="$(mktemp --suffix=.tar)"
|
|
while IFS= read -r img; do
|
|
if [ "$PULL_ALWAYS" = "true" ] || ! "$CONTAINER_TOOL" image inspect "$img" >/dev/null 2>&1; then
|
|
log "pulling $img on the host"
|
|
"$CONTAINER_TOOL" pull -q ${platform:+--platform "$platform"} "$img" >/dev/null \
|
|
|| { rm -f "$archive"; die "could not pull $img"; }
|
|
fi
|
|
log "loading $img into kind cluster '$CLUSTER_NAME'"
|
|
"$CONTAINER_TOOL" save "${save_args[@]}" -o "$archive" "$img" \
|
|
&& kind load image-archive --name "$CLUSTER_NAME" "$archive" >/dev/null \
|
|
|| { rm -f "$archive"; die "loading $img into kind failed"; }
|
|
done < <(demo_images)
|
|
rm -f "$archive"
|
|
}
|
|
|
|
install_operator() {
|
|
log "installing terdut-operator into namespace $OPERATOR_NAMESPACE"
|
|
helm upgrade --install "$RELEASE_NAME" "$CHART_DIR" \
|
|
--namespace "$OPERATOR_NAMESPACE" --create-namespace \
|
|
--wait --timeout "$HELM_TIMEOUT" \
|
|
|| die "helm install of terdut-operator did not become ready"
|
|
}
|
|
|
|
apply_demo() {
|
|
log "creating namespace $NAMESPACE"
|
|
kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
|
log "applying demo CRs (00-05) into $NAMESPACE"
|
|
kubectl apply -n "$NAMESPACE" -k "$SCRIPT_DIR" >/dev/null
|
|
}
|
|
|
|
wait_for_objects() {
|
|
local obj
|
|
for obj in "$@"; do
|
|
log "waiting for $obj to become Ready"
|
|
kubectl wait --for=condition=Ready --timeout "$WAIT_TIMEOUT" -n "$NAMESPACE" "$obj" >/dev/null \
|
|
|| die "timed out waiting for $obj -- try: kubectl describe -n $NAMESPACE $obj"
|
|
done
|
|
}
|
|
|
|
# Only the server: the teams cannot reach Ready until alice exists (their
|
|
# escalation ladders name her, and terdut-server resolves usernames when the
|
|
# ladder is applied), and alice is created below, through the server itself.
|
|
wait_for_server_ready() {
|
|
wait_for_objects "terdutserver/terdut-operator-demo"
|
|
}
|
|
|
|
# Everything that was waiting on alice to exist.
|
|
wait_for_remaining_ready() {
|
|
wait_for_objects \
|
|
"terdutteam/terdutteam-platform" \
|
|
"terdutteam/terdutteam-payments" \
|
|
"terdutalertsource/terdutalertsource-platform" \
|
|
"terdutalertsource/terdutalertsource-payments"
|
|
}
|
|
|
|
start_port_forward() {
|
|
# A stale pidfile from an earlier run would otherwise collide with us on
|
|
# $LOCAL_PORT -- if that pid is still alive, stop it first.
|
|
if [ -f "$PF_PIDFILE" ]; then
|
|
local old_pid
|
|
old_pid="$(cat "$PF_PIDFILE" 2>/dev/null || true)"
|
|
if [ -n "$old_pid" ] && kill -0 "$old_pid" 2>/dev/null; then
|
|
log "stopping stale port-forward from a previous run (pid $old_pid)"
|
|
kill "$old_pid" 2>/dev/null || true
|
|
sleep 1
|
|
fi
|
|
rm -f "$PF_PIDFILE"
|
|
fi
|
|
|
|
log "starting port-forward svc/terdut-operator-demo ${LOCAL_PORT}:8080"
|
|
kubectl -n "$NAMESPACE" port-forward svc/terdut-operator-demo "${LOCAL_PORT}:8080" \
|
|
>"$PF_LOGFILE" 2>&1 &
|
|
STARTED_PF_PID=$!
|
|
echo "$STARTED_PF_PID" > "$PF_PIDFILE"
|
|
|
|
local tries=0
|
|
until curl -sf -o /dev/null "http://localhost:${LOCAL_PORT}/healthz"; do
|
|
tries=$((tries + 1))
|
|
if [ "$tries" -ge 30 ]; then
|
|
die "port-forward never became ready -- see $PF_LOGFILE"
|
|
fi
|
|
sleep 1
|
|
done
|
|
log "port-forward ready (pid $STARTED_PF_PID, log $PF_LOGFILE)"
|
|
}
|
|
|
|
# Creates alice as the install's first user and administrator through
|
|
# /api/bootstrap, which is open until a first user exists -- the operator
|
|
# authenticates with its own seeded key and never uses it, so this is how a
|
|
# person gets in. An administrator may manage any team, so alice's own key is
|
|
# enough to add her to both; the teams' own status.teamID is set as soon as
|
|
# the operator has created each team, even while they still wait on her.
|
|
create_alice_and_join_teams() {
|
|
# A re-run: she exists already (and was added to the teams the first time).
|
|
local login_code
|
|
login_code="$(curl -sS -o /dev/null -w '%{http_code}' \
|
|
-X POST "${BASE_URL}/api/login" -H 'Content-Type: application/json' \
|
|
-d "$(jq -n --arg u "$ALICE_USERNAME" --arg p "$DEMO_PASSWORD" '{username: $u, password: $p}')")"
|
|
if [ "$login_code" = "200" ]; then
|
|
log "account ${ALICE_USERNAME} already exists and can sign in, skipping creation (re-run detected)"
|
|
return 0
|
|
fi
|
|
|
|
log "creating ${ALICE_USERNAME} as the first user (administrator)"
|
|
local resp_file code alice_key
|
|
resp_file="$(mktemp)"
|
|
code="$(curl -sS -o "$resp_file" -w '%{http_code}' \
|
|
-X POST "${BASE_URL}/api/bootstrap" -H 'Content-Type: application/json' \
|
|
-d "$(jq -n --arg u "$ALICE_USERNAME" --arg e "$ALICE_EMAIL" --arg p "$DEMO_PASSWORD" \
|
|
'{username: $u, email: $e, password: $p}')")"
|
|
[ "$code" = "201" ] || die "bootstrap failed (HTTP $code): $(cat "$resp_file")"
|
|
alice_key="$(jq -r '.api_key.key' "$resp_file")"
|
|
rm -f "$resp_file"
|
|
|
|
local team team_id
|
|
for team in terdutteam-platform terdutteam-payments; do
|
|
team_id=""
|
|
local tries=0
|
|
until team_id="$(kubectl -n "$NAMESPACE" get terdutteam "$team" -o jsonpath='{.status.teamID}' 2>/dev/null)" \
|
|
&& [ -n "$team_id" ]; do
|
|
tries=$((tries + 1))
|
|
[ "$tries" -lt 60 ] || die "$team never reported status.teamID -- kubectl describe it"
|
|
sleep 1
|
|
done
|
|
local alice_id
|
|
alice_id="$(curl -sS -H "Authorization: Bearer $alice_key" "${BASE_URL}/api/users" \
|
|
| jq -r --arg u "$ALICE_USERNAME" '.[] | select(.username == $u) | .id')"
|
|
code="$(curl -sS -o /dev/null -w '%{http_code}' \
|
|
-X POST "${BASE_URL}/api/teams/${team_id}/members" \
|
|
-H "Authorization: Bearer $alice_key" -H 'Content-Type: application/json' \
|
|
-d "$(jq -n --argjson id "$alice_id" '{user_id: $id, role: "owner"}')")"
|
|
[ "$code" = "204" ] || die "adding ${ALICE_USERNAME} to ${team} failed (HTTP $code)"
|
|
log "added ${ALICE_USERNAME} to ${team}"
|
|
done
|
|
}
|
|
|
|
fire_demo_alerts() {
|
|
log "firing representative demo alerts"
|
|
# Two clusters, so the queue shows the cluster chip and offers its filter.
|
|
# high-cpu fires in both: the same alert in two clusters is two incidents.
|
|
local fire="$SCRIPT_DIR/fire-alerts.sh"
|
|
CLUSTER=prod-eu NAMESPACE="$NAMESPACE" BASE_URL="$BASE_URL" "$fire" platform high-cpu
|
|
CLUSTER=prod-us NAMESPACE="$NAMESPACE" BASE_URL="$BASE_URL" "$fire" platform high-cpu
|
|
CLUSTER=prod-us NAMESPACE="$NAMESPACE" BASE_URL="$BASE_URL" "$fire" platform disk-full
|
|
CLUSTER=prod-eu NAMESPACE="$NAMESPACE" BASE_URL="$BASE_URL" "$fire" payments pod-crash
|
|
}
|
|
|
|
print_summary() {
|
|
cat <<EOF
|
|
|
|
terdut demo is up.
|
|
|
|
Web UI: http://localhost:${LOCAL_PORT}
|
|
Login: ${ALICE_USERNAME} / ${DEMO_PASSWORD}
|
|
Cluster: kind-${CLUSTER_NAME}
|
|
Namespace: ${NAMESPACE}
|
|
Port-forward pid: ${STARTED_PF_PID:-$(cat "$PF_PIDFILE" 2>/dev/null || echo unknown)} (log: ${PF_LOGFILE})
|
|
stop it with: kill \$(cat ${PF_PIDFILE})
|
|
|
|
Fire more alerts:
|
|
export NAMESPACE=${NAMESPACE} BASE_URL=${BASE_URL}
|
|
CLUSTER=prod-eu ./fire-alerts.sh platform high-cpu
|
|
CLUSTER=prod-eu ./fire-alerts.sh platform high-cpu resolve
|
|
./fire-alerts.sh payments heartbeat # send repeatedly (e.g. every minute)
|
|
# to keep a dead man's switch alive;
|
|
# stop sending it and, 15 minutes
|
|
# later, terdut-server opens a
|
|
# critical incident on its own.
|
|
|
|
Tear down:
|
|
$0 --teardown
|
|
# add TEARDOWN_CLUSTER=true to also delete the kind cluster itself
|
|
EOF
|
|
}
|
|
|
|
teardown() {
|
|
if [ -f "$PF_PIDFILE" ]; then
|
|
local pid
|
|
pid="$(cat "$PF_PIDFILE" 2>/dev/null || true)"
|
|
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
|
|
log "stopping port-forward (pid $pid)"
|
|
kill "$pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$PF_PIDFILE"
|
|
fi
|
|
|
|
log "deleting namespace $NAMESPACE"
|
|
kubectl delete namespace "$NAMESPACE" --ignore-not-found --wait=true --timeout "$WAIT_TIMEOUT"
|
|
|
|
if [ "$TEARDOWN_CLUSTER" = "true" ]; then
|
|
log "deleting kind cluster $CLUSTER_NAME"
|
|
kind delete cluster --name "$CLUSTER_NAME"
|
|
else
|
|
log "leaving kind cluster '$CLUSTER_NAME' and the operator install in place" \
|
|
"(set TEARDOWN_CLUSTER=true to also delete the cluster)"
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
main() {
|
|
case "${1:-}" in
|
|
--teardown)
|
|
preflight
|
|
teardown
|
|
trap - EXIT
|
|
exit 0
|
|
;;
|
|
--load-images)
|
|
preflight
|
|
ensure_kind_cluster
|
|
load_images
|
|
log "images loaded; pods stuck in ImagePullBackOff retry on their own," \
|
|
"or delete them to restart now"
|
|
trap - EXIT
|
|
exit 0
|
|
;;
|
|
--help|-h)
|
|
usage
|
|
trap - EXIT
|
|
exit 0
|
|
;;
|
|
"") ;;
|
|
*)
|
|
usage
|
|
die "unknown argument: $1"
|
|
;;
|
|
esac
|
|
|
|
preflight
|
|
ensure_kind_cluster
|
|
if [ "$PRELOAD_IMAGES" = "true" ]; then
|
|
load_images
|
|
fi
|
|
install_operator
|
|
apply_demo
|
|
wait_for_server_ready
|
|
start_port_forward
|
|
create_alice_and_join_teams
|
|
wait_for_remaining_ready
|
|
fire_demo_alerts
|
|
print_summary
|
|
|
|
# Success: leave the port-forward running, don't let the EXIT trap kill it.
|
|
trap - EXIT
|
|
}
|
|
|
|
main "$@"
|