From 7caafbaf807f47b4dab3b5345fd0ee95ee03d209 Mon Sep 17 00:00:00 2001 From: Niklas Ye Date: Fri, 7 Aug 2026 10:57:56 +0200 Subject: [PATCH] chart: back up the database through a python sidecar The image is FROM scratch, so there is no interpreter to run a k8up backupcommand in, and the database runs in WAL mode, where a file-level copy of the volume is not crash-consistent. Also switches to strategy: Recreate. The data PVC is ReadWriteOnce, so a RollingUpdate deadlocks the new pod against the old one holding it. --- README.md | 15 +++++ .../terdut-server/templates/deployment.yaml | 59 +++++++++++++++++++ charts/terdut-server/values.yaml | 12 ++++ 3 files changed, 86 insertions(+) diff --git a/README.md b/README.md index 71bd32d..b3e2095 100644 --- a/README.md +++ b/README.md @@ -71,10 +71,25 @@ than an `Ingress`. TLS is terminated at the gateway, so the server itself never | `networking.listener` | `""` | Gateway listener (`sectionName`) to bind to. Empty attaches to every matching listener, **including plaintext HTTP** — set it to the HTTPS listener's name to serve TLS only | | `networking.servicePort` | `8080` | Port the route forwards to; keep in sync with `service.port` | | `bootstrap.enabled` | `true` | Runs a post-install hook that creates the first user and stores its API key in the `-admin-key` Secret. Already-bootstrapped servers are left alone | +| `backupSidecar.enabled` | `true` | Adds an idle `python` sidecar and the [k8up](https://k8up.io/) annotations that dump the database through it | The API key travels in an `Authorization: Bearer` header, so set `networking.listener` whenever the hostname is reachable outside a trusted network. +#### Backups + +The server image is `FROM scratch` — the binary and nothing else — so there is no interpreter to +run a database dump in, and the database runs in WAL mode, where a file-level copy of the volume is +not crash-consistent. The chart therefore ships an idle `python:*-alpine` sidecar that shares the +data volume, and points k8up's `backupcommand` at it with `k8up.io/backupcommand-container`. Without +that annotation k8up execs into `.spec.containers[0]` and the dump fails. + +The dump is buffered and sanity-checked before its first byte reaches stdout, because k8up streams +stdout straight into Restic: a dump that dies partway is otherwise stored as a silently truncated +snapshot that k8up still reports as successful. + +Set `backupSidecar.enabled=false` if you back the volume up some other way. + --- ## Configuration diff --git a/charts/terdut-server/templates/deployment.yaml b/charts/terdut-server/templates/deployment.yaml index 566e21f..f237029 100644 --- a/charts/terdut-server/templates/deployment.yaml +++ b/charts/terdut-server/templates/deployment.yaml @@ -10,10 +10,50 @@ spec: selector: matchLabels: {{- include "terdut-server.selectorLabels" . | nindent 6 }} + # The data PVC is ReadWriteOnce, so a RollingUpdate deadlocks: the new pod + # cannot attach the volume until the old one releases it, and the old one is + # not torn down until the new one is ready. + strategy: + type: Recreate template: metadata: labels: {{- include "terdut-server.selectorLabels" . | nindent 8 }} + {{- if .Values.backupSidecar.enabled }} + annotations: + # Dumps the whole database: incidents, alerts, users, API key hashes, + # the schedule and the notification outbox. + # + # Runs in the `backup` sidecar, NOT in the app container: the server + # image is FROM scratch and has no interpreter at all. k8up execs into + # .spec.containers[0] unless told otherwise, hence the explicit + # k8up.io/backupcommand-container. + # + # Buffered and sanity-checked before the first byte reaches stdout: k8up + # streams stdout straight into restic, so a dump that dies partway is + # stored as a silently-truncated snapshot that k8up still reports as + # Succeeded. The check counts users rather than incidents -- incidents + # are swept and archived, so an empty incidents table is a legitimate + # state, whereas a database with no users never is. + # + # The connection is read-only but the mount is not: the database runs in + # WAL mode, and opening it mode=ro still needs write access to the -shm + # wal-index. + # + # chr(10), not '\n': k8up parses this annotation with go-shellquote. + k8up.io/backupcommand-container: backup + k8up.io/backupcommand: >- + python3 -c "import sqlite3, sys; + con = sqlite3.connect('file:/data/terdut.db?mode=ro', uri=True); + con.execute('BEGIN'); + users = con.execute('SELECT count(*) FROM users').fetchone()[0]; + out = chr(10).join(con.iterdump()) + chr(10); + (users > 0 and out.rstrip().endswith('COMMIT;')) + or sys.exit('terdut: db dump failed sanity checks'); + sys.stdout.write(out)" + k8up.io/file-extension: ".sql" + k8up.io/backup: "true" + {{- end }} spec: enableServiceLinks: false containers: @@ -63,6 +103,25 @@ spec: path: /healthz port: http initialDelaySeconds: 5 + + {{- if .Values.backupSidecar.enabled }} + # Idle sidecar. It exists only so k8up has a container with a sqlite3 + # module to exec the backupcommand in. Mounted read-write on purpose: + # see the note on the backupcommand annotation above. + - name: backup + image: "{{ .Values.backupSidecar.image.repository }}:{{ .Values.backupSidecar.image.tag }}" + imagePullPolicy: {{ .Values.backupSidecar.image.pullPolicy }} + command: ["sleep", "infinity"] + volumeMounts: + - name: data + mountPath: /data + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "64Mi" + {{- end }} volumes: - name: data persistentVolumeClaim: diff --git a/charts/terdut-server/values.yaml b/charts/terdut-server/values.yaml index 7087923..de4817f 100644 --- a/charts/terdut-server/values.yaml +++ b/charts/terdut-server/values.yaml @@ -50,6 +50,18 @@ notify: name: "" key: token +# The server image is FROM scratch — just the binary, with no shell, no sqlite3 +# and no python — so a k8up backupcommand cannot run in the app container. This +# idle sidecar shares the data volume and is selected with +# k8up.io/backupcommand-container. Only the stdlib sqlite3 module is used, so any +# python image works. +backupSidecar: + enabled: true + image: + repository: python + tag: "3.13-alpine" + pullPolicy: IfNotPresent + bootstrap: enabled: true username: admin