Files
terdut-server/charts/terdut-server/templates/deployment.yaml
T
Niklas Ye 42180948d1 Guard the archiver and notifier passes with a Postgres advisory lock
Both background loops run unconditionally on every instance with no
coordination between them, which the chart's replicas: 1 + strategy:
Recreate exists specifically to paper over: with more than one replica,
every one of them would sweep and deliver notifications independently,
and two overlapping during a rollout would both page for the same
incident.

Add withAdvisoryLock, which takes a Postgres advisory lock on a
dedicated connection and runs a pass only if it gets the lock,
otherwise skipping until the next tick. Wire StartArchiver and
StartNotifier through it with their own lock keys, so Sweep and
NotifySweep themselves are untouched and every existing test calling
them directly keeps working unchanged.

This also closes the notifier's double-delivery race in passing: two
replicas can no longer both be inside deliverPending at once, since
only one can hold notifierLockKey at a time.

Deliberately not addressed here, and still blocking a replica count
above 1: the in-memory login rate limiter, the unlocked migration
runner, and the new-incident-insert race on a webhook for a brand-new
groupKey. Noted in the updated chart comment.

Co-authored-by: Claude <noreply@anthropic.com>
2026-10-03 11:36:48 +02:00

145 lines
6.2 KiB
YAML

---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "terdut-server.fullname" . }}
labels:
{{- include "terdut-server.labels" . | nindent 4 }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "terdut-server.selectorLabels" . | nindent 6 }}
# Recreate, not RollingUpdate, even though the PVC that forced it is gone: the
# sweeper and notifier now take a Postgres advisory lock for each pass, so two
# replicas overlapping during a rollout no longer both page for the same
# incident, but DB migrations and new-incident creation on first webhook are
# still unguarded — a second replica starting concurrently with the first can
# still race either of those.
strategy:
type: Recreate
template:
metadata:
labels:
{{- include "terdut-server.selectorLabels" . | nindent 8 }}
spec:
enableServiceLinks: false
{{- if .Values.database.waitForPostgres.enabled }}
initContainers:
- name: wait-for-postgres
image: "{{ .Values.database.waitForPostgres.image.repository }}:{{ .Values.database.waitForPostgres.image.tag }}"
imagePullPolicy: {{ .Values.database.waitForPostgres.image.pullPolicy }}
env:
- name: TERDUT_DB_DSN
value: {{ required "database.dsn is required" .Values.database.dsn | quote }}
command:
- sh
- -c
- |
until pg_isready -d "$TERDUT_DB_DSN"; do
echo "wait-for-postgres: not ready yet, retrying in 2s"
sleep 2
done
{{- end }}
containers:
- name: terdut-server
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
env:
- name: TERDUT_ADDR
value: ":{{ .Values.service.port }}"
- name: TERDUT_DB_DSN
value: {{ required "database.dsn is required" .Values.database.dsn | quote }}
{{- if .Values.database.passwordSecret.name }}
# The password reaches pgx through libpq's environment variable
# rather than through the DSN, so it stays out of the rendered
# manifest. pgx fills in from PG* whatever the DSN leaves out.
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.database.passwordSecret.name }}
key: {{ .Values.database.passwordSecret.key }}
{{- end }}
- name: TERDUT_STALE_AFTER
value: "{{ .Values.sweeper.staleAfter }}"
- name: TERDUT_ARCHIVE_AFTER
value: "{{ .Values.sweeper.archiveAfter }}"
- name: TERDUT_DEADMAN_MATCHERS
value: "{{ .Values.deadman.matchers }}"
- name: TERDUT_DEADMAN_TIMEOUT
value: "{{ .Values.deadman.timeout }}"
- name: TERDUT_DEADMAN_SEVERITY
value: "{{ .Values.deadman.severity }}"
{{- if .Values.notify.ntfyUrl }}
- name: TERDUT_NTFY_URL
value: "{{ .Values.notify.ntfyUrl }}"
- name: TERDUT_NTFY_FALLBACK_TOPIC
value: "{{ .Values.notify.fallbackTopic }}"
- name: TERDUT_NOTIFY_REPEAT
value: "{{ .Values.notify.repeatEvery }}"
{{- if .Values.notify.tokenSecret.name }}
- name: TERDUT_NTFY_TOKEN
valueFrom:
secretKeyRef:
name: {{ .Values.notify.tokenSecret.name }}
key: {{ .Values.notify.tokenSecret.key }}
{{- end }}
{{- end }}
# Set whether or not ntfy is: single sign-on builds its redirect URI
# from it, and sessions use it to decide the cookie's Secure flag.
- name: TERDUT_PUBLIC_URL
value: "{{ .Values.notify.publicUrl | default (printf "https://%s" .Values.networking.hostname) }}"
- name: TERDUT_PASSWORD_LOGIN
value: {{ .Values.passwordLogin | quote }}
- name: TERDUT_OPERATOR_MODE
value: {{ .Values.operatorMode | quote }}
{{- if .Values.oidc.enabled }}
- name: TERDUT_OIDC_ISSUER
value: {{ required "oidc.issuer is required when oidc.enabled" .Values.oidc.issuer | quote }}
- name: TERDUT_OIDC_CLIENT_ID
value: {{ required "oidc.clientId is required when oidc.enabled" .Values.oidc.clientId | quote }}
- name: TERDUT_OIDC_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: {{ required "oidc.clientSecret.name is required when oidc.enabled" .Values.oidc.clientSecret.name }}
key: {{ .Values.oidc.clientSecret.key }}
- name: TERDUT_OIDC_NAME
value: {{ .Values.oidc.name | quote }}
- name: TERDUT_OIDC_SCOPES
value: {{ .Values.oidc.scopes | quote }}
- name: TERDUT_OIDC_USERNAME_CLAIM
value: {{ .Values.oidc.usernameClaim | quote }}
- name: TERDUT_OIDC_EMAIL_CLAIM
value: {{ .Values.oidc.emailClaim | quote }}
- name: TERDUT_OIDC_GROUPS_CLAIM
value: {{ .Values.oidc.groupsClaim | quote }}
- name: TERDUT_OIDC_TRUST_EMAIL
value: {{ .Values.oidc.trustEmail | quote }}
- name: TERDUT_OIDC_SESSION_MAX_AGE
value: {{ .Values.oidc.sessionMaxAge | quote }}
{{- if .Values.oidc.allowedGroups }}
- name: TERDUT_OIDC_ALLOWED_GROUPS
value: {{ join "," .Values.oidc.allowedGroups | quote }}
{{- end }}
{{- if .Values.oidc.adminGroup }}
- name: TERDUT_OIDC_ADMIN_GROUP
value: {{ .Values.oidc.adminGroup | quote }}
{{- end }}
{{- end }}
livenessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 5
readinessProbe:
httpGet:
path: /healthz
port: http
initialDelaySeconds: 5