42180948d1
Both background loops run unconditionally on every instance with no coordination between them, which the chart's replicas: 1 + strategy: Recreate exists specifically to paper over: with more than one replica, every one of them would sweep and deliver notifications independently, and two overlapping during a rollout would both page for the same incident. Add withAdvisoryLock, which takes a Postgres advisory lock on a dedicated connection and runs a pass only if it gets the lock, otherwise skipping until the next tick. Wire StartArchiver and StartNotifier through it with their own lock keys, so Sweep and NotifySweep themselves are untouched and every existing test calling them directly keeps working unchanged. This also closes the notifier's double-delivery race in passing: two replicas can no longer both be inside deliverPending at once, since only one can hold notifierLockKey at a time. Deliberately not addressed here, and still blocking a replica count above 1: the in-memory login rate limiter, the unlocked migration runner, and the new-incident-insert race on a webhook for a brand-new groupKey. Noted in the updated chart comment. Co-authored-by: Claude <noreply@anthropic.com>
145 lines
6.2 KiB
YAML
145 lines
6.2 KiB
YAML
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: {{ include "terdut-server.fullname" . }}
|
|
labels:
|
|
{{- include "terdut-server.labels" . | nindent 4 }}
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
{{- include "terdut-server.selectorLabels" . | nindent 6 }}
|
|
# Recreate, not RollingUpdate, even though the PVC that forced it is gone: the
|
|
# sweeper and notifier now take a Postgres advisory lock for each pass, so two
|
|
# replicas overlapping during a rollout no longer both page for the same
|
|
# incident, but DB migrations and new-incident creation on first webhook are
|
|
# still unguarded — a second replica starting concurrently with the first can
|
|
# still race either of those.
|
|
strategy:
|
|
type: Recreate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "terdut-server.selectorLabels" . | nindent 8 }}
|
|
spec:
|
|
enableServiceLinks: false
|
|
{{- if .Values.database.waitForPostgres.enabled }}
|
|
initContainers:
|
|
- name: wait-for-postgres
|
|
image: "{{ .Values.database.waitForPostgres.image.repository }}:{{ .Values.database.waitForPostgres.image.tag }}"
|
|
imagePullPolicy: {{ .Values.database.waitForPostgres.image.pullPolicy }}
|
|
env:
|
|
- name: TERDUT_DB_DSN
|
|
value: {{ required "database.dsn is required" .Values.database.dsn | quote }}
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
until pg_isready -d "$TERDUT_DB_DSN"; do
|
|
echo "wait-for-postgres: not ready yet, retrying in 2s"
|
|
sleep 2
|
|
done
|
|
{{- end }}
|
|
containers:
|
|
- name: terdut-server
|
|
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
|
ports:
|
|
- name: http
|
|
containerPort: {{ .Values.service.port }}
|
|
protocol: TCP
|
|
env:
|
|
- name: TERDUT_ADDR
|
|
value: ":{{ .Values.service.port }}"
|
|
- name: TERDUT_DB_DSN
|
|
value: {{ required "database.dsn is required" .Values.database.dsn | quote }}
|
|
{{- if .Values.database.passwordSecret.name }}
|
|
# The password reaches pgx through libpq's environment variable
|
|
# rather than through the DSN, so it stays out of the rendered
|
|
# manifest. pgx fills in from PG* whatever the DSN leaves out.
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.database.passwordSecret.name }}
|
|
key: {{ .Values.database.passwordSecret.key }}
|
|
{{- end }}
|
|
- name: TERDUT_STALE_AFTER
|
|
value: "{{ .Values.sweeper.staleAfter }}"
|
|
- name: TERDUT_ARCHIVE_AFTER
|
|
value: "{{ .Values.sweeper.archiveAfter }}"
|
|
- name: TERDUT_DEADMAN_MATCHERS
|
|
value: "{{ .Values.deadman.matchers }}"
|
|
- name: TERDUT_DEADMAN_TIMEOUT
|
|
value: "{{ .Values.deadman.timeout }}"
|
|
- name: TERDUT_DEADMAN_SEVERITY
|
|
value: "{{ .Values.deadman.severity }}"
|
|
{{- if .Values.notify.ntfyUrl }}
|
|
- name: TERDUT_NTFY_URL
|
|
value: "{{ .Values.notify.ntfyUrl }}"
|
|
- name: TERDUT_NTFY_FALLBACK_TOPIC
|
|
value: "{{ .Values.notify.fallbackTopic }}"
|
|
- name: TERDUT_NOTIFY_REPEAT
|
|
value: "{{ .Values.notify.repeatEvery }}"
|
|
{{- if .Values.notify.tokenSecret.name }}
|
|
- name: TERDUT_NTFY_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.notify.tokenSecret.name }}
|
|
key: {{ .Values.notify.tokenSecret.key }}
|
|
{{- end }}
|
|
{{- end }}
|
|
# Set whether or not ntfy is: single sign-on builds its redirect URI
|
|
# from it, and sessions use it to decide the cookie's Secure flag.
|
|
- name: TERDUT_PUBLIC_URL
|
|
value: "{{ .Values.notify.publicUrl | default (printf "https://%s" .Values.networking.hostname) }}"
|
|
- name: TERDUT_PASSWORD_LOGIN
|
|
value: {{ .Values.passwordLogin | quote }}
|
|
- name: TERDUT_OPERATOR_MODE
|
|
value: {{ .Values.operatorMode | quote }}
|
|
{{- if .Values.oidc.enabled }}
|
|
- name: TERDUT_OIDC_ISSUER
|
|
value: {{ required "oidc.issuer is required when oidc.enabled" .Values.oidc.issuer | quote }}
|
|
- name: TERDUT_OIDC_CLIENT_ID
|
|
value: {{ required "oidc.clientId is required when oidc.enabled" .Values.oidc.clientId | quote }}
|
|
- name: TERDUT_OIDC_CLIENT_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ required "oidc.clientSecret.name is required when oidc.enabled" .Values.oidc.clientSecret.name }}
|
|
key: {{ .Values.oidc.clientSecret.key }}
|
|
- name: TERDUT_OIDC_NAME
|
|
value: {{ .Values.oidc.name | quote }}
|
|
- name: TERDUT_OIDC_SCOPES
|
|
value: {{ .Values.oidc.scopes | quote }}
|
|
- name: TERDUT_OIDC_USERNAME_CLAIM
|
|
value: {{ .Values.oidc.usernameClaim | quote }}
|
|
- name: TERDUT_OIDC_EMAIL_CLAIM
|
|
value: {{ .Values.oidc.emailClaim | quote }}
|
|
- name: TERDUT_OIDC_GROUPS_CLAIM
|
|
value: {{ .Values.oidc.groupsClaim | quote }}
|
|
- name: TERDUT_OIDC_TRUST_EMAIL
|
|
value: {{ .Values.oidc.trustEmail | quote }}
|
|
- name: TERDUT_OIDC_SESSION_MAX_AGE
|
|
value: {{ .Values.oidc.sessionMaxAge | quote }}
|
|
{{- if .Values.oidc.allowedGroups }}
|
|
- name: TERDUT_OIDC_ALLOWED_GROUPS
|
|
value: {{ join "," .Values.oidc.allowedGroups | quote }}
|
|
{{- end }}
|
|
{{- if .Values.oidc.adminGroup }}
|
|
- name: TERDUT_OIDC_ADMIN_GROUP
|
|
value: {{ .Values.oidc.adminGroup | quote }}
|
|
{{- end }}
|
|
{{- end }}
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: http
|
|
initialDelaySeconds: 5
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: http
|
|
initialDelaySeconds: 5
|
|
|
|
|