From 059c70d9c6fe7d5eebf6f10fce2fa5f3bafee589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3recki?= Date: Thu, 20 Aug 2026 13:56:39 +0200 Subject: [PATCH] Add default readiness/liveness probes to the RabbitMQ StatefulSet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The StatefulSet rendered probes only from values (`rabbitmq.livenessProbe` / `readinessProbe`, both `{}` by default), so no probes reached the cluster: the broker pod counted as Ready the moment the container process started, not when it accepted AMQP connections. The whole hook ordering rests on the premise that the infra release installed with `wait: true` is really ready before the app release's migration hook runs — for RabbitMQ that premise was hollow. Defaults follow the official RabbitMQ guidance (https://www.rabbitmq.com/docs/monitoring#health-checks): - readiness: TCP check on the AMQP port 5672. The AMQP listener opens as one of the last boot steps, so this is exactly "the broker accepts connections"; it is the upstream Kubernetes Operator's default and the docs explicitly call CLI-based checks unsuitable as readiness probes. - liveness: the stage-1 `rabbitmq-diagnostics -q ping` with a generous budget (60 s initial delay, 3 x 30 s, 15 s timeout) because it spawns an Erlang VM and a false positive costs a full broker restart. Heavier checks (check_running, check_local_alarms) are deliberately not used for liveness. Probes are rendered with the redis `if/else` pattern instead of the `shopsys.containerSettings` helper so that setting a probe in values replaces the default wholesale — map deep-merge would otherwise make defaults impossible to remove. The container `securityContext` is rendered inline for the same reason (it was the helper's only other output here). Co-Authored-By: Claude Fable 5 --- .../templates/statefulset-rabbitmq.yaml | 40 ++++++++++- charts/shopsys-infra/tests/infra_test.yaml | 70 +++++++++++++++++++ charts/shopsys-infra/values.yaml | 3 + docs/migrating-from-shopsys-deployment.md | 11 +++ docs/values.md | 2 +- .../basic-production/expected/continuous.yaml | 18 ++++- .../expected/first-deploy-with-demo-data.yaml | 18 ++++- .../expected/first-deploy.yaml | 18 ++++- .../expected/continuous.yaml | 18 ++++- .../expected/first-deploy-with-demo-data.yaml | 18 ++++- .../expected/first-deploy.yaml | 18 ++++- .../expected/continuous.yaml | 18 ++++- .../expected/first-deploy-with-demo-data.yaml | 18 ++++- .../expected/first-deploy.yaml | 18 ++++- .../escaping-env/expected/continuous.yaml | 18 ++++- .../expected/first-deploy-with-demo-data.yaml | 18 ++++- .../escaping-env/expected/first-deploy.yaml | 18 ++++- .../expected/continuous.yaml | 18 ++++- .../expected/first-deploy-with-demo-data.yaml | 18 ++++- .../expected/first-deploy.yaml | 18 ++++- 20 files changed, 379 insertions(+), 17 deletions(-) diff --git a/charts/shopsys-infra/templates/statefulset-rabbitmq.yaml b/charts/shopsys-infra/templates/statefulset-rabbitmq.yaml index 7effd95..baaf499 100644 --- a/charts/shopsys-infra/templates/statefulset-rabbitmq.yaml +++ b/charts/shopsys-infra/templates/statefulset-rabbitmq.yaml @@ -53,7 +53,45 @@ spec: {{- end }} resources: {{- toYaml .Values.rabbitmq.resources | nindent 12 }} - {{- include "shopsys.containerSettings" .Values.rabbitmq | nindent 10 }} + {{- with .Values.rabbitmq.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.rabbitmq.livenessProbe }} + livenessProbe: + {{- toYaml .Values.rabbitmq.livenessProbe | nindent 12 }} + {{- else }} + {{- /* Stage-1 health check (https://www.rabbitmq.com/docs/monitoring#health-checks): + validates the runtime is up and CLI authentication works. rabbitmq-diagnostics + spawns an Erlang VM, so the budget is deliberately generous - a false positive + here costs a full broker restart. */}} + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + {{- end }} + {{- if .Values.rabbitmq.readinessProbe }} + readinessProbe: + {{- toYaml .Values.rabbitmq.readinessProbe | nindent 12 }} + {{- else }} + {{- /* The AMQP listener opens as one of the last boot steps, so a TCP check on 5672 + is exactly "the broker accepts connections" - the upstream Kubernetes + Operator's default and the practice recommended by the RabbitMQ docs (CLI + checks are explicitly discouraged as readiness probes). */}} + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + {{- end }} volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/charts/shopsys-infra/tests/infra_test.yaml b/charts/shopsys-infra/tests/infra_test.yaml index b8b6df2..6ff1ffc 100644 --- a/charts/shopsys-infra/tests/infra_test.yaml +++ b/charts/shopsys-infra/tests/infra_test.yaml @@ -44,6 +44,76 @@ tests: name: rabbitmq-credentials key: user + - it: gives rabbitmq default probes so infra readiness means an accepting broker + template: templates/statefulset-rabbitmq.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + + - it: replaces the default rabbitmq probes wholesale when they are set + template: templates/statefulset-rabbitmq.yaml + set: + rabbitmq: + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - check_running + periodSeconds: 20 + readinessProbe: + httpGet: + path: /api/health/checks/alarms + port: 15672 + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + exec: + command: + - rabbitmq-diagnostics + - -q + - check_running + periodSeconds: 20 + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + httpGet: + path: /api/health/checks/alarms + port: 15672 + + - it: renders the rabbitmq container securityContext next to the probes + template: templates/statefulset-rabbitmq.yaml + set: + rabbitmq: + securityContext: + runAsNonRoot: true + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.runAsNonRoot + value: true + - exists: + path: spec.template.spec.containers[0].readinessProbe + - it: derives the management hostname from the first domain template: templates/ingress-rabbitmq.yaml set: diff --git a/charts/shopsys-infra/values.yaml b/charts/shopsys-infra/values.yaml index 79cbe3c..a4beea5 100644 --- a/charts/shopsys-infra/values.yaml +++ b/charts/shopsys-infra/values.yaml @@ -131,6 +131,9 @@ rabbitmq: extraEnv: [] extraVolumes: [] extraVolumeMounts: [] + # Empty = the chart defaults render (liveness: `rabbitmq-diagnostics -q ping`, + # readiness: TCP check on the AMQP port 5672). Setting a probe REPLACES the default + # wholesale - there is no deep merge, so an override must be complete. livenessProbe: {} readinessProbe: {} priorityClassName: "" diff --git a/docs/migrating-from-shopsys-deployment.md b/docs/migrating-from-shopsys-deployment.md index c2e2d1f..3a06065 100644 --- a/docs/migrating-from-shopsys-deployment.md +++ b/docs/migrating-from-shopsys-deployment.md @@ -124,3 +124,14 @@ Intentional differences of the phase-1 rewrite; everything else is a 1:1 port. env vars (works with any registry — GCR/GAR via username `_json_key` and the service account JSON as the password); the GitLab-flavored `CI_REGISTRY`/`DEPLOY_REGISTER_*` variables keep working as a fallback. +21. **RabbitMQ has default probes** (legacy declared none, so the pod counted as Ready the + moment the container process started): `readinessProbe` is a TCP check on the AMQP port + 5672 — the upstream Kubernetes Operator's default and the practice recommended by the + [RabbitMQ docs](https://www.rabbitmq.com/docs/monitoring#health-checks), because the AMQP + listener opens as one of the last boot steps — and `livenessProbe` is the stage-1 + `rabbitmq-diagnostics -q ping` with a deliberately generous budget (60 s initial delay, + 3 × 30 s). This makes the infra release's `wait: true` mean "the broker accepts + connections" — the premise the migration hook's ordering relies on. Both are overridable + wholesale via `rabbitmq.livenessProbe` / `rabbitmq.readinessProbe` (set = replace, no + deep merge). Consequence: a broker that never finishes booting now fails the deploy + instead of letting the migration run against an unreachable broker. diff --git a/docs/values.md b/docs/values.md index fd69463..871928e 100644 --- a/docs/values.md +++ b/docs/values.md @@ -28,7 +28,7 @@ Every workload component (`webserver`, `storefront`, `cron`, `consumers.defaults | `nodeSelector` / `tolerations` / `affinity` / `priorityClassName` | scheduling | | `extraEnv` | extra env entries (raw list, supports `valueFrom`) | | `extraVolumes` / `extraVolumeMounts` | additional volumes | -| `livenessProbe` / `readinessProbe` | probe overrides | +| `livenessProbe` / `readinessProbe` | probe overrides; empty = the chart default where one exists (`redis`, `rabbitmq`), and setting one REPLACES the default wholesale (no deep merge) | | `securityContext` / `podSecurityContext` | security contexts | | `terminationGracePeriodSeconds`, `lifecycle` | shutdown behavior | diff --git a/tests/golden/scenarios/basic-production/expected/continuous.yaml b/tests/golden/scenarios/basic-production/expected/continuous.yaml index fda309d..77b8543 100644 --- a/tests/golden/scenarios/basic-production/expected/continuous.yaml +++ b/tests/golden/scenarios/basic-production/expected/continuous.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/basic-production/expected/first-deploy-with-demo-data.yaml b/tests/golden/scenarios/basic-production/expected/first-deploy-with-demo-data.yaml index 4fa723f..d903edf 100644 --- a/tests/golden/scenarios/basic-production/expected/first-deploy-with-demo-data.yaml +++ b/tests/golden/scenarios/basic-production/expected/first-deploy-with-demo-data.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/basic-production/expected/first-deploy.yaml b/tests/golden/scenarios/basic-production/expected/first-deploy.yaml index 6e59144..b7cf812 100644 --- a/tests/golden/scenarios/basic-production/expected/first-deploy.yaml +++ b/tests/golden/scenarios/basic-production/expected/first-deploy.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/development-single-domain/expected/continuous.yaml b/tests/golden/scenarios/development-single-domain/expected/continuous.yaml index d7a06fd..3e78bfe 100644 --- a/tests/golden/scenarios/development-single-domain/expected/continuous.yaml +++ b/tests/golden/scenarios/development-single-domain/expected/continuous.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: "0.01" - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/development-single-domain/expected/first-deploy-with-demo-data.yaml b/tests/golden/scenarios/development-single-domain/expected/first-deploy-with-demo-data.yaml index f24e535..70c166c 100644 --- a/tests/golden/scenarios/development-single-domain/expected/first-deploy-with-demo-data.yaml +++ b/tests/golden/scenarios/development-single-domain/expected/first-deploy-with-demo-data.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: "0.01" - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/development-single-domain/expected/first-deploy.yaml b/tests/golden/scenarios/development-single-domain/expected/first-deploy.yaml index c742794..fc38e2b 100644 --- a/tests/golden/scenarios/development-single-domain/expected/first-deploy.yaml +++ b/tests/golden/scenarios/development-single-domain/expected/first-deploy.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: "0.01" - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/development-with-cloudflare/expected/continuous.yaml b/tests/golden/scenarios/development-with-cloudflare/expected/continuous.yaml index 49a4eb4..73047fa 100644 --- a/tests/golden/scenarios/development-with-cloudflare/expected/continuous.yaml +++ b/tests/golden/scenarios/development-with-cloudflare/expected/continuous.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: "0.01" - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy-with-demo-data.yaml b/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy-with-demo-data.yaml index 2fe4f09..4d69464 100644 --- a/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy-with-demo-data.yaml +++ b/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy-with-demo-data.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: "0.01" - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy.yaml b/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy.yaml index ebcc1ea..d658ef6 100644 --- a/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy.yaml +++ b/tests/golden/scenarios/development-with-cloudflare/expected/first-deploy.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: "0.01" - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/escaping-env/expected/continuous.yaml b/tests/golden/scenarios/escaping-env/expected/continuous.yaml index f7d5168..a1e28c1 100644 --- a/tests/golden/scenarios/escaping-env/expected/continuous.yaml +++ b/tests/golden/scenarios/escaping-env/expected/continuous.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/escaping-env/expected/first-deploy-with-demo-data.yaml b/tests/golden/scenarios/escaping-env/expected/first-deploy-with-demo-data.yaml index 10aa7d7..6561bc5 100644 --- a/tests/golden/scenarios/escaping-env/expected/first-deploy-with-demo-data.yaml +++ b/tests/golden/scenarios/escaping-env/expected/first-deploy-with-demo-data.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/escaping-env/expected/first-deploy.yaml b/tests/golden/scenarios/escaping-env/expected/first-deploy.yaml index 050be36..e60e871 100644 --- a/tests/golden/scenarios/escaping-env/expected/first-deploy.yaml +++ b/tests/golden/scenarios/escaping-env/expected/first-deploy.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/production-with-cloudflare/expected/continuous.yaml b/tests/golden/scenarios/production-with-cloudflare/expected/continuous.yaml index e15485e..5794ee4 100644 --- a/tests/golden/scenarios/production-with-cloudflare/expected/continuous.yaml +++ b/tests/golden/scenarios/production-with-cloudflare/expected/continuous.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy-with-demo-data.yaml b/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy-with-demo-data.yaml index ba649cd..c31e2ac 100644 --- a/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy-with-demo-data.yaml +++ b/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy-with-demo-data.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq diff --git a/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy.yaml b/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy.yaml index 370b55e..104aef9 100644 --- a/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy.yaml +++ b/tests/golden/scenarios/production-with-cloudflare/expected/first-deploy.yaml @@ -323,7 +323,23 @@ spec: resources: requests: cpu: 20m - + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 15 + failureThreshold: 3 + readinessProbe: + tcpSocket: + port: 5672 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 volumeMounts: - name: rabbitmq-data mountPath: /var/lib/rabbitmq