From ad76a8eac210b138cff3dd03064364bbfeb3a688 Mon Sep 17 00:00:00 2001 From: Ian Furst Date: Wed, 19 Aug 2026 07:30:46 -0400 Subject: [PATCH] feat(chart): treat storage encryption keys as durable dependencies Stop self-provisioning WAL and JetStream encryption keys in the Helm chart. Treat them as out-of-band dependencies matching S3/Anthropic patterns (D-31). - Default `create` flags to `false`. - Source keys from `.env` in Tilt to ensure persistence across dev restarts and prevent data loss from accidental rotation. - Add Helm template validation tests to verify secret generation posture. --- .env.example | 11 ++ .../thump/templates/deployment-clank.yaml | 2 +- .../thump/templates/deployment-hiss.yaml | 2 +- .../thump/templates/deployment-rattle.yaml | 2 +- .../thump/templates/deployment-thump.yaml | 2 +- deploy/chart/thump/templates/nats.yaml | 2 +- deploy/chart/thump/templates/secret.yaml | 38 +++--- deploy/chart/thump/values.yaml | 26 ++-- docs/design-decisions.md | 10 ++ docs/dev-environment.md | 3 + docs/threat-model.md | 2 +- internal/beat/secret_chart_test.go | 113 ++++++++++++++++++ scripts/doctor.sh | 7 +- tilt/deploy.Tiltfile | 22 +--- tilt/infra.Tiltfile | 55 +++------ 15 files changed, 206 insertions(+), 91 deletions(-) create mode 100644 internal/beat/secret_chart_test.go diff --git a/.env.example b/.env.example index 6f8e9f3b..73b4c305 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,17 @@ S3_BUCKET=thump-wal S3_ACCESS_KEY=dummy-access-key S3_SECRET_KEY=dummy-secret-key +# A 32-byte base64-encoded AES-256 key used to seal WAL segments and +# transcripts before writing to S3. Offline CLI commands (calipers corpus/unseal) +# and live broker mode require this. Under Tilt, if left unset, a key is generated +# and saved here automatically on first run. +THUMP_SEAL_KEY= + +# A 32-byte base64-encoded key used by NATS JetStream for on-disk storage encryption. +# Under Tilt, if left unset, a key is generated and saved here automatically on first run. +THUMP_NATS_JS_KEY= + + ## --- rattle (Detect) --- # Required unconditionally — rattle is the first hop, nothing upstream of it. diff --git a/deploy/chart/thump/templates/deployment-clank.yaml b/deploy/chart/thump/templates/deployment-clank.yaml index 3453d32a..f8c3537c 100644 --- a/deploy/chart/thump/templates/deployment-clank.yaml +++ b/deploy/chart/thump/templates/deployment-clank.yaml @@ -100,7 +100,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.seal.secretName }} - key: key + key: {{ .Values.seal.secretKey }} - name: PROM_URL value: {{ .Values.prometheus.url | quote }} - name: LOKI_URL diff --git a/deploy/chart/thump/templates/deployment-hiss.yaml b/deploy/chart/thump/templates/deployment-hiss.yaml index 75b94ad3..3ea253b0 100644 --- a/deploy/chart/thump/templates/deployment-hiss.yaml +++ b/deploy/chart/thump/templates/deployment-hiss.yaml @@ -104,7 +104,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.seal.secretName }} - key: key + key: {{ .Values.seal.secretKey }} - name: HISS_POLICY value: /etc/thump/hiss/policy.yaml volumeMounts: diff --git a/deploy/chart/thump/templates/deployment-rattle.yaml b/deploy/chart/thump/templates/deployment-rattle.yaml index 21eaacb2..c8c91373 100644 --- a/deploy/chart/thump/templates/deployment-rattle.yaml +++ b/deploy/chart/thump/templates/deployment-rattle.yaml @@ -100,7 +100,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.seal.secretName }} - key: key + key: {{ .Values.seal.secretKey }} - name: PROM_URL value: {{ .Values.prometheus.url | quote }} - name: WHIR_CATALOG diff --git a/deploy/chart/thump/templates/deployment-thump.yaml b/deploy/chart/thump/templates/deployment-thump.yaml index e2b2e4a5..3d662a12 100644 --- a/deploy/chart/thump/templates/deployment-thump.yaml +++ b/deploy/chart/thump/templates/deployment-thump.yaml @@ -100,7 +100,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.seal.secretName }} - key: key + key: {{ .Values.seal.secretKey }} - name: ACTION_CATALOG value: /etc/thump/actions/catalog.yaml - name: THUMP_EXECUTOR diff --git a/deploy/chart/thump/templates/nats.yaml b/deploy/chart/thump/templates/nats.yaml index 05e0f273..44e53ef1 100644 --- a/deploy/chart/thump/templates/nats.yaml +++ b/deploy/chart/thump/templates/nats.yaml @@ -259,7 +259,7 @@ spec: valueFrom: secretKeyRef: name: {{ .Values.nats.jetstream.secretName }} - key: key + key: {{ .Values.nats.jetstream.secretKey }} ports: - { containerPort: 4222, name: client } - { containerPort: 8222, name: monitor } diff --git a/deploy/chart/thump/templates/secret.yaml b/deploy/chart/thump/templates/secret.yaml index 90874cf4..e09f26e7 100644 --- a/deploy/chart/thump/templates/secret.yaml +++ b/deploy/chart/thump/templates/secret.yaml @@ -40,21 +40,14 @@ stringData: {{- end }} {{/* -Unlike anthropic/s3 above, seal.key isn't an external credential — nothing -outside the cluster ever needs to know it, so the chart can safely generate -it instead of demanding an out-of-band provisioning step. `lookup` finds the -Secret if a prior `helm install`/`upgrade` already created one and this block -no-ops, so the key stays stable across upgrades; on a genuinely first -install it's absent, so `randBytes 32` mints a fresh AES-256 key. `helm -install thump` needs nothing extra for this one. `key` stays available as a ---set override for anyone importing existing key material (e.g. rotating in -a key from a KMS). NOTE: `lookup` only sees a live cluster during a real -`helm install`/`upgrade` — it always reads empty under `helm template` -(which is what Tilt's helm() calls), so this block alone would mint a new -key on every Tiltfile reload under Tilt. See the Tiltfile's thump-seal-secret -local_resource for the dev-loop equivalent that keeps the key stable there. +Off by default: production expects `seal.secretName` (default: thump-seal) +to already exist out-of-band (SOPS, External Secrets Operator, or Vault). +This key encrypts all S3 WAL segments and transcripts — ephemeral generation +couples data durability to the namespace lifecycle (issue #186, D-31). +This template only fires with `--set seal.create=true --set seal.key=...` +as a local/Tilt dev override, never in production. */}} -{{- if not (lookup "v1" "Secret" .Release.Namespace .Values.seal.secretName) }} +{{- if .Values.seal.create }} --- apiVersion: v1 kind: Secret @@ -65,17 +58,17 @@ metadata: {{- include "thump.labels" . | nindent 4 }} type: Opaque stringData: - key: {{ .Values.seal.key | default (randBytes 32) | quote }} + {{ .Values.seal.secretKey }}: {{ .Values.seal.key | quote }} {{- end }} {{/* -Same self-provisioning posture as seal.key above: nats.jetstream.key is -internal-only storage-encryption material, not an external credential, so -the chart generates it itself rather than requiring out-of-band setup. -Same `lookup`-under-`helm template` caveat applies — see the Tiltfile's -thump-nats-js-key-secret local_resource for Tilt's dev-loop equivalent. +Off by default: production expects `nats.jetstream.secretName` (default: +nats-js-key) to already exist out-of-band (SOPS, ESO, Vault). +This key encrypts NATS JetStream storage at rest — ephemeral generation +breaks recovery across namespace deletion. Only fires with +`--set nats.jetstream.create=true --set nats.jetstream.key=...`. */}} -{{- if not (lookup "v1" "Secret" .Release.Namespace .Values.nats.jetstream.secretName) }} +{{- if .Values.nats.jetstream.create }} --- apiVersion: v1 kind: Secret @@ -86,5 +79,6 @@ metadata: {{- include "thump.labels" . | nindent 4 }} type: Opaque stringData: - key: {{ .Values.nats.jetstream.key | default (randBytes 32) | quote }} + {{ .Values.nats.jetstream.secretKey }}: {{ .Values.nats.jetstream.key | quote }} {{- end }} + diff --git a/deploy/chart/thump/values.yaml b/deploy/chart/thump/values.yaml index f188c7ea..988fbfc7 100644 --- a/deploy/chart/thump/values.yaml +++ b/deploy/chart/thump/values.yaml @@ -138,13 +138,16 @@ s3: # THUMP_SEAL_KEY — every beat's broker path Require()s this once NATS_URL is # set: the WAL shipper's EncryptingSink and clank's S3Store both seal under -# it before a PutObject. A 32-byte AES-256 key, base64. Unlike s3/anthropic -# above, this is internal-only key material, not an external credential — the -# chart self-provisions it on first install (secret.yaml's `lookup`-guarded -# block) rather than requiring out-of-band setup. `key` stays as a --set -# override for anyone importing existing key material. +# it before a PutObject. A 32-byte AES-256 key, base64. +# Same out-of-band posture as anthropic/s3 (D-31, issue #186): production +# expects secretName to already exist out-of-band. Ephemeral generation +# inside the namespace couples audit-trail durability to the namespace +# lifecycle. `create: false` by default; `create: true` + `key` is a +# --set-only escape hatch. seal: secretName: thump-seal + secretKey: key + create: false key: "" # Forge (GitOps source of record) integration — FORGE_REPO and FORGE_TOKEN. @@ -271,13 +274,16 @@ nats: maxFile: 2Gi # $JS_KEY, interpolated into nats.conf's jetstream{} block (R9a) — the # AES/ChaCha20 key JetStream seals message blocks and stream/consumer - # metadata under on disk. Same self-provisioning posture as seal.key - # above: internal-only key material, not an external credential, so the - # chart generates it on first install (secret.yaml's `lookup`-guarded - # block) instead of requiring out-of-band setup. `key` stays as a --set - # override for anyone importing existing key material. + # metadata under on disk. Same out-of-band posture as seal.key above (D-31, + # issue #186): production expects secretName to already exist out-of-band. + # Ephemeral generation inside the namespace breaks on-disk recovery across + # namespace deletion. `create: false` by default; `create: true` + `key` + # is a --set-only escape hatch. secretName: nats-js-key + secretKey: key + create: false key: "" + resources: requests: cpu: 50m diff --git a/docs/design-decisions.md b/docs/design-decisions.md index 740d6cd9..b71a2a12 100644 --- a/docs/design-decisions.md +++ b/docs/design-decisions.md @@ -707,6 +707,16 @@ On reversible, low- or med-blast, automatically self-undoing actions (`BandActRe **Why not preserve the model's veto everywhere?** The counter-argument is that a hedging model might perceive subtle hazards uncaptured by structured evidence queries. But on a bounded, self-reverting action with an active convergence watcher, measuring reality beats predicting it: thump applies the candidate, watches the SLO recovery window directly, and resolves the outcome (actions authored with `reversal.holdOnMiss: true`, like both flagd actions in `config/dev/actions/catalog.yaml:133, :162`, hold the undo for an operator via `ReversalWatcher` rather than firing blind; `internal/thump/reversal.go:55-58`). Where thump cannot land its own undo or where the blast radius is wide, model caution remains the last defense and retains its veto. +## D-31 · Storage encryption keys are durable out-of-band dependencies, not namespace-scoped ephemera — **Ratified** (2026-08-19) + +**The problem:** `deploy/chart/thump/templates/secret.yaml` previously self-provisioned `seal.key` (`thump-seal`) and `nats.jetstream.key` (`nats-js-key`) via `lookup ... | default (randBytes 32)` when absent. Deleting the `thump` namespace (e.g. `tilt down`/`up`, helm reinstall, or cluster migration) destroyed the Secret; the next apply minted a new AES-256 key, permanently orphaning all historical WAL segments and transcripts in S3/Ceph (`sealbox: open: cipher: message authentication failed`) and breaking on-disk JetStream PVC recovery. + +**We do:** storage encryption keys are treated as durable external dependencies, matching the existing out-of-band posture of `anthropic` and `s3` (`create: false` by default). The Helm chart references pre-existing `v1/Secret` objects (`thump-seal` and `nats-js-key`). +- In production and hybrid rigs (`thump-test`), keys are managed out-of-band (SOPS, External Secrets Operator, HashiCorp Vault, AWS/GCP KMS). +- In local dev (`tilt`), keys are sourced from `.env` (`THUMP_SEAL_KEY`, `THUMP_NATS_JS_KEY`), persisting stably across `tilt down`/`up` cycles and enabling offline CLI tools (`calipers corpus`, `calipers unseal`, `task calibrate`). + +**Why:** WAL segments in S3 and JetStream data on persistent volumes outlive the Kubernetes namespace and cluster. Coupling key minting to namespace instantiation turns routine operational actions — recreating a namespace, tearing down a dev cluster, or spinning up a test rig against existing buckets — into silent cryptographic data-loss events. + ## Departures from other source material The D-ledger above is indexed against one book, *Agentic Reliability diff --git a/docs/dev-environment.md b/docs/dev-environment.md index a3c93e44..ccddf3c4 100644 --- a/docs/dev-environment.md +++ b/docs/dev-environment.md @@ -109,6 +109,9 @@ task dev:transcripts # every run under transcripts/, same lay `` comes from a `"reasoned"` log line's `run_id` field. Both targets port-forward `svc/s3mock` to local 9091 (Tilt already owns 9090 for Prometheus) and tear it down on exit. +`THUMP_SEAL_KEY` and `THUMP_NATS_JS_KEY` are persisted stably in `.env` (seeded automatically +on first `tilt up` if absent, D-31), so transcript extraction and WAL decryption work across +routine `tilt down`/`up` cycles without key drift. ## acme, the third domain diff --git a/docs/threat-model.md b/docs/threat-model.md index 045aeee2..9002ed0e 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -16,7 +16,7 @@ in [`design-decisions.md`](design-decisions.md). | **Operator** (`calipers`) | Approve a held action; force one past the risk gate | Approval only releases what hiss already conditionally granted. `force` is attributed, audited, rendered `forced` everywhere, and still kill-switch-gated | | **Cluster admin** (RBAC on `ApprovalRequest`) | Approve a held action as an authenticated Kubernetes subject | `spec.decision` accepts `approve` and nothing else. The API server records the patch independently of this engine | | **The model** (Anthropic API) | Choose which catalogued action to propose, and argue for it | It cannot leave the catalog, invent a magnitude, or grant itself permission. See below | -| **Reader of the object store** | Read every WAL segment and reasoning transcript ever shipped | Sealed with AES-256-GCM in-process before upload, so bucket access alone yields ciphertext | +| **Reader of the object store** | Read every WAL segment and reasoning transcript ever shipped | Sealed with AES-256-GCM in-process before upload, so bucket access alone yields ciphertext. Key durability is managed out-of-band (SOPS, ESO, Vault) — see D-31 | | **Reader of etcd** | Read `ApprovalRequest` objects | Deliberately the only thing in etcd. Reasoning, evidence, and verdicts never go there — see D-14 | | **Reader of the forge** (read access to a `maintenanceRelease` contract's GitOps repo) | Read the full rendered `Set` for any release: subject identifiers, losing candidates, confidence, citations | Nothing in this engine — bounded only by the repo's own visibility, which is public on the rig's own test repo. See D-26 | diff --git a/internal/beat/secret_chart_test.go b/internal/beat/secret_chart_test.go new file mode 100644 index 00000000..346ef19f --- /dev/null +++ b/internal/beat/secret_chart_test.go @@ -0,0 +1,113 @@ +package beat_test + +import ( + "errors" + "io" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "gopkg.in/yaml.v3" +) + +// TestChart_SecretsOffByDefault verifies that neither thump-seal nor nats-js-key +// Secrets are rendered by default, preventing ephemeral in-namespace key generation (D-31). +func TestChart_SecretsOffByDefault(t *testing.T) { + t.Parallel() + + out, err := runInRepoRoot(t, "helm", "template", "./deploy/chart/thump") + if err != nil { + t.Fatalf("helm template: %v", err) + } + + var secretNames []string + dec := yaml.NewDecoder(strings.NewReader(out)) + for { + var doc struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + } `yaml:"metadata"` + } + if err := dec.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("decode rendered chart: %v", err) + } + if doc.Kind == "Secret" { + secretNames = append(secretNames, doc.Metadata.Name) + } + } + + var want []string + if diff := cmp.Diff(want, secretNames); diff != "" { + t.Errorf("chart renders unexpected secrets by default (-want +got):\n%s", diff) + } +} + +// TestChart_OptInSecretsRenderWhenExplicitlyEnabled verifies that dev overrides +// (--set seal.create=true --set seal.key=...) render the requested secret. +func TestChart_OptInSecretsRenderWhenExplicitlyEnabled(t *testing.T) { + t.Parallel() + + //nolint:gosec // G101: synthetic test key fixture, not real credentials + testCases := map[string]struct { + args []string + wantSecret string + wantKey string + wantVal string + }{ + "seal.create=true renders thump-seal with provided key": { + args: []string{"--set", "seal.create=true", "--set", "seal.key=dGVzdC1zZWFsLWtleQ=="}, + wantSecret: "thump-seal", + wantKey: "key", + wantVal: "dGVzdC1zZWFsLWtleQ==", + }, + "nats.jetstream.create=true renders nats-js-key with provided key": { + args: []string{"--set", "nats.jetstream.create=true", "--set", "nats.jetstream.key=dGVzdC1qcy1rZXk="}, + wantSecret: "nats-js-key", + wantKey: "key", + wantVal: "dGVzdC1qcy1rZXk=", + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + args := append([]string{"template", "./deploy/chart/thump"}, tc.args...) + out, err := runInRepoRoot(t, "helm", args...) + if err != nil { + t.Fatalf("helm template: %v", err) + } + + found := false + dec := yaml.NewDecoder(strings.NewReader(out)) + for { + var doc struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + } `yaml:"metadata"` + StringData map[string]string `yaml:"stringData"` + } + if err := dec.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("decode rendered chart: %v", err) + } + if doc.Kind == "Secret" && doc.Metadata.Name == tc.wantSecret { + found = true + if got := doc.StringData[tc.wantKey]; got != tc.wantVal { + t.Errorf("secret %s key %s: want %q, got %q", tc.wantSecret, tc.wantKey, tc.wantVal, got) + } + } + } + if !found { + t.Errorf("secret %s was not rendered", tc.wantSecret) + } + }) + } +} diff --git a/scripts/doctor.sh b/scripts/doctor.sh index ad0de8e7..82deb53b 100755 --- a/scripts/doctor.sh +++ b/scripts/doctor.sh @@ -144,7 +144,12 @@ if [ -f .env ]; then if grep -q "^THUMP_SEAL_KEY=" .env && ! grep -q "^THUMP_SEAL_KEY=\"\"" .env && ! grep -q "^THUMP_SEAL_KEY=''" .env && [ -n "$(grep "^THUMP_SEAL_KEY=" .env | cut -d= -f2-)" ]; then echo -e " [${GREEN}OK${RESET}] THUMP_SEAL_KEY configured (enables WAL unseal and corpus mining)" else - echo -e " [${DIM}INFO${RESET}] THUMP_SEAL_KEY unset (transcript unsealing will require key)" + echo -e " [${DIM}INFO${RESET}] THUMP_SEAL_KEY unset (auto-generated on first tilt up or needed for unseal)" + fi + if grep -q "^THUMP_NATS_JS_KEY=" .env && ! grep -q "^THUMP_NATS_JS_KEY=\"\"" .env && ! grep -q "^THUMP_NATS_JS_KEY=''" .env && [ -n "$(grep "^THUMP_NATS_JS_KEY=" .env | cut -d= -f2-)" ]; then + echo -e " [${GREEN}OK${RESET}] THUMP_NATS_JS_KEY configured (enables persistent NATS storage encryption)" + else + echo -e " [${DIM}INFO${RESET}] THUMP_NATS_JS_KEY unset (auto-generated on first tilt up)" fi else echo -e " [${DIM}INFO${RESET}] .env file not present (cp .env.example .env when testing live mode)" diff --git a/tilt/deploy.Tiltfile b/tilt/deploy.Tiltfile index 190afdb9..69e60e6d 100644 --- a/tilt/deploy.Tiltfile +++ b/tilt/deploy.Tiltfile @@ -33,22 +33,12 @@ def setup(cluster, cluster_name, domain_values, domains): # blamed it on Tilt re-triggering. rendered = helm("deploy/chart/thump", namespace = "thump", values = [cluster["values"]]) - # secret.yaml's thump-seal and nats-js-key Secrets are guarded by - # `{{- if not (lookup ...) }}` so a real `helm install` mints them once - # and leaves them alone. `lookup` always reads empty under `helm template` - # (what helm() above runs), so left in this manifest set, that guard is - # inert: every Tiltfile reload re-renders a *freshly random* key and - # k8s_yaml applies it over whatever's already live. That's what broke - # thump-test 2026-07-31 — an unrelated chart edit (OTel env vars) - # triggered a reload mid-session, silently rotated nats-js-key's value, - # and the running NATS pod could no longer decrypt its own on-disk - # JetStream store ("unable to recover keys" / stream "could not be - # recovered"). Strip both Secret objects out of what k8s_yaml ever sees — - # thump-seal-secret and thump-nats-js-key-secret in tilt/infra.Tiltfile - # (create-if-absent) become the *only* thing allowed to touch them under - # Tilt, enforcing the same once-only intent the chart's lookup guard has - # for a real `helm install`, by manifest filtering instead of a guard Tilt - # can't evaluate. + # secret.yaml's thump-seal and nats-js-key Secrets are off by default in + # the chart (D-31, matching anthropic and s3). If an override sets + # .Values.seal.create or .Values.nats.jetstream.create, strip both Secret + # objects out of what k8s_yaml sees — thump-seal-secret and + # thump-nats-js-key-secret in tilt/infra.Tiltfile source these durably from + # .env and remain the authority on their lifecycle under Tilt. _, rendered = filter_yaml(rendered, kind = "Secret", name = "thump-seal") _, rendered = filter_yaml(rendered, kind = "Secret", name = "nats-js-key") diff --git a/tilt/infra.Tiltfile b/tilt/infra.Tiltfile index 0fb852ec..a1401f67 100644 --- a/tilt/infra.Tiltfile +++ b/tilt/infra.Tiltfile @@ -115,50 +115,33 @@ def setup(cluster, cluster_name, h): s3_deps = [] h.kubectl_local("thump-s3-secret", s3_body, resource_deps = s3_deps) - # thump-seal-secret / thump-nats-js-key-secret: unlike - # thump-anthropic-secret and thump-s3-secret above, these two keys are - # pure internal material — no external system issues them, so there's - # nothing to source from .env. The chart's own secret.yaml self-provisions - # them on a real `helm install` via a `lookup`-guarded block, but `lookup` - # always reads empty under `helm template` (what Tilt's helm() runs) — the - # k8s_yaml() call in tilt/deploy.Tiltfile filter_yaml()s both Secret - # objects out of the chart's rendered manifests entirely, so that inert - # guard never gets a chance to matter under Tilt. These local_resources - # are the *only* thing that ever creates or touches either secret in a Tilt - # session: create-if-absent, never touch an existing one (unlike - # thump-s3-secret's dry-run-apply, which is meant to re-sync every run) — - # `kubectl get ... || kubectl create ...`, not `--dry-run=client -o yaml | - # apply`. - # - # `get || create` is the obvious shape for create-if-absent and it is the - # wrong one here: a `get` that fails because the API was unreachable is - # indistinguishable from one that fails because the secret is absent, and - # the `||` branch answers both by minting a fresh random key. On - # thump-seal that orphans every sealed WAL segment; on nats-js-key it - # orphans the on-disk JetStream store, which is exactly the failure - # recorded on 2026-07-31. So: only a literal NotFound authorises a create. - # Any other error returns non-zero so kubectl_local retries it, and says - # out loud that it declined to mint a key rather than doing it quietly. - def _ensure_key_secret(resource, secret): + # thump-seal-secret / thump-nats-js-key-secret: storage encryption keys + # (D-31, issue #186). Production expects these out-of-band (SOPS, ESO, + # Vault). Under Tilt, source them from .env so they stay stable across + # `tilt down`/`up` cycles rather than rotating on every namespace recreation. + # If absent from .env, generate once and persist to .env. + def _ensure_key_secret(resource, secret, env_var): kubectl = "kubectl --context " + cluster["context"] h.kubectl_local( resource, - h.ENSURE_NS - + " && out=$(" - + kubectl - + " -n thump get secret " - + secret - + ' 2>&1); if [ $? -eq 0 ]; then exit 0; fi; case "$out" in *NotFound*) ' + 'set -a; source .env 2>/dev/null; set +a; ' + + 'if [ -z "$' + env_var + '" ]; then ' + + ' ' + env_var + '=$(openssl rand -base64 32); ' + + ' echo "' + env_var + '=$' + env_var + '" >> .env; ' + + ' echo "thump: generated new ' + env_var + ' and saved to .env"; ' + + 'fi; ' + + h.ENSURE_NS + + " && " + kubectl + " -n thump create secret generic " + secret - + ' --from-literal=key="$(openssl rand -base64 32)" ;; *) echo "thump: cannot tell whether secret ' - + secret - + ' exists, refusing to mint a replacement key over data encrypted under the old one: $out" >&2; false ;; esac', + + ' --from-literal=key="$' + env_var + '" --dry-run=client -o yaml | ' + + kubectl + + " apply -f -", ) - _ensure_key_secret("thump-seal-secret", "thump-seal") - _ensure_key_secret("thump-nats-js-key-secret", "nats-js-key") + _ensure_key_secret("thump-seal-secret", "thump-seal", "THUMP_SEAL_KEY") + _ensure_key_secret("thump-nats-js-key-secret", "nats-js-key", "THUMP_NATS_JS_KEY") # The namespace is filtered out of the chart's manifests (see the # filter_yaml block in tilt/deploy.Tiltfile for why), so something else