IanEff · IanEff · Aug 19, 2026 · Aug 19, 2026
@@ -30,6 +30,17 @@ S3_BUCKET=thump-wal
 S3_ACCESS_KEY=dummy-access-key
 S3_SECRET_KEY=dummy-secret-key
 
+# A 32-byte base64-encoded AES-256 key used to seal WAL segments and
+# transcripts before writing to S3. Offline CLI commands (calipers corpus/unseal)
+# and live broker mode require this. Under Tilt, if left unset, a key is generated
+# and saved here automatically on first run.
+THUMP_SEAL_KEY=
+
+# A 32-byte base64-encoded key used by NATS JetStream for on-disk storage encryption.
+# Under Tilt, if left unset, a key is generated and saved here automatically on first run.
+THUMP_NATS_JS_KEY=
+
+
 ## --- rattle (Detect) ---
 
 # Required unconditionally — rattle is the first hop, nothing upstream of it.

@@ -100,7 +100,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.seal.secretName }}
-                  key: key
+                  key: {{ .Values.seal.secretKey }}
             - name: PROM_URL
               value: {{ .Values.prometheus.url | quote }}
             - name: LOKI_URL

@@ -104,7 +104,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.seal.secretName }}
-                  key: key
+                  key: {{ .Values.seal.secretKey }}
             - name: HISS_POLICY
               value: /etc/thump/hiss/policy.yaml
           volumeMounts:

@@ -100,7 +100,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.seal.secretName }}
-                  key: key
+                  key: {{ .Values.seal.secretKey }}
             - name: PROM_URL
               value: {{ .Values.prometheus.url | quote }}
             - name: WHIR_CATALOG

@@ -100,7 +100,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.seal.secretName }}
-                  key: key
+                  key: {{ .Values.seal.secretKey }}
             - name: ACTION_CATALOG
               value: /etc/thump/actions/catalog.yaml
             - name: THUMP_EXECUTOR

@@ -259,7 +259,7 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: {{ .Values.nats.jetstream.secretName }}
-                  key: key
+                  key: {{ .Values.nats.jetstream.secretKey }}
           ports:
             - { containerPort: 4222, name: client }
             - { containerPort: 8222, name: monitor }

@@ -40,21 +40,14 @@ stringData:
 {{- end }}
 
 {{/*
-Unlike anthropic/s3 above, seal.key isn't an external credential — nothing
-outside the cluster ever needs to know it, so the chart can safely generate
-it instead of demanding an out-of-band provisioning step. `lookup` finds the
-Secret if a prior `helm install`/`upgrade` already created one and this block
-no-ops, so the key stays stable across upgrades; on a genuinely first
-install it's absent, so `randBytes 32` mints a fresh AES-256 key. `helm
-install thump` needs nothing extra for this one. `key` stays available as a
---set override for anyone importing existing key material (e.g. rotating in
-a key from a KMS). NOTE: `lookup` only sees a live cluster during a real
-`helm install`/`upgrade` — it always reads empty under `helm template`
-(which is what Tilt's helm() calls), so this block alone would mint a new
-key on every Tiltfile reload under Tilt. See the Tiltfile's thump-seal-secret
-local_resource for the dev-loop equivalent that keeps the key stable there.
+Off by default: production expects `seal.secretName` (default: thump-seal)
+to already exist out-of-band (SOPS, External Secrets Operator, or Vault).
+This key encrypts all S3 WAL segments and transcripts — ephemeral generation
+couples data durability to the namespace lifecycle (issue #186, D-31).
+This template only fires with `--set seal.create=true --set seal.key=...`
+as a local/Tilt dev override, never in production.
 */}}
-{{- if not (lookup "v1" "Secret" .Release.Namespace .Values.seal.secretName) }}
+{{- if .Values.seal.create }}
 ---
 apiVersion: v1
 kind: Secret
@@ -65,17 +58,17 @@ metadata:
     {{- include "thump.labels" . | nindent 4 }}
 type: Opaque
 stringData:
-  key: {{ .Values.seal.key | default (randBytes 32) | quote }}
+  {{ .Values.seal.secretKey }}: {{ .Values.seal.key | quote }}
 {{- end }}
 
 {{/*
-Same self-provisioning posture as seal.key above: nats.jetstream.key is
-internal-only storage-encryption material, not an external credential, so
-the chart generates it itself rather than requiring out-of-band setup.
-Same `lookup`-under-`helm template` caveat applies — see the Tiltfile's
-thump-nats-js-key-secret local_resource for Tilt's dev-loop equivalent.
+Off by default: production expects `nats.jetstream.secretName` (default:
+nats-js-key) to already exist out-of-band (SOPS, ESO, Vault).
+This key encrypts NATS JetStream storage at rest — ephemeral generation
+breaks recovery across namespace deletion. Only fires with
+`--set nats.jetstream.create=true --set nats.jetstream.key=...`.
 */}}
-{{- if not (lookup "v1" "Secret" .Release.Namespace .Values.nats.jetstream.secretName) }}
+{{- if .Values.nats.jetstream.create }}
 ---
 apiVersion: v1
 kind: Secret
@@ -86,5 +79,6 @@ metadata:
     {{- include "thump.labels" . | nindent 4 }}
 type: Opaque
 stringData:
-  key: {{ .Values.nats.jetstream.key | default (randBytes 32) | quote }}
+  {{ .Values.nats.jetstream.secretKey }}: {{ .Values.nats.jetstream.key | quote }}
 {{- end }}
+
@@ -138,13 +138,16 @@ s3:
 
 # THUMP_SEAL_KEY — every beat's broker path Require()s this once NATS_URL is
 # set: the WAL shipper's EncryptingSink and clank's S3Store both seal under
-# it before a PutObject. A 32-byte AES-256 key, base64. Unlike s3/anthropic
-# above, this is internal-only key material, not an external credential — the
-# chart self-provisions it on first install (secret.yaml's `lookup`-guarded
-# block) rather than requiring out-of-band setup. `key` stays as a --set
-# override for anyone importing existing key material.
+# it before a PutObject. A 32-byte AES-256 key, base64.
+# Same out-of-band posture as anthropic/s3 (D-31, issue #186): production
+# expects secretName to already exist out-of-band. Ephemeral generation
+# inside the namespace couples audit-trail durability to the namespace
+# lifecycle. `create: false` by default; `create: true` + `key` is a
+# --set-only escape hatch.
 seal:
   secretName: thump-seal
+  secretKey: key
+  create: false
   key: ""
 
 # Forge (GitOps source of record) integration — FORGE_REPO and FORGE_TOKEN.
@@ -271,13 +274,16 @@ nats:
     maxFile: 2Gi
     # $JS_KEY, interpolated into nats.conf's jetstream{} block (R9a) — the
     # AES/ChaCha20 key JetStream seals message blocks and stream/consumer
-    # metadata under on disk. Same self-provisioning posture as seal.key
-    # above: internal-only key material, not an external credential, so the
-    # chart generates it on first install (secret.yaml's `lookup`-guarded
-    # block) instead of requiring out-of-band setup. `key` stays as a --set
-    # override for anyone importing existing key material.
+    # metadata under on disk. Same out-of-band posture as seal.key above (D-31,
+    # issue #186): production expects secretName to already exist out-of-band.
+    # Ephemeral generation inside the namespace breaks on-disk recovery across
+    # namespace deletion. `create: false` by default; `create: true` + `key`
+    # is a --set-only escape hatch.
     secretName: nats-js-key
+    secretKey: key
+    create: false
     key: ""
+
   resources:
     requests:
       cpu: 50m

@@ -707,6 +707,16 @@ On reversible, low- or med-blast, automatically self-undoing actions (`BandActRe
 
 **Why not preserve the model's veto everywhere?** The counter-argument is that a hedging model might perceive subtle hazards uncaptured by structured evidence queries. But on a bounded, self-reverting action with an active convergence watcher, measuring reality beats predicting it: thump applies the candidate, watches the SLO recovery window directly, and resolves the outcome (actions authored with `reversal.holdOnMiss: true`, like both flagd actions in `config/dev/actions/catalog.yaml:133, :162`, hold the undo for an operator via `ReversalWatcher` rather than firing blind; `internal/thump/reversal.go:55-58`). Where thump cannot land its own undo or where the blast radius is wide, model caution remains the last defense and retains its veto.
 
+## D-31 · Storage encryption keys are durable out-of-band dependencies, not namespace-scoped ephemera — **Ratified** (2026-08-19)
+
+**The problem:** `deploy/chart/thump/templates/secret.yaml` previously self-provisioned `seal.key` (`thump-seal`) and `nats.jetstream.key` (`nats-js-key`) via `lookup ... | default (randBytes 32)` when absent. Deleting the `thump` namespace (e.g. `tilt down`/`up`, helm reinstall, or cluster migration) destroyed the Secret; the next apply minted a new AES-256 key, permanently orphaning all historical WAL segments and transcripts in S3/Ceph (`sealbox: open: cipher: message authentication failed`) and breaking on-disk JetStream PVC recovery.
+
+**We do:** storage encryption keys are treated as durable external dependencies, matching the existing out-of-band posture of `anthropic` and `s3` (`create: false` by default). The Helm chart references pre-existing `v1/Secret` objects (`thump-seal` and `nats-js-key`).
+- In production and hybrid rigs (`thump-test`), keys are managed out-of-band (SOPS, External Secrets Operator, HashiCorp Vault, AWS/GCP KMS).
+- In local dev (`tilt`), keys are sourced from `.env` (`THUMP_SEAL_KEY`, `THUMP_NATS_JS_KEY`), persisting stably across `tilt down`/`up` cycles and enabling offline CLI tools (`calipers corpus`, `calipers unseal`, `task calibrate`).
+
+**Why:** WAL segments in S3 and JetStream data on persistent volumes outlive the Kubernetes namespace and cluster. Coupling key minting to namespace instantiation turns routine operational actions — recreating a namespace, tearing down a dev cluster, or spinning up a test rig against existing buckets — into silent cryptographic data-loss events.
+
 ## Departures from other source material
 
 The D-ledger above is indexed against one book, *Agentic Reliability

@@ -109,6 +109,9 @@ task dev:transcripts                    # every run under transcripts/, same lay
 
 `<run_id>` comes from a `"reasoned"` log line's `run_id` field. Both targets port-forward
 `svc/s3mock` to local 9091 (Tilt already owns 9090 for Prometheus) and tear it down on exit.
+`THUMP_SEAL_KEY` and `THUMP_NATS_JS_KEY` are persisted stably in `.env` (seeded automatically
+on first `tilt up` if absent, D-31), so transcript extraction and WAL decryption work across
+routine `tilt down`/`up` cycles without key drift.
 
 
 ## acme, the third domain

@@ -16,7 +16,7 @@ in [`design-decisions.md`](design-decisions.md).
 | **Operator** (`calipers`) | Approve a held action; force one past the risk gate | Approval only releases what hiss already conditionally granted. `force` is attributed, audited, rendered `forced` everywhere, and still kill-switch-gated |
 | **Cluster admin** (RBAC on `ApprovalRequest`) | Approve a held action as an authenticated Kubernetes subject | `spec.decision` accepts `approve` and nothing else. The API server records the patch independently of this engine |
 | **The model** (Anthropic API) | Choose which catalogued action to propose, and argue for it | It cannot leave the catalog, invent a magnitude, or grant itself permission. See below |
-| **Reader of the object store** | Read every WAL segment and reasoning transcript ever shipped | Sealed with AES-256-GCM in-process before upload, so bucket access alone yields ciphertext |
+| **Reader of the object store** | Read every WAL segment and reasoning transcript ever shipped | Sealed with AES-256-GCM in-process before upload, so bucket access alone yields ciphertext. Key durability is managed out-of-band (SOPS, ESO, Vault) — see D-31 |
 | **Reader of etcd** | Read `ApprovalRequest` objects | Deliberately the only thing in etcd. Reasoning, evidence, and verdicts never go there — see D-14 |
 | **Reader of the forge** (read access to a `maintenanceRelease` contract's GitOps repo) | Read the full rendered `Set` for any release: subject identifiers, losing candidates, confidence, citations | Nothing in this engine — bounded only by the repo's own visibility, which is public on the rig's own test repo. See D-26 |
 

@@ -0,0 +1,113 @@
+package beat_test
+
+import (
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"gopkg.in/yaml.v3"
+)
+
+// TestChart_SecretsOffByDefault verifies that neither thump-seal nor nats-js-key
+// Secrets are rendered by default, preventing ephemeral in-namespace key generation (D-31).
+func TestChart_SecretsOffByDefault(t *testing.T) {
+	t.Parallel()
+
+	out, err := runInRepoRoot(t, "helm", "template", "./deploy/chart/thump")
+	if err != nil {
+		t.Fatalf("helm template: %v", err)
+	}
+
+	var secretNames []string
+	dec := yaml.NewDecoder(strings.NewReader(out))
+	for {
+		var doc struct {
+			Kind     string `yaml:"kind"`
+			Metadata struct {
+				Name string `yaml:"name"`
+			} `yaml:"metadata"`
+		}
+		if err := dec.Decode(&doc); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			t.Fatalf("decode rendered chart: %v", err)
+		}
+		if doc.Kind == "Secret" {
+			secretNames = append(secretNames, doc.Metadata.Name)
+		}
+	}
+
+	var want []string
+	if diff := cmp.Diff(want, secretNames); diff != "" {
+		t.Errorf("chart renders unexpected secrets by default (-want +got):\n%s", diff)
+	}
+}
+
+// TestChart_OptInSecretsRenderWhenExplicitlyEnabled verifies that dev overrides
+// (--set seal.create=true --set seal.key=...) render the requested secret.
+func TestChart_OptInSecretsRenderWhenExplicitlyEnabled(t *testing.T) {
+	t.Parallel()
+
+	//nolint:gosec // G101: synthetic test key fixture, not real credentials
+	testCases := map[string]struct {
+		args       []string
+		wantSecret string
+		wantKey    string
+		wantVal    string
+	}{
+		"seal.create=true renders thump-seal with provided key": {
+			args:       []string{"--set", "seal.create=true", "--set", "seal.key=dGVzdC1zZWFsLWtleQ=="},
+			wantSecret: "thump-seal",
+			wantKey:    "key",
+			wantVal:    "dGVzdC1zZWFsLWtleQ==",
+		},
+		"nats.jetstream.create=true renders nats-js-key with provided key": {
+			args:       []string{"--set", "nats.jetstream.create=true", "--set", "nats.jetstream.key=dGVzdC1qcy1rZXk="},
+			wantSecret: "nats-js-key",
+			wantKey:    "key",
+			wantVal:    "dGVzdC1qcy1rZXk=",
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			args := append([]string{"template", "./deploy/chart/thump"}, tc.args...)
+			out, err := runInRepoRoot(t, "helm", args...)
+			if err != nil {
+				t.Fatalf("helm template: %v", err)
+			}
+
+			found := false
+			dec := yaml.NewDecoder(strings.NewReader(out))
+			for {
+				var doc struct {
+					Kind     string `yaml:"kind"`
+					Metadata struct {
+						Name string `yaml:"name"`
+					} `yaml:"metadata"`
+					StringData map[string]string `yaml:"stringData"`
+				}
+				if err := dec.Decode(&doc); err != nil {
+					if errors.Is(err, io.EOF) {
+						break
+					}
+					t.Fatalf("decode rendered chart: %v", err)
+				}
+				if doc.Kind == "Secret" && doc.Metadata.Name == tc.wantSecret {
+					found = true
+					if got := doc.StringData[tc.wantKey]; got != tc.wantVal {
+						t.Errorf("secret %s key %s: want %q, got %q", tc.wantSecret, tc.wantKey, tc.wantVal, got)
+					}
+				}
+			}
+			if !found {
+				t.Errorf("secret %s was not rendered", tc.wantSecret)
+			}
+		})
+	}
+}
@@ -144,7 +144,12 @@ if [ -f .env ]; then
   if grep -q "^THUMP_SEAL_KEY=" .env && ! grep -q "^THUMP_SEAL_KEY=\"\"" .env && ! grep -q "^THUMP_SEAL_KEY=''" .env && [ -n "$(grep "^THUMP_SEAL_KEY=" .env | cut -d= -f2-)" ]; then
     echo -e "  [${GREEN}OK${RESET}] THUMP_SEAL_KEY configured (enables WAL unseal and corpus mining)"
   else
-    echo -e "  [${DIM}INFO${RESET}] THUMP_SEAL_KEY unset (transcript unsealing will require key)"
+    echo -e "  [${DIM}INFO${RESET}] THUMP_SEAL_KEY unset (auto-generated on first tilt up or needed for unseal)"
+  fi
+  if grep -q "^THUMP_NATS_JS_KEY=" .env && ! grep -q "^THUMP_NATS_JS_KEY=\"\"" .env && ! grep -q "^THUMP_NATS_JS_KEY=''" .env && [ -n "$(grep "^THUMP_NATS_JS_KEY=" .env | cut -d= -f2-)" ]; then
+    echo -e "  [${GREEN}OK${RESET}] THUMP_NATS_JS_KEY configured (enables persistent NATS storage encryption)"
+  else
+    echo -e "  [${DIM}INFO${RESET}] THUMP_NATS_JS_KEY unset (auto-generated on first tilt up)"
   fi
 else
   echo -e "  [${DIM}INFO${RESET}] .env file not present (cp .env.example .env when testing live mode)"

@@ -33,22 +33,12 @@ def setup(cluster, cluster_name, domain_values, domains):
     # blamed it on Tilt re-triggering.
     rendered = helm("deploy/chart/thump", namespace = "thump", values = [cluster["values"]])
 
-    # secret.yaml's thump-seal and nats-js-key Secrets are guarded by
-    # `{{- if not (lookup ...) }}` so a real `helm install` mints them once
-    # and leaves them alone. `lookup` always reads empty under `helm template`
-    # (what helm() above runs), so left in this manifest set, that guard is
-    # inert: every Tiltfile reload re-renders a *freshly random* key and
-    # k8s_yaml applies it over whatever's already live. That's what broke
-    # thump-test 2026-07-31 — an unrelated chart edit (OTel env vars)
-    # triggered a reload mid-session, silently rotated nats-js-key's value,
-    # and the running NATS pod could no longer decrypt its own on-disk
-    # JetStream store ("unable to recover keys" / stream "could not be
-    # recovered"). Strip both Secret objects out of what k8s_yaml ever sees —
-    # thump-seal-secret and thump-nats-js-key-secret in tilt/infra.Tiltfile
-    # (create-if-absent) become the *only* thing allowed to touch them under
-    # Tilt, enforcing the same once-only intent the chart's lookup guard has
-    # for a real `helm install`, by manifest filtering instead of a guard Tilt
-    # can't evaluate.
+    # secret.yaml's thump-seal and nats-js-key Secrets are off by default in
+    # the chart (D-31, matching anthropic and s3). If an override sets
+    # .Values.seal.create or .Values.nats.jetstream.create, strip both Secret
+    # objects out of what k8s_yaml sees — thump-seal-secret and
+    # thump-nats-js-key-secret in tilt/infra.Tiltfile source these durably from
+    # .env and remain the authority on their lifecycle under Tilt.
     _, rendered = filter_yaml(rendered, kind = "Secret", name = "thump-seal")
     _, rendered = filter_yaml(rendered, kind = "Secret", name = "nats-js-key")