From 42746ff459369775c27f11a8b5ba29f0cd163844 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:17:46 -0500 Subject: [PATCH 1/8] Decide what a published record may disclose, from declared types The record exporter must not import a bounded context: it lives at cora.infrastructure so that tach's `depends_on = ["cora.shared"]` ENFORCES the zero-import property its standalone-verifiability claim rests on. But deciding whether a field's annotation is a StrEnum, a str alias, or a value object needs the defining module imported. Resolving at runtime is therefore not available to the code that needs the answer. So resolve at build time and commit the answer as inert data, the way `make openapi-snapshot` already does. The generator lives outside src/ on purpose: nothing shippable can import it, so the exporter cannot quietly acquire the dependency the design forbids. Resolving real type objects rather than annotation strings is what makes this tractable. An AST census left 88 unclassifiable fields across 48 annotation names; runtime resolution collapses plain aliases for free and leaves 2, both genuine modelling questions rather than gaps. The generator ABORTS on anything it cannot classify, because an unrecognised type is a question about the model, not a row to skip. The default is drop. A new event with a bare `str` field is excluded with nobody editing a list, which is the whole fail-closed property; the committed diff is the list of what a published record would disclose, and the drift test fails until someone regenerates and reads it. Two predictions from the design memo came out of the artifact rather than out of argument: FacilityCode resolves to a value object and drops, and LogbookOpened.kind drops because it is a bare `str` with no enum, which is the pressure to type it properly that F5 said it would be. One correction to the review record: PresenceMode and VisitType are NOT published. Both are declared `str` with the enum named only in a comment, so both drop. The round-4 threat model listed them as kept. Its conclusion is unchanged, since full-precision timestamps and ActorKind still compose with a public beamtime schedule, but the premise was wrong and the reviews memo says so now. Refs: project_record_export_v3.md F5, project_record_export_build_brief.md Co-Authored-By: Claude Opus 5 (1M context) --- Makefile | 26 +- apps/api/pyproject.toml | 7 +- .../infrastructure/record_export/__init__.py | 25 + .../record_export/_dispositions.py | 1819 +++++++++++++++++ .../test_record_dispositions_drift.py | 67 + apps/api/tools/gen_record_dispositions.py | 316 +++ 6 files changed, 2252 insertions(+), 8 deletions(-) create mode 100644 apps/api/src/cora/infrastructure/record_export/__init__.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_dispositions.py create mode 100644 apps/api/tests/architecture/test_record_dispositions_drift.py create mode 100644 apps/api/tools/gen_record_dispositions.py diff --git a/Makefile b/Makefile index 5b9e01ec738..d2423f627e9 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ test-noio test-db test-coverage store-durations diff-coverage fmt clean help \ migrate-status migrate-apply migrate-new migrate-hash precommit precommit-run \ arch-check arch-show docs-stage docs-build docs-serve openapi-snapshot \ - mutmut-audit mutmut-browse restore-drill + record-dispositions mutmut-audit mutmut-browse restore-drill API_DIR := apps/api COMPOSE := docker compose -f infra/docker-compose.yml @@ -36,6 +36,7 @@ help: @echo " arch-check Tach dependency contract + architecture fitness-function tests" @echo " arch-show Open the dependency graph (tach show)" @echo " openapi-snapshot Regenerate apps/api/openapi.json from create_app()" + @echo " record-dispositions Regenerate the record-export redaction table" @echo " mutmut-audit Run mutmut against Access BC deciders/evolver (audit cadence, ~5-15 min)" @echo " mutmut-browse Open mutmut's interactive TUI to triage surviving mutants" @echo " precommit Install pre-commit hooks (one-time per clone)" @@ -62,15 +63,15 @@ db-reset: $(COMPOSE) up -d postgres lint: - cd $(API_DIR) && uv run ruff check src tests - cd $(API_DIR) && uv run ruff format --check src tests + cd $(API_DIR) && uv run ruff check src tests tools + cd $(API_DIR) && uv run ruff format --check src tests tools fmt: - cd $(API_DIR) && uv run ruff check --fix src tests - cd $(API_DIR) && uv run ruff format src tests + cd $(API_DIR) && uv run ruff check --fix src tests tools + cd $(API_DIR) && uv run ruff format src tests tools typecheck: - cd $(API_DIR) && uv run pyright src tests + cd $(API_DIR) && uv run pyright src tests tools # pytest-xdist with `--dist=worksteal -n 4`: worksteal is the # scheduler-of-choice for mixed-duration suites (50ms unit alongside @@ -138,6 +139,19 @@ openapi-snapshot: cd $(API_DIR) && APP_ENV=test uv run python -c "import json; from cora.api.main import create_app; \ f = open('openapi.json', 'w'); json.dump(create_app().openapi(), f, indent=2, sort_keys=True); f.write('\n'); f.close()" +# Regenerate the record-export redaction disposition table after adding +# an event type, adding a field, or changing a field's declared type. +# The drift test (tests/architecture/test_record_dispositions_drift.py) +# fails until this is run and the diff is reviewed in the PR, and that +# diff is the list of what a published record would disclose. +# +# The generator ABORTS on an annotation it cannot classify. That is the +# design, not a bug: an unrecognised type is a question about the model. +# Pass --survey to list every such annotation in one pass instead of +# hitting them one exception at a time. +record-dispositions: + cd $(API_DIR) && uv run python tools/gen_record_dispositions.py + # Mutation testing audit. Pure-logic scope via the CLI wildcard pattern # (Access BC deciders + evolver only). [tool.mutmut] in apps/api/pyproject.toml # carries the test-selection + runner config. Audit-only — not per-PR. diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index b6d4a50440d..3935ebf9d3e 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -246,7 +246,10 @@ packages = ["src/cora"] [tool.ruff] line-length = 100 target-version = "py313" -src = ["src", "tests"] +# `tools` holds build-time generators that MAY import every bounded +# context; they live outside `src` so nothing shippable can import them, +# but they are still first-party code and get linted and type-checked. +src = ["src", "tests", "tools"] # Parked directories are work-in-progress slices the user explicitly # shelved (suffix `.parked`); mirrors the same exclusion in [tool.pyright]. extend-exclude = ["**/*.parked"] @@ -298,7 +301,7 @@ indent-style = "space" known-first-party = ["cora", "tests"] [tool.pyright] -include = ["src", "tests"] +include = ["src", "tests", "tools"] # Parked directories are work-in-progress slices the user explicitly # shelved (suffix `.parked`); they shouldn't block type-check on the # active codebase. Same convention applies to ruff (below). diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py new file mode 100644 index 00000000000..3fdc1e38ed5 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -0,0 +1,25 @@ +"""Record export: CORA's record as a hashed, offline-verifiable artifact. + +The design is `project_record_export_v3.md`, locked for the no-beam +commissioning scope. This package holds the exporter and the generated +redaction disposition table it reads. + +It lives at `cora.infrastructure`, whose `tach.toml` entry allows +`cora.shared` and nothing else, and that constraint is the guarantee +rather than the obstacle: the exporter composes nothing (column-driven +over raw rows, a registry of `str -> str` pairs, and a table read as +data), so the layering rule ENFORCES the zero-bounded-context-import +property that makes the standalone-verifiability claim honest. If a +change here wants to import an aggregate, the design has drifted. + +Two things this package must never do, both learned from review rather +than from first principles: + +- Drive extraction, redaction or parity from `LogbookSchema`. + `LogbookFieldType` is closed over six scalars, so no schema can name a + jsonb column, and jsonb is where the doing lives. The schema travels + with the export as documentation. +- Describe the published record as anonymous. It is pseudonymous: + timestamps plus a facility's own published beamtime schedule + re-identify without the token map. +""" diff --git a/apps/api/src/cora/infrastructure/record_export/_dispositions.py b/apps/api/src/cora/infrastructure/record_export/_dispositions.py new file mode 100644 index 00000000000..a1eef305e12 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_dispositions.py @@ -0,0 +1,1819 @@ +"""Generated redaction dispositions. DO NOT EDIT BY HAND. + +Regenerate with `make record-dispositions`; +`tests/architecture/test_record_dispositions_drift.py` fails until you do. + +One entry per event type, one disposition per declared field, resolved +from the field's real type by `tools/gen_record_dispositions.py`. The +vocabulary: + + keep:enum: closed value set, provably reviewable. The enum is + NAMED because a human signs off the value set, and + swapping one enum for another must read as drift. + keep:number int / float / bool + keep:time datetime + token:uuid replaced with a per-export random surrogate + drop:text free text, no finite range, dropped by default + drop:opaque a dict with no declared keys, nothing to allowlist + by-value the slot is polymorphic across scalars and objects, + so no static answer exists. Apply the tier-2 leaf + rule at export time: numbers and booleans keep, + UUID-shaped strings token, other strings drop. + +A nested mapping is a value object recursed into. A mapping whose sole +key is `[]` is a fixed-length heterogeneous tuple, and its value lists +one disposition per position. + +Redaction iterates the STORED payload's keys and looks each up here. A +key absent from its event's entry is dropped; an event type absent from +this table aborts the export. The canonical hash of this mapping is the +redaction profile hash recorded in the export manifest. +""" + +from typing import Any + +DISPOSITIONS: dict[str, dict[str, Any]] = { + "AcquisitionRecorded": { + "acquisition_id": "token:uuid", + "captured_at": "keep:time", + "dataset_id": "token:uuid", + "evidence": "drop:opaque", + "occurred_at": "keep:time", + "producing_asset_id": "token:uuid", + "producing_run_id": "token:uuid", + "recorded_by": "token:uuid", + "settings": "drop:opaque", + }, + "ActorDeactivated": {"actor_id": "token:uuid", "occurred_at": "keep:time"}, + "ActorProfileForgotten": {"actor_id": "token:uuid", "occurred_at": "keep:time"}, + "ActorReactivated": {"actor_id": "token:uuid", "occurred_at": "keep:time"}, + "ActorRegistered": { + "actor_id": "token:uuid", + "kind": "keep:enum:ActorKind", + "occurred_at": "keep:time", + }, + "AgentBudgetUpdated": { + "agent_id": "token:uuid", + "daily_token_cap": "keep:number", + "monthly_usd_cap": "keep:number", + "occurred_at": "keep:time", + }, + "AgentDefined": { + "agent_id": "token:uuid", + "canonical_uri": "drop:text", + "capabilities": "drop:text", + "daily_token_cap": "keep:number", + "description": "drop:text", + "kind": "drop:text", + "model_ref": {"model": "drop:text", "provider": "drop:text", "snapshot_pin": "drop:text"}, + "monthly_usd_cap": "keep:number", + "name": "drop:text", + "occurred_at": "keep:time", + "prompt_template_id": "token:uuid", + "tools": "drop:text", + "version": "drop:text", + }, + "AgentDeprecated": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AgentResumed": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "resumed_by": "token:uuid", + }, + "AgentSuspended": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "suspended_by": "token:uuid", + }, + "AgentTargetPlanUpdated": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "target_plan_id": "token:uuid", + }, + "AgentToolGranted": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "tool_name": "drop:text", + }, + "AgentToolRevoked": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "tool_name": "drop:text", + }, + "AgentVersioned": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "version": "drop:text", + }, + "AllocationActivated": { + "activated_by": "token:uuid", + "allocation_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AllocationCeilingUpdated": { + "allocation_id": "token:uuid", + "ceiling_usd": "keep:number", + "occurred_at": "keep:time", + }, + "AllocationGranted": { + "allocation_id": "token:uuid", + "campaign_id": "token:uuid", + "ceiling_usd": "keep:number", + "granted_by": "token:uuid", + "note": "drop:text", + "occurred_at": "keep:time", + }, + "AllocationSealed": { + "allocation_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "sealed_by": "token:uuid", + "spent_usd": "keep:number", + }, + "AllocationVoided": { + "allocation_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AssemblyDefined": { + "assembly_id": "token:uuid", + "content_hash": "drop:text", + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "name": {"value": "drop:text"}, + "occurred_at": "keep:time", + "parameter_overrides_schema": "drop:opaque", + "presents_as": "token:uuid", + "required_slots": { + "cardinality": "keep:enum:SlotCardinality", + "default_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "default_settings": "drop:opaque", + "required_family_ids": "token:uuid", + "slot_name": {"value": "drop:text"}, + }, + "required_sub_assemblies": { + "content_hash": "drop:text", + "slot_name": {"value": "drop:text"}, + "sub_assembly_id": "token:uuid", + }, + "required_wires": { + "source_port_name": "drop:text", + "source_slot_name": "drop:text", + "target_port_name": "drop:text", + "target_slot_name": "drop:text", + }, + "version": "drop:text", + }, + "AssemblyDeprecated": { + "assembly_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AssemblyPresentsAsAdded": { + "assembly_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "AssemblyPresentsAsRemoved": { + "assembly_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "AssemblyVersioned": { + "assembly_id": "token:uuid", + "content_hash": "drop:text", + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "name": {"value": "drop:text"}, + "occurred_at": "keep:time", + "parameter_overrides_schema": "drop:opaque", + "presents_as": "token:uuid", + "previous_content_hash": "drop:text", + "required_slots": { + "cardinality": "keep:enum:SlotCardinality", + "default_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "default_settings": "drop:opaque", + "required_family_ids": "token:uuid", + "slot_name": {"value": "drop:text"}, + }, + "required_sub_assemblies": { + "content_hash": "drop:text", + "slot_name": {"value": "drop:text"}, + "sub_assembly_id": "token:uuid", + }, + "required_wires": { + "source_port_name": "drop:text", + "source_slot_name": "drop:text", + "target_port_name": "drop:text", + "target_slot_name": "drop:text", + }, + "version": "drop:text", + }, + "AssetActivated": {"asset_id": "token:uuid", "occurred_at": "keep:time"}, + "AssetAlternateIdentifierAdded": { + "alternate_identifier": {"kind": "keep:enum:AlternateIdentifierKind", "value": "drop:text"}, + "asset_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetAlternateIdentifierRemoved": { + "alternate_identifier": {"kind": "keep:enum:AlternateIdentifierKind", "value": "drop:text"}, + "asset_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetAttachedToFixture": { + "asset_id": "token:uuid", + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetDecommissioned": { + "asset_id": "token:uuid", + "decommissioned_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AssetDegraded": {"asset_id": "token:uuid", "occurred_at": "keep:time", "reason": "drop:text"}, + "AssetDetachedFromFixture": { + "asset_id": "token:uuid", + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetFacilityCodeAssigned": { + "asset_id": "token:uuid", + "assigned_by": "token:uuid", + "facility_code": {"value": "drop:text"}, + "occurred_at": "keep:time", + }, + "AssetFamilyAdded": { + "asset_id": "token:uuid", + "family_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetFamilyRemoved": { + "asset_id": "token:uuid", + "family_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetFaulted": {"asset_id": "token:uuid", "occurred_at": "keep:time", "reason": "drop:text"}, + "AssetMaintenanceEntered": {"asset_id": "token:uuid", "occurred_at": "keep:time"}, + "AssetMaintenanceExited": {"asset_id": "token:uuid", "occurred_at": "keep:time"}, + "AssetOwnerAdded": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "owner": { + "contact": {"value": "drop:text"}, + "identifier": {"value": "drop:text"}, + "identifier_type": {"value": "drop:text"}, + "name": {"value": "drop:text"}, + }, + }, + "AssetOwnerRemoved": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "owner_name": {"value": "drop:text"}, + }, + "AssetPartitionRuleUpdated": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "partition_rule": { + "aggregator_kind": "keep:enum:AggregatorKind", + "calibration_id": "token:uuid", + "calibration_revision_id": "token:uuid", + "constituent_count": "keep:number", + "extrapolation_kind": "keep:enum:ExtrapolationKind", + "gain": "keep:number", + "interpolation_kind": "keep:enum:InterpolationKind", + "invertible": "keep:number", + "kind": "keep:enum", + "offset": "keep:number", + "partition_kind": "keep:enum:PartitionKind", + "partition_parameters": {"[]": ["drop:text", "keep:number"]}, + "readback_aggregator_kind": "keep:enum:ReadbackAggregatorKind", + "residual_tolerance_limit": "keep:number", + "singularity_threshold": "keep:number", + "solver_id": "drop:text", + "solver_transport_kind": "keep:enum:SolverTransportKind", + "solver_version": "drop:text", + "unit_in": "drop:text", + "unit_out": "drop:text", + }, + }, + "AssetPersistentIdAssigned": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "persistent_id_scheme": "drop:text", + "persistent_id_value": "drop:text", + }, + "AssetPortAdded": { + "asset_id": "token:uuid", + "direction": "drop:text", + "occurred_at": "keep:time", + "port_name": "drop:text", + "signal_type": "drop:text", + }, + "AssetPortRemoved": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "port_name": "drop:text", + }, + "AssetRegistered": { + "alternate_identifiers": { + "kind": "keep:enum:AlternateIdentifierKind", + "value": "drop:text", + }, + "asset_id": "token:uuid", + "commissioned_by": "token:uuid", + "controller_id": "token:uuid", + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "facility_code": {"value": "drop:text"}, + "located_in_enclosure_id": "token:uuid", + "model_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "owners": { + "contact": {"value": "drop:text"}, + "identifier": {"value": "drop:text"}, + "identifier_type": {"value": "drop:text"}, + "name": {"value": "drop:text"}, + }, + "parent_id": "token:uuid", + "tier": "drop:text", + }, + "AssetRelocated": { + "asset_id": "token:uuid", + "from_parent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "to_parent_id": "token:uuid", + }, + "AssetRestored": {"asset_id": "token:uuid", "occurred_at": "keep:time", "reason": "drop:text"}, + "AssetSettingsUpdated": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "settings": "drop:opaque", + }, + "AttestationRecorded": { + "attestation_id": "token:uuid", + "attested_by": "token:uuid", + "dataset_id": "token:uuid", + "distribution_id": "token:uuid", + "evidence": "drop:opaque", + "kind": "drop:text", + "occurred_at": "keep:time", + "outcome": "drop:text", + }, + "CalibrationDefined": { + "calibration_id": "token:uuid", + "defined_by": "token:uuid", + "description": "drop:text", + "occurred_at": "keep:time", + "operating_point": "drop:opaque", + "quantity": "drop:text", + "target_id": "token:uuid", + }, + "CalibrationRevisionAppended": { + "asserted_by": "token:uuid", + "calibration_id": "token:uuid", + "content_hash": "drop:text", + "decided_by_decision_id": "token:uuid", + "established_at": "keep:time", + "established_by": "token:uuid", + "occurred_at": "keep:time", + "revision_id": "token:uuid", + "source_dataset_id": "token:uuid", + "source_procedure_id": "token:uuid", + "status": "keep:enum:CalibrationStatus", + "supersedes_revision_id": "token:uuid", + "value": "drop:opaque", + }, + "CalibrationRevisionPublished": { + "calibration_id": "token:uuid", + "occurred_at": "keep:time", + "outbound_permit_id": "token:uuid", + "publication_status": "drop:text", + "published_at": "keep:time", + "published_by": "token:uuid", + "receipt_id": "token:uuid", + "revision_id": "token:uuid", + "signature_bytes_hex": "drop:text", + "signature_envelope_kind": "drop:text", + "signature_kid": "drop:text", + "signing_version": "drop:text", + }, + "CampaignAbandoned": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "CampaignClosed": {"campaign_id": "token:uuid", "occurred_at": "keep:time"}, + "CampaignHeld": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "CampaignRegistered": { + "campaign_id": "token:uuid", + "description": "drop:text", + "external_id": "drop:text", + "external_refs": {"scheme": "drop:text", "value": "drop:text"}, + "intent": "drop:text", + "lead_actor_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "subject_id": "token:uuid", + "tags": "drop:text", + }, + "CampaignResumed": {"campaign_id": "token:uuid", "occurred_at": "keep:time"}, + "CampaignRunAdded": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "CampaignRunRemoved": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "CampaignStarted": {"campaign_id": "token:uuid", "occurred_at": "keep:time"}, + "CampaignSteeringDeclared": { + "campaign_id": "token:uuid", + "objective": { + "kind": "keep:enum:SteeringObjectiveKind", + "target_measurement_name": "drop:text", + "target_value": "keep:number", + }, + "occurred_at": "keep:time", + "space": { + "axes": { + "choices": "drop:opaque", + "lower": "keep:number", + "name": "drop:text", + "upper": "keep:number", + } + }, + }, + "CapabilityDefined": { + "capability_id": "token:uuid", + "code": "drop:text", + "description": "drop:text", + "executor_shapes": "keep:enum:ExecutorShape", + "name": "drop:text", + "occurred_at": "keep:time", + "parameters_schema": "drop:opaque", + "required_affordances": "keep:enum:Affordance", + }, + "CapabilityDeprecated": { + "capability_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "replaced_by_capability_id": "token:uuid", + }, + "CapabilitySuggestedRolesUpdated": { + "capability_id": "token:uuid", + "occurred_at": "keep:time", + "suggested_role_ids": "token:uuid", + }, + "CapabilityVersioned": { + "capability_id": "token:uuid", + "description": "drop:text", + "executor_shapes": "keep:enum:ExecutorShape", + "occurred_at": "keep:time", + "parameters_schema": "drop:opaque", + "required_affordances": "keep:enum:Affordance", + "version_tag": "drop:text", + }, + "CautionAcknowledgement": { + "category": "drop:text", + "caution_id": "token:uuid", + "severity": "drop:text", + "target_id": "token:uuid", + "target_kind": "drop:text", + "text_excerpt": "drop:text", + "workaround_excerpt": "drop:text", + }, + "CautionRegistered": { + "authored_by": "token:uuid", + "category": "drop:text", + "caution_id": "token:uuid", + "expires_at": "keep:time", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "propagate_to_children": "keep:number", + "severity": "drop:text", + "tags": "drop:text", + "target": {"asset_id": "token:uuid", "procedure_id": "token:uuid"}, + "text": "drop:text", + "workaround": "drop:text", + }, + "CautionRetired": { + "caution_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "CautionSuperseded": { + "caution_id": "token:uuid", + "occurred_at": "keep:time", + "superseded_by_caution_id": "token:uuid", + }, + "ClearanceActivated": {"clearance_id": "token:uuid", "occurred_at": "keep:time"}, + "ClearanceApproved": { + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + "valid_from": "keep:time", + "valid_until": "keep:time", + }, + "ClearanceExpired": { + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "ClearanceRegistered": { + "bindings": "drop:opaque", + "clearance_id": "token:uuid", + "declarations": "drop:opaque", + "external_id": "drop:text", + "facility_code": "drop:text", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "risk_band": "drop:text", + "template_code": "drop:text", + "template_id": "token:uuid", + "title": "drop:text", + "valid_from": "keep:time", + "valid_until": "keep:time", + }, + "ClearanceRejected": { + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "ClearanceReviewStarted": { + "clearance_id": "token:uuid", + "first_reviewer_role": "drop:text", + "occurred_at": "keep:time", + }, + "ClearanceReviewStepAppended": { + "clearance_id": "token:uuid", + "decided_at": "keep:time", + "decided_by": "token:uuid", + "decision": "drop:text", + "notes": "drop:text", + "occurred_at": "keep:time", + "role": "drop:text", + "step_index": "keep:number", + }, + "ClearanceSubmitted": {"clearance_id": "token:uuid", "occurred_at": "keep:time"}, + "ClearanceSuperseded": { + "by_clearance_id": "token:uuid", + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ClearanceTemplateActivated": { + "activated_by": "token:uuid", + "occurred_at": "keep:time", + "template_id": "token:uuid", + }, + "ClearanceTemplateDefined": { + "code": "drop:text", + "defined_by": "token:uuid", + "external_ref": "drop:text", + "facility_code": "drop:text", + "occurred_at": "keep:time", + "supersedes_template_id": "token:uuid", + "template_id": "token:uuid", + "title": "drop:text", + "version": "keep:number", + }, + "ClearanceTemplateDeprecated": { + "deprecated_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "template_id": "token:uuid", + }, + "ClearanceTemplateVersioned": { + "new_version": "keep:number", + "occurred_at": "keep:time", + "supersedes_template_id": "token:uuid", + "template_id": "token:uuid", + "versioned_by": "token:uuid", + }, + "ClearanceTemplateWithdrawn": { + "occurred_at": "keep:time", + "reason": "drop:text", + "template_id": "token:uuid", + "withdrawn_by": "token:uuid", + }, + "ConduitDefined": { + "conduit_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "source_zone_id": "token:uuid", + "target_zone_id": "token:uuid", + }, + "ConduitLogbookClosed": { + "conduit_id": "token:uuid", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ConduitLogbookOpened": { + "conduit_id": "token:uuid", + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "CredentialRegistered": { + "audience": "drop:text", + "credential_id": "token:uuid", + "expires_at": "keep:time", + "facility_code": {"value": "drop:text"}, + "occurred_at": "keep:time", + "public_material_ref": "drop:text", + "purpose": "keep:enum:CredentialPurpose", + "registered_by": "token:uuid", + "secret_ref": "drop:text", + }, + "CredentialRevoked": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "revoked_by": "token:uuid", + }, + "CredentialRotationAborted": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "rotation_aborted_by": "token:uuid", + }, + "CredentialRotationCompleted": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "rotation_completed_by": "token:uuid", + }, + "CredentialRotationStarted": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "pending_public_material_ref": "drop:text", + "pending_secret_ref": "drop:text", + "rotation_started_by": "token:uuid", + }, + "DatasetDemoted": { + "dataset_id": "token:uuid", + "demoted_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "DatasetDiscarded": { + "dataset_id": "token:uuid", + "discarded_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "DatasetPromoted": { + "dataset_id": "token:uuid", + "occurred_at": "keep:time", + "promoted_by": "token:uuid", + "reason": "drop:text", + }, + "DatasetRegistered": { + "byte_size": "keep:number", + "checksum_algorithm": "drop:text", + "checksum_value": "drop:text", + "conforms_to": "drop:text", + "dataset_id": "token:uuid", + "derived_from": "token:uuid", + "intent": "drop:text", + "media_type": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "producing_actuation_kind": "drop:text", + "producing_procedure_id": "token:uuid", + "producing_run_end_state": "drop:text", + "producing_run_id": "token:uuid", + "registered_by": "token:uuid", + "subject_id": "token:uuid", + "uri": "drop:text", + "used_calibration_ids": "token:uuid", + }, + "DecisionDebriefRequested": { + "debriefer_agent_id": "token:uuid", + "debriefer_kind": "drop:text", + "occurred_at": "keep:time", + "run_id": "token:uuid", + "terminal_event_id": "token:uuid", + }, + "DecisionLogbookClosed": { + "decision_id": "token:uuid", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + }, + "DecisionLogbookOpened": { + "decision_id": "token:uuid", + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "DecisionRated": { + "comment": "drop:text", + "confidence_at_rating": "keep:number", + "decision_id": "token:uuid", + "occurred_at": "keep:time", + "rated_at": "keep:time", + "rated_by": "token:uuid", + "rating": "keep:enum:DecisionRating", + }, + "DecisionRegistered": { + "alternatives": "drop:text", + "choice": "drop:text", + "confidence": "keep:number", + "confidence_source": "keep:enum:DecisionConfidenceSource", + "context": "drop:text", + "decided_by": "token:uuid", + "decision_id": "token:uuid", + "inputs": "drop:opaque", + "occurred_at": "keep:time", + "override_kind": "keep:enum", + "parent_id": "token:uuid", + "reasoning": "drop:text", + "reasoning_signature": "drop:text", + "rule": "drop:text", + }, + "DistributionDiscarded": { + "discarded_by": "token:uuid", + "distribution_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "DistributionRegistered": { + "access_protocol": "drop:text", + "byte_size": "keep:number", + "checksum_algorithm": "drop:text", + "checksum_value": "drop:text", + "conforms_to": "drop:text", + "dataset_id": "token:uuid", + "distribution_id": "token:uuid", + "media_type": "drop:text", + "occurred_at": "keep:time", + "registered_by": "token:uuid", + "supply_id": "token:uuid", + "uri": "drop:text", + }, + "EditionDatasetAdded": { + "added_by": "token:uuid", + "dataset_id": "token:uuid", + "edition_id": "token:uuid", + "occurred_at": "keep:time", + }, + "EditionDatasetRemoved": { + "dataset_id": "token:uuid", + "edition_id": "token:uuid", + "occurred_at": "keep:time", + "removed_by": "token:uuid", + }, + "EditionPublished": { + "edition_id": "token:uuid", + "external_pid_scheme": "drop:text", + "external_pid_value": "drop:text", + "occurred_at": "keep:time", + "published_by": "token:uuid", + "published_content_hash": "drop:text", + }, + "EditionRegistered": { + "creators": "drop:opaque", + "dataset_ids": "token:uuid", + "edition_id": "token:uuid", + "kind": "drop:text", + "license": "drop:text", + "occurred_at": "keep:time", + "publication_year": "keep:number", + "publisher_facility_code": "drop:text", + "registered_by": "token:uuid", + "title": "drop:text", + }, + "EditionSealed": { + "content_hash": "drop:text", + "edition_id": "token:uuid", + "license": "drop:text", + "occurred_at": "keep:time", + "publication_year": "keep:number", + "publisher_facility_code": "drop:text", + "sealed_by": "token:uuid", + "sealed_dataset_ids": "token:uuid", + }, + "EditionWithdrawn": { + "edition_id": "token:uuid", + "occurred_at": "keep:time", + "withdrawal_reason": "drop:text", + "withdrawn_by": "token:uuid", + }, + "EnclosureDecommissioned": { + "enclosure_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "triggered_by": "token:uuid", + }, + "EnclosurePermitObserved": { + "enclosure_id": "token:uuid", + "from_status": "drop:text", + "monitor_ref": "drop:text", + "observed_at": "keep:time", + "occurred_at": "keep:time", + "reason": "drop:text", + "to_status": "drop:text", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "EnclosureRegistered": { + "enclosure_id": "token:uuid", + "facility_code": {"value": "drop:text"}, + "name": "drop:text", + "occurred_at": "keep:time", + "registered_by": "token:uuid", + }, + "FacilityDecommissioned": { + "decommissioned_by": "token:uuid", + "facility_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "FacilityRegistered": { + "alternate_identifiers": { + "kind": "keep:enum:AlternateIdentifierKind", + "value": "drop:text", + }, + "code": {"value": "drop:text"}, + "display_name": "drop:text", + "facility_id": "token:uuid", + "kind": "keep:enum:FacilityKind", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "registered_by": "token:uuid", + }, + "FacilityTrustAnchorCredentialAdded": { + "added_by": "token:uuid", + "credential_id": "token:uuid", + "facility_id": "token:uuid", + "occurred_at": "keep:time", + }, + "FacilityTrustAnchorCredentialRemoved": { + "credential_id": "token:uuid", + "facility_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "removed_by": "token:uuid", + }, + "FamilyDefined": { + "affordances": "keep:enum:Affordance", + "family_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + }, + "FamilyDeprecated": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "FamilyPresentsAsAdded": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "FamilyPresentsAsRemoved": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "FamilySettingsSchemaUpdated": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "settings_schema": "drop:opaque", + }, + "FamilyVersioned": { + "affordances": "keep:enum:Affordance", + "family_id": "token:uuid", + "occurred_at": "keep:time", + "version_tag": "drop:text", + }, + "FixturePersistentIdAssigned": { + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + "persistent_id_scheme": "drop:text", + "persistent_id_value": "drop:text", + }, + "FixtureRegistered": { + "assembly_content_hash": "drop:text", + "assembly_id": "token:uuid", + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + "parameter_overrides": "drop:opaque", + "registered_by": "token:uuid", + "slot_asset_bindings": {"asset_id": "token:uuid", "slot_name": "drop:text"}, + "surface_id": "token:uuid", + }, + "FrameDecommissioned": { + "frame_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "FramePlacementUpdated": { + "frame_id": "token:uuid", + "new_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "occurred_at": "keep:time", + "survey": "drop:opaque", + }, + "FrameRegistered": { + "frame_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "supersedes": { + "predecessor_frame_id": "token:uuid", + "transform_from_predecessor": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + }, + }, + "HoldClaimReleased": { + "cause": "drop:text", + "claim_id": "token:uuid", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "LanguageModelApproved": {"language_model_id": "token:uuid", "occurred_at": "keep:time"}, + "LanguageModelDefined": { + "archivability": "drop:text", + "cost_basis": "drop:opaque", + "data_tier": "drop:text", + "endpoint_note": "drop:text", + "language_model_id": "token:uuid", + "model": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "provider": "drop:text", + "served_via": "drop:text", + "snapshot_pin": "drop:text", + }, + "LanguageModelDeprecated": { + "language_model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "LanguageModelRetired": { + "language_model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "LanguageModelRetirementAnnounced": { + "effective_at": "keep:time", + "language_model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MethodDefined": { + "capability_id": "token:uuid", + "execution_pattern": "keep:enum:ExecutionPattern", + "method_id": "token:uuid", + "monotone_quality": "keep:number", + "name": "drop:text", + "needed_assembly_ids": "token:uuid", + "needed_family_ids": "token:uuid", + "needed_input_kinds": "drop:text", + "needed_supplies": "drop:text", + "occurred_at": "keep:time", + "resumable_from_checkpoint": "keep:number", + }, + "MethodDeprecated": { + "method_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MethodLaunchSpecUpdated": { + "launch_spec": "drop:opaque", + "method_id": "token:uuid", + "occurred_at": "keep:time", + }, + "MethodParametersSchemaUpdated": { + "method_id": "token:uuid", + "occurred_at": "keep:time", + "parameters_schema": "drop:opaque", + }, + "MethodRequiredRoleAdded": { + "family_id": "token:uuid", + "method_id": "token:uuid", + "occurred_at": "keep:time", + "optional": "keep:number", + "required_ports": "drop:opaque", + "role_kind": "token:uuid", + "role_name": "drop:text", + }, + "MethodRequiredRoleRemoved": { + "method_id": "token:uuid", + "occurred_at": "keep:time", + "role_name": "drop:text", + }, + "MethodVersioned": { + "content_hash": "drop:text", + "method_id": "token:uuid", + "occurred_at": "keep:time", + "version_tag": "drop:text", + }, + "ModelDefined": { + "declared_family_ids": "token:uuid", + "manufacturer": { + "identifier": {"value": "drop:text"}, + "identifier_type": "keep:enum:ManufacturerIdentifierType", + "name": {"value": "drop:text"}, + }, + "model_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "part_number": "drop:text", + "version_tag": "drop:text", + }, + "ModelDeprecated": { + "model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "ModelFamilyAdded": { + "family_id": "token:uuid", + "model_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ModelFamilyRemoved": { + "family_id": "token:uuid", + "model_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ModelVersioned": { + "declared_family_ids": "token:uuid", + "manufacturer": { + "identifier": {"value": "drop:text"}, + "identifier_type": "keep:enum:ManufacturerIdentifierType", + "name": {"value": "drop:text"}, + }, + "model_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "part_number": "drop:text", + "version_tag": "drop:text", + }, + "MountAssetInstalled": { + "asset_id": "token:uuid", + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "previously_installed_asset_id": "token:uuid", + }, + "MountAssetUninstalled": { + "asset_id": "token:uuid", + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MountDecommissioned": { + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MountPlacementUpdated": { + "mount_id": "token:uuid", + "new_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "occurred_at": "keep:time", + "survey": "drop:opaque", + }, + "MountRegistered": { + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "slot_code": "drop:text", + }, + "PermitActivated": { + "activated_by": "token:uuid", + "occurred_at": "keep:time", + "permit_id": "token:uuid", + }, + "PermitDefined": { + "abi_tier_floor": "keep:enum:AbiTier", + "allowed_artifact_kinds": "drop:text", + "allowed_credential_ids": "token:uuid", + "allowed_payload_types": "drop:text", + "defined_by": "token:uuid", + "direction": "keep:enum:Direction", + "expires_at": "keep:time", + "occurred_at": "keep:time", + "peer_facility_code": {"value": "drop:text"}, + "permit_id": "token:uuid", + "terms": { + "accepted_canonicalization_versions": "drop:text", + "inbound_allowed_artifact_kinds": "drop:text", + "onward_action_scope": "keep:enum:OnwardActionScope", + "publisher_grant_correlation_handle": "drop:text", + "read_scope": "keep:enum:ReadScope", + "required_receipt_kinds": "keep:enum:ReceiptKind", + "scopes": {"kind": "drop:text", "name": "drop:text", "qualifier": "drop:text"}, + }, + }, + "PermitResumed": { + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "resumed_by": "token:uuid", + }, + "PermitRevoked": { + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "reason": "drop:text", + "revoked_by": "token:uuid", + }, + "PermitSuspended": { + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "reason": "drop:text", + "suspended_by": "token:uuid", + }, + "PlanDefaultParametersUpdated": { + "default_parameters": "drop:opaque", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + }, + "PlanDefined": { + "asset_families_snapshot": "drop:opaque", + "asset_ids": "token:uuid", + "method_id": "token:uuid", + "method_needed_family_ids_snapshot": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "practice_id": "token:uuid", + }, + "PlanDeprecated": {"occurred_at": "keep:time", "plan_id": "token:uuid", "reason": "drop:text"}, + "PlanRoleBound": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "role_name": "drop:text", + }, + "PlanRoleUnbound": { + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "role_name": "drop:text", + }, + "PlanVersioned": { + "content_hash": "drop:text", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "version_tag": "drop:text", + }, + "PlanWireAdded": { + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "source_asset_id": "token:uuid", + "source_port_name": "drop:text", + "target_asset_id": "token:uuid", + "target_port_name": "drop:text", + }, + "PlanWireRemoved": { + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "source_asset_id": "token:uuid", + "source_port_name": "drop:text", + "target_asset_id": "token:uuid", + "target_port_name": "drop:text", + }, + "PolicyDefined": { + "conduit_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "permitted_commands": "drop:text", + "permitted_principal_ids": "token:uuid", + "policy_id": "token:uuid", + "surface_id": "token:uuid", + }, + "PolicyGrantRevoked": { + "occurred_at": "keep:time", + "policy_id": "token:uuid", + "principal_id": "token:uuid", + "reason": "drop:text", + "revoked_by": "token:uuid", + }, + "PracticeDefined": { + "method_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "practice_id": "token:uuid", + "site_id": "token:uuid", + }, + "PracticeDeprecated": { + "occurred_at": "keep:time", + "practice_id": "token:uuid", + "reason": "drop:text", + }, + "PracticeVersioned": { + "occurred_at": "keep:time", + "practice_id": "token:uuid", + "version_tag": "drop:text", + }, + "ProcedureAborted": { + "actuation_kind": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + }, + "ProcedureActivitiesLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "ProcedureCompleted": { + "actuation_kind": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + }, + "ProcedureDiagnosticLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "ProcedureHeld": { + "actuation_kind": "drop:text", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + }, + "ProcedureIterationEnded": { + "advised_next_point": "drop:opaque", + "advised_stop": "keep:number", + "alternatives": "drop:text", + "confidence": "keep:number", + "confidence_source": "keep:enum:DecisionConfidenceSource", + "converged": "keep:number", + "iteration_index": "keep:number", + "model_ref": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + "reasoning": "drop:text", + }, + "ProcedureIterationStarted": { + "iteration_index": "keep:number", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + }, + "ProcedureOutcomeLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "ProcedureRegistered": { + "capability_id": "token:uuid", + "kind": "drop:text", + "max_consecutive_unconverged_iterations": "keep:number", + "name": "drop:text", + "occurred_at": "keep:time", + "parent_run_id": "token:uuid", + "procedure_id": "token:uuid", + "recipe_id": "token:uuid", + "target_asset_ids": "token:uuid", + }, + "ProcedureResumed": { + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "re_establishment_boundary": "keep:number", + }, + "ProcedureStarted": {"occurred_at": "keep:time", "procedure_id": "token:uuid"}, + "ProcedureTruncated": { + "interrupted_at": "keep:time", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + }, + "PublicationReceiptRecorded": { + "content_hash": "drop:text", + "home_artifact_id": "token:uuid", + "home_stream_id": "token:uuid", + "home_stream_type": "drop:text", + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "receipt_id": "token:uuid", + "recorded_at": "keep:time", + }, + "RatificationDenied": { + "occurred_at": "keep:time", + "ratification_id": "token:uuid", + "reason": "drop:text", + }, + "RatificationGranted": {"occurred_at": "keep:time", "ratification_id": "token:uuid"}, + "RatificationRequested": { + "command_name": "drop:text", + "consequence_class": "drop:text", + "occurred_at": "keep:time", + "ratification_id": "token:uuid", + "requested_by": "token:uuid", + "target_action_id": "token:uuid", + }, + "RecipeDefined": { + "capability_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "recipe_id": "token:uuid", + "steps": { + "address": "drop:text", + "capture_name": "drop:text", + "command": "drop:text", + "criterion": "drop:opaque", + "input_uris": "by-value", + "name": "drop:text", + "output_ref_name": "drop:text", + "output_uri": "drop:text", + "parameters": "drop:opaque", + "params": "drop:opaque", + "value": "by-value", + "verify": "keep:number", + }, + }, + "RecipeDeprecated": { + "occurred_at": "keep:time", + "reason": "drop:text", + "recipe_id": "token:uuid", + "replaced_by_recipe_id": "token:uuid", + }, + "RecipeExpansionRecorded": { + "bindings": "drop:opaque", + "bindings_hash": "drop:text", + "capability_id": "token:uuid", + "capability_version": "drop:text", + "expansion_port_version": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "recipe_id": "token:uuid", + "recipe_version": "drop:text", + "step_count": "keep:number", + "steps_hash": "drop:text", + }, + "RecipeVersioned": { + "occurred_at": "keep:time", + "recipe_id": "token:uuid", + "steps": { + "address": "drop:text", + "capture_name": "drop:text", + "command": "drop:text", + "criterion": "drop:opaque", + "input_uris": "by-value", + "name": "drop:text", + "output_ref_name": "drop:text", + "output_uri": "drop:text", + "parameters": "drop:opaque", + "params": "drop:opaque", + "value": "by-value", + "verify": "keep:number", + }, + "version_tag": "drop:text", + }, + "ResolvedStepsRecorded": { + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "resolved_steps": "drop:opaque", + "step_count": "keep:number", + }, + "RoleDefined": { + "consumes": "drop:text", + "docstring": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "optional_affordances": "keep:enum:Affordance", + "produces": "drop:text", + "required_affordances": "keep:enum:Affordance", + "role_id": "token:uuid", + }, + "RunAborted": { + "actuation_kind": "drop:text", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "producing_job_id": "drop:text", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunAddedToCampaign": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "RunAdjusted": { + "adjusted_by": "token:uuid", + "decided_by_decision_id": "token:uuid", + "effective_parameters": "drop:opaque", + "occurred_at": "keep:time", + "parameters_patch": "drop:opaque", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunCompleted": { + "actuation_kind": "drop:text", + "artifact_uri": "drop:text", + "occurred_at": "keep:time", + "producing_job_id": "drop:text", + "run_id": "token:uuid", + }, + "RunHeld": { + "cause": "drop:text", + "claim_id": "token:uuid", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "RunObservationLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "RunRemovedFromCampaign": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunResumed": { + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "released_claim_id": "token:uuid", + "run_id": "token:uuid", + }, + "RunStarted": { + "acknowledged_cautions": { + "category": "drop:text", + "caution_id": "token:uuid", + "severity": "drop:text", + "target_id": "token:uuid", + "target_kind": "drop:text", + "text_excerpt": "drop:text", + "workaround_excerpt": "drop:text", + }, + "campaign_id": "token:uuid", + "decided_by_decision_id": "token:uuid", + "effective_parameters": "drop:opaque", + "external_refs": "drop:opaque", + "input_dataset_ids": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "override_parameters": "drop:opaque", + "pinned_calibration_ids": "token:uuid", + "plan_id": "token:uuid", + "raid": "drop:text", + "run_id": "token:uuid", + "subject_id": "token:uuid", + "trigger_source": "drop:text", + }, + "RunStopped": { + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunTruncated": { + "decided_by_decision_id": "token:uuid", + "interrupted_at": "keep:time", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "SealInitialized": { + "facility_code": {"value": "drop:text"}, + "initialized_by": "token:uuid", + "occurred_at": "keep:time", + "offline_credential_id": "token:uuid", + "online_credential_id": "token:uuid", + }, + "SealOnlineKeyRotated": { + "facility_code": {"value": "drop:text"}, + "new_online_credential_id": "token:uuid", + "occurred_at": "keep:time", + "rotated_by": "token:uuid", + "signed_by_offline_root": "keep:number", + }, + "SealPointerSigned": { + "facility_code": {"value": "drop:text"}, + "head_hash": "drop:text", + "occurred_at": "keep:time", + "sequence_number": "keep:number", + "signed_at": "keep:time", + "signed_by": "token:uuid", + }, + "SealRepublishingCompleted": { + "completed_by": "token:uuid", + "facility_code": {"value": "drop:text"}, + "new_head_hash": "drop:text", + "new_sequence_number": "keep:number", + "occurred_at": "keep:time", + }, + "SealRepublishingStarted": { + "facility_code": {"value": "drop:text"}, + "occurred_at": "keep:time", + "reason": "drop:text", + "started_by": "token:uuid", + }, + "SubjectDiscarded": { + "discarded_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "subject_id": "token:uuid", + }, + "SubjectDismounted": { + "dismounted_by": "token:uuid", + "from_asset_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "subject_id": "token:uuid", + }, + "SubjectMeasured": { + "measured_by": "token:uuid", + "occurred_at": "keep:time", + "subject_id": "token:uuid", + }, + "SubjectMounted": { + "asset_id": "token:uuid", + "mounted_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "subject_id": "token:uuid", + }, + "SubjectRegistered": { + "name": "drop:text", + "occurred_at": "keep:time", + "registered_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SubjectRemoved": { + "occurred_at": "keep:time", + "removed_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SubjectReturned": { + "occurred_at": "keep:time", + "returned_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SubjectStored": { + "occurred_at": "keep:time", + "stored_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SupplyDegraded": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyDeregistered": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyMarkedAvailable": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyMarkedRecovering": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyMarkedUnavailable": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyRegistered": { + "containing_asset_id": "token:uuid", + "facility_code": {"value": "drop:text"}, + "kind": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyRestored": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SurfaceDefined": { + "kind": "keep:enum:SurfaceKind", + "name": "drop:text", + "occurred_at": "keep:time", + "surface_id": "token:uuid", + }, + "VisitAborted": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "VisitArrived": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitCancelled": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "VisitCheckedIn": { + "actor_id": "token:uuid", + "mode": "drop:text", + "occurred_at": "keep:time", + "visit_id": "token:uuid", + }, + "VisitCheckedOut": { + "actor_id": "token:uuid", + "occurred_at": "keep:time", + "visit_id": "token:uuid", + }, + "VisitCompleted": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitHeld": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "VisitPresenceClosed": { + "actor_id": "token:uuid", + "occurred_at": "keep:time", + "visit_id": "token:uuid", + }, + "VisitRegistered": { + "external_refs": {"scheme": "drop:text", "value": "drop:text"}, + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "planned_end_at": "keep:time", + "planned_start_at": "keep:time", + "policy_id": "token:uuid", + "surface_id": "token:uuid", + "type": "drop:text", + "visit_id": "token:uuid", + }, + "VisitResumed": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitStarted": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitSurfaceControlReleased": { + "occurred_at": "keep:time", + "surface_id": "token:uuid", + "visit_id": "token:uuid", + }, + "VisitSurfaceControlTaken": { + "occurred_at": "keep:time", + "surface_id": "token:uuid", + "visit_id": "token:uuid", + }, + "VisitVoided": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "ZoneDefined": {"name": "drop:text", "occurred_at": "keep:time", "zone_id": "token:uuid"}, +} diff --git a/apps/api/tests/architecture/test_record_dispositions_drift.py b/apps/api/tests/architecture/test_record_dispositions_drift.py new file mode 100644 index 00000000000..a80b86f2078 --- /dev/null +++ b/apps/api/tests/architecture/test_record_dispositions_drift.py @@ -0,0 +1,67 @@ +"""Redaction disposition table drift test. + +Asserts that the committed +`src/cora/infrastructure/record_export/_dispositions.py` matches what +`tools/gen_record_dispositions.py` produces right now. Adding an event +type, adding a field, or changing a field's declared type fails this +test until the table is regenerated and the diff reviewed. + +That review is the point. The table decides what a published record +discloses, so a field's arrival has to be a line in a pull request +rather than a silent behaviour change. Regenerate with: + + make record-dispositions + +The generator runs in a SUBPROCESS rather than being imported. It lives +outside `src/` precisely so nothing shippable can import it, and this +test declining to import it keeps that true. It also means the test +exercises the same entry point a developer runs. + +Fail-closed is NOT provided by this test. An unlisted field drops at +export time by the rule itself; this test only catches staleness early. +Deleting it would make the table rot silently, not make the exporter +leak. + +Running the generator in a subprocess hides this test's real dependency +(every `events.py` in the tree) from pytest-tach's impact analysis, so +`pytest --tach` would skip it after an event-only change. CI runs the +suite without that flag, so it is covered today; anyone turning impact +analysis on in CI has to exempt this test. +""" + +import subprocess +import sys +from pathlib import Path + +_API_ROOT = Path(__file__).resolve().parents[2] +_GENERATOR = _API_ROOT / "tools" / "gen_record_dispositions.py" +_TABLE = _API_ROOT / "src" / "cora" / "infrastructure" / "record_export" / "_dispositions.py" + + +def test_committed_disposition_table_matches_generator() -> None: + """Committed `_dispositions.py` equals a fresh generator run.""" + before = _TABLE.read_text(encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(_GENERATOR)], + cwd=_API_ROOT, + capture_output=True, + text=True, + timeout=120, + ) + + after = _TABLE.read_text(encoding="utf-8") + if before != after: + _TABLE.write_text(before, encoding="utf-8") + + assert result.returncode == 0, ( + "The disposition generator refused to produce a table. An " + "annotation it cannot classify ABORTS the run by design, because " + "an unrecognised type is a question about the design rather than " + f"a row to skip. Run `make record-dispositions`.\n{result.stderr}" + ) + assert before == after, ( + f"Disposition table drift detected at {_TABLE.name}. Run " + "`make record-dispositions`, then review the diff: it is the " + "list of what a published record would disclose." + ) diff --git a/apps/api/tools/gen_record_dispositions.py b/apps/api/tools/gen_record_dispositions.py new file mode 100644 index 00000000000..fe7e2c0d6c9 --- /dev/null +++ b/apps/api/tools/gen_record_dispositions.py @@ -0,0 +1,316 @@ +"""Generate the record-export redaction disposition table. + +The record exporter decides what to publish from a field's DECLARED TYPE +(see the F5 section of `project_record_export_v3.md`). Answering "is this +annotation a StrEnum, a str alias, or a value object" needs the defining +module imported, and the exporter lives at `cora.infrastructure`, where +`tach.toml` allows `cora.shared` and nothing else. So the question is +answered HERE, at build time, by a tool that may import every bounded +context, and the answer is committed as inert data the exporter reads. + +This mirrors `make openapi-snapshot`: a generated artifact in the tree, +a drift test that fails until it is regenerated, and a diff a reviewer +can read. Living OUTSIDE `src/` is deliberate and structural, not +cosmetic: nothing the application ships can import this module, so the +exporter cannot acquire the BC dependency the design forbids it. + +Run it with: + + make record-dispositions + +Resolution happens on real type objects rather than on annotation +strings, which is the point. A plain alias (`ActorId = UUID`) collapses +to its target for free; only `NewType` and genuine value objects need +handling. An annotation this tool cannot classify ABORTS the run: an +unrecognized type is a question about the design, not a row to skip. +""" + +from __future__ import annotations + +import dataclasses +import enum +import importlib +import inspect +import json +import subprocess +import sys +import types +import typing +from collections.abc import Iterable, Mapping, MutableMapping, Sequence +from datetime import datetime +from pathlib import Path +from typing import Any, Literal, NewType, Union, get_args, get_origin +from uuid import UUID + +_API_ROOT = Path(__file__).resolve().parents[1] +_SRC = _API_ROOT / "src" +_OUT = _SRC / "cora" / "infrastructure" / "record_export" / "_dispositions.py" + +KEEP_ENUM = "keep:enum" +KEEP_NUMBER = "keep:number" +KEEP_TIME = "keep:time" +TOKEN_UUID = "token:uuid" +DROP_TEXT = "drop:text" +DROP_OPAQUE = "drop:opaque" +BY_VALUE = "by-value" + +_SCALAR_KEEP: Mapping[type, str] = { + bool: KEEP_NUMBER, + int: KEEP_NUMBER, + float: KEEP_NUMBER, + datetime: KEEP_TIME, +} + +_CONTAINER_ORIGINS = (tuple, frozenset, set, list) + + +class UnclassifiedAnnotationError(Exception): + """An annotation no rule in F5 covers. Aborts the run by design.""" + + def __init__(self, event: str, field: str, annotation: object) -> None: + super().__init__( + f"{event}.{field}: cannot classify {annotation!r}. " + "Add a rule to gen_record_dispositions.py, or change the " + "declared type. Do NOT silently drop it." + ) + + +def _union_variants(annotation: Any) -> list[Any] | None: + """Non-None members of a union, or None if this is not a union.""" + origin = get_origin(annotation) + if origin is not Union and origin is not types.UnionType: + return None + return [a for a in get_args(annotation) if a is not type(None)] + + +def _merge( + variants: Sequence[str | dict[str, Any]], event: str, field: str, ann: Any +) -> str | dict[str, Any]: + """Fold the dispositions of a union's variants into one. + + Scalars must agree. Value objects are MERGED, because a discriminated + union stores whichever arm's keys, so the published rule has to cover + every arm. A key two arms disagree on is a real ambiguity and aborts. + """ + if all(isinstance(v, str) for v in variants): + if len({typing.cast("str", v) for v in variants}) == 1: + return typing.cast("str", variants[0]) + return BY_VALUE + if not all(isinstance(v, dict) for v in variants): + # A slot spanning scalars AND value objects (a Recipe setpoint is + # a number, a channel string, or a binding reference) cannot be + # decided statically. Defer to the same leaf rule the tier-2 jsonb + # columns already use, which is fail-closed on strings. + return BY_VALUE + merged: dict[str, Any] = {} + for variant in variants: + for key, disposition in typing.cast("dict[str, Any]", variant).items(): + if key in merged and merged[key] != disposition: + raise UnclassifiedAnnotationError(event, field, ann) + merged[key] = disposition + return merged + + +def _is_value_object(annotation: Any) -> bool: + """True for a frozen dataclass, which has declared fields to recurse into.""" + if not inspect.isclass(annotation) or not dataclasses.is_dataclass(annotation): + return False + params: Any = getattr(annotation, "__dataclass_params__", None) + return bool(params is not None and params.frozen) + + +def _classify(annotation: Any, event: str, field: str) -> str | dict[str, Any]: + """Map one resolved annotation to a disposition, per the F5 table.""" + if isinstance(annotation, typing.TypeAliasType): + return _classify(annotation.__value__, event, field) + + variants = _union_variants(annotation) + if variants is not None: + if len(variants) == 1: + return _classify(variants[0], event, field) + return _merge([_classify(v, event, field) for v in variants], event, field, annotation) + + if annotation is Any: + return DROP_OPAQUE + + if isinstance(annotation, NewType): + return _classify(annotation.__supertype__, event, field) + + if get_origin(annotation) is Literal: + args = get_args(annotation) + if all(isinstance(a, str) for a in args): + return KEEP_ENUM + if all(isinstance(a, bool | int | float) for a in args): + return KEEP_NUMBER + raise UnclassifiedAnnotationError(event, field, annotation) + + if inspect.isclass(annotation): + if issubclass(annotation, enum.Enum): + # Name the enum. The value set is what a human signs off, so + # the disposition has to say WHICH set, and swapping one enum + # for another must show up as drift rather than as no change. + return f"{KEEP_ENUM}:{annotation.__name__}" + if annotation is UUID: + return TOKEN_UUID + if annotation is str: + return DROP_TEXT + for scalar, disposition in _SCALAR_KEEP.items(): + if annotation is scalar: + return disposition + if _is_value_object(annotation): + return _resolve_fields(annotation) + + origin = get_origin(annotation) + if origin in _CONTAINER_ORIGINS: + raw = get_args(annotation) + args = [a for a in raw if a is not Ellipsis] + if not args: + raise UnclassifiedAnnotationError(event, field, annotation) + inner = [_classify(a, event, field) for a in args] + if origin is tuple and Ellipsis not in raw and len(inner) > 1: + # A fixed-length heterogeneous tuple is a positional record, + # not a collection: `tuple[str, float]` is (name, value) and + # its two slots get different answers. Emit one per position. + return {"[]": inner} + if any(candidate != inner[0] for candidate in inner[1:]): + raise UnclassifiedAnnotationError(event, field, annotation) + return inner[0] + if origin in (dict, Mapping, MutableMapping): + return DROP_OPAQUE + + raise UnclassifiedAnnotationError(event, field, annotation) + + +def _resolve_fields(cls: type) -> dict[str, Any]: + """Disposition per field of one dataclass, recursing into value objects.""" + hints = typing.get_type_hints(cls) + out: dict[str, Any] = {} + for spec in dataclasses.fields(cls): + out[spec.name] = _classify(hints[spec.name], cls.__name__, spec.name) + return out + + +def _event_modules() -> list[str]: + """Dotted names of every `events.py` under the cora package.""" + names: list[str] = [] + for path in sorted((_SRC / "cora").rglob("events.py")): + names.append(".".join(path.relative_to(_SRC).with_suffix("").parts)) + return names + + +def _event_classes(module_name: str) -> Iterable[type]: + """Frozen dataclasses DEFINED in this module, deduplicated by identity. + + A single-member union alias (`AcquisitionEvent = AcquisitionRecorded`) + binds one class under two module-level names, so `getmembers` yields + it twice. Dedupe on identity, not on name. + """ + module = importlib.import_module(module_name) + seen: set[int] = set() + for _, obj in inspect.getmembers(module, inspect.isclass): + if obj.__module__ != module_name or not _is_value_object(obj): + continue + if id(obj) in seen: + continue + seen.add(id(obj)) + yield obj + + +def build_table(survey: bool = False) -> tuple[dict[str, dict[str, Any]], list[str]]: + """Disposition per (event type, field) across every bounded context. + + With `survey`, collect every unclassified annotation instead of + aborting on the first. The generator still refuses to write a table + while any remain; the flag exists so the tail can be read in one + pass rather than one exception at a time. + """ + table: dict[str, dict[str, Any]] = {} + unclassified: list[str] = [] + for module_name in _event_modules(): + for cls in _event_classes(module_name): + if cls.__name__ in table: + raise RuntimeError( + f"Duplicate event class name {cls.__name__!r}; the table is " + "keyed on the bare name because that is what `events.event_type` " + "stores. Rename one, or key on the qualified name." + ) + if not survey: + table[cls.__name__] = _resolve_fields(cls) + continue + try: + table[cls.__name__] = _resolve_fields(cls) + except UnclassifiedAnnotationError as exc: + unclassified.append(str(exc).split(".", 1)[0] + ": " + str(exc)) + return dict(sorted(table.items())), unclassified + + +def render(table: Mapping[str, Mapping[str, Any]]) -> str: + """Render the table as a committed Python module.""" + body = json.dumps(table, indent=4, sort_keys=True) + return f'''"""Generated redaction dispositions. DO NOT EDIT BY HAND. + +Regenerate with `make record-dispositions`; +`tests/architecture/test_record_dispositions_drift.py` fails until you do. + +One entry per event type, one disposition per declared field, resolved +from the field's real type by `tools/gen_record_dispositions.py`. The +vocabulary: + + keep:enum: closed value set, provably reviewable. The enum is + NAMED because a human signs off the value set, and + swapping one enum for another must read as drift. + keep:number int / float / bool + keep:time datetime + token:uuid replaced with a per-export random surrogate + drop:text free text, no finite range, dropped by default + drop:opaque a dict with no declared keys, nothing to allowlist + by-value the slot is polymorphic across scalars and objects, + so no static answer exists. Apply the tier-2 leaf + rule at export time: numbers and booleans keep, + UUID-shaped strings token, other strings drop. + +A nested mapping is a value object recursed into. A mapping whose sole +key is `[]` is a fixed-length heterogeneous tuple, and its value lists +one disposition per position. + +Redaction iterates the STORED payload's keys and looks each up here. A +key absent from its event's entry is dropped; an event type absent from +this table aborts the export. The canonical hash of this mapping is the +redaction profile hash recorded in the export manifest. +""" + +from typing import Any + +DISPOSITIONS: dict[str, dict[str, Any]] = {body} +''' + + +def main() -> int: + survey = "--survey" in sys.argv + table, unclassified = build_table(survey=survey) + if unclassified: + print(f"{len(unclassified)} unclassified annotations:", file=sys.stderr) + for line in unclassified: + print(f" {line}", file=sys.stderr) + return 1 + _OUT.parent.mkdir(parents=True, exist_ok=True) + _OUT.write_text(render(table), encoding="utf-8") + # Format in place so the committed artifact is what `make lint` + # expects. Without this the generator and the formatter disagree and + # the drift test can never be green at the same time as lint. + formatted = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--quiet", str(_OUT)], + capture_output=True, + text=True, + check=False, + ) + if formatted.returncode != 0: + print(formatted.stderr, file=sys.stderr) + return 1 + fields = sum(len(v) for v in table.values()) + print(f"{len(table)} event types, {fields} fields -> {_OUT.relative_to(_API_ROOT)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 4d3f4a3c5963addcca9916d10965035c99a47eb4 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:10:07 -0500 Subject: [PATCH 2/8] Register every entries table by the kind its logbook envelope carries The exporter walks the stream and resolves each *LogbookOpened envelope's kind through one registry to the entries table it opened, per project_record_export_v3.md F0/F2. Eight entries, not six: six resolve a kind from an envelope; entries_run_feed_heartbeats and entries_enclosure_permit_probes have no envelope and are declared here explicitly, since a table with no envelope stays invisible to envelope-driven traversal until something declares it. Order key lives per kind because sampled_at exists on only four of the eight tables. An unknown kind raises rather than returning None or being skipped, and both the fitness test (every *LogbookOpened class under src/cora is named by the registry, and vice versa) and the information_schema integration test (renames defeat AST enumeration; three of these tables have already been renamed) exist to keep the registry from rotting quietly the way the frame memo's own census did. Co-Authored-By: Claude Sonnet 5 --- .../infrastructure/record_export/__init__.py | 18 ++ .../infrastructure/record_export/_registry.py | 189 ++++++++++++++++++ ...est_record_export_registry_completeness.py | 82 ++++++++ .../test_record_export_registry_postgres.py | 75 +++++++ .../record_export/test_registry.py | 79 ++++++++ 5 files changed, 443 insertions(+) create mode 100644 apps/api/src/cora/infrastructure/record_export/_registry.py create mode 100644 apps/api/tests/architecture/test_record_export_registry_completeness.py create mode 100644 apps/api/tests/integration/test_record_export_registry_postgres.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_registry.py diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py index 3fdc1e38ed5..d564788630e 100644 --- a/apps/api/src/cora/infrastructure/record_export/__init__.py +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -23,3 +23,21 @@ timestamps plus a facility's own published beamtime schedule re-identify without the token map. """ + +from cora.infrastructure.record_export._registry import ( + EntriesReader, + EntriesTableSpec, + UnknownLogbookKindError, + all_specs, + registered_envelope_classes, + resolve, +) + +__all__ = [ + "EntriesReader", + "EntriesTableSpec", + "UnknownLogbookKindError", + "all_specs", + "registered_envelope_classes", + "resolve", +] diff --git a/apps/api/src/cora/infrastructure/record_export/_registry.py b/apps/api/src/cora/infrastructure/record_export/_registry.py new file mode 100644 index 00000000000..028329df983 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_registry.py @@ -0,0 +1,189 @@ +"""Registry: logbook ``kind`` -> (table, order key, reader). + +Per `project_record_export_v3.md` F0/F2 and +`project_record_is_two_tier.md`. The exporter walks the event stream and, +on each ``LogbookOpened`` envelope, reads the envelope's ``kind`` (a +bare ``str``, not a closed enum) and resolves it through this ONE +registry to the entries table that holds the fine-grained doing. An +unknown ``kind`` refuses loudly rather than being skipped. + +Eight entries, not six. Six kinds come from an envelope event on the +main stream; two tables, `entries_run_feed_heartbeats` and +`entries_enclosure_permit_probes`, have no envelope at all and are +declared here explicitly, with `envelope_class` set to `None`, per +`project_record_is_two_tier.md`'s "declare or exclude, in writing" +finding. Whether the exporter actually pulls their rows into a published +bundle, or excludes them as operational telemetry, is a separate, +still-open call for the exporter step; this registry only has to make +both tables reachable and refuse to silently drop either. + +The order key lives per kind because `sampled_at` exists on only four of +the eight tables (activity, diagnostic, outcome, observation). The other +four order by `event_id` alone: CORA mints it with `UUIDv7Generator`, so +it is total and insertion-ordered without a separate timestamp column, +and `occurred_at` is never the tiebreak because it ties across a whole +append batch (one Clock read per handler call). + +The six envelope-driven tables are scoped by `logbook_id`, the join +column the envelope carries. Heartbeats and probes are not +Logbook-and-Entry instances (no `logbook_id` column at all) and are +scoped by their owning aggregate's id instead: `run_id` for heartbeats, +`enclosure_id` for probes. +""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from uuid import UUID + +import asyncpg + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# asyncpg's stubs are loose; suppress at module level, matching the other +# entries-table readers (e.g. postgres_procedure_activity_lookup.py). + +EntriesReader = Callable[[asyncpg.Connection, UUID], Awaitable[list[asyncpg.Record]]] + + +@dataclass(frozen=True, slots=True) +class EntriesTableSpec: + """One registry row: how to read one kind's entries-tier rows. + + `envelope_class` names the `*LogbookOpened` class that carries this + kind on the main stream, or `None` for the two tables with no + envelope. `scope_column` is the column `reader` filters on: the + envelope's `logbook_id` for the six logbook-backed kinds, the owning + aggregate's id for the other two. + """ + + kind: str + table: str + envelope_class: str | None + scope_column: str + order_by: tuple[str, ...] + reader: EntriesReader + + +class UnknownLogbookKindError(LookupError): + """A `kind` with no registry entry. Refuses loudly rather than skipping.""" + + def __init__(self, kind: str) -> None: + super().__init__( + f"no entries-table registry entry for logbook kind {kind!r}; " + "an unknown kind must not be skipped" + ) + self.kind = kind + + +def _make_reader(table: str, scope_column: str, order_by: tuple[str, ...]) -> EntriesReader: + # table, scope_column and order_by are all registry-declared constants + # below, never caller input, so the interpolation cannot carry + # attacker-controlled SQL. + sql = f"SELECT * FROM {table} WHERE {scope_column} = $1 ORDER BY {', '.join(order_by)}" + + async def read(conn: asyncpg.Connection, scope_id: UUID) -> list[asyncpg.Record]: + return await conn.fetch(sql, scope_id) + + return read + + +def _spec( + *, + kind: str, + table: str, + envelope_class: str | None, + scope_column: str, + order_by: tuple[str, ...], +) -> EntriesTableSpec: + return EntriesTableSpec( + kind=kind, + table=table, + envelope_class=envelope_class, + scope_column=scope_column, + order_by=order_by, + reader=_make_reader(table, scope_column, order_by), + ) + + +_ENTRIES: tuple[EntriesTableSpec, ...] = ( + _spec( + kind="verdict", + table="entries_conduit_verdicts", + envelope_class="ConduitLogbookOpened", + scope_column="logbook_id", + order_by=("event_id",), + ), + _spec( + kind="inference", + table="entries_decision_inferences", + envelope_class="DecisionLogbookOpened", + scope_column="logbook_id", + order_by=("event_id",), + ), + _spec( + kind="activity", + table="entries_operation_procedure_activities", + envelope_class="ProcedureActivitiesLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="diagnostic", + table="entries_operation_procedure_diagnostics", + envelope_class="ProcedureDiagnosticLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="outcome", + table="entries_operation_procedure_outcomes", + envelope_class="ProcedureOutcomeLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="observation", + table="entries_run_observations", + envelope_class="RunObservationLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="heartbeat", + table="entries_run_feed_heartbeats", + envelope_class=None, + scope_column="run_id", + order_by=("event_id",), + ), + _spec( + kind="probe", + table="entries_enclosure_permit_probes", + envelope_class=None, + scope_column="enclosure_id", + order_by=("event_id",), + ), +) + +_REGISTRY: dict[str, EntriesTableSpec] = {spec.kind: spec for spec in _ENTRIES} + + +def resolve(kind: str) -> EntriesTableSpec: + """Look up `kind`'s table spec. + + Raises `UnknownLogbookKindError` rather than returning `None`: an + envelope carrying a kind this registry has never heard of must stop + the export, not be skipped. + """ + try: + return _REGISTRY[kind] + except KeyError: + raise UnknownLogbookKindError(kind) from None + + +def all_specs() -> tuple[EntriesTableSpec, ...]: + """Every registered spec, in declaration order.""" + return _ENTRIES + + +def registered_envelope_classes() -> frozenset[str]: + """Every `*LogbookOpened` class name named by a registry entry.""" + return frozenset(spec.envelope_class for spec in _ENTRIES if spec.envelope_class is not None) diff --git a/apps/api/tests/architecture/test_record_export_registry_completeness.py b/apps/api/tests/architecture/test_record_export_registry_completeness.py new file mode 100644 index 00000000000..aa8d1db6bc0 --- /dev/null +++ b/apps/api/tests/architecture/test_record_export_registry_completeness.py @@ -0,0 +1,82 @@ +"""Every `*LogbookOpened` class is named by the entries-tier registry. + +`cora.infrastructure.record_export._registry` resolves a logbook +envelope's `kind` to the `entries_*` table it opened. Per +`project_record_is_two_tier.md`, the envelope never names its own table, +so the registry is the one place that link is written down, and a new +`*LogbookOpened` class that forgets to register itself would silently +narrow envelope-driven traversal the same way the eighth entries table +(`entries_enclosure_permit_probes`) silently narrowed it before this +registry existed. + +This fitness function AST-discovers every `*LogbookOpened` class under +`src/cora` (git-tracked, so pre-commit's tracked-file staging sees the +same set) and pins it against `registered_envelope_classes()`, both +directions: a new envelope class with no registry entry, and a registry +entry naming a class that no longer exists (e.g. after a rename), both +fail loudly instead of rotting quietly. +""" + +import ast +from pathlib import Path + +import pytest + +from cora.infrastructure.record_export import registered_envelope_classes +from tests.architecture.conftest import tracked_python_files + + +def _find_logbook_opened_classes(tree: ast.Module) -> list[str]: + return [ + node.name + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) and node.name.endswith("LogbookOpened") + ] + + +def _discover_logbook_opened_classes() -> dict[str, Path]: + """Map every discovered `*LogbookOpened` class name to its defining file.""" + found: dict[str, Path] = {} + for path in tracked_python_files(): + if path.name != "events.py": + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + for class_name in _find_logbook_opened_classes(tree): + found[class_name] = path + return found + + +@pytest.mark.architecture +def test_every_logbook_opened_class_is_in_the_registry() -> None: + discovered = _discover_logbook_opened_classes() + registered = registered_envelope_classes() + + unregistered = set(discovered) - registered + assert not unregistered, ( + f"{sorted(unregistered)} define a *LogbookOpened envelope with no " + "entry in cora.infrastructure.record_export._registry. Envelope-" + "driven traversal silently stops covering the new table until " + "one is added. Files: " + f"{ {name: str(discovered[name]) for name in sorted(unregistered)} }" + ) + + +@pytest.mark.architecture +def test_no_registry_entry_names_a_class_that_no_longer_exists() -> None: + discovered = _discover_logbook_opened_classes() + registered = registered_envelope_classes() + + stale = registered - set(discovered) + assert not stale, ( + f"{sorted(stale)} are named by the registry's envelope_class field " + "but no *LogbookOpened class by that name exists under src/cora " + "anymore (renamed?). Update _registry.py's envelope_class." + ) + + +@pytest.mark.architecture +def test_exactly_six_logbook_opened_classes_exist() -> None: + """Pins the count so a seventh envelope class is a deliberate, reviewed + registry addition rather than a silent drift. Bump alongside a new + _registry.py entry, never on its own.""" + assert len(_discover_logbook_opened_classes()) == 6 diff --git a/apps/api/tests/integration/test_record_export_registry_postgres.py b/apps/api/tests/integration/test_record_export_registry_postgres.py new file mode 100644 index 00000000000..4b8e8767551 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_registry_postgres.py @@ -0,0 +1,75 @@ +"""Integration test: the entries-tier registry against a live schema. + +AST enumeration can prove a `*LogbookOpened` class exists; it cannot +prove the table or columns the registry names for it are real, because +renames defeat it. Three of these eight tables have already been +renamed (see `project_record_is_two_tier.md`), so this test queries +`information_schema` directly against the fully-migrated template +database and fails if `_registry.py` still points at a name from before +a rename. + +Also exercises the reader end to end: for a table with zero rows scoped +to a random id, the reader returns an empty list rather than raising, +which is the shape the exporter needs to distinguish "this stream opened +a logbook with no entries yet" (Conduit's eager-open pattern) from a +registry or SQL error. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from uuid import uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.record_export import EntriesTableSpec, all_specs + + +async def _columns(conn: asyncpg.Connection, table: str) -> set[str]: + rows = await conn.fetch( + "SELECT column_name FROM information_schema.columns WHERE table_name = $1", + table, + ) + return {row["column_name"] for row in rows} + + +@pytest.mark.integration +@pytest.mark.parametrize("spec", all_specs(), ids=lambda spec: spec.kind) +async def test_registered_table_exists_with_its_declared_columns( + db_pool: asyncpg.Pool, spec: EntriesTableSpec +) -> None: + async with db_pool.acquire() as conn: + # PoolConnectionProxy and Connection expose the same runtime API; + # narrowed here to match the `_columns` / EntriesReader signature, + # per the postgres_profile_store.py convention. + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exists = await pg_conn.fetchval( + "SELECT count(*) FROM information_schema.tables WHERE table_name = $1", + spec.table, + ) + assert exists == 1, ( + f"kind={spec.kind!r} names table {spec.table!r}, which does not " + "exist in the live schema. A migration rename that forgot to " + "update _registry.py is exactly the failure mode this test " + "exists to catch." + ) + + columns = await _columns(pg_conn, spec.table) + needed = {spec.scope_column, *spec.order_by} + missing = needed - columns + assert not missing, ( + f"kind={spec.kind!r}: {spec.table!r} is missing column(s) " + f"{sorted(missing)} that _registry.py's scope_column/order_by " + "declare for it." + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("spec", all_specs(), ids=lambda spec: spec.kind) +async def test_reader_returns_empty_list_for_an_unscoped_id( + db_pool: asyncpg.Pool, spec: EntriesTableSpec +) -> None: + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + rows = await spec.reader(pg_conn, uuid4()) + assert rows == [] diff --git a/apps/api/tests/unit/infrastructure/record_export/test_registry.py b/apps/api/tests/unit/infrastructure/record_export/test_registry.py new file mode 100644 index 00000000000..6a84a185a3d --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_registry.py @@ -0,0 +1,79 @@ +"""Unit tests for the entries-tier registry: `kind -> (table, order key, reader)`. + +See `cora.infrastructure.record_export._registry` for the design: eight +entries, six resolved from a `*LogbookOpened` envelope's `kind` and two +(`heartbeat`, `probe`) declared explicitly because they have no envelope. +""" + +import pytest + +from cora.infrastructure.record_export import ( + UnknownLogbookKindError, + all_specs, + registered_envelope_classes, + resolve, +) + +_ENVELOPE_DRIVEN_KINDS = ( + "verdict", + "inference", + "activity", + "diagnostic", + "outcome", + "observation", +) +_DECLARED_KINDS = ("heartbeat", "probe") + + +def test_registry_has_eight_entries_not_six() -> None: + assert len(all_specs()) == 8 + + +@pytest.mark.parametrize("kind", [*_ENVELOPE_DRIVEN_KINDS, *_DECLARED_KINDS]) +def test_resolve_finds_every_registered_kind(kind: str) -> None: + spec = resolve(kind) + assert spec.kind == kind + + +@pytest.mark.parametrize("kind", _ENVELOPE_DRIVEN_KINDS) +def test_envelope_driven_kinds_carry_their_envelope_class(kind: str) -> None: + spec = resolve(kind) + assert spec.envelope_class is not None + assert spec.envelope_class.endswith("LogbookOpened") + assert spec.scope_column == "logbook_id" + + +@pytest.mark.parametrize("kind", _DECLARED_KINDS) +def test_declared_kinds_have_no_envelope_class(kind: str) -> None: + spec = resolve(kind) + assert spec.envelope_class is None + assert spec.scope_column != "logbook_id" + + +def test_registered_envelope_classes_has_exactly_the_six_logbook_kinds() -> None: + assert registered_envelope_classes() == { + "ConduitLogbookOpened", + "DecisionLogbookOpened", + "ProcedureActivitiesLogbookOpened", + "ProcedureDiagnosticLogbookOpened", + "ProcedureOutcomeLogbookOpened", + "RunObservationLogbookOpened", + } + + +def test_unknown_kind_refuses_loudly_instead_of_returning_none() -> None: + with pytest.raises(UnknownLogbookKindError) as excinfo: + resolve("steps") # the pre-rename name; must not silently resurrect it + assert excinfo.value.kind == "steps" + + +def test_order_by_uses_sampled_at_on_exactly_the_four_tables_that_have_it() -> None: + with_sampled_at = {spec.kind for spec in all_specs() if spec.order_by[0] == "sampled_at"} + assert with_sampled_at == {"activity", "diagnostic", "outcome", "observation"} + without_sampled_at = {spec.kind for spec in all_specs() if spec.order_by == ("event_id",)} + assert without_sampled_at == {"verdict", "inference", "heartbeat", "probe"} + + +def test_table_names_are_unique_across_the_registry() -> None: + tables = [spec.table for spec in all_specs()] + assert len(tables) == len(set(tables)) From bb78bc09354eec951867f991a63e24d26a4468f9 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:09:13 -0500 Subject: [PATCH 3/8] Walk the event store and follow every logbook envelope into its entries export_record captures one pg_snapshot_xmin watermark up front (not re-evaluated per row like the projection worker's catch-up query, so one export sees one consistent snapshot), walks the whole events table under it in (transaction_id, position) order, and on each row whose event_type names one of Step 1's six *LogbookOpened classes, pulls kind/logbook_id straight out of the already-decoded payload and follows it through Step 1's registry into the matching entries rows. Payload UUID/datetime fields are already plain strings by the time they reach jsonb (every to_payload() converts them before the row is written, confirmed against Step 0's own generated disposition table), so F6 rendering only has to touch the outer typed columns asyncpg hands back as uuid.UUID / tz-aware datetime / bytes -- not recurse into payload/metadata. An unknown stream_type refuses immediately rather than being skipped, against a closed set collected from the 42 distinct _STREAM_TYPE literals scattered across the tree (no such registry existed before; Step 2's own acceptance criteria required one). Zero exported rows is an error, not an empty result: a freshly migrated database is not actually empty, since two migrations seed bootstrap events, so the zero-rows test empties events explicitly on its own disposable per-test database rather than relying on that lucky-empty state. Heartbeats and probes are deliberately not walked here: neither has an envelope event to trigger from, and that is written down as an explicit deferral rather than an omission, per the build brief's open question about excluding them as operational telemetry. Co-Authored-By: Claude Sonnet 5 --- .../infrastructure/record_export/__init__.py | 21 ++ .../infrastructure/record_export/_export.py | 133 +++++++++++ .../infrastructure/record_export/_render.py | 58 +++++ .../record_export/_stream_types.py | 88 +++++++ ...record_export_stream_types_completeness.py | 58 +++++ .../test_record_export_exporter_postgres.py | 220 ++++++++++++++++++ .../record_export/test_render.py | 73 ++++++ .../record_export/test_stream_types.py | 25 ++ 8 files changed, 676 insertions(+) create mode 100644 apps/api/src/cora/infrastructure/record_export/_export.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_render.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_stream_types.py create mode 100644 apps/api/tests/architecture/test_record_export_stream_types_completeness.py create mode 100644 apps/api/tests/integration/test_record_export_exporter_postgres.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_render.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_stream_types.py diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py index d564788630e..cce95ab6e42 100644 --- a/apps/api/src/cora/infrastructure/record_export/__init__.py +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -24,6 +24,12 @@ re-identify without the token map. """ +from cora.infrastructure.record_export._export import ( + EmptyExportError, + ExportedRecord, + capture_watermark, + export_record, +) from cora.infrastructure.record_export._registry import ( EntriesReader, EntriesTableSpec, @@ -32,12 +38,27 @@ registered_envelope_classes, resolve, ) +from cora.infrastructure.record_export._render import render_row, render_value +from cora.infrastructure.record_export._stream_types import ( + KNOWN_STREAM_TYPES, + UnknownStreamTypeError, + ensure_stream_type_known, +) __all__ = [ + "KNOWN_STREAM_TYPES", + "EmptyExportError", "EntriesReader", "EntriesTableSpec", + "ExportedRecord", "UnknownLogbookKindError", + "UnknownStreamTypeError", "all_specs", + "capture_watermark", + "ensure_stream_type_known", + "export_record", "registered_envelope_classes", + "render_row", + "render_value", "resolve", ] diff --git a/apps/api/src/cora/infrastructure/record_export/_export.py b/apps/api/src/cora/infrastructure/record_export/_export.py new file mode 100644 index 00000000000..fb505187a89 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_export.py @@ -0,0 +1,133 @@ +"""Walk the event store and follow every logbook envelope into its entries. + +Per `project_record_export_v3.md` F0/F2: a single stream query over the +whole `events` table, ordered by `(transaction_id, position)`, bounded by +one `pg_snapshot_xmin(pg_current_snapshot())` watermark captured up front +(not re-evaluated per row, unlike the projection worker's catch-up query, +which this SQL otherwise mirrors) so the export sees one consistent +snapshot. On each row whose `event_type` names a `*LogbookOpened` class +(`registered_envelope_classes()`, from Step 1's `_registry`), `kind` and +`logbook_id` come straight out of the already-decoded `payload` dict and +resolve through Step 1's `resolve()` to the matching entries-tier reader. + +Deliberately does not: write bundle files (that is Step 3/4's job, once +hashing and the manifest exist), or touch `entries_run_feed_heartbeats` / +`entries_enclosure_permit_probes` (they have no envelope event to trigger +them from an envelope-driven walk; see `project_record_export_build_brief.md` +step 2 notes -- an explicit, written deferral, not an omission). +""" + +from dataclasses import dataclass +from uuid import UUID + +import asyncpg + +from cora.infrastructure.record_export._registry import registered_envelope_classes, resolve +from cora.infrastructure.record_export._render import render_row +from cora.infrastructure.record_export._stream_types import ensure_stream_type_known + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# asyncpg's stubs are loose; suppress at module level, matching the other +# entries-table readers. + +_WATERMARK_SQL = "SELECT pg_snapshot_xmin(pg_current_snapshot())::text" + +# Same column list as postgres_event_store.py's _LOAD_SQL and +# projection/worker.py's _ADVANCE_SQL. transaction_id is xid8; asyncpg has +# no output codec for it (same gap those two modules work around), so it +# is cast to text and aliased back onto its own name. +_STREAM_SQL = """ +SELECT position, event_id, stream_type, stream_id, version, event_type, + schema_version, payload, metadata, correlation_id, causation_id, + principal_id, occurred_at, recorded_at, + signature, signature_kid, signature_version, + transaction_id::text AS transaction_id +FROM events +WHERE transaction_id < $1::xid8 +ORDER BY transaction_id, position +""" + + +class EmptyExportError(RuntimeError): + """Zero rows exported. + + Per the build brief's acceptance criteria: a bundle claiming to be + the record with nothing in it is a bug in the caller (wrong + database, unmigrated database, watermark set before anything was + written), never a legitimate empty result to hand back quietly. + """ + + +@dataclass(frozen=True, slots=True) +class ExportedRecord: + """One export's two tiers, rendered but not yet hashed or redacted. + + `streams` is every `events` row below the captured watermark, in + `(transaction_id, position)` order. `logbooks` groups every + entries-tier row pulled via an envelope by `kind`, in each kind's own + registry order-by order; rows are UNFOLDED (one dict per row), never + aggregated. + """ + + streams: tuple[dict[str, object], ...] + logbooks: dict[str, tuple[dict[str, object], ...]] + + +async def capture_watermark(conn: asyncpg.Connection) -> int: + """The xmin bound for one consistent export snapshot. + + Returns a plain `int`: asyncpg has no xid8 codec on either side, so + (mirroring `projection/worker.py`'s bookmark parameter, the existing + precedent for binding a value against an `xid8` column) the value is + cast to `text` on the way out here and to `int` on the way back in, + then bound as `$1::xid8` by the caller -- never compared as a string. + + Callers must pass the SAME returned value to `export_record`'s + underlying query exactly once; capturing it here rather than letting + the stream query re-evaluate `pg_snapshot_xmin` per row is what makes + one export see one snapshot instead of a moving target. + """ + value = await conn.fetchval(_WATERMARK_SQL) + assert value is not None, "pg_snapshot_xmin(pg_current_snapshot()) returned NULL" + return int(value) + + +async def export_record(conn: asyncpg.Connection) -> ExportedRecord: + """Walk the whole event store and its envelope-linked logbooks. + + Raises `EmptyExportError` on zero stream rows, + `cora.infrastructure.record_export.UnknownStreamTypeError` on a + `stream_type` outside the declared closed set (refuses immediately; + never skips), and `UnknownLogbookKindError` on an envelope's `kind` + with no registry entry. + """ + watermark = await capture_watermark(conn) + rows = await conn.fetch(_STREAM_SQL, watermark) + if not rows: + raise EmptyExportError( + "export_record found zero events.rows below the captured " + f"watermark ({watermark!r}); an empty record is never a " + "valid export." + ) + + envelope_classes = registered_envelope_classes() + streams: list[dict[str, object]] = [] + logbooks: dict[str, list[dict[str, object]]] = {} + + for row in rows: + ensure_stream_type_known(row["stream_type"]) + streams.append(render_row(row)) + + if row["event_type"] not in envelope_classes: + continue + payload = row["payload"] + kind = payload["kind"] + logbook_id = UUID(payload["logbook_id"]) + spec = resolve(kind) + entries = await spec.reader(conn, logbook_id) + logbooks.setdefault(kind, []).extend(render_row(entry) for entry in entries) + + return ExportedRecord( + streams=tuple(streams), + logbooks={kind: tuple(entries) for kind, entries in logbooks.items()}, + ) diff --git a/apps/api/src/cora/infrastructure/record_export/_render.py b/apps/api/src/cora/infrastructure/record_export/_render.py new file mode 100644 index 00000000000..5a3a1aba472 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_render.py @@ -0,0 +1,58 @@ +"""F6 rendering: the one table for turning a raw row into export bytes-ready form. + +Per `project_record_export_v3.md` F6: UUID to string, datetime to UTC +ISO-8601, bytes to hex. UTC normalization is measured, not stylistic: +the same instant at `-05:00` and `+00:00` hashes differently, and +Postgres returns `timestamptz` columns in the session's configured +offset while payload timestamps were already written as UTC strings. + +This only has to touch the OUTER `events` / `entries_*` row columns +that asyncpg hands back as typed Python objects (`uuid.UUID`, tz-aware +`datetime`, `bytes`). It must NOT recurse into `payload` / `metadata`: +every `to_payload()` in the tree already converts UUID and datetime +fields to plain strings before the row is written (verified against +`_dispositions.py`, the Step 0 generated table, whose entries for all +six `*LogbookOpened` classes show `logbook_id` as a flat `token:uuid` +field rather than a nested one), and the asyncpg pool's jsonb codec +(`cora.infrastructure.postgres.pool`) decodes straight to `dict` / `str` +/ etc, never back to `UUID` or `datetime`. Rendering jsonb contents a +second time here would be a no-op at best and silently wrong if a future +payload ever carried a real (not pre-stringified) UUID or datetime. +""" + +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from uuid import UUID + +if TYPE_CHECKING: + import asyncpg + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# asyncpg's stubs are loose; suppress at module level, matching the other +# entries-table readers. + + +def render_value(value: object) -> object: + """Render one column value for export. + + `UUID -> str`, `datetime -> UTC ISO-8601 str`, `bytes`-like -> hex + `str`. Everything else (str, int, float, bool, None, and jsonb-decoded + dict/list contents) passes through unchanged. + """ + if isinstance(value, UUID): + return str(value) + if isinstance(value, datetime): + # asyncpg returns `timestamptz` columns as timezone-aware in the + # session's configured offset; astimezone(UTC) normalizes + # regardless of what that offset was. + return value.astimezone(UTC).isoformat() + if isinstance(value, (bytes, bytearray, memoryview)): + return value.hex() + return value + + +def render_row(row: "dict[str, object] | asyncpg.Record") -> dict[str, object]: + """Render every column of one asyncpg `Record` (production) or plain + `dict` (unit tests -- `Record` isn't constructible outside asyncpg's + own protocol machinery).""" + return {key: render_value(value) for key, value in row.items()} diff --git a/apps/api/src/cora/infrastructure/record_export/_stream_types.py b/apps/api/src/cora/infrastructure/record_export/_stream_types.py new file mode 100644 index 00000000000..677b5682cf5 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_stream_types.py @@ -0,0 +1,88 @@ +"""The closed set of `events.stream_type` values. + +No such set exists anywhere else in the code: each BC's aggregate/slice +declares its own `_STREAM_TYPE = "X"` module constant (42 distinct +literals across 211 declarations at last count), and `stream_type` on +`events` is a bare `text` column with no DB or Python enum closing it. + +The exporter needs one anyway: per `project_record_export_build_brief.md` +step 2's acceptance criteria, "an unknown `stream_type` refuses rather +than skips ... silently skipping is pass-while-differing." So this +module declares the closed set explicitly, the same shape as +`_registry.py`'s `kind` registry, and +`test_record_export_stream_types_completeness.py` AST-discovers every +`_STREAM_TYPE` literal under `src/cora` and pins it against +`KNOWN_STREAM_TYPES` in both directions so a new BC cannot silently fall +through `ensure_stream_type_known` as "skip" instead of "refuse". +""" + +KNOWN_STREAM_TYPES: frozenset[str] = frozenset( + { + "Acquisition", + "Actor", + "Agent", + "Allocation", + "Assembly", + "Asset", + "Attestation", + "Calibration", + "Campaign", + "Capability", + "Caution", + "Clearance", + "ClearanceTemplate", + "Conduit", + "Credential", + "Dataset", + "Decision", + "Distribution", + "Edition", + "Enclosure", + "Facility", + "Family", + "Fixture", + "Frame", + "LanguageModel", + "Method", + "Model", + "Mount", + "Permit", + "Plan", + "Policy", + "Practice", + "Procedure", + "Ratification", + "Recipe", + "Role", + "Run", + "Seal", + "Subject", + "Supply", + "Surface", + "Visit", + "Zone", + } +) + + +class UnknownStreamTypeError(LookupError): + """A `stream_type` with no entry in `KNOWN_STREAM_TYPES`. + + Refuses loudly rather than being silently skipped: skipping an + unrecognised stream is "pass while differing" per the build brief's + trap list, not a safe default. + """ + + def __init__(self, stream_type: str) -> None: + super().__init__( + f"unknown events.stream_type {stream_type!r}; not in " + "KNOWN_STREAM_TYPES. A new BC's stream_type must be added " + "here before its streams can be exported." + ) + self.stream_type = stream_type + + +def ensure_stream_type_known(stream_type: str) -> None: + """Raise `UnknownStreamTypeError` unless `stream_type` is declared.""" + if stream_type not in KNOWN_STREAM_TYPES: + raise UnknownStreamTypeError(stream_type) diff --git a/apps/api/tests/architecture/test_record_export_stream_types_completeness.py b/apps/api/tests/architecture/test_record_export_stream_types_completeness.py new file mode 100644 index 00000000000..9527261724d --- /dev/null +++ b/apps/api/tests/architecture/test_record_export_stream_types_completeness.py @@ -0,0 +1,58 @@ +"""Every `_STREAM_TYPE = "X"` literal under `src/cora` is in `KNOWN_STREAM_TYPES`. + +`events.stream_type` is a bare `text` column with no DB or Python enum +closing it; each BC/slice declares its own `_STREAM_TYPE` module +constant instead (211 declarations across 42 distinct literals at last +count). `cora.infrastructure.record_export._stream_types` declares that +set explicitly so the exporter can refuse an unknown `stream_type` +rather than silently skip it. This fitness function AST-discovers every +`_STREAM_TYPE = "X"` assignment and pins it against `KNOWN_STREAM_TYPES` +both directions: a new BC that forgets to add its stream_type here would +otherwise pass every other test and still make `export_record` refuse +on the very first row of its stream. +""" + +import ast + +import pytest + +from cora.infrastructure.record_export import KNOWN_STREAM_TYPES +from tests.architecture.conftest import tracked_python_files + + +def _discover_stream_type_literals() -> set[str]: + found: set[str] = set() + for path in tracked_python_files(): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if not (isinstance(target, ast.Name) and target.id == "_STREAM_TYPE"): + continue + if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + found.add(node.value.value) + return found + + +@pytest.mark.architecture +def test_every_declared_stream_type_literal_is_known() -> None: + discovered = _discover_stream_type_literals() + unknown = discovered - KNOWN_STREAM_TYPES + assert not unknown, ( + f"{sorted(unknown)} appear as `_STREAM_TYPE = ...` literals under " + "src/cora but are not in KNOWN_STREAM_TYPES " + "(cora.infrastructure.record_export._stream_types). export_record " + "would refuse on the first row of that stream." + ) + + +@pytest.mark.architecture +def test_no_known_stream_type_is_unused() -> None: + discovered = _discover_stream_type_literals() + stale = KNOWN_STREAM_TYPES - discovered + assert not stale, ( + f"{sorted(stale)} are declared in KNOWN_STREAM_TYPES but no " + "`_STREAM_TYPE = ...` literal under src/cora uses them anymore " + "(renamed or removed BC?). Update _stream_types.py." + ) diff --git a/apps/api/tests/integration/test_record_export_exporter_postgres.py b/apps/api/tests/integration/test_record_export_exporter_postgres.py new file mode 100644 index 00000000000..bdef404e556 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_exporter_postgres.py @@ -0,0 +1,220 @@ +"""Acceptance test for Step 2 of the record exporter build brief. + +Per `project_record_export_build_brief.md` step 2: a Procedure stream +folds to `Running` with its activity rows beside it, unfolded; an +unknown `stream_type` refuses rather than skips; zero rows exported is +an error. + +The Procedure is built the same way +`test_append_activities_handler_postgres.py` does it: `ProcedureRegistered` ++ `ProcedureStarted` seeded directly into the event store (bypassing +`register_procedure`/`start_procedure`'s cross-aggregate validation, +which is not this test's concern), then the real `append_activities` +handler so `ProcedureActivitiesLogbookOpened` fires for real and rows +land in `entries_operation_procedure_activities`. + +The "folds to Running" assertion reconstructs `StoredEvent`s from +`export_record`'s own rendered `streams` output (not from a fresh +`event_store.load`), because the point of the acceptance test is that +the EXPORTED representation is complete and correct enough to fold, the +same property the standalone verifier (step 5) will depend on. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC as _UTC +from datetime import datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.ports.event_store import StoredEvent +from cora.infrastructure.record_export import ( + EmptyExportError, + UnknownStreamTypeError, + export_record, +) +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + ProcedureStatus, + event_type_name, + fold, + from_stored, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=_UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +async def _seed_running_procedure(event_store: object, procedure_id: UUID) -> None: + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await event_store.append( # type: ignore[attr-defined] + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + +def _stored_event_from_rendered_row(row: dict[str, object]) -> StoredEvent: + """Reconstruct a `StoredEvent` from one of `export_record`'s rendered + `streams` rows, undoing F6 rendering. This is exactly the + reconstruction the standalone verifier (step 5) will need to do.""" + return StoredEvent( + position=row["position"], # type: ignore[arg-type] + event_id=UUID(row["event_id"]), # type: ignore[arg-type] + stream_type=row["stream_type"], # type: ignore[arg-type] + stream_id=UUID(row["stream_id"]), # type: ignore[arg-type] + version=row["version"], # type: ignore[arg-type] + event_type=row["event_type"], # type: ignore[arg-type] + schema_version=row["schema_version"], # type: ignore[arg-type] + payload=row["payload"], # type: ignore[arg-type] + metadata=row["metadata"], # type: ignore[arg-type] + correlation_id=UUID(row["correlation_id"]), # type: ignore[arg-type] + causation_id=UUID(row["causation_id"]) if row["causation_id"] else None, # type: ignore[arg-type] + occurred_at=datetime.fromisoformat(row["occurred_at"]), # type: ignore[arg-type] + recorded_at=datetime.fromisoformat(row["recorded_at"]), # type: ignore[arg-type] + transaction_id=int(row["transaction_id"]), # type: ignore[arg-type] + principal_id=UUID(row["principal_id"]) if row["principal_id"] else None, # type: ignore[arg-type] + signature=bytes.fromhex(row["signature"]) if row["signature"] else None, # type: ignore[arg-type] + signature_kid=row["signature_kid"], # type: ignore[arg-type] + signature_version=row["signature_version"], # type: ignore[arg-type] + ) + + +async def _read_activity_rows(db_pool: asyncpg.Pool, procedure_id: UUID) -> list[asyncpg.Record]: + async with db_pool.acquire() as conn: + return await conn.fetch( + """ + SELECT event_id, procedure_id, logbook_id, step_kind, payload, + sampled_at, occurred_at, recorded_at + FROM entries_operation_procedure_activities + WHERE procedure_id = $1 + ORDER BY sampled_at, event_id + """, + procedure_id, + ) + + +@pytest.mark.integration +async def test_procedure_stream_folds_to_running_with_activity_rows_beside_it( + db_pool: asyncpg.Pool, +) -> None: + procedure_id = UUID("01900000-0000-7000-8000-0000020a0a01") + logbook_id = UUID("01900000-0000-7000-8000-0000020a0a02") + open_event_id = UUID("01900000-0000-7000-8000-0000020a0a03") + + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + await _seed_running_procedure(deps.event_store, procedure_id) + + step_store = PostgresActivityStore(db_pool) + handler = bind_append(deps, step_store=step_store) + sampled_a = datetime(2026, 5, 15, 12, 0, 1, tzinfo=_UTC) + sampled_b = datetime(2026, 5, 15, 12, 0, 2, tzinfo=_UTC) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=UUID("01900000-0000-7000-8000-0000020a0b01"), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0}, + sampled_at=sampled_a, + ), + ActivityInput( + event_id=UUID("01900000-0000-7000-8000-0000020a0b02"), + step_kind="check", + payload={"channel": "T_oven", "passed": True}, + sampled_at=sampled_b, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + result = await export_record(pg_conn) + + procedure_rows = [row for row in result.streams if row["stream_id"] == str(procedure_id)] + assert len(procedure_rows) == 3 # Registered, Started, ActivitiesLogbookOpened + procedure_rows.sort(key=lambda row: int(row["version"])) # type: ignore[arg-type] + stored = [_stored_event_from_rendered_row(row) for row in procedure_rows] + state = fold([from_stored(s) for s in stored]) + assert state is not None + assert state.status == ProcedureStatus.RUNNING + assert state.activity_logbook_id == logbook_id + + exported_activity_rows = result.logbooks["activity"] + assert len(exported_activity_rows) == 2 + live_rows = await _read_activity_rows(db_pool, procedure_id) + assert {row["event_id"] for row in exported_activity_rows} == { + str(row["event_id"]) for row in live_rows + } + by_kind = {row["step_kind"]: row for row in exported_activity_rows} + assert by_kind["setpoint"]["payload"] == {"channel": "T_oven", "target_value": 423.0} + assert by_kind["check"]["payload"] == {"channel": "T_oven", "passed": True} + # Unfolded: two distinct rows, not a summary of the logbook. + assert by_kind["setpoint"] is not by_kind["check"] + + +@pytest.mark.integration +async def test_unknown_stream_type_refuses_rather_than_skips(db_pool: asyncpg.Pool) -> None: + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + await pg_conn.execute( + """ + INSERT INTO events (event_id, stream_type, stream_id, version, + event_type, payload, correlation_id, occurred_at) + VALUES ($1, 'Widget', $2, 1, 'WidgetRegistered', '{}'::jsonb, $3, now()) + """, + uuid4(), + uuid4(), + uuid4(), + ) + with pytest.raises(UnknownStreamTypeError) as excinfo: + await export_record(pg_conn) + assert excinfo.value.stream_type == "Widget" + + +@pytest.mark.integration +async def test_zero_rows_exported_is_an_error(db_pool: asyncpg.Pool) -> None: + """A freshly migrated template is NOT empty: two seed migrations + (`20260519000000_seed_bootstrap_policy.sql`, + `20260519200000_seed_default_surfaces_and_v2_policy.sql`) insert + bootstrap events. Emptying `events` explicitly is what actually + exercises the zero-rows refusal, on this test's own disposable + per-test database.""" + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + await pg_conn.execute("DELETE FROM events") + with pytest.raises(EmptyExportError): + await export_record(pg_conn) diff --git a/apps/api/tests/unit/infrastructure/record_export/test_render.py b/apps/api/tests/unit/infrastructure/record_export/test_render.py new file mode 100644 index 00000000000..f6c35bd9e6d --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_render.py @@ -0,0 +1,73 @@ +"""Unit tests for F6 rendering: `render_value` / `render_row`. + +See `cora.infrastructure.record_export._render` for why this only +touches typed outer-row columns and must not recurse into jsonb. +""" + +from datetime import UTC, datetime, timedelta, timezone +from uuid import UUID + +from cora.infrastructure.record_export import render_row, render_value + +_SOME_UUID = UUID("12345678-1234-5678-1234-567812345678") + + +def test_uuid_renders_as_its_string_form() -> None: + assert render_value(_SOME_UUID) == "12345678-1234-5678-1234-567812345678" + + +def test_utc_datetime_renders_as_iso8601() -> None: + dt = datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC) + assert render_value(dt) == "2026-05-31T12:00:00+00:00" + + +def test_session_local_offset_datetime_normalizes_to_utc() -> None: + """The same instant at -05:00 must render identically to +00:00: this + is the measured hazard F6 exists to close, not a stylistic choice.""" + minus_five = datetime(2026, 5, 31, 7, 0, 0, tzinfo=timezone(timedelta(hours=-5))) + utc = datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC) + assert render_value(minus_five) == render_value(utc) == "2026-05-31T12:00:00+00:00" + + +def test_bytes_renders_as_hex() -> None: + assert render_value(b"\x00\xff\xab") == "00ffab" + + +def test_bytearray_and_memoryview_render_as_hex() -> None: + assert render_value(bytearray(b"\x01\x02")) == "0102" + assert render_value(memoryview(b"\x01\x02")) == "0102" + + +def test_plain_primitives_pass_through_unchanged() -> None: + assert render_value("channel-a") == "channel-a" + assert render_value(42) == 42 + assert render_value(3.14) == 3.14 + assert render_value(True) is True + assert render_value(None) is None + + +def test_jsonb_decoded_dict_and_list_pass_through_unrendered() -> None: + """payload/metadata already arrive as JSON-safe primitives (every + to_payload() pre-converts UUID/datetime to strings before the row is + written); render_value must not try to walk into them.""" + payload = {"logbook_id": "12345678-1234-5678-1234-567812345678", "kind": "activity"} + assert render_value(payload) is payload + assert render_value([1, "a", None]) == [1, "a", None] + + +def test_render_row_applies_render_value_to_every_column() -> None: + row: dict[str, object] = { + "event_id": _SOME_UUID, + "occurred_at": datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC), + "signature": b"\xde\xad", + "event_type": "ProcedureRegistered", + "payload": {"a": 1}, + } + rendered = render_row(row) + assert rendered == { + "event_id": "12345678-1234-5678-1234-567812345678", + "occurred_at": "2026-05-31T12:00:00+00:00", + "signature": "dead", + "event_type": "ProcedureRegistered", + "payload": {"a": 1}, + } diff --git a/apps/api/tests/unit/infrastructure/record_export/test_stream_types.py b/apps/api/tests/unit/infrastructure/record_export/test_stream_types.py new file mode 100644 index 00000000000..5b2567bca72 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_stream_types.py @@ -0,0 +1,25 @@ +"""Unit tests for the closed `stream_type` set.""" + +import pytest + +from cora.infrastructure.record_export import ( + KNOWN_STREAM_TYPES, + UnknownStreamTypeError, + ensure_stream_type_known, +) + + +@pytest.mark.parametrize("stream_type", sorted(KNOWN_STREAM_TYPES)) +def test_ensure_stream_type_known_accepts_every_declared_stream_type(stream_type: str) -> None: + ensure_stream_type_known(stream_type) # must not raise + + +def test_ensure_stream_type_known_refuses_an_undeclared_stream_type() -> None: + with pytest.raises(UnknownStreamTypeError) as excinfo: + ensure_stream_type_known("Widget") + assert excinfo.value.stream_type == "Widget" + + +def test_known_stream_types_has_no_empty_or_duplicate_entries() -> None: + assert "" not in KNOWN_STREAM_TYPES + assert len(KNOWN_STREAM_TYPES) == len({s.strip() for s in KNOWN_STREAM_TYPES}) From e6b3404192d9b39a80e8de1e89f0d2f25b3d220a Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:00:41 -0500 Subject: [PATCH 4/8] Hash an exported record with content_hash's canonicalization, no exclusions hash_record covers both tiers of an ExportedRecord in one compute_content_hash call (SHA-256 over DSSE-PAE-wrapped, NFC-normalized, sorted-key JSON) with no exclusions, per F2: re-exporting an unchanged database reproduces the same hash, and any single differing byte in either tier changes it. hash_streams/hash_logbooks hash each tier alone, each pinned to its own payload type from the design's Naming section so identical bodies under different types never collide. Deliberately reuses content_hash's existing canonicalization rather than canonical_json, which is a different recipe: the committed file must be the bytes the hash covers, and cora.infrastructure's tach dependency on cora.shared is exactly what permits this import. Co-Authored-By: Claude Sonnet 5 --- .../infrastructure/record_export/__init__.py | 14 ++ .../infrastructure/record_export/_hashing.py | 76 +++++++++++ .../test_record_export_hashing_postgres.py | 120 ++++++++++++++++++ .../record_export/test_hashing.py | 109 ++++++++++++++++ 4 files changed, 319 insertions(+) create mode 100644 apps/api/src/cora/infrastructure/record_export/_hashing.py create mode 100644 apps/api/tests/integration/test_record_export_hashing_postgres.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_hashing.py diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py index cce95ab6e42..1756f92a2de 100644 --- a/apps/api/src/cora/infrastructure/record_export/__init__.py +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -30,6 +30,14 @@ capture_watermark, export_record, ) +from cora.infrastructure.record_export._hashing import ( + LOGBOOKS_PAYLOAD_TYPE, + RECORD_PAYLOAD_TYPE, + STREAMS_PAYLOAD_TYPE, + hash_logbooks, + hash_record, + hash_streams, +) from cora.infrastructure.record_export._registry import ( EntriesReader, EntriesTableSpec, @@ -47,6 +55,9 @@ __all__ = [ "KNOWN_STREAM_TYPES", + "LOGBOOKS_PAYLOAD_TYPE", + "RECORD_PAYLOAD_TYPE", + "STREAMS_PAYLOAD_TYPE", "EmptyExportError", "EntriesReader", "EntriesTableSpec", @@ -57,6 +68,9 @@ "capture_watermark", "ensure_stream_type_known", "export_record", + "hash_logbooks", + "hash_record", + "hash_streams", "registered_envelope_classes", "render_row", "render_value", diff --git a/apps/api/src/cora/infrastructure/record_export/_hashing.py b/apps/api/src/cora/infrastructure/record_export/_hashing.py new file mode 100644 index 00000000000..48626ba0e79 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_hashing.py @@ -0,0 +1,76 @@ +"""Hash an exported record with `cora.shared.content_hash`, no exclusions. + +Per `project_record_export_v3.md` F6: one canonicalization, `content_hash`'s +profile, for both serialization and hashing. `canonical_json` (used +elsewhere in the codebase) is a DIFFERENT recipe; mixing the two means the +committed file is not the bytes the hash covers. `cora.shared` is exactly +what `tach.toml` allows `cora.infrastructure` to depend on, so this import +is the dependency the layering rule exists to permit. + +No exclusions: every rendered stream row and every rendered logbook row +goes into the hashed body. Re-exporting the same database twice is +measured stable (`test_hashing.py`'s integration test); a regenerated +record hashing differently across genuinely different content is content +addressing working, not a defect, and cross-record comparison is the +shape layer's job, not this module's. + +Payload types follow the scheme +`application/vnd.cora.+json` (`content_hash.py`'s own +convention) and the three names `project_record_export_v3.md`'s "Naming" +section already picked: `record`, `record-streams`, `record-logbooks`. +That section calls `record+json` the manifest's type; nothing writes a +manifest yet (step 4), so there is no live collision today. Used here for +the whole-bundle hash because `{streams, logbooks}` together *is* the +record the manifest will describe -- if step 4 wants manifest.json to +carry its own distinct wrapping type when it starts writing bytes to +disk, that is a decision for whoever builds it, made with an actual +manifest body in hand rather than guessed at here. +""" + +from cora.infrastructure.record_export._export import ExportedRecord +from cora.shared.content_hash import compute_content_hash + +RECORD_PAYLOAD_TYPE = "application/vnd.cora.record+json" +STREAMS_PAYLOAD_TYPE = "application/vnd.cora.record-streams+json" +LOGBOOKS_PAYLOAD_TYPE = "application/vnd.cora.record-logbooks+json" + + +def hash_streams(streams: tuple[dict[str, object], ...]) -> str: + """SHA-256 content hash over the streams tier alone, in its own order.""" + return compute_content_hash(STREAMS_PAYLOAD_TYPE, list(streams)) + + +def hash_logbooks(logbooks: dict[str, tuple[dict[str, object], ...]]) -> str: + """SHA-256 content hash over the logbooks tier alone. + + Kind keys are NFC-normalized and sort themselves via + `json.dumps(sort_keys=True)`; each kind's row order is preserved + (the registry's own order-by), never re-sorted. + """ + body = {kind: list(rows) for kind, rows in logbooks.items()} + return compute_content_hash(LOGBOOKS_PAYLOAD_TYPE, body) + + +def hash_record(record: ExportedRecord) -> str: + """SHA-256 content hash over the whole bundle, both tiers, no exclusions. + + This is THE record hash: per F2, it covers everything `export_record` + produced. Re-running the export against an unchanged database + reproduces this value exactly; any single differing byte anywhere in + either tier changes it. + """ + body = { + "streams": list(record.streams), + "logbooks": {kind: list(rows) for kind, rows in record.logbooks.items()}, + } + return compute_content_hash(RECORD_PAYLOAD_TYPE, body) + + +__all__ = [ + "LOGBOOKS_PAYLOAD_TYPE", + "RECORD_PAYLOAD_TYPE", + "STREAMS_PAYLOAD_TYPE", + "hash_logbooks", + "hash_record", + "hash_streams", +] diff --git a/apps/api/tests/integration/test_record_export_hashing_postgres.py b/apps/api/tests/integration/test_record_export_hashing_postgres.py new file mode 100644 index 00000000000..5dfa6d30e39 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_hashing_postgres.py @@ -0,0 +1,120 @@ +"""Acceptance test for Step 3 of the record exporter build brief. + +Per `project_record_export_build_brief.md` step 3: same DB exported +twice, identical hash; a flipped byte fails. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.record_export import ExportedRecord, export_record, hash_record +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + event_type_name, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +async def _seed_running_procedure_with_activity(db_pool: asyncpg.Pool, procedure_id: UUID) -> None: + logbook_id = uuid4() + open_event_id = uuid4() + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await deps.event_store.append( + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + handler = bind_append(deps, step_store=PostgresActivityStore(db_pool)) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=uuid4(), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0}, + sampled_at=_NOW, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + +@pytest.mark.integration +async def test_same_database_exported_twice_hashes_identically(db_pool: asyncpg.Pool) -> None: + await _seed_running_procedure_with_activity(db_pool, uuid4()) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + first = await export_record(pg_conn) + async with db_pool.acquire() as conn: + pg_conn = conn # type: ignore[assignment] + second = await export_record(pg_conn) + + assert hash_record(first) == hash_record(second) + + +@pytest.mark.integration +async def test_a_flipped_byte_in_the_exported_record_changes_the_hash( + db_pool: asyncpg.Pool, +) -> None: + await _seed_running_procedure_with_activity(db_pool, uuid4()) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + baseline_hash = hash_record(exported) + + tampered_first_row = dict(exported.streams[0]) + original_event_type = tampered_first_row["event_type"] + assert isinstance(original_event_type, str) and original_event_type + # Flip exactly one character of one field. + tampered_first_row["event_type"] = original_event_type[:-1] + ( + "X" if original_event_type[-1] != "X" else "Y" + ) + tampered = ExportedRecord( + streams=(tampered_first_row, *exported.streams[1:]), + logbooks=exported.logbooks, + ) + + assert hash_record(tampered) != baseline_hash diff --git a/apps/api/tests/unit/infrastructure/record_export/test_hashing.py b/apps/api/tests/unit/infrastructure/record_export/test_hashing.py new file mode 100644 index 00000000000..3deaf669e63 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_hashing.py @@ -0,0 +1,109 @@ +"""Unit tests for hashing an exported record. + +Per `project_record_export_v3.md` F2: no exclusions, stable across +re-exports of the same content, sensitive to any single differing byte. +These tests exercise that purely as a function of Python data (no DB +needed); the DB-backed "same database exported twice" acceptance test +lives in `tests/integration/test_record_export_hashing_postgres.py`. +""" + +from cora.infrastructure.record_export import ( + ExportedRecord, + hash_logbooks, + hash_record, + hash_streams, +) + +_STREAM_ROW: dict[str, object] = { + "event_id": "12345678-1234-5678-1234-567812345678", + "event_type": "ProcedureRegistered", + "version": 1, +} +_ACTIVITY_ROW: dict[str, object] = { + "event_id": "aaaaaaaa-1234-5678-1234-567812345678", + "step_kind": "setpoint", + "payload": {"channel": "T_oven", "target_value": 423.0}, +} + + +def _record() -> ExportedRecord: + return ExportedRecord( + streams=(_STREAM_ROW,), + logbooks={"activity": (_ACTIVITY_ROW,)}, + ) + + +def test_hash_record_is_stable_across_independently_built_equal_records() -> None: + """Two structurally identical ExportedRecords (fresh dict/tuple copies, + not the same objects) must hash identically -- this is what "the same + database exported twice" reduces to at the data level.""" + first = ExportedRecord( + streams=(dict(_STREAM_ROW),), logbooks={"activity": (dict(_ACTIVITY_ROW),)} + ) + second = ExportedRecord( + streams=(dict(_STREAM_ROW),), logbooks={"activity": (dict(_ACTIVITY_ROW),)} + ) + assert hash_record(first) == hash_record(second) + + +def test_hash_record_changes_on_a_single_field_change_anywhere() -> None: + baseline = hash_record(_record()) + + changed_stream = ExportedRecord( + streams=({**_STREAM_ROW, "version": 2},), + logbooks={"activity": (_ACTIVITY_ROW,)}, + ) + assert hash_record(changed_stream) != baseline + + changed_logbook_row = dict(_ACTIVITY_ROW) + changed_logbook_row["payload"] = {"channel": "T_oven", "target_value": 424.0} + changed_logbook = ExportedRecord( + streams=(_STREAM_ROW,), + logbooks={"activity": (changed_logbook_row,)}, + ) + assert hash_record(changed_logbook) != baseline + + +def test_hash_record_is_sensitive_to_stream_row_order() -> None: + """Stream order is significant (it is the replay order); reordering + two rows must change the hash even though the SET of rows is the + same.""" + row_a = {**_STREAM_ROW, "version": 1} + row_b = {**_STREAM_ROW, "version": 2, "event_id": "bbbbbbbb-1234-5678-1234-567812345678"} + forward = ExportedRecord(streams=(row_a, row_b), logbooks={}) + backward = ExportedRecord(streams=(row_b, row_a), logbooks={}) + assert hash_record(forward) != hash_record(backward) + + +def test_hash_record_is_insensitive_to_logbook_kind_key_order() -> None: + """Kind keys sort themselves via json.dumps(sort_keys=True); insertion + order into the `logbooks` dict must not affect the hash.""" + forward = ExportedRecord( + streams=(), + logbooks={"activity": (_ACTIVITY_ROW,), "outcome": ({"a": 1},)}, + ) + backward = ExportedRecord( + streams=(), + logbooks={"outcome": ({"a": 1},), "activity": (_ACTIVITY_ROW,)}, + ) + assert hash_record(forward) == hash_record(backward) + + +def test_hash_record_covers_both_tiers_not_just_one() -> None: + """A record hash must differ from hashing either tier alone -- proof + that hash_record is not accidentally ignoring one of the two.""" + record = _record() + assert hash_record(record) != hash_streams(record.streams) + assert hash_record(record) != hash_logbooks(record.logbooks) + + +def test_empty_record_hashes_deterministically() -> None: + empty = ExportedRecord(streams=(), logbooks={}) + assert hash_record(empty) == hash_record(ExportedRecord(streams=(), logbooks={})) + + +def test_hash_streams_and_hash_logbooks_are_pinned_to_distinct_payload_types() -> None: + """Same body, different payload_type, must hash differently: this is + the whole point of binding payload_type into the PAE wrap.""" + record = _record() + assert hash_streams(record.streams) != hash_logbooks({"x": record.streams}) From 38fac389c9dd74ba10e1cbb954c200a810d8322b Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:29:40 -0500 Subject: [PATCH 5/8] Assemble the export manifest: commit, watermark, both profile hashes build_manifest is pure: git_commit and watermark are captured by the caller (capture_git_commit, step 2's capture_watermark) and passed in, so the function itself does no I/O. Carries the two profile hashes namable before step 6 exists -- record_hash (H1, step 3) and redaction_profile_hash (H2, step 0's disposition table, computed for the first time here since nothing needed it before) -- plus row_count_by_logbook_kind and max_schema_version_by_event_type as reader-facing sanity checks a reader can recompute and compare against. is_simulated rolls up every observation row's own flag, vacuously True when an export carries none. expansion_digest_presence_by_run answers F8 per run: a run has a pinned digest iff at least one child Procedure was registered via register_procedure_from_recipe (carries RecipeExpansionRecorded on its own stream), derived by cross-referencing ProcedureRegistered.parent_run_id against RecipeExpansionRecorded rows rather than assuming anything about how a run's Procedures were made. Naming-r3 review renamed four names before commit: xmin_watermark -> watermark (parity with capture_watermark), logbook_row_counts -> row_count_by_logbook_kind and has_expansion_digest_by_run -> expansion_digest_presence_by_run (both were dict fields wearing a scalar's naming shape), _str -> _require_str (matches the repo's _require_* assert-and-return convention). Co-Authored-By: Claude Sonnet 5 --- .../infrastructure/record_export/__init__.py | 8 + .../infrastructure/record_export/_hashing.py | 18 +++ .../infrastructure/record_export/_manifest.py | 150 ++++++++++++++++++ .../test_record_export_manifest_postgres.py | 105 ++++++++++++ .../record_export/test_manifest.py | 139 ++++++++++++++++ 5 files changed, 420 insertions(+) create mode 100644 apps/api/src/cora/infrastructure/record_export/_manifest.py create mode 100644 apps/api/tests/integration/test_record_export_manifest_postgres.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_manifest.py diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py index 1756f92a2de..33f4d5744d5 100644 --- a/apps/api/src/cora/infrastructure/record_export/__init__.py +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -33,11 +33,14 @@ from cora.infrastructure.record_export._hashing import ( LOGBOOKS_PAYLOAD_TYPE, RECORD_PAYLOAD_TYPE, + REDACTION_PROFILE_PAYLOAD_TYPE, STREAMS_PAYLOAD_TYPE, hash_logbooks, hash_record, + hash_redaction_profile, hash_streams, ) +from cora.infrastructure.record_export._manifest import Manifest, build_manifest, capture_git_commit from cora.infrastructure.record_export._registry import ( EntriesReader, EntriesTableSpec, @@ -57,19 +60,24 @@ "KNOWN_STREAM_TYPES", "LOGBOOKS_PAYLOAD_TYPE", "RECORD_PAYLOAD_TYPE", + "REDACTION_PROFILE_PAYLOAD_TYPE", "STREAMS_PAYLOAD_TYPE", "EmptyExportError", "EntriesReader", "EntriesTableSpec", "ExportedRecord", + "Manifest", "UnknownLogbookKindError", "UnknownStreamTypeError", "all_specs", + "build_manifest", + "capture_git_commit", "capture_watermark", "ensure_stream_type_known", "export_record", "hash_logbooks", "hash_record", + "hash_redaction_profile", "hash_streams", "registered_envelope_classes", "render_row", diff --git a/apps/api/src/cora/infrastructure/record_export/_hashing.py b/apps/api/src/cora/infrastructure/record_export/_hashing.py index 48626ba0e79..2934c8df9ee 100644 --- a/apps/api/src/cora/infrastructure/record_export/_hashing.py +++ b/apps/api/src/cora/infrastructure/record_export/_hashing.py @@ -27,12 +27,14 @@ manifest body in hand rather than guessed at here. """ +from cora.infrastructure.record_export._dispositions import DISPOSITIONS from cora.infrastructure.record_export._export import ExportedRecord from cora.shared.content_hash import compute_content_hash RECORD_PAYLOAD_TYPE = "application/vnd.cora.record+json" STREAMS_PAYLOAD_TYPE = "application/vnd.cora.record-streams+json" LOGBOOKS_PAYLOAD_TYPE = "application/vnd.cora.record-logbooks+json" +REDACTION_PROFILE_PAYLOAD_TYPE = "application/vnd.cora.record-redaction-profile+json" def hash_streams(streams: tuple[dict[str, object], ...]) -> str: @@ -66,11 +68,27 @@ def hash_record(record: ExportedRecord) -> str: return compute_content_hash(RECORD_PAYLOAD_TYPE, body) +def hash_redaction_profile() -> str: + """SHA-256 content hash over Step 0's generated disposition table. + + This IS the redaction profile hash (H2): the disposition table + decides what a published record discloses, so its hash is what + step 6's fail-closed switch checks and what the manifest names + alongside `record_hash` (H1). Regenerating the table via + `make record-dispositions` after a real event-model change is + expected to change this value; `test_record_dispositions_drift.py` + is the guard against an UNREVIEWED change slipping through instead. + """ + return compute_content_hash(REDACTION_PROFILE_PAYLOAD_TYPE, DISPOSITIONS) + + __all__ = [ "LOGBOOKS_PAYLOAD_TYPE", "RECORD_PAYLOAD_TYPE", + "REDACTION_PROFILE_PAYLOAD_TYPE", "STREAMS_PAYLOAD_TYPE", "hash_logbooks", "hash_record", + "hash_redaction_profile", "hash_streams", ] diff --git a/apps/api/src/cora/infrastructure/record_export/_manifest.py b/apps/api/src/cora/infrastructure/record_export/_manifest.py new file mode 100644 index 00000000000..7bbdb8fa65f --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_manifest.py @@ -0,0 +1,150 @@ +"""The export manifest: git commit, watermark, both profile hashes, and +the per-kind / per-event-type / per-run facts a reader needs before +trusting the bundle. + +Per `project_record_export_build_brief.md` step 4 and +`project_record_export_v3.md` F8. "Both profile hashes" are the two +namable today: `record_hash` (H1, the whole unredacted bundle, built in +step 3) and `redaction_profile_hash` (H2, step 0's generated disposition +table -- "the table's canonical hash IS the redaction profile hash", +computed here because nothing computed it before this step existed). H3 +(the published/redacted record's hash) does not exist until step 6 +builds redaction. + +`build_manifest` is pure: every input it needs (`git_commit`, +`watermark`) is captured by the caller first and passed in, so the +function itself does no I/O and is trivial to test with synthetic +`ExportedRecord`s. +""" + +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from cora.infrastructure.record_export._export import ExportedRecord +from cora.infrastructure.record_export._hashing import hash_record, hash_redaction_profile + + +@dataclass(frozen=True, slots=True) +class Manifest: + """One export's provenance and shape, independent of its bytes on disk. + + `row_count_by_logbook_kind` and `max_schema_version_by_event_type` + are reader-facing sanity checks: a reader can recompute both from + the bundle itself and compare, catching truncation or a stale + generator without needing to trust this manifest blindly. + """ + + git_commit: str + watermark: int + record_hash: str + redaction_profile_hash: str + row_count_by_logbook_kind: dict[str, int] + max_schema_version_by_event_type: dict[str, int] + is_simulated: bool + expansion_digest_presence_by_run: dict[str, bool] + + +def capture_git_commit(*, cwd: Path | str | None = None) -> str: + """The exporting checkout's HEAD commit SHA. + + No caching, no fallback: a failure here (detached submodule, no + `.git`, corrupt repo) should stop the export rather than write a + manifest that lies about provenance. + """ + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def _require_str(value: object) -> str: + assert isinstance(value, str) + return value + + +def _payload(row: dict[str, object]) -> dict[str, object]: + payload = row["payload"] + assert isinstance(payload, dict) + return cast("dict[str, object]", payload) + + +def _row_count_by_logbook_kind(record: ExportedRecord) -> dict[str, int]: + return {kind: len(rows) for kind, rows in record.logbooks.items()} + + +def _max_schema_version_by_event_type(record: ExportedRecord) -> dict[str, int]: + versions: dict[str, int] = {} + for row in record.streams: + event_type = row["event_type"] + schema_version = row["schema_version"] + assert isinstance(event_type, str) + assert isinstance(schema_version, int) + if schema_version > versions.get(event_type, 0): + versions[event_type] = schema_version + return versions + + +def _is_simulated(record: ExportedRecord) -> bool: + """True unless an observation row explicitly says otherwise. + + Vacuously True when the export carries no observation rows at all: + nothing in the bundle contradicts "this is a simulated record". A + mixed result (some True, some False) reports as False rather than + raising -- the manifest's job is to report the fact, not gate on it. + """ + observations = record.logbooks.get("observation", ()) + return all(row["is_simulated"] is True for row in observations) + + +def _expansion_digest_presence_by_run(record: ExportedRecord) -> dict[str, bool]: + """Per F8: a run has a pinned expansion digest iff at least one of its + child Procedures was registered via `register_procedure_from_recipe` + (carries a `RecipeExpansionRecorded` on its own stream). A Procedure + registered directly, or a run recorded by observing an external + scan, has no digest to compare against; that is correct, not a gap. + """ + run_ids = { + _require_str(row["stream_id"]) for row in record.streams if row["stream_type"] == "Run" + } + + parent_run_by_procedure: dict[str, str | None] = {} + expanded_procedures: set[str] = set() + for row in record.streams: + if row["event_type"] == "ProcedureRegistered": + payload = _payload(row) + parent_run_id = payload["parent_run_id"] + parent_run_by_procedure[_require_str(payload["procedure_id"])] = ( + None if parent_run_id is None else _require_str(parent_run_id) + ) + elif row["event_type"] == "RecipeExpansionRecorded": + expanded_procedures.add(_require_str(_payload(row)["procedure_id"])) + return { + run_id: any( + parent_run_id == run_id and procedure_id in expanded_procedures + for procedure_id, parent_run_id in parent_run_by_procedure.items() + ) + for run_id in run_ids + } + + +def build_manifest(record: ExportedRecord, *, watermark: int, git_commit: str) -> Manifest: + """Assemble the manifest for one already-exported, already-rendered record.""" + return Manifest( + git_commit=git_commit, + watermark=watermark, + record_hash=hash_record(record), + redaction_profile_hash=hash_redaction_profile(), + row_count_by_logbook_kind=_row_count_by_logbook_kind(record), + max_schema_version_by_event_type=_max_schema_version_by_event_type(record), + is_simulated=_is_simulated(record), + expansion_digest_presence_by_run=_expansion_digest_presence_by_run(record), + ) + + +__all__ = ["Manifest", "build_manifest", "capture_git_commit"] diff --git a/apps/api/tests/integration/test_record_export_manifest_postgres.py b/apps/api/tests/integration/test_record_export_manifest_postgres.py new file mode 100644 index 00000000000..d562aee1363 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_manifest_postgres.py @@ -0,0 +1,105 @@ +"""Manifest built from a real `export_record` result, not synthetic fixtures. + +`build_manifest` itself is pure and unit-tested against hand-built +`ExportedRecord`s in `tests/unit/infrastructure/record_export/test_manifest.py`. +This test exists because a real rendered row's shape could diverge from +those synthetic fixtures in ways only a live export would surface (e.g. +if `event_type_name()` or `to_payload()` ever changed a key name). +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.record_export import ( + build_manifest, + capture_git_commit, + export_record, + hash_record, +) +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + event_type_name, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +@pytest.mark.integration +async def test_manifest_built_from_a_real_export(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + logbook_id = uuid4() + open_event_id = uuid4() + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await deps.event_store.append( + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + handler = bind_append(deps, step_store=PostgresActivityStore(db_pool)) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=uuid4(), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0}, + sampled_at=_NOW, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + manifest = build_manifest(exported, watermark=1, git_commit=capture_git_commit()) + + assert manifest.record_hash == hash_record(exported) + assert len(manifest.redaction_profile_hash) == 64 + assert manifest.row_count_by_logbook_kind["activity"] == 1 + assert manifest.max_schema_version_by_event_type["ProcedureRegistered"] >= 1 + # No Run stream in this fixture (parent_run_id=None, no RunStarted + # seeded), so the per-run map must be empty, not crash. + assert manifest.expansion_digest_presence_by_run == {} + # No observation rows in this fixture: vacuously simulated. + assert manifest.is_simulated is True diff --git a/apps/api/tests/unit/infrastructure/record_export/test_manifest.py b/apps/api/tests/unit/infrastructure/record_export/test_manifest.py new file mode 100644 index 00000000000..334864a5b6a --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_manifest.py @@ -0,0 +1,139 @@ +"""Unit tests for the export manifest. + +`build_manifest` is pure (no I/O): every input is passed in, so these +tests construct synthetic `ExportedRecord`s directly rather than going +through a live database. The DB-backed acceptance path (via a real +Procedure + Run) lives in +`tests/integration/test_record_export_manifest_postgres.py`. +""" + +import re + +from cora.infrastructure.record_export import ( + ExportedRecord, + build_manifest, + capture_git_commit, + hash_record, + hash_redaction_profile, +) + +_RUN_A = "01900000-0000-7000-8000-0000000000a1" +_RUN_B = "01900000-0000-7000-8000-0000000000a2" +_PROC_EXPANDED = "01900000-0000-7000-8000-0000000000b1" +_PROC_DIRECT = "01900000-0000-7000-8000-0000000000b2" +_PROC_NO_RUN = "01900000-0000-7000-8000-0000000000b3" + + +def _stream_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "stream_type": "Procedure", + "stream_id": _PROC_EXPANDED, + "event_type": "ProcedureStarted", + "schema_version": 1, + "payload": {}, + } + row.update(overrides) + return row + + +def _procedure_registered( + procedure_id: str, parent_run_id: str | None, *, schema_version: int = 1 +) -> dict[str, object]: + return _stream_row( + stream_id=procedure_id, + event_type="ProcedureRegistered", + schema_version=schema_version, + payload={"procedure_id": procedure_id, "parent_run_id": parent_run_id}, + ) + + +def _recipe_expansion_recorded(procedure_id: str) -> dict[str, object]: + return _stream_row( + stream_id=procedure_id, + event_type="RecipeExpansionRecorded", + payload={"procedure_id": procedure_id}, + ) + + +def _run_started(run_id: str) -> dict[str, object]: + return _stream_row(stream_type="Run", stream_id=run_id, event_type="RunStarted", payload={}) + + +def _record() -> ExportedRecord: + streams = ( + _run_started(_RUN_A), + _run_started(_RUN_B), + _procedure_registered(_PROC_EXPANDED, _RUN_A), + _recipe_expansion_recorded(_PROC_EXPANDED), + _procedure_registered(_PROC_DIRECT, _RUN_B), + _procedure_registered(_PROC_NO_RUN, None), + _procedure_registered("01900000-0000-7000-8000-0000000000b9", None, schema_version=2), + ) + logbooks: dict[str, tuple[dict[str, object], ...]] = { + "activity": ({"step_kind": "setpoint"}, {"step_kind": "check"}), + "observation": ({"is_simulated": True}, {"is_simulated": True}), + } + return ExportedRecord(streams=streams, logbooks=logbooks) + + +def test_logbook_row_counts_match_each_kinds_length() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert manifest.row_count_by_logbook_kind == {"activity": 2, "observation": 2} + + +def test_max_schema_version_takes_the_max_per_event_type() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + # Two ProcedureRegistered rows in the fixture: schema_version 1 (the + # three _procedure_registered() calls) and 2 (the last row). + assert manifest.max_schema_version_by_event_type["ProcedureRegistered"] == 2 + assert manifest.max_schema_version_by_event_type["RunStarted"] == 1 + + +def test_is_simulated_true_when_every_observation_says_so() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert manifest.is_simulated is True + + +def test_is_simulated_false_on_a_single_dissenting_observation() -> None: + record = ExportedRecord( + streams=(), + logbooks={"observation": ({"is_simulated": True}, {"is_simulated": False})}, + ) + manifest = build_manifest(record, watermark=100, git_commit="deadbeef") + assert manifest.is_simulated is False + + +def test_is_simulated_vacuously_true_with_no_observations() -> None: + record = ExportedRecord(streams=(), logbooks={}) + manifest = build_manifest(record, watermark=100, git_commit="deadbeef") + assert manifest.is_simulated is True + + +def test_expansion_digest_present_only_for_the_run_whose_child_was_expanded() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert manifest.expansion_digest_presence_by_run == {_RUN_A: True, _RUN_B: False} + + +def test_expansion_digest_ignores_procedures_with_no_parent_run() -> None: + """_PROC_NO_RUN has parent_run_id=None; it must not create a phantom + run entry or affect either real run's result.""" + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert set(manifest.expansion_digest_presence_by_run) == {_RUN_A, _RUN_B} + + +def test_manifest_hashes_match_calling_the_hash_functions_directly() -> None: + record = _record() + manifest = build_manifest(record, watermark=100, git_commit="deadbeef") + assert manifest.record_hash == hash_record(record) + assert manifest.redaction_profile_hash == hash_redaction_profile() + + +def test_manifest_carries_the_watermark_and_commit_verbatim() -> None: + manifest = build_manifest(_record(), watermark=4242, git_commit="cafef00d") + assert manifest.watermark == 4242 + assert manifest.git_commit == "cafef00d" + + +def test_capture_git_commit_returns_a_full_sha() -> None: + commit = capture_git_commit() + assert re.fullmatch(r"[0-9a-f]{40}", commit) From 3c937679f92de37e692d4f563710bb44d80487e0 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:08:14 -0500 Subject: [PATCH 6/8] Add a standalone, zero-cora-import verifier for a record export's hash verify_record_hash.py reimplements compute_content_hash's canonicalization (NFC-normalized, sorted-key JSON, DSSE-PAE wrap, SHA-256) using only the stdlib, at repo-root scripts/ rather than inside apps/api: the whole point is running on a machine that has never installed CORA, and apps/api's ruff/pyright/tach coverage never reaches that directory today, which is the correct trade for a file with zero third-party dependencies by design. Deliberately narrower than cora.shared.content_hash: skips the dataclass/Pydantic/set branches, since an exported record body is already reduced to JSON primitives by the time it reaches this tool (render_row in step 2). A test cross-checks the two implementations byte-for-byte over a corpus including composed/decomposed NFC forms and floats, loading this script via the same importlib bridge tests/unit/deployments/test_beamline_descriptor.py already uses for scripts/ modules. A separate genuine-subprocess test proves a flipped byte in the input file changes the verdict from OK to MISMATCH with a nonzero exit. Naming-r3 review renamed the check subcommand to verify (matching the script's own name and docstring), --expected to --expected-hash (parity with its sibling --payload-type), and json_file to body_file (every other symbol in the file calls this the body). Co-Authored-By: Claude Sonnet 5 --- .../record_export/test_standalone_verifier.py | 190 ++++++++++++++++++ scripts/verify_record_hash.py | 146 ++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py create mode 100644 scripts/verify_record_hash.py diff --git a/apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py b/apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py new file mode 100644 index 00000000000..269157bd7ec --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py @@ -0,0 +1,190 @@ +"""Tests for the standalone, zero-cora-import verifier at +`scripts/verify_record_hash.py`. + +Per `project_record_export_build_brief.md` step 5's acceptance: a unit +test cross-checking it byte-for-byte against `cora.shared.content_hash` +over a corpus including NFC and float cases; a subprocess test that +fails on a flipped byte. + +The script is loaded via `importlib` for the byte-for-byte cross-check +(same dynamic-import bridge `tests/unit/deployments/test_beamline_descriptor.py` +uses for other `scripts/` modules, since `scripts/` is not on the +type-checker's or the `cora` package's path) and invoked as a genuine +subprocess for the CLI / flipped-byte tests, so at least one test proves +the file runs as an actual standalone OS process, not just an +importable module. +""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from cora.shared.content_hash import compute_content_hash as cora_compute_content_hash + +if TYPE_CHECKING: + from types import ModuleType + +_REPO_ROOT = Path(__file__).resolve().parents[6] +_SCRIPT = _REPO_ROOT / "scripts" / "verify_record_hash.py" +_PAYLOAD_TYPE = "application/vnd.cora.record-test+json" + +# Precomposed 'e-acute' (single codepoint U+00E9) vs decomposed ('e' +# U+0065 + combining acute accent U+0301). Both display as "e" with an +# accent but are different byte sequences until NFC-normalized. +_PRECOMPOSED_E_ACUTE = "café" +_DECOMPOSED_E_ACUTE = "café" + + +def _load_verifier() -> ModuleType: + spec = importlib.util.spec_from_file_location("verify_record_hash", _SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load verify_record_hash from {_SCRIPT}") + module = importlib.util.module_from_spec(spec) + sys.modules["verify_record_hash"] = module + spec.loader.exec_module(module) + return module + + +_verifier = _load_verifier() + +_CORPUS: list[Any] = [ + {"a": 1, "b": "text"}, + {"target_value": 423.0, "tolerance": 3.14159, "ramp_rate": 0.0}, + {"name": _PRECOMPOSED_E_ACUTE}, + {"name": _DECOMPOSED_E_ACUTE}, + {"flag": True, "missing": None}, + {"items": [{"k": "v"}, {"k2": [1, 2, 3]}], "nested": {"a": {"b": {"c": None}}}}, + {"turkish": "dotless ı vs dotted i İ", "emoji": "\U0001f52c"}, # noqa: RUF001 + [1, "two", 3.0, None, True, {"five": 5}], + {}, + [], + "bare string body", + 42, +] + + +def test_script_has_zero_cora_imports() -> None: + source = _SCRIPT.read_text(encoding="utf-8") + for line in source.splitlines(): + stripped = line.strip() + assert not stripped.startswith("import cora"), ( + f"{_SCRIPT} imports cora ({stripped!r}); the whole point of this " + "file is running on a machine with no CORA installed." + ) + assert not stripped.startswith("from cora"), ( + f"{_SCRIPT} imports cora ({stripped!r}); the whole point of this " + "file is running on a machine with no CORA installed." + ) + + +@pytest.mark.parametrize("body", _CORPUS, ids=range(len(_CORPUS))) +def test_compute_content_hash_matches_cora_byte_for_byte(body: Any) -> None: + assert _verifier.compute_content_hash(_PAYLOAD_TYPE, body) == cora_compute_content_hash( + _PAYLOAD_TYPE, body + ) + + +def test_composed_and_decomposed_nfc_forms_hash_identically() -> None: + assert _PRECOMPOSED_E_ACUTE != _DECOMPOSED_E_ACUTE # different bytes pre-normalization + composed = {"name": _PRECOMPOSED_E_ACUTE} + decomposed = {"name": _DECOMPOSED_E_ACUTE} + assert _verifier.compute_content_hash( + _PAYLOAD_TYPE, composed + ) == _verifier.compute_content_hash(_PAYLOAD_TYPE, decomposed) + + +def test_cli_hash_subcommand_prints_the_matching_digest(tmp_path: Path) -> None: + body = {"a": 1, "b": _PRECOMPOSED_E_ACUTE} + body_file = tmp_path / "body.json" + body_file.write_text(json.dumps(body), encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(_SCRIPT), "hash", "--payload-type", _PAYLOAD_TYPE, str(body_file)], + capture_output=True, + text=True, + check=True, + ) + + expected = cora_compute_content_hash(_PAYLOAD_TYPE, body) + assert result.stdout.strip() == expected + + +def test_cli_verify_subcommand_exits_zero_on_a_match(tmp_path: Path) -> None: + body = {"a": 1, "b": _PRECOMPOSED_E_ACUTE} + body_file = tmp_path / "body.json" + body_file.write_text(json.dumps(body), encoding="utf-8") + expected = cora_compute_content_hash(_PAYLOAD_TYPE, body) + + result = subprocess.run( + [ + sys.executable, + str(_SCRIPT), + "verify", + "--payload-type", + _PAYLOAD_TYPE, + "--expected-hash", + expected, + str(body_file), + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "OK" in result.stdout + + +def test_cli_verify_subcommand_fails_on_a_flipped_byte(tmp_path: Path) -> None: + """The acceptance test named explicitly: a subprocess run against a + tampered file must FAIL (nonzero exit), proving the recomputed hash + is sensitive to the tamper rather than silently passing.""" + body = {"a": 1, "b": _PRECOMPOSED_E_ACUTE, "target_value": 423.0} + body_file = tmp_path / "body.json" + body_file.write_text(json.dumps(body), encoding="utf-8") + expected = cora_compute_content_hash(_PAYLOAD_TYPE, body) + + def _verify() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(_SCRIPT), + "verify", + "--payload-type", + _PAYLOAD_TYPE, + "--expected-hash", + expected, + str(body_file), + ], + capture_output=True, + text=True, + ) + + before = _verify() + assert before.returncode == 0 + + # Flip exactly one character in the file's bytes on disk. + original_text = body_file.read_text(encoding="utf-8") + tampered_text = original_text.replace("423.0", "424.0") + assert tampered_text != original_text + body_file.write_text(tampered_text, encoding="utf-8") + + after = _verify() + assert after.returncode == 1 + assert "MISMATCH" in after.stderr + + +def test_cli_reports_a_clean_error_on_unreadable_input(tmp_path: Path) -> None: + missing = tmp_path / "does_not_exist.json" + result = subprocess.run( + [sys.executable, str(_SCRIPT), "hash", "--payload-type", _PAYLOAD_TYPE, str(missing)], + capture_output=True, + text=True, + ) + assert result.returncode == 2 diff --git a/scripts/verify_record_hash.py b/scripts/verify_record_hash.py new file mode 100644 index 00000000000..a77fe83404a --- /dev/null +++ b/scripts/verify_record_hash.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Standalone content-hash verifier for a CORA record export. + +Per `project_record_export_build_brief.md` step 5 and +`project_record_export_v3.md` F4: "sha256 and a JSON reader, no CORA" was +the earlier (falsified) claim -- `compute_content_hash` is SHA-256 over +DSSE-PAE-wrapped, NFC-normalized, sorted-key JSON, so a standalone +checker has to reimplement that canonicalization, not just hash raw +bytes. This file is that reimplementation: stdlib only (`json`, +`hashlib`, `unicodedata`, `argparse`), zero imports of `cora` or any +third-party package, so it runs with any Python 3.13 interpreter on a +machine that has never installed CORA. + +This is deliberately NOT the full `cora.shared.content_hash` port. That +module also canonicalizes dataclasses, Pydantic models, and sets/ +frozensets, because it hashes live domain objects elsewhere in CORA. A +record export never contains any of those: `render_row` (step 2) +already reduces every column to a JSON primitive before it reaches +`hash_record` (step 3), and the disposition table `hash_redaction_profile` +hashes (step 4) is itself a plain nested dict/str structure. So this +file's `_canonicalize` only has to handle what actually appears in an +exported body: `str`, `dict`, `list`/`tuple`, and JSON scalars passed +through unchanged. `tests/unit/infrastructure/record_export/ +test_standalone_verifier.py` cross-checks this file byte-for-byte +against `cora.shared.content_hash.compute_content_hash` to keep the two +recipes from drifting apart. + +Usage: + python3 verify_record_hash.py hash --payload-type TYPE body.json + python3 verify_record_hash.py verify --payload-type TYPE --expected-hash HASH body.json + +Exit codes: 0 success (hash printed, or verify matched); 1 verify +mismatch; 2 the input file could not be read or parsed as JSON. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# `_canonicalize` deliberately takes `Any`, mirroring +# cora.shared.content_hash._canonicalize; suppressed the same way there. + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import unicodedata +from pathlib import Path +from typing import Any + + +def _canonicalize(value: Any) -> Any: + """Recursively normalize a value into a JSON-stable Python structure. + + Mirrors `cora.shared.content_hash._canonicalize`'s str/Mapping/list + branches exactly (NFC normalization of strings and dict keys, list + recursion); omits the dataclass/Pydantic/set branches, which never + apply to an already-rendered export body (see module docstring). + """ + if isinstance(value, str): + return unicodedata.normalize("NFC", value) + if isinstance(value, dict): + return {unicodedata.normalize("NFC", str(k)): _canonicalize(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_canonicalize(item) for item in value] + return value + + +def canonical_body_bytes(body: Any) -> bytes: + """Produce canonical UTF-8 JSON bytes, byte-for-byte identical to + `cora.shared.content_hash.canonical_body_bytes` for any body an + export can actually contain.""" + canonical = _canonicalize(body) + return json.dumps( + canonical, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def pae_bytes(payload_type: str, body: bytes) -> bytes: + """DSSE Pre-Authentication Encoding, identical recipe to + `cora.shared.content_hash.pae_bytes`. LEN is BYTE length, not + character length: matters for a non-ASCII `payload_type`.""" + payload_type_bytes = payload_type.encode("utf-8") + return b"DSSEv1 %d %b %d %b" % ( + len(payload_type_bytes), + payload_type_bytes, + len(body), + body, + ) + + +def compute_content_hash(payload_type: str, body: Any) -> str: + """SHA-256 content hash, 64-char lowercase hex. Identical pipeline to + `cora.shared.content_hash.compute_content_hash`.""" + body_bytes = canonical_body_bytes(body) + pae = pae_bytes(payload_type, body_bytes) + return hashlib.sha256(pae).hexdigest() + + +def _load_body(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Standalone content-hash verifier for a CORA record export." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + hash_parser = subparsers.add_parser("hash", help="Print the content hash of a JSON body.") + hash_parser.add_argument("--payload-type", required=True) + hash_parser.add_argument("body_file", type=Path) + + verify_parser = subparsers.add_parser( + "verify", help="Verify a JSON body's hash matches an expected value." + ) + verify_parser.add_argument("--payload-type", required=True) + verify_parser.add_argument("--expected-hash", required=True) + verify_parser.add_argument("body_file", type=Path) + + args = parser.parse_args(argv) + + try: + body = _load_body(args.body_file) + except (OSError, json.JSONDecodeError) as exc: + print(f"cannot read {args.body_file} as JSON: {exc}", file=sys.stderr) + return 2 + + digest = compute_content_hash(args.payload_type, body) + + if args.command == "hash": + print(digest) + return 0 + + if digest == args.expected_hash: + print(f"OK {digest}") + return 0 + print(f"MISMATCH: expected {args.expected_hash}, computed {digest}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 269a72f6ec0f46d61179324cf665f2311cb44bac Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:03:55 -0500 Subject: [PATCH 7/8] Redact both tiers of an exported record per F5's disposition tables redact_record is the fail-closed switch: it computes hash_redaction_profile() itself and compares against a caller-supplied expected hash before touching a single row, refusing under a stale or substituted disposition table rather than redacting silently under the wrong rules. Tier 1 iterates the STORED payload's own keys against Step 0's generated DISPOSITIONS table (never the profile's keys, which is the whole fail-closed property), corrected against an internal wording conflict in the design docs: an event_type entirely absent from the table aborts (the table is stale relative to the code), but a known event type's unlisted field drops (the graceful path for schema evolution, and what "a bare str on a new event drops with nobody editing a list" actually describes). position/version re-index densely; transaction_id maps to a dense monotone int; the five UUID columns token through one shared TokenMap so tier-1/tier-2 joins survive within an export without the surrogate being derivable from its source (never a hash). Tier 2 is hand-transcribed from F5's three text-column tables (28 columns/8 kinds) plus the five jsonb columns (four recursing through a shared generic leaf-rule walker, one dropped whole), since F5's own argument for not generating this table is that 103 columns is small enough to enumerate by hand. The recursion walker never branches on which step_kind or Recipe step type produced a value, which is what kept a real drift (ProcedureActivity.step_kind grew from 3 to 5 values since the design was written, and two unenumerated step kinds write payload shapes the design never saw) from being a correctness problem: unenumerated fields already drop by the same default every other unlisted field drops by. Also built: an unfired-clearance check, catching a declared jsonb string-leaf clearance that never matched a row in a kind present in the export -- most useful as a completeness signal against a realistic fixture (a narrow one, like a single-step-kind rehearsal activity, legitimately fails it, which the redaction acceptance test's fixture was widened to avoid rather than the check being loosened). Naming-r3 review renamed six names before commit: reveal_map -> surrogate_by_source, looks_like_uuid -> is_uuid_shaped, DROPPED -> OMITTED (collided with tier-2's own DROP disposition constant), RECURSE -> BY_VALUE (tier-1's locked, generated vocabulary already calls this dispatch by-value), TIER2_JSONB_DROP_WHOLE -> TIER2_JSONB_DROPPED_COLUMNS, RedactionResult.redacted -> redacted_record. Co-Authored-By: Claude Sonnet 5 --- .../infrastructure/record_export/__init__.py | 34 +++ .../record_export/_leaf_rule.py | 115 ++++++++ .../record_export/_redact_tier1.py | 181 +++++++++++++ .../record_export/_redact_tier2.py | 249 ++++++++++++++++++ .../record_export/_redaction.py | 102 +++++++ .../infrastructure/record_export/_tokens.py | 53 ++++ .../test_record_export_redaction_postgres.py | 249 ++++++++++++++++++ .../record_export/test_redact_tier1.py | 163 ++++++++++++ .../record_export/test_redact_tier2.py | 153 +++++++++++ .../record_export/test_tokens.py | 52 ++++ 10 files changed, 1351 insertions(+) create mode 100644 apps/api/src/cora/infrastructure/record_export/_leaf_rule.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_redact_tier1.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_redact_tier2.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_redaction.py create mode 100644 apps/api/src/cora/infrastructure/record_export/_tokens.py create mode 100644 apps/api/tests/integration/test_record_export_redaction_postgres.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_tokens.py diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py index 33f4d5744d5..d55e37a010a 100644 --- a/apps/api/src/cora/infrastructure/record_export/__init__.py +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -41,6 +41,25 @@ hash_streams, ) from cora.infrastructure.record_export._manifest import Manifest, build_manifest, capture_git_commit +from cora.infrastructure.record_export._redact_tier1 import ( + Tier1Redactor, + UnknownEventTypeError, + redact_tier1_payload, +) +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + TIER2_JSONB_CLEARED_POINTERS, + TIER2_JSONB_DROPPED_COLUMNS, + UnfiredClearanceError, + ensure_all_clearances_fired, + redact_tier2_row, +) +from cora.infrastructure.record_export._redaction import ( + RedactedRecord, + RedactionProfileMismatchError, + RedactionResult, + redact_record, +) from cora.infrastructure.record_export._registry import ( EntriesReader, EntriesTableSpec, @@ -55,6 +74,7 @@ UnknownStreamTypeError, ensure_stream_type_known, ) +from cora.infrastructure.record_export._tokens import TokenMap __all__ = [ "KNOWN_STREAM_TYPES", @@ -62,23 +82,37 @@ "RECORD_PAYLOAD_TYPE", "REDACTION_PROFILE_PAYLOAD_TYPE", "STREAMS_PAYLOAD_TYPE", + "TIER2_DISPOSITIONS", + "TIER2_JSONB_CLEARED_POINTERS", + "TIER2_JSONB_DROPPED_COLUMNS", "EmptyExportError", "EntriesReader", "EntriesTableSpec", "ExportedRecord", "Manifest", + "RedactedRecord", + "RedactionProfileMismatchError", + "RedactionResult", + "Tier1Redactor", + "TokenMap", + "UnfiredClearanceError", + "UnknownEventTypeError", "UnknownLogbookKindError", "UnknownStreamTypeError", "all_specs", "build_manifest", "capture_git_commit", "capture_watermark", + "ensure_all_clearances_fired", "ensure_stream_type_known", "export_record", "hash_logbooks", "hash_record", "hash_redaction_profile", "hash_streams", + "redact_record", + "redact_tier1_payload", + "redact_tier2_row", "registered_envelope_classes", "render_row", "render_value", diff --git a/apps/api/src/cora/infrastructure/record_export/_leaf_rule.py b/apps/api/src/cora/infrastructure/record_export/_leaf_rule.py new file mode 100644 index 00000000000..b8685c34ce6 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_leaf_rule.py @@ -0,0 +1,115 @@ +"""The generic recursive leaf rule. + +Shared by tier-1's `by-value` polymorphic fields (a slot that can hold a +scalar or a value object -- `_dispositions.py`'s `by-value` disposition) +and tier-2's four recursing jsonb columns (`activities.payload`, +`diagnostics.payload`, `outcomes.point`, `outcomes.measurements`). Per +`project_record_export_v3.md` F5: numeric and boolean leaves KEEP, +UUID-shaped string leaves TOKEN, every other string leaf DROPS unless +its `(column, json-pointer)` is on a caller-supplied cleared list. +Object KEYS are always published -- in practice they are field names +and they carry the structure -- only VALUES are ever redacted. + +This walker does not know, or need to know, what produced the value it +is walking (which `step_kind`, which Recipe step type, ...). That is +deliberate: a nested shape nobody enumerated a clearance for simply +drops by the same default every other unlisted field drops by, rather +than needing a special case per producer. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# Walks `Any`-typed, already-rendered JSON values; suppressed the same +# way cora.shared.content_hash._canonicalize is, for the same reason. + +import re +from typing import Any + +from cora.infrastructure.record_export._tokens import TokenMap + +OMITTED = object() +"""Sentinel: the caller must omit this key (dict) or this element (list) +entirely, not store `None` in its place.""" + +_UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def is_uuid_shaped(value: str) -> bool: + return bool(_UUID_RE.match(value)) + + +def _join_pointer(pointer: str, segment: str) -> str: + return segment if not pointer else f"{pointer}/{segment}" + + +def apply_leaf_rule( + value: Any, + *, + token_map: TokenMap, + cleared_pointers: frozenset[str] = frozenset(), + pointer: str = "", + fired_pointers: set[str] | None = None, +) -> Any: + """Recursively redact `value`. + + Returns a redacted `dict`/`list` (same shape, dropped keys/elements + omitted) for a container, a redacted scalar for a leaf, or `OMITTED` + if the top-level value itself is a leaf that must be omitted by the + caller (dict/list values are never themselves `OMITTED` -- an empty + dict/list is a valid redacted result and is returned as such). + + `fired_pointers`, when supplied, collects every `pointer` whose + value was actually kept via `cleared_pointers` (not via the + unconditional number/bool/None/UUID branches) -- the record needed + to check a cleared list ever matched anything, per F5's + unfired-clearance rejection. + """ + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, sub_value in value.items(): + child_pointer = _join_pointer(pointer, str(key)) + redacted = apply_leaf_rule( + sub_value, + token_map=token_map, + cleared_pointers=cleared_pointers, + pointer=child_pointer, + fired_pointers=fired_pointers, + ) + if redacted is not OMITTED: + result[key] = redacted + return result + + if isinstance(value, (list, tuple)): + child_pointer = _join_pointer(pointer, "*") + out: list[Any] = [] + for item in value: + redacted = apply_leaf_rule( + item, + token_map=token_map, + cleared_pointers=cleared_pointers, + pointer=child_pointer, + fired_pointers=fired_pointers, + ) + if redacted is not OMITTED: + out.append(redacted) + return out + + # Leaf values. + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + if is_uuid_shaped(value): + return token_map.token_uuid(value) + if pointer in cleared_pointers: + if fired_pointers is not None: + fired_pointers.add(pointer) + return value + return OMITTED + + # An already-rendered export body (per step 2's F6) never contains + # anything else; fail closed rather than guess at a new type's intent. + return OMITTED + + +__all__ = ["OMITTED", "apply_leaf_rule", "is_uuid_shaped"] diff --git a/apps/api/src/cora/infrastructure/record_export/_redact_tier1.py b/apps/api/src/cora/infrastructure/record_export/_redact_tier1.py new file mode 100644 index 00000000000..5d95da780ef --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_redact_tier1.py @@ -0,0 +1,181 @@ +"""Redact `events` rows: fixed columns plus disposition-table-driven payload. + +Per `project_record_export_build_brief.md` step 6 and +`project_record_export_v3.md` F5. + +Two DIFFERENT failure modes, easy to conflate (the brief's own step-6 +notes and its terser Rejections list read ambiguously against each +other on this point; this resolves it the way that keeps schema +evolution possible): + +- `event_type` absent from `_dispositions.DISPOSITIONS` entirely -> + ABORT (`UnknownEventTypeError`). Step 0's generator is exhaustive + over every currently-declared event class; a stream carrying a type + the table has never heard of means the table is stale relative to + the code, which is a build problem, not a per-row one. +- A payload KEY present on a row but absent from + `DISPOSITIONS[event_type]`'s own field list -> DROP (omit the key). + This is what makes schema evolution survivable: an OLDER + `schema_version`'s row can carry a field the CURRENT dataclass no + longer declares (removed in a later version), and dropping it rather + than aborting the whole export is the graceful path. This is also + exactly what "a bare str on a NEW event drops with nobody editing a + list" is really about at the OTHER end: Step 0's generator + auto-classifies every new bare `str` field as `drop:text` the moment + it is generated, so the field already has a rule (drop) with no + manual list-editing required; it is not the "missing key" case. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# Dispatches on `Any`-typed disposition values and already-rendered +# JSON payloads; suppressed the same way cora.shared.content_hash is. + +from typing import Any + +from cora.infrastructure.record_export._dispositions import DISPOSITIONS +from cora.infrastructure.record_export._leaf_rule import OMITTED, apply_leaf_rule +from cora.infrastructure.record_export._tokens import TokenMap + +_FIXED_KEEP_COLUMNS = ( + "schema_version", + "stream_type", + "event_type", + "occurred_at", + "recorded_at", +) +_FIXED_TOKEN_COLUMNS = ( + "stream_id", + "correlation_id", + "causation_id", + "event_id", + "principal_id", +) +_FIXED_DROP_COLUMNS = ("metadata", "signature", "signature_kid", "signature_version") + + +class UnknownEventTypeError(LookupError): + """`event_type` has no entry in `_dispositions.DISPOSITIONS` at all. + + Refuses loudly: the disposition table is exhaustive over every + currently-declared event class, so this means the table is stale + relative to the code, not that this one row should be skipped. + """ + + def __init__(self, event_type: str) -> None: + super().__init__( + f"event_type {event_type!r} has no entry in DISPOSITIONS; the " + "table is stale relative to the code (run `make record-dispositions`)." + ) + self.event_type = event_type + + +def _apply_field_disposition(disposition: Any, value: Any, *, token_map: TokenMap) -> Any: + if isinstance(disposition, dict): + if "[]" in disposition: + # Fixed-length heterogeneous tuple: one disposition per position. + per_position = disposition["[]"] + if not isinstance(value, (list, tuple)): + return OMITTED + return [ + _apply_field_disposition(pos_disposition, item, token_map=token_map) + for pos_disposition, item in zip(per_position, value, strict=True) + ] + # A recursed value object: apply this same per-key logic one level down. + if not isinstance(value, dict): + return OMITTED + result: dict[str, Any] = {} + for key, sub_disposition in disposition.items(): + if key not in value: + continue + redacted = _apply_field_disposition(sub_disposition, value[key], token_map=token_map) + if redacted is not OMITTED: + result[key] = redacted + return result + + if disposition == "by-value": + return apply_leaf_rule(value, token_map=token_map) + if disposition.startswith("keep:"): + return value + if disposition == "token:uuid": + # A UUID-collection field (e.g. `target_asset_ids`) carries + # `token:uuid` too, per Step 0's census: the disposition names + # the ELEMENT type, not the field's own cardinality. + if isinstance(value, (list, tuple)): + return [token_map.token_uuid(item) for item in value] + return token_map.token_uuid(value) + if disposition.startswith("drop:"): + return OMITTED + return OMITTED + + +def redact_tier1_payload( + event_type: str, payload: dict[str, Any], *, token_map: TokenMap +) -> dict[str, Any]: + """Redact one event's `payload`, iterating the STORED payload's own + keys (never the disposition table's), per F5's fail-closed property.""" + if event_type not in DISPOSITIONS: + raise UnknownEventTypeError(event_type) + field_dispositions = DISPOSITIONS[event_type] + + result: dict[str, Any] = {} + for key, value in payload.items(): + disposition = field_dispositions.get(key) + if disposition is None: + continue # known event type, unlisted field: schema-evolution DROP + redacted = _apply_field_disposition(disposition, value, token_map=token_map) + if redacted is not OMITTED: + result[key] = redacted + return result + + +class Tier1Redactor: + """Stateful per-export redactor for `events` rows. + + Must be fed rows in the SAME order `export_record` produced them + (`(transaction_id, position)`): `position`/`transaction_id` + re-indexing depends on that order, and `version` re-indexing depends + on rows for one `stream_id` arriving in stream order (already true + of the exporter's global order). + """ + + def __init__(self, token_map: TokenMap) -> None: + self._token_map = token_map + self._next_position = 1 + self._next_version_by_stream: dict[str, int] = {} + self._transaction_id_index: dict[str, int] = {} + self._next_transaction_id = 1 + + def _dense_version(self, raw_stream_id: str) -> int: + version = self._next_version_by_stream.get(raw_stream_id, 1) + self._next_version_by_stream[raw_stream_id] = version + 1 + return version + + def _dense_transaction_id(self, raw_transaction_id: object) -> int: + key = str(raw_transaction_id) + if key not in self._transaction_id_index: + self._transaction_id_index[key] = self._next_transaction_id + self._next_transaction_id += 1 + return self._transaction_id_index[key] + + def redact_row(self, row: dict[str, Any]) -> dict[str, Any]: + raw_stream_id = row["stream_id"] + redacted: dict[str, Any] = { + "position": self._next_position, + "version": self._dense_version(raw_stream_id), + "transaction_id": self._dense_transaction_id(row["transaction_id"]), + } + self._next_position += 1 + + for column in _FIXED_KEEP_COLUMNS: + redacted[column] = row[column] + for column in _FIXED_TOKEN_COLUMNS: + redacted[column] = self._token_map.token_uuid(row[column]) + # _FIXED_DROP_COLUMNS (metadata, signature*) intentionally absent. + + redacted["payload"] = redact_tier1_payload( + row["event_type"], row["payload"], token_map=self._token_map + ) + return redacted + + +__all__ = ["Tier1Redactor", "UnknownEventTypeError", "redact_tier1_payload"] diff --git a/apps/api/src/cora/infrastructure/record_export/_redact_tier2.py b/apps/api/src/cora/infrastructure/record_export/_redact_tier2.py new file mode 100644 index 00000000000..2167d52382c --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_redact_tier2.py @@ -0,0 +1,249 @@ +"""Redact `entries_*` rows: a hand-authored disposition table per kind. + +Transcribed from `project_record_export_v3.md` F5's three tables (28 +`text` columns across 8 tables, split PROVED CLOSED / JUDGED LOW RISK / +DROPPED; the 5 jsonb columns, 4 BY_VALUE + 1 dropped-whole). Unlike +tier 1, there is no generator here: F5 itself says tier 2 is small +enough (103 columns total) to hand-enumerate, and that hand-enumeration +is exactly what a fitness test elsewhere +(`test_record_export_registry_completeness.py`-style AST discovery does +not cover, since this table has no source annotation to derive from) +makes `test_redact_tier2.py` responsible for keeping honest against +`_registry.py`'s own table list. + +`BY_VALUE` reuses the exact string tier 1's locked, generated vocabulary +already uses for the same dispatch (`_dispositions.py`'s `"by-value"`): +both name the same operation (apply the generic leaf rule), and tier 2 +is the side free to conform since it is hand-authored, not generated. + +`kind` here is the Step 1 registry's kind (`"activity"`, `"verdict"`, +...), matching `_registry.EntriesTableSpec.kind`, not the raw table name. +""" + +from typing import Any + +from cora.infrastructure.record_export._leaf_rule import OMITTED, apply_leaf_rule +from cora.infrastructure.record_export._tokens import TokenMap + +KEEP = "keep" +TOKEN = "token" +DROP = "drop" +BY_VALUE = "by-value" + +# Every column of every kind's entries table. `BY_VALUE` columns are +# jsonb and are handled via TIER2_JSONB_CLEARED_POINTERS / +# TIER2_JSONB_DROPPED_COLUMNS below, never via this dict's own value. +TIER2_DISPOSITIONS: dict[str, dict[str, str]] = { + "verdict": { + "event_id": TOKEN, + "conduit_id": TOKEN, + "logbook_id": TOKEN, + "actor_id": TOKEN, + "command_name": KEEP, # judged low risk: written from code literals + "decision": KEEP, # proved closed: DB CHECK (decision IN ('Allow','Deny')) + "reason": DROP, # P0-4: builds f"Principal {principal_id} not in policy..." + "correlation_id": TOKEN, + "causation_id": TOKEN, + "occurred_at": KEEP, + "recorded_at": KEEP, + }, + "inference": { + "event_id": TOKEN, + "decision_id": TOKEN, + "logbook_id": TOKEN, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "occurred_at": KEEP, + "duration": KEEP, + "operation_name": KEEP, # judged low risk: names an operation, not a person + "provider_name": KEEP, + "request_model": KEEP, + "response_id": DROP, # correlator into an external vendor's records + "response_model": KEEP, + "request_temperature": KEEP, + "request_top_p": KEEP, + "request_max_tokens": KEEP, + "output_type": KEEP, + "finish_reasons": DROP, # text[]: array of str, drop-unless-cleared + "input_tokens": KEEP, + "output_tokens": KEEP, + "agent_id": DROP, # OTel gen_ai.agent.id; correlator, fail closed + "agent_name": DROP, # operator-authored free text + "agent_description": DROP, + "conversation_id": DROP, + "tool_name": KEEP, # judged low risk: names a tool, not a person + "tool_call_id": DROP, + "tool_type": KEEP, + "messages": BY_VALUE, # dropped whole, see TIER2_JSONB_DROPPED_COLUMNS + "recorded_at": KEEP, + "cost_usd": KEEP, + }, + "activity": { + "event_id": TOKEN, + "procedure_id": TOKEN, + "logbook_id": TOKEN, + "actor_id": TOKEN, + "command_name": KEEP, # judged low risk (3-of-8 group) + "step_kind": KEEP, # proved closed: Literal + STEP_KIND_VALUES + "payload": BY_VALUE, + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + }, + "diagnostic": { + "event_id": TOKEN, + "procedure_id": TOKEN, + "logbook_id": TOKEN, + "iteration_index": KEEP, + "model_ref": KEEP, # judged low risk: fed from advice.model_ref, "unknown" fallback + "payload": BY_VALUE, + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + }, + "outcome": { + "event_id": TOKEN, + "procedure_id": TOKEN, + "logbook_id": TOKEN, + "iteration_index": KEEP, + "point": BY_VALUE, + "measurements": BY_VALUE, + "succeeded": KEEP, + "actuation_kind": KEEP, # judged low risk: rehearsal-vs-live gate, closed by its writers + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + }, + "observation": { + "event_id": TOKEN, + "run_id": TOKEN, + "logbook_id": TOKEN, + "actor_id": TOKEN, + "command_name": KEEP, # judged low risk (3-of-8 group) + "channel_name": KEEP, # EPICS channel address, facility-fixed, already public + "value": KEEP, + "units": KEEP, # judged low risk: unvalidated, low risk, not closed + "sampling_procedure": KEEP, # proved closed: Literal + SAMPLING_PROCEDURE_VALUES + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + "is_simulated": KEEP, + }, + "heartbeat": { + "event_id": TOKEN, + "run_id": TOKEN, + "source_id": KEEP, # EPICS channel address, facility-fixed, already public + "heartbeat_at": KEEP, + "recorded_at": KEEP, + }, + "probe": { + "event_id": TOKEN, + "enclosure_id": TOKEN, + "source_kind": KEEP, # judged low risk, same standard as heartbeat.source_id + "source_id": DROP, # NOT cleared alongside its heartbeat twin, deliberately: + # pairing a reachability failure with the exact substrate address is + # closer to a security disclosure about a safety system than to science. + "reach_tier": KEEP, # proved closed: ReachTier StrEnum + "status_claimed": KEEP, # grouped with reach_tier; needs its own human pass (watch item) + "recorded_at": KEEP, + }, +} + +# (kind, column) -> the set of string-leaf json-pointers cleared inside +# that jsonb column. Absent entries default to an empty set (every +# string leaf drops). Pointers use "*" for "any list element", matching +# F5's own notation (`outcomes.measurements/*/name`). +TIER2_JSONB_CLEARED_POINTERS: dict[tuple[str, str], frozenset[str]] = { + ("activity", "payload"): frozenset({"channel", "action_name", "units"}), + ("outcome", "measurements"): frozenset({"*/name", "*/units", "*/kind", "*/quality"}), +} + +# (kind, column) pairs whose jsonb value drops WHOLE rather than recursing. +TIER2_JSONB_DROPPED_COLUMNS: frozenset[tuple[str, str]] = frozenset({("inference", "messages")}) + + +class UnfiredClearanceError(RuntimeError): + """A declared tier-2 jsonb clearance never matched a row in a kind + that WAS present in this export. + + Per F5's Rejections list: an unfired rule inside a type that was + exported is a leak-shaped gap (most likely a typo in this file's + own `TIER2_JSONB_CLEARED_POINTERS`), not a benign no-op. + """ + + def __init__(self, kind: str, column: str, pointer: str) -> None: + super().__init__( + f"declared clearance ({kind!r}, {column!r})/{pointer!r} never matched " + f"any row while redacting {kind!r} rows present in this export." + ) + self.kind = kind + self.column = column + self.pointer = pointer + + +def redact_tier2_row( + kind: str, + row: dict[str, Any], + *, + token_map: TokenMap, + fired_pointers: dict[tuple[str, str], set[str]], +) -> dict[str, Any]: + """Redact one entries row of the given registry `kind`.""" + column_dispositions = TIER2_DISPOSITIONS[kind] + result: dict[str, Any] = {} + for column, value in row.items(): + disposition = column_dispositions.get(column) + if disposition is None: + # An entries column not enumerated above. Fail closed rather + # than silently pass it through: this is this file's own + # drift-detection surface (see test_redact_tier2.py). + continue + if disposition == KEEP: + result[column] = value + elif disposition == TOKEN: + result[column] = token_map.token_uuid(value) + elif disposition == DROP: + continue + elif disposition == BY_VALUE: + if (kind, column) in TIER2_JSONB_DROPPED_COLUMNS: + continue + cleared = TIER2_JSONB_CLEARED_POINTERS.get((kind, column), frozenset()) + fired = fired_pointers.setdefault((kind, column), set()) + redacted = apply_leaf_rule( + value, token_map=token_map, cleared_pointers=cleared, fired_pointers=fired + ) + if redacted is not OMITTED: + result[column] = redacted + return result + + +def ensure_all_clearances_fired( + fired_pointers: dict[tuple[str, str], set[str]], *, kinds_present: frozenset[str] +) -> None: + """Raise `UnfiredClearanceError` for a declared clearance that never + matched, scoped to kinds actually present in this export (an unused + clearance for a kind with zero rows is not a leak -- nothing to leak).""" + for (kind, column), cleared in TIER2_JSONB_CLEARED_POINTERS.items(): + if kind not in kinds_present: + continue + fired = fired_pointers.get((kind, column), set()) + for pointer in cleared - fired: + raise UnfiredClearanceError(kind, column, pointer) + + +__all__ = [ + "TIER2_DISPOSITIONS", + "TIER2_JSONB_CLEARED_POINTERS", + "TIER2_JSONB_DROPPED_COLUMNS", + "UnfiredClearanceError", + "ensure_all_clearances_fired", + "redact_tier2_row", +] diff --git a/apps/api/src/cora/infrastructure/record_export/_redaction.py b/apps/api/src/cora/infrastructure/record_export/_redaction.py new file mode 100644 index 00000000000..6a16ccf93e6 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_redaction.py @@ -0,0 +1,102 @@ +"""Redact a whole `ExportedRecord`: the fail-closed switch plus both tiers. + +Per `project_record_export_v3.md` F5's fail-closed switch: the exporter +refuses to produce a bundle marked publishable unless a redaction +profile is supplied and its hash matches. `redact_record` computes +`hash_redaction_profile()` itself and compares against the caller- +supplied `expected_redaction_profile_hash` BEFORE touching a single row, +so a stale or substituted disposition table aborts rather than silently +redacting under the wrong rules. +""" + +from dataclasses import dataclass + +from cora.infrastructure.record_export._export import ExportedRecord +from cora.infrastructure.record_export._hashing import hash_redaction_profile +from cora.infrastructure.record_export._redact_tier1 import Tier1Redactor, UnknownEventTypeError +from cora.infrastructure.record_export._redact_tier2 import ( + ensure_all_clearances_fired, + redact_tier2_row, +) +from cora.infrastructure.record_export._tokens import TokenMap + +__all__ = [ + "RedactedRecord", + "RedactionProfileMismatchError", + "RedactionResult", + "UnknownEventTypeError", + "redact_record", +] + + +class RedactionProfileMismatchError(RuntimeError): + """`expected_redaction_profile_hash` does not match the disposition + table this checkout would actually redact with. + + Refuses rather than redacting under an unverified or stale table: + the exact failure mode the fail-closed switch exists to catch. + """ + + def __init__(self, expected: str, actual: str) -> None: + super().__init__( + f"redaction profile mismatch: expected {expected!r}, this checkout's " + f"disposition table hashes to {actual!r}. Refusing to redact." + ) + self.expected = expected + self.actual = actual + + +@dataclass(frozen=True, slots=True) +class RedactedRecord: + """The published projection of an `ExportedRecord`: same two-tier + shape, every value passed through F5's dispositions.""" + + streams: tuple[dict[str, object], ...] + logbooks: dict[str, tuple[dict[str, object], ...]] + + +@dataclass(frozen=True, slots=True) +class RedactionResult: + """`redacted_record` is safe to hash as the published record. + `token_map` is an artifact of THIS export, retained separately under + H1's obligation; it must never be shipped or fed into anything + hashed as the published record.""" + + redacted_record: RedactedRecord + token_map: TokenMap + + +def redact_record( + record: ExportedRecord, *, expected_redaction_profile_hash: str +) -> RedactionResult: + """Redact both tiers of `record` under one shared `TokenMap`. + + Raises `RedactionProfileMismatchError` before touching any row if + the hash does not match; `UnknownEventTypeError` if a stream row's + `event_type` has no entry in the disposition table at all; + `UnfiredClearanceError` if a declared tier-2 jsonb clearance never + matched a row in a kind present in this export. + """ + actual_hash = hash_redaction_profile() + if expected_redaction_profile_hash != actual_hash: + raise RedactionProfileMismatchError(expected_redaction_profile_hash, actual_hash) + + token_map = TokenMap() + + tier1 = Tier1Redactor(token_map) + redacted_streams = tuple(tier1.redact_row(row) for row in record.streams) + + fired_pointers: dict[tuple[str, str], set[str]] = {} + redacted_logbooks = { + kind: tuple( + redact_tier2_row(kind, row, token_map=token_map, fired_pointers=fired_pointers) + for row in rows + ) + for kind, rows in record.logbooks.items() + } + ensure_all_clearances_fired(fired_pointers, kinds_present=frozenset(record.logbooks)) + + return RedactionResult( + redacted_record=RedactedRecord(streams=redacted_streams, logbooks=redacted_logbooks), + token_map=token_map, + ) diff --git a/apps/api/src/cora/infrastructure/record_export/_tokens.py b/apps/api/src/cora/infrastructure/record_export/_tokens.py new file mode 100644 index 00000000000..ba912ecbdad --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_tokens.py @@ -0,0 +1,53 @@ +"""The per-export UUID surrogate map. + +Per `project_record_export_v3.md` F5: a TOKEN is a random surrogate, +never a hash. A hash is deterministic, so against a known candidate set +(a ten-person roster, a facility's proposal list) it is brute-forceable +in as many guesses as there are candidates -- the same defect as +publishing a signature beside a redacted payload. `TokenMap` mints a +fresh `uuid4()` per distinct source string the first time it is seen +and returns that same surrogate on every later lookup within the same +export, which is what preserves joins (both within tier 1 and across +the tier-1/tier-2 seam, per F5) without making the surrogate +derivable from its source. + +The map itself is an artifact of the export, not the record: F5 says it +"never ships" and sits under the same retention obligation as the +unredacted record. `surrogate_by_source` exists so a caller can persist +it separately; nothing in this package ever feeds it back into anything +that gets hashed as "the published record". +""" + +from uuid import uuid4 + + +class TokenMap: + """Mint-once, reuse-after per-export UUID surrogates. Never a hash.""" + + def __init__(self) -> None: + self._surrogates: dict[str, str] = {} + + def token_uuid(self, source: str | None) -> str | None: + """Surrogate for `source`, or `None` if `source` is `None`. + + `None` is a legitimate value for an optional UUID column + (`causation_id`, `principal_id`, ...) and must pass through + as `None` rather than being tokenized or dropped. + """ + if source is None: + return None + if source not in self._surrogates: + self._surrogates[source] = str(uuid4()) + return self._surrogates[source] + + @property + def surrogate_by_source(self) -> dict[str, str]: + """Source UUID -> surrogate, for retention under H1's obligation. + + A copy, not a live view: callers must not be able to mutate the + map this instance uses for tokenization by holding this dict. + """ + return dict(self._surrogates) + + +__all__ = ["TokenMap"] diff --git a/apps/api/tests/integration/test_record_export_redaction_postgres.py b/apps/api/tests/integration/test_record_export_redaction_postgres.py new file mode 100644 index 00000000000..48b4cc62e50 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_redaction_postgres.py @@ -0,0 +1,249 @@ +"""Acceptance test for Step 6 of the record exporter build brief. + +Per `project_record_export_build_brief.md` step 6: a bare `str` on a NEW +event drops with nobody editing a list; an event type absent from the +disposition table aborts; no raw store identifier survives the export +(NOT "no join key" -- timestamps remain one, deliberately); a rehearsal +export contains no real principal, ASSERTED rather than assumed. + +Built the same way steps 2-4's acceptance tests were: a real Procedure +through Running with a real activity appended, via the real handlers, +then `export_record` + `redact_record` against the live database. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +import dataclasses +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.record_export import ( + RedactionProfileMismatchError, + UnknownEventTypeError, + export_record, + hash_redaction_profile, + redact_record, +) +from cora.infrastructure.record_export._redact_tier2 import TIER2_DISPOSITIONS +from cora.infrastructure.record_export._registry import all_specs +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + event_type_name, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +def _collect_string_leaves(value: object, out: set[str]) -> None: + if isinstance(value, dict): + for sub in value.values(): + _collect_string_leaves(sub, out) + elif isinstance(value, (list, tuple)): + for item in value: + _collect_string_leaves(item, out) + elif isinstance(value, str): + out.add(value) + + +async def _seed_running_procedure_with_activity(db_pool: asyncpg.Pool, procedure_id: UUID) -> None: + logbook_id = uuid4() + open_event_id = uuid4() + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await deps.event_store.append( + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + # A single setpoint-only activity would leave the activity kind's + # `action_name` and `units` clearances unfired: a real whole-database + # export accumulates many activities across many procedures over + # time and would plausibly exercise every step kind, so this mirrors + # that with one of each (same 3-entry shape as step 2's fixture) + # rather than under-representing a real export. + handler = bind_append(deps, step_store=PostgresActivityStore(db_pool)) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=uuid4(), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0, "units": "K"}, + sampled_at=_NOW, + ), + ActivityInput( + event_id=uuid4(), + step_kind="action", + payload={"action_name": "open_valve", "params": {"valve": "V12"}}, + sampled_at=_NOW, + ), + ActivityInput( + event_id=uuid4(), + step_kind="check", + payload={"channel": "T_oven", "passed": True}, + sampled_at=_NOW, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + +@pytest.mark.integration +async def test_bare_str_field_drops_and_no_real_principal_survives(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + result = redact_record(exported, expected_redaction_profile_hash=hash_redaction_profile()) + redacted = result.redacted_record + + registered_rows = [ + row for row in redacted.streams if row["event_type"] == "ProcedureRegistered" + ] + assert len(registered_rows) == 1 + registered_payload = registered_rows[0]["payload"] + assert isinstance(registered_payload, dict) + # ProcedureRegistered.name and .kind are drop:text in the real, + # generated table -- no test-only disposition edited for this. + assert "name" not in registered_payload + assert "kind" not in registered_payload + + # ASSERTED, not assumed: the real principal/actor UUID used to build + # this fixture must not appear anywhere in the redacted output. + leaves: set[str] = set() + for row in redacted.streams: + _collect_string_leaves(row, leaves) + for rows in redacted.logbooks.values(): + for row in rows: + _collect_string_leaves(row, leaves) + assert str(_PRINCIPAL_ID) not in leaves + assert str(procedure_id) not in leaves + assert str(_CORRELATION_ID) not in leaves + + +@pytest.mark.integration +async def test_event_type_absent_from_disposition_table_aborts(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + tampered_streams = list(exported.streams) + tampered_streams[0] = {**tampered_streams[0], "event_type": "ThisEventTypeDoesNotExist"} + tampered = dataclasses.replace(exported, streams=tuple(tampered_streams)) + + with pytest.raises(UnknownEventTypeError) as excinfo: + redact_record(tampered, expected_redaction_profile_hash=hash_redaction_profile()) + assert excinfo.value.event_type == "ThisEventTypeDoesNotExist" + + +@pytest.mark.integration +async def test_no_raw_uuid_survives_the_redacted_export(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + raw_uuids: set[str] = set() + for row in exported.streams: + for key in ("event_id", "stream_id", "correlation_id", "causation_id", "principal_id"): + value = row.get(key) + if isinstance(value, str): + raw_uuids.add(value) + for rows in exported.logbooks.values(): + for row in rows: + for value in row.values(): + if isinstance(value, str) and len(value) == 36 and value.count("-") == 4: + raw_uuids.add(value) + + result = redact_record(exported, expected_redaction_profile_hash=hash_redaction_profile()) + + redacted_leaves: set[str] = set() + for row in result.redacted_record.streams: + _collect_string_leaves(row, redacted_leaves) + for rows in result.redacted_record.logbooks.values(): + for row in rows: + _collect_string_leaves(row, redacted_leaves) + + assert raw_uuids, "fixture produced no UUIDs to check -- test would pass vacuously" + assert raw_uuids.isdisjoint(redacted_leaves) + + +@pytest.mark.integration +async def test_wrong_redaction_profile_hash_refuses_before_redacting(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + with pytest.raises(RedactionProfileMismatchError): + redact_record(exported, expected_redaction_profile_hash="0" * 64) + + +@pytest.mark.integration +@pytest.mark.parametrize("spec", all_specs(), ids=lambda spec: spec.kind) +async def test_tier2_disposition_table_columns_match_live_schema( + db_pool: asyncpg.Pool, spec: object +) -> None: + """Drift guard: TIER2_DISPOSITIONS[kind] must enumerate every column + the live table actually has, nothing more, nothing less -- a new + migration adding a column must fail this test until this file is + updated, the same posture step 1's registry completeness test takes + for the registry itself.""" + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + rows = await pg_conn.fetch( + "SELECT column_name FROM information_schema.columns WHERE table_name = $1", + spec.table, # type: ignore[attr-defined] + ) + live_columns = {row["column_name"] for row in rows} + declared_columns = set(TIER2_DISPOSITIONS[spec.kind].keys()) # type: ignore[attr-defined] + assert declared_columns == live_columns, ( + f"kind={spec.kind!r}: TIER2_DISPOSITIONS declares {declared_columns} but " # type: ignore[attr-defined] + f"{spec.table!r} actually has {live_columns}" # type: ignore[attr-defined] + ) diff --git a/apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py new file mode 100644 index 00000000000..20e06b8ff35 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py @@ -0,0 +1,163 @@ +"""Unit tests for tier-1 (`events`) redaction. + +Uses `AgentDefined`'s real, generated disposition entry (not a +hand-tuned test-only one) for the payload-shape tests, so "a bare str +drops with nobody editing a list" is demonstrated against the actual +committed table. +""" + +from uuid import uuid4 + +import pytest + +from cora.infrastructure.record_export import TokenMap, UnknownEventTypeError, redact_tier1_payload +from cora.infrastructure.record_export._redact_tier1 import Tier1Redactor + +_AGENT_ID = "01900000-0000-7000-8000-0000000000c1" +_PROMPT_TEMPLATE_ID = "01900000-0000-7000-8000-0000000000c2" + + +def _agent_defined_payload() -> dict[str, object]: + return { + "agent_id": _AGENT_ID, + "canonical_uri": "https://internal/agents/foo", + "capabilities": "read,write", + "daily_token_cap": 1000, + "description": "an agent that does things", + "kind": "assistant", + "model_ref": {"model": "claude-x", "provider": "anthropic", "snapshot_pin": "2026-01-01"}, + "monthly_usd_cap": 50.0, + "name": "Foo Agent", + "occurred_at": "2026-05-15T12:00:00+00:00", + "prompt_template_id": _PROMPT_TEMPLATE_ID, + "tools": "search,browse", + "version": "v1", + } + + +def test_bare_str_drop_text_field_drops_with_no_list_edited() -> None: + """AgentDefined.name is drop:text in the real generated table.""" + redacted = redact_tier1_payload("AgentDefined", _agent_defined_payload(), token_map=TokenMap()) + assert "name" not in redacted + assert "canonical_uri" not in redacted + assert "kind" not in redacted + assert "tools" not in redacted + assert "version" not in redacted + + +def test_keep_number_and_keep_time_fields_survive_unchanged() -> None: + payload = _agent_defined_payload() + redacted = redact_tier1_payload("AgentDefined", payload, token_map=TokenMap()) + assert redacted["daily_token_cap"] == 1000 + assert redacted["monthly_usd_cap"] == 50.0 + assert redacted["occurred_at"] == "2026-05-15T12:00:00+00:00" + + +def test_token_uuid_fields_become_a_distinct_surrogate() -> None: + token_map = TokenMap() + redacted = redact_tier1_payload("AgentDefined", _agent_defined_payload(), token_map=token_map) + assert redacted["agent_id"] == token_map.token_uuid(_AGENT_ID) + assert redacted["agent_id"] != _AGENT_ID + assert redacted["prompt_template_id"] != _PROMPT_TEMPLATE_ID + + +def test_recursed_value_object_drops_its_own_drop_text_subfields() -> None: + """model_ref is a recursed VO whose 3 fields are all drop:text.""" + redacted = redact_tier1_payload("AgentDefined", _agent_defined_payload(), token_map=TokenMap()) + assert "model_ref" not in redacted or redacted["model_ref"] == {} + + +def test_a_payload_key_absent_from_a_known_events_field_list_drops() -> None: + """Schema evolution: an older schema_version's row can carry a field + the current dataclass no longer declares. Must drop, not abort -- + aborting would make every export containing legacy-schema rows fail.""" + payload = _agent_defined_payload() + payload["a_field_removed_in_a_later_schema_version"] = "still in an old row" + redacted = redact_tier1_payload("AgentDefined", payload, token_map=TokenMap()) + assert "a_field_removed_in_a_later_schema_version" not in redacted + # The known fields are still processed normally. + assert redacted["daily_token_cap"] == 1000 + + +def test_unknown_event_type_aborts() -> None: + with pytest.raises(UnknownEventTypeError) as excinfo: + redact_tier1_payload("TotallyMadeUpEventType", {}, token_map=TokenMap()) + assert excinfo.value.event_type == "TotallyMadeUpEventType" + + +def _stream_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "position": 999, + "version": 999, + "transaction_id": "12345", + "schema_version": 1, + "stream_type": "Agent", + "event_type": "AgentDefined", + "occurred_at": "2026-05-15T12:00:00+00:00", + "recorded_at": "2026-05-15T12:00:01+00:00", + "stream_id": str(uuid4()), + "correlation_id": str(uuid4()), + "causation_id": None, + "event_id": str(uuid4()), + "principal_id": str(uuid4()), + "metadata": {"some": "metadata"}, + "signature": "deadbeef", + "signature_kid": "key-1", + "signature_version": "v1", + "payload": _agent_defined_payload(), + } + row.update(overrides) + return row + + +def test_metadata_and_signature_columns_are_absent_from_the_redacted_row() -> None: + redacted = Tier1Redactor(TokenMap()).redact_row(_stream_row()) + assert "metadata" not in redacted + assert "signature" not in redacted + assert "signature_kid" not in redacted + assert "signature_version" not in redacted + + +def test_position_is_dense_from_one_across_calls() -> None: + redactor = Tier1Redactor(TokenMap()) + first = redactor.redact_row(_stream_row(position=4400)) + second = redactor.redact_row(_stream_row(position=9100)) + assert first["position"] == 1 + assert second["position"] == 2 + + +def test_version_is_dense_per_stream_and_independent_across_streams() -> None: + redactor = Tier1Redactor(TokenMap()) + stream_a = str(uuid4()) + stream_b = str(uuid4()) + a1 = redactor.redact_row(_stream_row(stream_id=stream_a, version=1)) + b1 = redactor.redact_row(_stream_row(stream_id=stream_b, version=1)) + a2 = redactor.redact_row(_stream_row(stream_id=stream_a, version=2)) + assert a1["version"] == 1 + assert b1["version"] == 1 + assert a2["version"] == 2 + + +def test_transaction_id_is_a_small_monotone_int_not_the_raw_value() -> None: + redactor = Tier1Redactor(TokenMap()) + first = redactor.redact_row(_stream_row(transaction_id="99999999")) + second_same_tx = redactor.redact_row(_stream_row(transaction_id="99999999")) + third_new_tx = redactor.redact_row(_stream_row(transaction_id="100000000")) + assert first["transaction_id"] == 1 + assert second_same_tx["transaction_id"] == 1 # same raw tx -> same dense id + assert third_new_tx["transaction_id"] == 2 + assert third_new_tx["transaction_id"] != 100000000 + + +def test_stream_id_correlation_id_and_event_id_are_tokened() -> None: + token_map = TokenMap() + raw_stream_id = str(uuid4()) + row = _stream_row(stream_id=raw_stream_id) + redacted = Tier1Redactor(token_map).redact_row(row) + assert redacted["stream_id"] == token_map.token_uuid(raw_stream_id) + assert redacted["stream_id"] != raw_stream_id + + +def test_causation_id_none_stays_none() -> None: + redacted = Tier1Redactor(TokenMap()).redact_row(_stream_row(causation_id=None)) + assert redacted["causation_id"] is None diff --git a/apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py new file mode 100644 index 00000000000..3829d3c2ef2 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py @@ -0,0 +1,153 @@ +"""Unit tests for tier-2 (`entries_*`) redaction: the hand-authored +per-kind disposition table, jsonb recursion, and the unfired-clearance +check.""" + +from uuid import uuid4 + +import pytest + +from cora.infrastructure.record_export import TokenMap +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + UnfiredClearanceError, + ensure_all_clearances_fired, + redact_tier2_row, +) + + +def test_proved_closed_text_column_survives() -> None: + """conduit_verdicts.decision: DB CHECK (decision IN ('Allow','Deny')).""" + row = {"event_id": str(uuid4()), "decision": "Allow"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert redacted["decision"] == "Allow" + + +def test_judged_low_risk_text_column_survives() -> None: + row = {"event_id": str(uuid4()), "command_name": "AppendActivities"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert redacted["command_name"] == "AppendActivities" + + +def test_dropped_text_column_is_omitted() -> None: + """conduit_verdicts.reason: P0-4, builds free text that can republish + a tokened UUID.""" + row = {"event_id": str(uuid4()), "reason": "Principal ... not in policy"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert "reason" not in redacted + + +def test_uuid_column_is_tokened() -> None: + token_map = TokenMap() + raw = str(uuid4()) + row = {"event_id": raw} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=token_map, fired_pointers=fired) + assert redacted["event_id"] == token_map.token_uuid(raw) + assert redacted["event_id"] != raw + + +def test_a_column_absent_from_the_disposition_table_is_omitted() -> None: + """Fail closed: an entries column this file never enumerated drops, + same posture as tier 1's missing-key rule.""" + row = {"event_id": str(uuid4()), "a_column_nobody_listed": "surprise"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert "a_column_nobody_listed" not in redacted + + +def test_activities_payload_keeps_cleared_string_leaves_and_drops_others() -> None: + row = { + "event_id": str(uuid4()), + "payload": { + "channel": "T_oven", + "target_value": 423.0, + "units": "K", + "action_name": "open_valve", + "an_uncleared_free_text_field": "should drop", + }, + } + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("activity", row, token_map=TokenMap(), fired_pointers=fired) + payload = redacted["payload"] + assert payload["channel"] == "T_oven" + assert payload["units"] == "K" + assert payload["action_name"] == "open_valve" + assert payload["target_value"] == 423.0 + assert "an_uncleared_free_text_field" not in payload + + +def test_activities_payload_tokens_a_uuid_shaped_string_leaf() -> None: + token_map = TokenMap() + raw = str(uuid4()) + row = {"event_id": str(uuid4()), "payload": {"asset_id": raw}} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("activity", row, token_map=token_map, fired_pointers=fired) + assert redacted["payload"]["asset_id"] == token_map.token_uuid(raw) + + +def test_outcomes_measurements_keeps_cleared_pointers_and_drops_quality_detail() -> None: + row = { + "event_id": str(uuid4()), + "measurements": [ + { + "name": "flux", + "value": 1.23, + "kind": "Scalar", + "quality": "Good", + "quality_detail": "opaque forensic string", + "units": "cps", + } + ], + } + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("outcome", row, token_map=TokenMap(), fired_pointers=fired) + measurement = redacted["measurements"][0] + assert measurement["name"] == "flux" + assert measurement["units"] == "cps" + assert measurement["kind"] == "Scalar" + assert measurement["quality"] == "Good" + assert measurement["value"] == 1.23 + assert "quality_detail" not in measurement + + +def test_decision_inferences_messages_drops_whole() -> None: + row = {"event_id": str(uuid4()), "messages": [{"role": "user", "content": "secret prompt"}]} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("inference", row, token_map=TokenMap(), fired_pointers=fired) + assert "messages" not in redacted + + +@pytest.mark.parametrize("kind", sorted(TIER2_DISPOSITIONS)) +def test_every_declared_kind_has_at_least_one_uuid_scope_column(kind: str) -> None: + """Sanity check on the hand-authored table itself: every kind has at + least one `token` column (its logbook_id/run_id/enclosure_id scope + column at minimum).""" + assert "token" in TIER2_DISPOSITIONS[kind].values() + + +def test_unfired_clearance_raises_when_a_declared_pointer_never_matched() -> None: + # No channel/action_name/units in this payload. + row = {"event_id": str(uuid4()), "payload": {"target_value": 423.0}} + fired: dict[tuple[str, str], set[str]] = {} + redact_tier2_row("activity", row, token_map=TokenMap(), fired_pointers=fired) + with pytest.raises(UnfiredClearanceError): + ensure_all_clearances_fired(fired, kinds_present=frozenset({"activity"})) + + +def test_unfired_clearance_does_not_raise_for_a_kind_not_present() -> None: + """An unused clearance for a kind with zero rows in this export is + not a leak -- nothing exported to leak.""" + ensure_all_clearances_fired({}, kinds_present=frozenset({"verdict"})) + + +def test_all_declared_clearances_fire_when_every_pointer_is_exercised() -> None: + row = { + "event_id": str(uuid4()), + "payload": {"channel": "T_oven", "action_name": "open_valve", "units": "K"}, + } + fired: dict[tuple[str, str], set[str]] = {} + redact_tier2_row("activity", row, token_map=TokenMap(), fired_pointers=fired) + ensure_all_clearances_fired(fired, kinds_present=frozenset({"activity"})) # must not raise diff --git a/apps/api/tests/unit/infrastructure/record_export/test_tokens.py b/apps/api/tests/unit/infrastructure/record_export/test_tokens.py new file mode 100644 index 00000000000..b756b7aa379 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_tokens.py @@ -0,0 +1,52 @@ +"""Unit tests for the per-export UUID surrogate map.""" + +from cora.infrastructure.record_export import TokenMap + +_SOURCE_A = "01900000-0000-7000-8000-0000000000a1" +_SOURCE_B = "01900000-0000-7000-8000-0000000000a2" + + +def test_none_passes_through_as_none() -> None: + assert TokenMap().token_uuid(None) is None + + +def test_same_source_returns_the_same_surrogate_within_one_map() -> None: + token_map = TokenMap() + first = token_map.token_uuid(_SOURCE_A) + second = token_map.token_uuid(_SOURCE_A) + assert first == second + + +def test_distinct_sources_get_distinct_surrogates() -> None: + token_map = TokenMap() + assert token_map.token_uuid(_SOURCE_A) != token_map.token_uuid(_SOURCE_B) + + +def test_surrogate_is_never_equal_to_its_source() -> None: + token_map = TokenMap() + assert token_map.token_uuid(_SOURCE_A) != _SOURCE_A + + +def test_surrogate_is_not_derivable_from_its_source() -> None: + """Two independent TokenMaps produce DIFFERENT surrogates for the + SAME source: a hash would be deterministic (same input, same + output, every time) and therefore brute-forceable against a known + candidate roster. A random mint is not.""" + first_map = TokenMap() + second_map = TokenMap() + assert first_map.token_uuid(_SOURCE_A) != second_map.token_uuid(_SOURCE_A) + + +def test_surrogate_by_source_reflects_every_source_tokenized_so_far() -> None: + token_map = TokenMap() + a = token_map.token_uuid(_SOURCE_A) + b = token_map.token_uuid(_SOURCE_B) + assert token_map.surrogate_by_source == {_SOURCE_A: a, _SOURCE_B: b} + + +def test_surrogate_by_source_is_a_copy_not_a_live_view() -> None: + token_map = TokenMap() + token_map.token_uuid(_SOURCE_A) + snapshot = token_map.surrogate_by_source + snapshot["injected"] = "not-real" + assert "injected" not in token_map.surrogate_by_source From 87733e08eae52bbbd9548d1f2a595209a7f40a29 Mon Sep 17 00:00:00 2001 From: xmap <16776958+xmap@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:30:06 -0500 Subject: [PATCH 8/8] Cover tier-2 redaction tables in the redaction profile hash (H2) Step 7's single security-lens re-review (project_record_export_v3.md F5's "How to apply" item 7) found hash_redaction_profile() hashing only Step 0's generated tier-1 DISPOSITIONS table. Step 6's hand- authored tier-2 tables (TIER2_DISPOSITIONS, TIER2_JSONB_CLEARED_POINTERS, TIER2_JSONB_DROPPED_COLUMNS) were never part of H2, so redact_record's fail-closed switch could not detect a tier-2 disposition weakened after the fact (e.g. conduit_verdicts.reason DROP -> KEEP): expected_redaction_profile_hash would still match. Widen the hash body to cover all four tables, flattening the two tuple-keyed dicts to "kind/column" string keys for a reviewable canonical form. Regression tests pin that each of the four tables actually moves the hash. --- .../infrastructure/record_export/_hashing.py | 53 +++++++++++++++---- .../test_hash_redaction_profile.py | 50 +++++++++++++++++ 2 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py diff --git a/apps/api/src/cora/infrastructure/record_export/_hashing.py b/apps/api/src/cora/infrastructure/record_export/_hashing.py index 2934c8df9ee..4a5dea26eef 100644 --- a/apps/api/src/cora/infrastructure/record_export/_hashing.py +++ b/apps/api/src/cora/infrastructure/record_export/_hashing.py @@ -29,6 +29,11 @@ from cora.infrastructure.record_export._dispositions import DISPOSITIONS from cora.infrastructure.record_export._export import ExportedRecord +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + TIER2_JSONB_CLEARED_POINTERS, + TIER2_JSONB_DROPPED_COLUMNS, +) from cora.shared.content_hash import compute_content_hash RECORD_PAYLOAD_TYPE = "application/vnd.cora.record+json" @@ -69,17 +74,47 @@ def hash_record(record: ExportedRecord) -> str: def hash_redaction_profile() -> str: - """SHA-256 content hash over Step 0's generated disposition table. - - This IS the redaction profile hash (H2): the disposition table - decides what a published record discloses, so its hash is what - step 6's fail-closed switch checks and what the manifest names - alongside `record_hash` (H1). Regenerating the table via - `make record-dispositions` after a real event-model change is + """SHA-256 content hash over every table that decides what a + published record discloses: tier 1's generated `DISPOSITIONS` AND + tier 2's hand-authored `TIER2_DISPOSITIONS` / + `TIER2_JSONB_CLEARED_POINTERS` / `TIER2_JSONB_DROPPED_COLUMNS`. + + This IS the redaction profile hash (H2). Step 7's security re-review + found the tier-2 tables missing from this hash: the fail-closed + switch (`redact_record`'s `expected_redaction_profile_hash` check) + was fail-closed for tier 1 only, silently blind to a tier-2 table + edit that weakened a disposition (e.g. `conduit_verdicts.reason` + `DROP` -> `KEEP`) or dropped a jsonb clearance restriction. Both + tiers must be in H2, or "the hash matches" does not mean what + `RedactionProfileMismatchError`'s docstring claims it means. + + Tuple-keyed dicts (`TIER2_JSONB_CLEARED_POINTERS` / + `TIER2_JSONB_DROPPED_COLUMNS`) are flattened to `"kind/column"` + string keys before hashing rather than relying on + `compute_content_hash`'s `str(key)` fallback for non-string Mapping + keys, which would hash a Python tuple's `repr()` instead of a + reviewable string. + + Regenerating tier 1's table via `make record-dispositions` after a + real event-model change, or hand-editing tier 2's tables, is expected to change this value; `test_record_dispositions_drift.py` - is the guard against an UNREVIEWED change slipping through instead. + guards tier 1's generator output specifically, and + `test_redact_tier2.py`'s live-schema drift test guards tier 2's + column coverage, but only THIS hash is what a caller's + `expected_redaction_profile_hash` actually pins. """ - return compute_content_hash(REDACTION_PROFILE_PAYLOAD_TYPE, DISPOSITIONS) + body = { + "tier1": DISPOSITIONS, + "tier2_dispositions": TIER2_DISPOSITIONS, + "tier2_jsonb_cleared_pointers": { + f"{kind}/{column}": sorted(pointers) + for (kind, column), pointers in TIER2_JSONB_CLEARED_POINTERS.items() + }, + "tier2_jsonb_dropped_columns": sorted( + f"{kind}/{column}" for kind, column in TIER2_JSONB_DROPPED_COLUMNS + ), + } + return compute_content_hash(REDACTION_PROFILE_PAYLOAD_TYPE, body) __all__ = [ diff --git a/apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py b/apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py new file mode 100644 index 00000000000..da73751e88f --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py @@ -0,0 +1,50 @@ +"""Unit tests for `hash_redaction_profile` (H2). + +Step 7's security re-review found tier 2's hand-authored tables +(`TIER2_DISPOSITIONS`, `TIER2_JSONB_CLEARED_POINTERS`, +`TIER2_JSONB_DROPPED_COLUMNS`) missing from H2: the hash covered only +tier 1's generated `DISPOSITIONS`, so `redact_record`'s fail-closed +switch could not detect a tier-2 table edit that weakened a +disposition. These tests pin the fix: every one of the four tables H2 +is supposed to cover must actually move the hash. +""" + +import pytest + +from cora.infrastructure.record_export import hash_redaction_profile +from cora.infrastructure.record_export._dispositions import DISPOSITIONS +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + TIER2_JSONB_CLEARED_POINTERS, +) + + +def test_changing_a_tier1_disposition_changes_the_hash(monkeypatch: pytest.MonkeyPatch) -> None: + baseline = hash_redaction_profile() + some_event_type = next(iter(DISPOSITIONS)) + some_field = next(iter(DISPOSITIONS[some_event_type])) + monkeypatch.setitem(DISPOSITIONS[some_event_type], some_field, "keep:enum:Tampered") + assert hash_redaction_profile() != baseline + + +def test_changing_a_tier2_disposition_changes_the_hash(monkeypatch: pytest.MonkeyPatch) -> None: + """The gap Step 7 found: this used to be a no-op on the hash.""" + baseline = hash_redaction_profile() + monkeypatch.setitem(TIER2_DISPOSITIONS["verdict"], "reason", "keep") + assert hash_redaction_profile() != baseline + + +def test_widening_a_tier2_jsonb_clearance_changes_the_hash( + monkeypatch: pytest.MonkeyPatch, +) -> None: + baseline = hash_redaction_profile() + monkeypatch.setitem( + TIER2_JSONB_CLEARED_POINTERS, + ("activity", "payload"), + frozenset({"channel", "action_name", "units", "an_extra_leaked_pointer"}), + ) + assert hash_redaction_profile() != baseline + + +def test_hash_redaction_profile_is_stable_across_repeated_calls() -> None: + assert hash_redaction_profile() == hash_redaction_profile()