diff --git a/Makefile b/Makefile index 5b9e01ec738..d2423f627e9 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ test-noio test-db test-coverage store-durations diff-coverage fmt clean help \ migrate-status migrate-apply migrate-new migrate-hash precommit precommit-run \ arch-check arch-show docs-stage docs-build docs-serve openapi-snapshot \ - mutmut-audit mutmut-browse restore-drill + record-dispositions mutmut-audit mutmut-browse restore-drill API_DIR := apps/api COMPOSE := docker compose -f infra/docker-compose.yml @@ -36,6 +36,7 @@ help: @echo " arch-check Tach dependency contract + architecture fitness-function tests" @echo " arch-show Open the dependency graph (tach show)" @echo " openapi-snapshot Regenerate apps/api/openapi.json from create_app()" + @echo " record-dispositions Regenerate the record-export redaction table" @echo " mutmut-audit Run mutmut against Access BC deciders/evolver (audit cadence, ~5-15 min)" @echo " mutmut-browse Open mutmut's interactive TUI to triage surviving mutants" @echo " precommit Install pre-commit hooks (one-time per clone)" @@ -62,15 +63,15 @@ db-reset: $(COMPOSE) up -d postgres lint: - cd $(API_DIR) && uv run ruff check src tests - cd $(API_DIR) && uv run ruff format --check src tests + cd $(API_DIR) && uv run ruff check src tests tools + cd $(API_DIR) && uv run ruff format --check src tests tools fmt: - cd $(API_DIR) && uv run ruff check --fix src tests - cd $(API_DIR) && uv run ruff format src tests + cd $(API_DIR) && uv run ruff check --fix src tests tools + cd $(API_DIR) && uv run ruff format src tests tools typecheck: - cd $(API_DIR) && uv run pyright src tests + cd $(API_DIR) && uv run pyright src tests tools # pytest-xdist with `--dist=worksteal -n 4`: worksteal is the # scheduler-of-choice for mixed-duration suites (50ms unit alongside @@ -138,6 +139,19 @@ openapi-snapshot: cd $(API_DIR) && APP_ENV=test uv run python -c "import json; from cora.api.main import create_app; \ f = open('openapi.json', 'w'); json.dump(create_app().openapi(), f, indent=2, sort_keys=True); f.write('\n'); f.close()" +# Regenerate the record-export redaction disposition table after adding +# an event type, adding a field, or changing a field's declared type. +# The drift test (tests/architecture/test_record_dispositions_drift.py) +# fails until this is run and the diff is reviewed in the PR, and that +# diff is the list of what a published record would disclose. +# +# The generator ABORTS on an annotation it cannot classify. That is the +# design, not a bug: an unrecognised type is a question about the model. +# Pass --survey to list every such annotation in one pass instead of +# hitting them one exception at a time. +record-dispositions: + cd $(API_DIR) && uv run python tools/gen_record_dispositions.py + # Mutation testing audit. Pure-logic scope via the CLI wildcard pattern # (Access BC deciders + evolver only). [tool.mutmut] in apps/api/pyproject.toml # carries the test-selection + runner config. Audit-only — not per-PR. diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index b6d4a50440d..3935ebf9d3e 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -246,7 +246,10 @@ packages = ["src/cora"] [tool.ruff] line-length = 100 target-version = "py313" -src = ["src", "tests"] +# `tools` holds build-time generators that MAY import every bounded +# context; they live outside `src` so nothing shippable can import them, +# but they are still first-party code and get linted and type-checked. +src = ["src", "tests", "tools"] # Parked directories are work-in-progress slices the user explicitly # shelved (suffix `.parked`); mirrors the same exclusion in [tool.pyright]. extend-exclude = ["**/*.parked"] @@ -298,7 +301,7 @@ indent-style = "space" known-first-party = ["cora", "tests"] [tool.pyright] -include = ["src", "tests"] +include = ["src", "tests", "tools"] # Parked directories are work-in-progress slices the user explicitly # shelved (suffix `.parked`); they shouldn't block type-check on the # active codebase. Same convention applies to ruff (below). diff --git a/apps/api/src/cora/infrastructure/record_export/__init__.py b/apps/api/src/cora/infrastructure/record_export/__init__.py new file mode 100644 index 00000000000..d55e37a010a --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/__init__.py @@ -0,0 +1,120 @@ +"""Record export: CORA's record as a hashed, offline-verifiable artifact. + +The design is `project_record_export_v3.md`, locked for the no-beam +commissioning scope. This package holds the exporter and the generated +redaction disposition table it reads. + +It lives at `cora.infrastructure`, whose `tach.toml` entry allows +`cora.shared` and nothing else, and that constraint is the guarantee +rather than the obstacle: the exporter composes nothing (column-driven +over raw rows, a registry of `str -> str` pairs, and a table read as +data), so the layering rule ENFORCES the zero-bounded-context-import +property that makes the standalone-verifiability claim honest. If a +change here wants to import an aggregate, the design has drifted. + +Two things this package must never do, both learned from review rather +than from first principles: + +- Drive extraction, redaction or parity from `LogbookSchema`. + `LogbookFieldType` is closed over six scalars, so no schema can name a + jsonb column, and jsonb is where the doing lives. The schema travels + with the export as documentation. +- Describe the published record as anonymous. It is pseudonymous: + timestamps plus a facility's own published beamtime schedule + re-identify without the token map. +""" + +from cora.infrastructure.record_export._export import ( + EmptyExportError, + ExportedRecord, + capture_watermark, + export_record, +) +from cora.infrastructure.record_export._hashing import ( + LOGBOOKS_PAYLOAD_TYPE, + RECORD_PAYLOAD_TYPE, + REDACTION_PROFILE_PAYLOAD_TYPE, + STREAMS_PAYLOAD_TYPE, + hash_logbooks, + hash_record, + hash_redaction_profile, + hash_streams, +) +from cora.infrastructure.record_export._manifest import Manifest, build_manifest, capture_git_commit +from cora.infrastructure.record_export._redact_tier1 import ( + Tier1Redactor, + UnknownEventTypeError, + redact_tier1_payload, +) +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + TIER2_JSONB_CLEARED_POINTERS, + TIER2_JSONB_DROPPED_COLUMNS, + UnfiredClearanceError, + ensure_all_clearances_fired, + redact_tier2_row, +) +from cora.infrastructure.record_export._redaction import ( + RedactedRecord, + RedactionProfileMismatchError, + RedactionResult, + redact_record, +) +from cora.infrastructure.record_export._registry import ( + EntriesReader, + EntriesTableSpec, + UnknownLogbookKindError, + all_specs, + registered_envelope_classes, + resolve, +) +from cora.infrastructure.record_export._render import render_row, render_value +from cora.infrastructure.record_export._stream_types import ( + KNOWN_STREAM_TYPES, + UnknownStreamTypeError, + ensure_stream_type_known, +) +from cora.infrastructure.record_export._tokens import TokenMap + +__all__ = [ + "KNOWN_STREAM_TYPES", + "LOGBOOKS_PAYLOAD_TYPE", + "RECORD_PAYLOAD_TYPE", + "REDACTION_PROFILE_PAYLOAD_TYPE", + "STREAMS_PAYLOAD_TYPE", + "TIER2_DISPOSITIONS", + "TIER2_JSONB_CLEARED_POINTERS", + "TIER2_JSONB_DROPPED_COLUMNS", + "EmptyExportError", + "EntriesReader", + "EntriesTableSpec", + "ExportedRecord", + "Manifest", + "RedactedRecord", + "RedactionProfileMismatchError", + "RedactionResult", + "Tier1Redactor", + "TokenMap", + "UnfiredClearanceError", + "UnknownEventTypeError", + "UnknownLogbookKindError", + "UnknownStreamTypeError", + "all_specs", + "build_manifest", + "capture_git_commit", + "capture_watermark", + "ensure_all_clearances_fired", + "ensure_stream_type_known", + "export_record", + "hash_logbooks", + "hash_record", + "hash_redaction_profile", + "hash_streams", + "redact_record", + "redact_tier1_payload", + "redact_tier2_row", + "registered_envelope_classes", + "render_row", + "render_value", + "resolve", +] diff --git a/apps/api/src/cora/infrastructure/record_export/_dispositions.py b/apps/api/src/cora/infrastructure/record_export/_dispositions.py new file mode 100644 index 00000000000..a1eef305e12 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_dispositions.py @@ -0,0 +1,1819 @@ +"""Generated redaction dispositions. DO NOT EDIT BY HAND. + +Regenerate with `make record-dispositions`; +`tests/architecture/test_record_dispositions_drift.py` fails until you do. + +One entry per event type, one disposition per declared field, resolved +from the field's real type by `tools/gen_record_dispositions.py`. The +vocabulary: + + keep:enum: closed value set, provably reviewable. The enum is + NAMED because a human signs off the value set, and + swapping one enum for another must read as drift. + keep:number int / float / bool + keep:time datetime + token:uuid replaced with a per-export random surrogate + drop:text free text, no finite range, dropped by default + drop:opaque a dict with no declared keys, nothing to allowlist + by-value the slot is polymorphic across scalars and objects, + so no static answer exists. Apply the tier-2 leaf + rule at export time: numbers and booleans keep, + UUID-shaped strings token, other strings drop. + +A nested mapping is a value object recursed into. A mapping whose sole +key is `[]` is a fixed-length heterogeneous tuple, and its value lists +one disposition per position. + +Redaction iterates the STORED payload's keys and looks each up here. A +key absent from its event's entry is dropped; an event type absent from +this table aborts the export. The canonical hash of this mapping is the +redaction profile hash recorded in the export manifest. +""" + +from typing import Any + +DISPOSITIONS: dict[str, dict[str, Any]] = { + "AcquisitionRecorded": { + "acquisition_id": "token:uuid", + "captured_at": "keep:time", + "dataset_id": "token:uuid", + "evidence": "drop:opaque", + "occurred_at": "keep:time", + "producing_asset_id": "token:uuid", + "producing_run_id": "token:uuid", + "recorded_by": "token:uuid", + "settings": "drop:opaque", + }, + "ActorDeactivated": {"actor_id": "token:uuid", "occurred_at": "keep:time"}, + "ActorProfileForgotten": {"actor_id": "token:uuid", "occurred_at": "keep:time"}, + "ActorReactivated": {"actor_id": "token:uuid", "occurred_at": "keep:time"}, + "ActorRegistered": { + "actor_id": "token:uuid", + "kind": "keep:enum:ActorKind", + "occurred_at": "keep:time", + }, + "AgentBudgetUpdated": { + "agent_id": "token:uuid", + "daily_token_cap": "keep:number", + "monthly_usd_cap": "keep:number", + "occurred_at": "keep:time", + }, + "AgentDefined": { + "agent_id": "token:uuid", + "canonical_uri": "drop:text", + "capabilities": "drop:text", + "daily_token_cap": "keep:number", + "description": "drop:text", + "kind": "drop:text", + "model_ref": {"model": "drop:text", "provider": "drop:text", "snapshot_pin": "drop:text"}, + "monthly_usd_cap": "keep:number", + "name": "drop:text", + "occurred_at": "keep:time", + "prompt_template_id": "token:uuid", + "tools": "drop:text", + "version": "drop:text", + }, + "AgentDeprecated": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AgentResumed": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "resumed_by": "token:uuid", + }, + "AgentSuspended": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "suspended_by": "token:uuid", + }, + "AgentTargetPlanUpdated": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "target_plan_id": "token:uuid", + }, + "AgentToolGranted": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "tool_name": "drop:text", + }, + "AgentToolRevoked": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "tool_name": "drop:text", + }, + "AgentVersioned": { + "agent_id": "token:uuid", + "occurred_at": "keep:time", + "version": "drop:text", + }, + "AllocationActivated": { + "activated_by": "token:uuid", + "allocation_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AllocationCeilingUpdated": { + "allocation_id": "token:uuid", + "ceiling_usd": "keep:number", + "occurred_at": "keep:time", + }, + "AllocationGranted": { + "allocation_id": "token:uuid", + "campaign_id": "token:uuid", + "ceiling_usd": "keep:number", + "granted_by": "token:uuid", + "note": "drop:text", + "occurred_at": "keep:time", + }, + "AllocationSealed": { + "allocation_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "sealed_by": "token:uuid", + "spent_usd": "keep:number", + }, + "AllocationVoided": { + "allocation_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AssemblyDefined": { + "assembly_id": "token:uuid", + "content_hash": "drop:text", + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "name": {"value": "drop:text"}, + "occurred_at": "keep:time", + "parameter_overrides_schema": "drop:opaque", + "presents_as": "token:uuid", + "required_slots": { + "cardinality": "keep:enum:SlotCardinality", + "default_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "default_settings": "drop:opaque", + "required_family_ids": "token:uuid", + "slot_name": {"value": "drop:text"}, + }, + "required_sub_assemblies": { + "content_hash": "drop:text", + "slot_name": {"value": "drop:text"}, + "sub_assembly_id": "token:uuid", + }, + "required_wires": { + "source_port_name": "drop:text", + "source_slot_name": "drop:text", + "target_port_name": "drop:text", + "target_slot_name": "drop:text", + }, + "version": "drop:text", + }, + "AssemblyDeprecated": { + "assembly_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AssemblyPresentsAsAdded": { + "assembly_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "AssemblyPresentsAsRemoved": { + "assembly_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "AssemblyVersioned": { + "assembly_id": "token:uuid", + "content_hash": "drop:text", + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "name": {"value": "drop:text"}, + "occurred_at": "keep:time", + "parameter_overrides_schema": "drop:opaque", + "presents_as": "token:uuid", + "previous_content_hash": "drop:text", + "required_slots": { + "cardinality": "keep:enum:SlotCardinality", + "default_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "default_settings": "drop:opaque", + "required_family_ids": "token:uuid", + "slot_name": {"value": "drop:text"}, + }, + "required_sub_assemblies": { + "content_hash": "drop:text", + "slot_name": {"value": "drop:text"}, + "sub_assembly_id": "token:uuid", + }, + "required_wires": { + "source_port_name": "drop:text", + "source_slot_name": "drop:text", + "target_port_name": "drop:text", + "target_slot_name": "drop:text", + }, + "version": "drop:text", + }, + "AssetActivated": {"asset_id": "token:uuid", "occurred_at": "keep:time"}, + "AssetAlternateIdentifierAdded": { + "alternate_identifier": {"kind": "keep:enum:AlternateIdentifierKind", "value": "drop:text"}, + "asset_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetAlternateIdentifierRemoved": { + "alternate_identifier": {"kind": "keep:enum:AlternateIdentifierKind", "value": "drop:text"}, + "asset_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetAttachedToFixture": { + "asset_id": "token:uuid", + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetDecommissioned": { + "asset_id": "token:uuid", + "decommissioned_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "AssetDegraded": {"asset_id": "token:uuid", "occurred_at": "keep:time", "reason": "drop:text"}, + "AssetDetachedFromFixture": { + "asset_id": "token:uuid", + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetFacilityCodeAssigned": { + "asset_id": "token:uuid", + "assigned_by": "token:uuid", + "facility_code": {"value": "drop:text"}, + "occurred_at": "keep:time", + }, + "AssetFamilyAdded": { + "asset_id": "token:uuid", + "family_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetFamilyRemoved": { + "asset_id": "token:uuid", + "family_id": "token:uuid", + "occurred_at": "keep:time", + }, + "AssetFaulted": {"asset_id": "token:uuid", "occurred_at": "keep:time", "reason": "drop:text"}, + "AssetMaintenanceEntered": {"asset_id": "token:uuid", "occurred_at": "keep:time"}, + "AssetMaintenanceExited": {"asset_id": "token:uuid", "occurred_at": "keep:time"}, + "AssetOwnerAdded": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "owner": { + "contact": {"value": "drop:text"}, + "identifier": {"value": "drop:text"}, + "identifier_type": {"value": "drop:text"}, + "name": {"value": "drop:text"}, + }, + }, + "AssetOwnerRemoved": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "owner_name": {"value": "drop:text"}, + }, + "AssetPartitionRuleUpdated": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "partition_rule": { + "aggregator_kind": "keep:enum:AggregatorKind", + "calibration_id": "token:uuid", + "calibration_revision_id": "token:uuid", + "constituent_count": "keep:number", + "extrapolation_kind": "keep:enum:ExtrapolationKind", + "gain": "keep:number", + "interpolation_kind": "keep:enum:InterpolationKind", + "invertible": "keep:number", + "kind": "keep:enum", + "offset": "keep:number", + "partition_kind": "keep:enum:PartitionKind", + "partition_parameters": {"[]": ["drop:text", "keep:number"]}, + "readback_aggregator_kind": "keep:enum:ReadbackAggregatorKind", + "residual_tolerance_limit": "keep:number", + "singularity_threshold": "keep:number", + "solver_id": "drop:text", + "solver_transport_kind": "keep:enum:SolverTransportKind", + "solver_version": "drop:text", + "unit_in": "drop:text", + "unit_out": "drop:text", + }, + }, + "AssetPersistentIdAssigned": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "persistent_id_scheme": "drop:text", + "persistent_id_value": "drop:text", + }, + "AssetPortAdded": { + "asset_id": "token:uuid", + "direction": "drop:text", + "occurred_at": "keep:time", + "port_name": "drop:text", + "signal_type": "drop:text", + }, + "AssetPortRemoved": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "port_name": "drop:text", + }, + "AssetRegistered": { + "alternate_identifiers": { + "kind": "keep:enum:AlternateIdentifierKind", + "value": "drop:text", + }, + "asset_id": "token:uuid", + "commissioned_by": "token:uuid", + "controller_id": "token:uuid", + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "facility_code": {"value": "drop:text"}, + "located_in_enclosure_id": "token:uuid", + "model_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "owners": { + "contact": {"value": "drop:text"}, + "identifier": {"value": "drop:text"}, + "identifier_type": {"value": "drop:text"}, + "name": {"value": "drop:text"}, + }, + "parent_id": "token:uuid", + "tier": "drop:text", + }, + "AssetRelocated": { + "asset_id": "token:uuid", + "from_parent_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "to_parent_id": "token:uuid", + }, + "AssetRestored": {"asset_id": "token:uuid", "occurred_at": "keep:time", "reason": "drop:text"}, + "AssetSettingsUpdated": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "settings": "drop:opaque", + }, + "AttestationRecorded": { + "attestation_id": "token:uuid", + "attested_by": "token:uuid", + "dataset_id": "token:uuid", + "distribution_id": "token:uuid", + "evidence": "drop:opaque", + "kind": "drop:text", + "occurred_at": "keep:time", + "outcome": "drop:text", + }, + "CalibrationDefined": { + "calibration_id": "token:uuid", + "defined_by": "token:uuid", + "description": "drop:text", + "occurred_at": "keep:time", + "operating_point": "drop:opaque", + "quantity": "drop:text", + "target_id": "token:uuid", + }, + "CalibrationRevisionAppended": { + "asserted_by": "token:uuid", + "calibration_id": "token:uuid", + "content_hash": "drop:text", + "decided_by_decision_id": "token:uuid", + "established_at": "keep:time", + "established_by": "token:uuid", + "occurred_at": "keep:time", + "revision_id": "token:uuid", + "source_dataset_id": "token:uuid", + "source_procedure_id": "token:uuid", + "status": "keep:enum:CalibrationStatus", + "supersedes_revision_id": "token:uuid", + "value": "drop:opaque", + }, + "CalibrationRevisionPublished": { + "calibration_id": "token:uuid", + "occurred_at": "keep:time", + "outbound_permit_id": "token:uuid", + "publication_status": "drop:text", + "published_at": "keep:time", + "published_by": "token:uuid", + "receipt_id": "token:uuid", + "revision_id": "token:uuid", + "signature_bytes_hex": "drop:text", + "signature_envelope_kind": "drop:text", + "signature_kid": "drop:text", + "signing_version": "drop:text", + }, + "CampaignAbandoned": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "CampaignClosed": {"campaign_id": "token:uuid", "occurred_at": "keep:time"}, + "CampaignHeld": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "CampaignRegistered": { + "campaign_id": "token:uuid", + "description": "drop:text", + "external_id": "drop:text", + "external_refs": {"scheme": "drop:text", "value": "drop:text"}, + "intent": "drop:text", + "lead_actor_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "subject_id": "token:uuid", + "tags": "drop:text", + }, + "CampaignResumed": {"campaign_id": "token:uuid", "occurred_at": "keep:time"}, + "CampaignRunAdded": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "CampaignRunRemoved": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "CampaignStarted": {"campaign_id": "token:uuid", "occurred_at": "keep:time"}, + "CampaignSteeringDeclared": { + "campaign_id": "token:uuid", + "objective": { + "kind": "keep:enum:SteeringObjectiveKind", + "target_measurement_name": "drop:text", + "target_value": "keep:number", + }, + "occurred_at": "keep:time", + "space": { + "axes": { + "choices": "drop:opaque", + "lower": "keep:number", + "name": "drop:text", + "upper": "keep:number", + } + }, + }, + "CapabilityDefined": { + "capability_id": "token:uuid", + "code": "drop:text", + "description": "drop:text", + "executor_shapes": "keep:enum:ExecutorShape", + "name": "drop:text", + "occurred_at": "keep:time", + "parameters_schema": "drop:opaque", + "required_affordances": "keep:enum:Affordance", + }, + "CapabilityDeprecated": { + "capability_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "replaced_by_capability_id": "token:uuid", + }, + "CapabilitySuggestedRolesUpdated": { + "capability_id": "token:uuid", + "occurred_at": "keep:time", + "suggested_role_ids": "token:uuid", + }, + "CapabilityVersioned": { + "capability_id": "token:uuid", + "description": "drop:text", + "executor_shapes": "keep:enum:ExecutorShape", + "occurred_at": "keep:time", + "parameters_schema": "drop:opaque", + "required_affordances": "keep:enum:Affordance", + "version_tag": "drop:text", + }, + "CautionAcknowledgement": { + "category": "drop:text", + "caution_id": "token:uuid", + "severity": "drop:text", + "target_id": "token:uuid", + "target_kind": "drop:text", + "text_excerpt": "drop:text", + "workaround_excerpt": "drop:text", + }, + "CautionRegistered": { + "authored_by": "token:uuid", + "category": "drop:text", + "caution_id": "token:uuid", + "expires_at": "keep:time", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "propagate_to_children": "keep:number", + "severity": "drop:text", + "tags": "drop:text", + "target": {"asset_id": "token:uuid", "procedure_id": "token:uuid"}, + "text": "drop:text", + "workaround": "drop:text", + }, + "CautionRetired": { + "caution_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "CautionSuperseded": { + "caution_id": "token:uuid", + "occurred_at": "keep:time", + "superseded_by_caution_id": "token:uuid", + }, + "ClearanceActivated": {"clearance_id": "token:uuid", "occurred_at": "keep:time"}, + "ClearanceApproved": { + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + "valid_from": "keep:time", + "valid_until": "keep:time", + }, + "ClearanceExpired": { + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "ClearanceRegistered": { + "bindings": "drop:opaque", + "clearance_id": "token:uuid", + "declarations": "drop:opaque", + "external_id": "drop:text", + "facility_code": "drop:text", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "risk_band": "drop:text", + "template_code": "drop:text", + "template_id": "token:uuid", + "title": "drop:text", + "valid_from": "keep:time", + "valid_until": "keep:time", + }, + "ClearanceRejected": { + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "ClearanceReviewStarted": { + "clearance_id": "token:uuid", + "first_reviewer_role": "drop:text", + "occurred_at": "keep:time", + }, + "ClearanceReviewStepAppended": { + "clearance_id": "token:uuid", + "decided_at": "keep:time", + "decided_by": "token:uuid", + "decision": "drop:text", + "notes": "drop:text", + "occurred_at": "keep:time", + "role": "drop:text", + "step_index": "keep:number", + }, + "ClearanceSubmitted": {"clearance_id": "token:uuid", "occurred_at": "keep:time"}, + "ClearanceSuperseded": { + "by_clearance_id": "token:uuid", + "clearance_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ClearanceTemplateActivated": { + "activated_by": "token:uuid", + "occurred_at": "keep:time", + "template_id": "token:uuid", + }, + "ClearanceTemplateDefined": { + "code": "drop:text", + "defined_by": "token:uuid", + "external_ref": "drop:text", + "facility_code": "drop:text", + "occurred_at": "keep:time", + "supersedes_template_id": "token:uuid", + "template_id": "token:uuid", + "title": "drop:text", + "version": "keep:number", + }, + "ClearanceTemplateDeprecated": { + "deprecated_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "template_id": "token:uuid", + }, + "ClearanceTemplateVersioned": { + "new_version": "keep:number", + "occurred_at": "keep:time", + "supersedes_template_id": "token:uuid", + "template_id": "token:uuid", + "versioned_by": "token:uuid", + }, + "ClearanceTemplateWithdrawn": { + "occurred_at": "keep:time", + "reason": "drop:text", + "template_id": "token:uuid", + "withdrawn_by": "token:uuid", + }, + "ConduitDefined": { + "conduit_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "source_zone_id": "token:uuid", + "target_zone_id": "token:uuid", + }, + "ConduitLogbookClosed": { + "conduit_id": "token:uuid", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ConduitLogbookOpened": { + "conduit_id": "token:uuid", + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "CredentialRegistered": { + "audience": "drop:text", + "credential_id": "token:uuid", + "expires_at": "keep:time", + "facility_code": {"value": "drop:text"}, + "occurred_at": "keep:time", + "public_material_ref": "drop:text", + "purpose": "keep:enum:CredentialPurpose", + "registered_by": "token:uuid", + "secret_ref": "drop:text", + }, + "CredentialRevoked": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "revoked_by": "token:uuid", + }, + "CredentialRotationAborted": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "rotation_aborted_by": "token:uuid", + }, + "CredentialRotationCompleted": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "rotation_completed_by": "token:uuid", + }, + "CredentialRotationStarted": { + "credential_id": "token:uuid", + "occurred_at": "keep:time", + "pending_public_material_ref": "drop:text", + "pending_secret_ref": "drop:text", + "rotation_started_by": "token:uuid", + }, + "DatasetDemoted": { + "dataset_id": "token:uuid", + "demoted_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "DatasetDiscarded": { + "dataset_id": "token:uuid", + "discarded_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "DatasetPromoted": { + "dataset_id": "token:uuid", + "occurred_at": "keep:time", + "promoted_by": "token:uuid", + "reason": "drop:text", + }, + "DatasetRegistered": { + "byte_size": "keep:number", + "checksum_algorithm": "drop:text", + "checksum_value": "drop:text", + "conforms_to": "drop:text", + "dataset_id": "token:uuid", + "derived_from": "token:uuid", + "intent": "drop:text", + "media_type": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "producing_actuation_kind": "drop:text", + "producing_procedure_id": "token:uuid", + "producing_run_end_state": "drop:text", + "producing_run_id": "token:uuid", + "registered_by": "token:uuid", + "subject_id": "token:uuid", + "uri": "drop:text", + "used_calibration_ids": "token:uuid", + }, + "DecisionDebriefRequested": { + "debriefer_agent_id": "token:uuid", + "debriefer_kind": "drop:text", + "occurred_at": "keep:time", + "run_id": "token:uuid", + "terminal_event_id": "token:uuid", + }, + "DecisionLogbookClosed": { + "decision_id": "token:uuid", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + }, + "DecisionLogbookOpened": { + "decision_id": "token:uuid", + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "DecisionRated": { + "comment": "drop:text", + "confidence_at_rating": "keep:number", + "decision_id": "token:uuid", + "occurred_at": "keep:time", + "rated_at": "keep:time", + "rated_by": "token:uuid", + "rating": "keep:enum:DecisionRating", + }, + "DecisionRegistered": { + "alternatives": "drop:text", + "choice": "drop:text", + "confidence": "keep:number", + "confidence_source": "keep:enum:DecisionConfidenceSource", + "context": "drop:text", + "decided_by": "token:uuid", + "decision_id": "token:uuid", + "inputs": "drop:opaque", + "occurred_at": "keep:time", + "override_kind": "keep:enum", + "parent_id": "token:uuid", + "reasoning": "drop:text", + "reasoning_signature": "drop:text", + "rule": "drop:text", + }, + "DistributionDiscarded": { + "discarded_by": "token:uuid", + "distribution_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "DistributionRegistered": { + "access_protocol": "drop:text", + "byte_size": "keep:number", + "checksum_algorithm": "drop:text", + "checksum_value": "drop:text", + "conforms_to": "drop:text", + "dataset_id": "token:uuid", + "distribution_id": "token:uuid", + "media_type": "drop:text", + "occurred_at": "keep:time", + "registered_by": "token:uuid", + "supply_id": "token:uuid", + "uri": "drop:text", + }, + "EditionDatasetAdded": { + "added_by": "token:uuid", + "dataset_id": "token:uuid", + "edition_id": "token:uuid", + "occurred_at": "keep:time", + }, + "EditionDatasetRemoved": { + "dataset_id": "token:uuid", + "edition_id": "token:uuid", + "occurred_at": "keep:time", + "removed_by": "token:uuid", + }, + "EditionPublished": { + "edition_id": "token:uuid", + "external_pid_scheme": "drop:text", + "external_pid_value": "drop:text", + "occurred_at": "keep:time", + "published_by": "token:uuid", + "published_content_hash": "drop:text", + }, + "EditionRegistered": { + "creators": "drop:opaque", + "dataset_ids": "token:uuid", + "edition_id": "token:uuid", + "kind": "drop:text", + "license": "drop:text", + "occurred_at": "keep:time", + "publication_year": "keep:number", + "publisher_facility_code": "drop:text", + "registered_by": "token:uuid", + "title": "drop:text", + }, + "EditionSealed": { + "content_hash": "drop:text", + "edition_id": "token:uuid", + "license": "drop:text", + "occurred_at": "keep:time", + "publication_year": "keep:number", + "publisher_facility_code": "drop:text", + "sealed_by": "token:uuid", + "sealed_dataset_ids": "token:uuid", + }, + "EditionWithdrawn": { + "edition_id": "token:uuid", + "occurred_at": "keep:time", + "withdrawal_reason": "drop:text", + "withdrawn_by": "token:uuid", + }, + "EnclosureDecommissioned": { + "enclosure_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "triggered_by": "token:uuid", + }, + "EnclosurePermitObserved": { + "enclosure_id": "token:uuid", + "from_status": "drop:text", + "monitor_ref": "drop:text", + "observed_at": "keep:time", + "occurred_at": "keep:time", + "reason": "drop:text", + "to_status": "drop:text", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "EnclosureRegistered": { + "enclosure_id": "token:uuid", + "facility_code": {"value": "drop:text"}, + "name": "drop:text", + "occurred_at": "keep:time", + "registered_by": "token:uuid", + }, + "FacilityDecommissioned": { + "decommissioned_by": "token:uuid", + "facility_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "FacilityRegistered": { + "alternate_identifiers": { + "kind": "keep:enum:AlternateIdentifierKind", + "value": "drop:text", + }, + "code": {"value": "drop:text"}, + "display_name": "drop:text", + "facility_id": "token:uuid", + "kind": "keep:enum:FacilityKind", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "registered_by": "token:uuid", + }, + "FacilityTrustAnchorCredentialAdded": { + "added_by": "token:uuid", + "credential_id": "token:uuid", + "facility_id": "token:uuid", + "occurred_at": "keep:time", + }, + "FacilityTrustAnchorCredentialRemoved": { + "credential_id": "token:uuid", + "facility_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "removed_by": "token:uuid", + }, + "FamilyDefined": { + "affordances": "keep:enum:Affordance", + "family_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + }, + "FamilyDeprecated": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "FamilyPresentsAsAdded": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "FamilyPresentsAsRemoved": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "role_id": "token:uuid", + }, + "FamilySettingsSchemaUpdated": { + "family_id": "token:uuid", + "occurred_at": "keep:time", + "settings_schema": "drop:opaque", + }, + "FamilyVersioned": { + "affordances": "keep:enum:Affordance", + "family_id": "token:uuid", + "occurred_at": "keep:time", + "version_tag": "drop:text", + }, + "FixturePersistentIdAssigned": { + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + "persistent_id_scheme": "drop:text", + "persistent_id_value": "drop:text", + }, + "FixtureRegistered": { + "assembly_content_hash": "drop:text", + "assembly_id": "token:uuid", + "fixture_id": "token:uuid", + "occurred_at": "keep:time", + "parameter_overrides": "drop:opaque", + "registered_by": "token:uuid", + "slot_asset_bindings": {"asset_id": "token:uuid", "slot_name": "drop:text"}, + "surface_id": "token:uuid", + }, + "FrameDecommissioned": { + "frame_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "FramePlacementUpdated": { + "frame_id": "token:uuid", + "new_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "occurred_at": "keep:time", + "survey": "drop:opaque", + }, + "FrameRegistered": { + "frame_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "supersedes": { + "predecessor_frame_id": "token:uuid", + "transform_from_predecessor": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + }, + }, + "HoldClaimReleased": { + "cause": "drop:text", + "claim_id": "token:uuid", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "LanguageModelApproved": {"language_model_id": "token:uuid", "occurred_at": "keep:time"}, + "LanguageModelDefined": { + "archivability": "drop:text", + "cost_basis": "drop:opaque", + "data_tier": "drop:text", + "endpoint_note": "drop:text", + "language_model_id": "token:uuid", + "model": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "provider": "drop:text", + "served_via": "drop:text", + "snapshot_pin": "drop:text", + }, + "LanguageModelDeprecated": { + "language_model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "LanguageModelRetired": { + "language_model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "LanguageModelRetirementAnnounced": { + "effective_at": "keep:time", + "language_model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MethodDefined": { + "capability_id": "token:uuid", + "execution_pattern": "keep:enum:ExecutionPattern", + "method_id": "token:uuid", + "monotone_quality": "keep:number", + "name": "drop:text", + "needed_assembly_ids": "token:uuid", + "needed_family_ids": "token:uuid", + "needed_input_kinds": "drop:text", + "needed_supplies": "drop:text", + "occurred_at": "keep:time", + "resumable_from_checkpoint": "keep:number", + }, + "MethodDeprecated": { + "method_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MethodLaunchSpecUpdated": { + "launch_spec": "drop:opaque", + "method_id": "token:uuid", + "occurred_at": "keep:time", + }, + "MethodParametersSchemaUpdated": { + "method_id": "token:uuid", + "occurred_at": "keep:time", + "parameters_schema": "drop:opaque", + }, + "MethodRequiredRoleAdded": { + "family_id": "token:uuid", + "method_id": "token:uuid", + "occurred_at": "keep:time", + "optional": "keep:number", + "required_ports": "drop:opaque", + "role_kind": "token:uuid", + "role_name": "drop:text", + }, + "MethodRequiredRoleRemoved": { + "method_id": "token:uuid", + "occurred_at": "keep:time", + "role_name": "drop:text", + }, + "MethodVersioned": { + "content_hash": "drop:text", + "method_id": "token:uuid", + "occurred_at": "keep:time", + "version_tag": "drop:text", + }, + "ModelDefined": { + "declared_family_ids": "token:uuid", + "manufacturer": { + "identifier": {"value": "drop:text"}, + "identifier_type": "keep:enum:ManufacturerIdentifierType", + "name": {"value": "drop:text"}, + }, + "model_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "part_number": "drop:text", + "version_tag": "drop:text", + }, + "ModelDeprecated": { + "model_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "ModelFamilyAdded": { + "family_id": "token:uuid", + "model_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ModelFamilyRemoved": { + "family_id": "token:uuid", + "model_id": "token:uuid", + "occurred_at": "keep:time", + }, + "ModelVersioned": { + "declared_family_ids": "token:uuid", + "manufacturer": { + "identifier": {"value": "drop:text"}, + "identifier_type": "keep:enum:ManufacturerIdentifierType", + "name": {"value": "drop:text"}, + }, + "model_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "part_number": "drop:text", + "version_tag": "drop:text", + }, + "MountAssetInstalled": { + "asset_id": "token:uuid", + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "previously_installed_asset_id": "token:uuid", + }, + "MountAssetUninstalled": { + "asset_id": "token:uuid", + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MountDecommissioned": { + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + }, + "MountPlacementUpdated": { + "mount_id": "token:uuid", + "new_placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "occurred_at": "keep:time", + "survey": "drop:opaque", + }, + "MountRegistered": { + "drawing": { + "number": "drop:text", + "revision": "drop:text", + "system": "keep:enum:DrawingSystem", + }, + "mount_id": "token:uuid", + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "placement": { + "parent_frame_id": "token:uuid", + "reference_surface": "keep:enum:ReferenceSurface", + "rx": "keep:number", + "ry": "keep:number", + "rz": "keep:number", + "tol_rx": "keep:number", + "tol_ry": "keep:number", + "tol_rz": "keep:number", + "tol_x": "keep:number", + "tol_y": "keep:number", + "tol_z": "keep:number", + "units": "keep:enum:UnitSystem", + "x": "keep:number", + "y": "keep:number", + "z": "keep:number", + }, + "slot_code": "drop:text", + }, + "PermitActivated": { + "activated_by": "token:uuid", + "occurred_at": "keep:time", + "permit_id": "token:uuid", + }, + "PermitDefined": { + "abi_tier_floor": "keep:enum:AbiTier", + "allowed_artifact_kinds": "drop:text", + "allowed_credential_ids": "token:uuid", + "allowed_payload_types": "drop:text", + "defined_by": "token:uuid", + "direction": "keep:enum:Direction", + "expires_at": "keep:time", + "occurred_at": "keep:time", + "peer_facility_code": {"value": "drop:text"}, + "permit_id": "token:uuid", + "terms": { + "accepted_canonicalization_versions": "drop:text", + "inbound_allowed_artifact_kinds": "drop:text", + "onward_action_scope": "keep:enum:OnwardActionScope", + "publisher_grant_correlation_handle": "drop:text", + "read_scope": "keep:enum:ReadScope", + "required_receipt_kinds": "keep:enum:ReceiptKind", + "scopes": {"kind": "drop:text", "name": "drop:text", "qualifier": "drop:text"}, + }, + }, + "PermitResumed": { + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "resumed_by": "token:uuid", + }, + "PermitRevoked": { + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "reason": "drop:text", + "revoked_by": "token:uuid", + }, + "PermitSuspended": { + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "reason": "drop:text", + "suspended_by": "token:uuid", + }, + "PlanDefaultParametersUpdated": { + "default_parameters": "drop:opaque", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + }, + "PlanDefined": { + "asset_families_snapshot": "drop:opaque", + "asset_ids": "token:uuid", + "method_id": "token:uuid", + "method_needed_family_ids_snapshot": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "practice_id": "token:uuid", + }, + "PlanDeprecated": {"occurred_at": "keep:time", "plan_id": "token:uuid", "reason": "drop:text"}, + "PlanRoleBound": { + "asset_id": "token:uuid", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "role_name": "drop:text", + }, + "PlanRoleUnbound": { + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "role_name": "drop:text", + }, + "PlanVersioned": { + "content_hash": "drop:text", + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "version_tag": "drop:text", + }, + "PlanWireAdded": { + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "source_asset_id": "token:uuid", + "source_port_name": "drop:text", + "target_asset_id": "token:uuid", + "target_port_name": "drop:text", + }, + "PlanWireRemoved": { + "occurred_at": "keep:time", + "plan_id": "token:uuid", + "source_asset_id": "token:uuid", + "source_port_name": "drop:text", + "target_asset_id": "token:uuid", + "target_port_name": "drop:text", + }, + "PolicyDefined": { + "conduit_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "permitted_commands": "drop:text", + "permitted_principal_ids": "token:uuid", + "policy_id": "token:uuid", + "surface_id": "token:uuid", + }, + "PolicyGrantRevoked": { + "occurred_at": "keep:time", + "policy_id": "token:uuid", + "principal_id": "token:uuid", + "reason": "drop:text", + "revoked_by": "token:uuid", + }, + "PracticeDefined": { + "method_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "practice_id": "token:uuid", + "site_id": "token:uuid", + }, + "PracticeDeprecated": { + "occurred_at": "keep:time", + "practice_id": "token:uuid", + "reason": "drop:text", + }, + "PracticeVersioned": { + "occurred_at": "keep:time", + "practice_id": "token:uuid", + "version_tag": "drop:text", + }, + "ProcedureAborted": { + "actuation_kind": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + }, + "ProcedureActivitiesLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "ProcedureCompleted": { + "actuation_kind": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + }, + "ProcedureDiagnosticLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "ProcedureHeld": { + "actuation_kind": "drop:text", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + }, + "ProcedureIterationEnded": { + "advised_next_point": "drop:opaque", + "advised_stop": "keep:number", + "alternatives": "drop:text", + "confidence": "keep:number", + "confidence_source": "keep:enum:DecisionConfidenceSource", + "converged": "keep:number", + "iteration_index": "keep:number", + "model_ref": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + "reasoning": "drop:text", + }, + "ProcedureIterationStarted": { + "iteration_index": "keep:number", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + }, + "ProcedureOutcomeLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "ProcedureRegistered": { + "capability_id": "token:uuid", + "kind": "drop:text", + "max_consecutive_unconverged_iterations": "keep:number", + "name": "drop:text", + "occurred_at": "keep:time", + "parent_run_id": "token:uuid", + "procedure_id": "token:uuid", + "recipe_id": "token:uuid", + "target_asset_ids": "token:uuid", + }, + "ProcedureResumed": { + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "re_establishment_boundary": "keep:number", + }, + "ProcedureStarted": {"occurred_at": "keep:time", "procedure_id": "token:uuid"}, + "ProcedureTruncated": { + "interrupted_at": "keep:time", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "reason": "drop:text", + }, + "PublicationReceiptRecorded": { + "content_hash": "drop:text", + "home_artifact_id": "token:uuid", + "home_stream_id": "token:uuid", + "home_stream_type": "drop:text", + "occurred_at": "keep:time", + "permit_id": "token:uuid", + "receipt_id": "token:uuid", + "recorded_at": "keep:time", + }, + "RatificationDenied": { + "occurred_at": "keep:time", + "ratification_id": "token:uuid", + "reason": "drop:text", + }, + "RatificationGranted": {"occurred_at": "keep:time", "ratification_id": "token:uuid"}, + "RatificationRequested": { + "command_name": "drop:text", + "consequence_class": "drop:text", + "occurred_at": "keep:time", + "ratification_id": "token:uuid", + "requested_by": "token:uuid", + "target_action_id": "token:uuid", + }, + "RecipeDefined": { + "capability_id": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "recipe_id": "token:uuid", + "steps": { + "address": "drop:text", + "capture_name": "drop:text", + "command": "drop:text", + "criterion": "drop:opaque", + "input_uris": "by-value", + "name": "drop:text", + "output_ref_name": "drop:text", + "output_uri": "drop:text", + "parameters": "drop:opaque", + "params": "drop:opaque", + "value": "by-value", + "verify": "keep:number", + }, + }, + "RecipeDeprecated": { + "occurred_at": "keep:time", + "reason": "drop:text", + "recipe_id": "token:uuid", + "replaced_by_recipe_id": "token:uuid", + }, + "RecipeExpansionRecorded": { + "bindings": "drop:opaque", + "bindings_hash": "drop:text", + "capability_id": "token:uuid", + "capability_version": "drop:text", + "expansion_port_version": "drop:text", + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "recipe_id": "token:uuid", + "recipe_version": "drop:text", + "step_count": "keep:number", + "steps_hash": "drop:text", + }, + "RecipeVersioned": { + "occurred_at": "keep:time", + "recipe_id": "token:uuid", + "steps": { + "address": "drop:text", + "capture_name": "drop:text", + "command": "drop:text", + "criterion": "drop:opaque", + "input_uris": "by-value", + "name": "drop:text", + "output_ref_name": "drop:text", + "output_uri": "drop:text", + "parameters": "drop:opaque", + "params": "drop:opaque", + "value": "by-value", + "verify": "keep:number", + }, + "version_tag": "drop:text", + }, + "ResolvedStepsRecorded": { + "occurred_at": "keep:time", + "procedure_id": "token:uuid", + "resolved_steps": "drop:opaque", + "step_count": "keep:number", + }, + "RoleDefined": { + "consumes": "drop:text", + "docstring": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "optional_affordances": "keep:enum:Affordance", + "produces": "drop:text", + "required_affordances": "keep:enum:Affordance", + "role_id": "token:uuid", + }, + "RunAborted": { + "actuation_kind": "drop:text", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "producing_job_id": "drop:text", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunAddedToCampaign": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "RunAdjusted": { + "adjusted_by": "token:uuid", + "decided_by_decision_id": "token:uuid", + "effective_parameters": "drop:opaque", + "occurred_at": "keep:time", + "parameters_patch": "drop:opaque", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunCompleted": { + "actuation_kind": "drop:text", + "artifact_uri": "drop:text", + "occurred_at": "keep:time", + "producing_job_id": "drop:text", + "run_id": "token:uuid", + }, + "RunHeld": { + "cause": "drop:text", + "claim_id": "token:uuid", + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + }, + "RunObservationLogbookOpened": { + "kind": "drop:text", + "logbook_id": "token:uuid", + "occurred_at": "keep:time", + "run_id": "token:uuid", + "schema": {"description": "drop:text", "fields": "drop:opaque"}, + }, + "RunRemovedFromCampaign": { + "campaign_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunResumed": { + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "released_claim_id": "token:uuid", + "run_id": "token:uuid", + }, + "RunStarted": { + "acknowledged_cautions": { + "category": "drop:text", + "caution_id": "token:uuid", + "severity": "drop:text", + "target_id": "token:uuid", + "target_kind": "drop:text", + "text_excerpt": "drop:text", + "workaround_excerpt": "drop:text", + }, + "campaign_id": "token:uuid", + "decided_by_decision_id": "token:uuid", + "effective_parameters": "drop:opaque", + "external_refs": "drop:opaque", + "input_dataset_ids": "token:uuid", + "name": "drop:text", + "occurred_at": "keep:time", + "override_parameters": "drop:opaque", + "pinned_calibration_ids": "token:uuid", + "plan_id": "token:uuid", + "raid": "drop:text", + "run_id": "token:uuid", + "subject_id": "token:uuid", + "trigger_source": "drop:text", + }, + "RunStopped": { + "decided_by_decision_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "RunTruncated": { + "decided_by_decision_id": "token:uuid", + "interrupted_at": "keep:time", + "occurred_at": "keep:time", + "reason": "drop:text", + "run_id": "token:uuid", + }, + "SealInitialized": { + "facility_code": {"value": "drop:text"}, + "initialized_by": "token:uuid", + "occurred_at": "keep:time", + "offline_credential_id": "token:uuid", + "online_credential_id": "token:uuid", + }, + "SealOnlineKeyRotated": { + "facility_code": {"value": "drop:text"}, + "new_online_credential_id": "token:uuid", + "occurred_at": "keep:time", + "rotated_by": "token:uuid", + "signed_by_offline_root": "keep:number", + }, + "SealPointerSigned": { + "facility_code": {"value": "drop:text"}, + "head_hash": "drop:text", + "occurred_at": "keep:time", + "sequence_number": "keep:number", + "signed_at": "keep:time", + "signed_by": "token:uuid", + }, + "SealRepublishingCompleted": { + "completed_by": "token:uuid", + "facility_code": {"value": "drop:text"}, + "new_head_hash": "drop:text", + "new_sequence_number": "keep:number", + "occurred_at": "keep:time", + }, + "SealRepublishingStarted": { + "facility_code": {"value": "drop:text"}, + "occurred_at": "keep:time", + "reason": "drop:text", + "started_by": "token:uuid", + }, + "SubjectDiscarded": { + "discarded_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "subject_id": "token:uuid", + }, + "SubjectDismounted": { + "dismounted_by": "token:uuid", + "from_asset_id": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "subject_id": "token:uuid", + }, + "SubjectMeasured": { + "measured_by": "token:uuid", + "occurred_at": "keep:time", + "subject_id": "token:uuid", + }, + "SubjectMounted": { + "asset_id": "token:uuid", + "mounted_by": "token:uuid", + "occurred_at": "keep:time", + "reason": "drop:text", + "subject_id": "token:uuid", + }, + "SubjectRegistered": { + "name": "drop:text", + "occurred_at": "keep:time", + "registered_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SubjectRemoved": { + "occurred_at": "keep:time", + "removed_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SubjectReturned": { + "occurred_at": "keep:time", + "returned_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SubjectStored": { + "occurred_at": "keep:time", + "stored_by": "token:uuid", + "subject_id": "token:uuid", + }, + "SupplyDegraded": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyDeregistered": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyMarkedAvailable": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyMarkedRecovering": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyMarkedUnavailable": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyRegistered": { + "containing_asset_id": "token:uuid", + "facility_code": {"value": "drop:text"}, + "kind": "drop:text", + "name": "drop:text", + "occurred_at": "keep:time", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SupplyRestored": { + "from_status": "drop:text", + "monitor_ref": "drop:text", + "occurred_at": "keep:time", + "reason": "drop:text", + "supply_id": "token:uuid", + "trigger": "drop:text", + "triggered_by": "token:uuid", + }, + "SurfaceDefined": { + "kind": "keep:enum:SurfaceKind", + "name": "drop:text", + "occurred_at": "keep:time", + "surface_id": "token:uuid", + }, + "VisitAborted": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "VisitArrived": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitCancelled": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "VisitCheckedIn": { + "actor_id": "token:uuid", + "mode": "drop:text", + "occurred_at": "keep:time", + "visit_id": "token:uuid", + }, + "VisitCheckedOut": { + "actor_id": "token:uuid", + "occurred_at": "keep:time", + "visit_id": "token:uuid", + }, + "VisitCompleted": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitHeld": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "VisitPresenceClosed": { + "actor_id": "token:uuid", + "occurred_at": "keep:time", + "visit_id": "token:uuid", + }, + "VisitRegistered": { + "external_refs": {"scheme": "drop:text", "value": "drop:text"}, + "occurred_at": "keep:time", + "parent_id": "token:uuid", + "planned_end_at": "keep:time", + "planned_start_at": "keep:time", + "policy_id": "token:uuid", + "surface_id": "token:uuid", + "type": "drop:text", + "visit_id": "token:uuid", + }, + "VisitResumed": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitStarted": {"occurred_at": "keep:time", "visit_id": "token:uuid"}, + "VisitSurfaceControlReleased": { + "occurred_at": "keep:time", + "surface_id": "token:uuid", + "visit_id": "token:uuid", + }, + "VisitSurfaceControlTaken": { + "occurred_at": "keep:time", + "surface_id": "token:uuid", + "visit_id": "token:uuid", + }, + "VisitVoided": {"occurred_at": "keep:time", "reason": "drop:text", "visit_id": "token:uuid"}, + "ZoneDefined": {"name": "drop:text", "occurred_at": "keep:time", "zone_id": "token:uuid"}, +} diff --git a/apps/api/src/cora/infrastructure/record_export/_export.py b/apps/api/src/cora/infrastructure/record_export/_export.py new file mode 100644 index 00000000000..fb505187a89 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_export.py @@ -0,0 +1,133 @@ +"""Walk the event store and follow every logbook envelope into its entries. + +Per `project_record_export_v3.md` F0/F2: a single stream query over the +whole `events` table, ordered by `(transaction_id, position)`, bounded by +one `pg_snapshot_xmin(pg_current_snapshot())` watermark captured up front +(not re-evaluated per row, unlike the projection worker's catch-up query, +which this SQL otherwise mirrors) so the export sees one consistent +snapshot. On each row whose `event_type` names a `*LogbookOpened` class +(`registered_envelope_classes()`, from Step 1's `_registry`), `kind` and +`logbook_id` come straight out of the already-decoded `payload` dict and +resolve through Step 1's `resolve()` to the matching entries-tier reader. + +Deliberately does not: write bundle files (that is Step 3/4's job, once +hashing and the manifest exist), or touch `entries_run_feed_heartbeats` / +`entries_enclosure_permit_probes` (they have no envelope event to trigger +them from an envelope-driven walk; see `project_record_export_build_brief.md` +step 2 notes -- an explicit, written deferral, not an omission). +""" + +from dataclasses import dataclass +from uuid import UUID + +import asyncpg + +from cora.infrastructure.record_export._registry import registered_envelope_classes, resolve +from cora.infrastructure.record_export._render import render_row +from cora.infrastructure.record_export._stream_types import ensure_stream_type_known + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# asyncpg's stubs are loose; suppress at module level, matching the other +# entries-table readers. + +_WATERMARK_SQL = "SELECT pg_snapshot_xmin(pg_current_snapshot())::text" + +# Same column list as postgres_event_store.py's _LOAD_SQL and +# projection/worker.py's _ADVANCE_SQL. transaction_id is xid8; asyncpg has +# no output codec for it (same gap those two modules work around), so it +# is cast to text and aliased back onto its own name. +_STREAM_SQL = """ +SELECT position, event_id, stream_type, stream_id, version, event_type, + schema_version, payload, metadata, correlation_id, causation_id, + principal_id, occurred_at, recorded_at, + signature, signature_kid, signature_version, + transaction_id::text AS transaction_id +FROM events +WHERE transaction_id < $1::xid8 +ORDER BY transaction_id, position +""" + + +class EmptyExportError(RuntimeError): + """Zero rows exported. + + Per the build brief's acceptance criteria: a bundle claiming to be + the record with nothing in it is a bug in the caller (wrong + database, unmigrated database, watermark set before anything was + written), never a legitimate empty result to hand back quietly. + """ + + +@dataclass(frozen=True, slots=True) +class ExportedRecord: + """One export's two tiers, rendered but not yet hashed or redacted. + + `streams` is every `events` row below the captured watermark, in + `(transaction_id, position)` order. `logbooks` groups every + entries-tier row pulled via an envelope by `kind`, in each kind's own + registry order-by order; rows are UNFOLDED (one dict per row), never + aggregated. + """ + + streams: tuple[dict[str, object], ...] + logbooks: dict[str, tuple[dict[str, object], ...]] + + +async def capture_watermark(conn: asyncpg.Connection) -> int: + """The xmin bound for one consistent export snapshot. + + Returns a plain `int`: asyncpg has no xid8 codec on either side, so + (mirroring `projection/worker.py`'s bookmark parameter, the existing + precedent for binding a value against an `xid8` column) the value is + cast to `text` on the way out here and to `int` on the way back in, + then bound as `$1::xid8` by the caller -- never compared as a string. + + Callers must pass the SAME returned value to `export_record`'s + underlying query exactly once; capturing it here rather than letting + the stream query re-evaluate `pg_snapshot_xmin` per row is what makes + one export see one snapshot instead of a moving target. + """ + value = await conn.fetchval(_WATERMARK_SQL) + assert value is not None, "pg_snapshot_xmin(pg_current_snapshot()) returned NULL" + return int(value) + + +async def export_record(conn: asyncpg.Connection) -> ExportedRecord: + """Walk the whole event store and its envelope-linked logbooks. + + Raises `EmptyExportError` on zero stream rows, + `cora.infrastructure.record_export.UnknownStreamTypeError` on a + `stream_type` outside the declared closed set (refuses immediately; + never skips), and `UnknownLogbookKindError` on an envelope's `kind` + with no registry entry. + """ + watermark = await capture_watermark(conn) + rows = await conn.fetch(_STREAM_SQL, watermark) + if not rows: + raise EmptyExportError( + "export_record found zero events.rows below the captured " + f"watermark ({watermark!r}); an empty record is never a " + "valid export." + ) + + envelope_classes = registered_envelope_classes() + streams: list[dict[str, object]] = [] + logbooks: dict[str, list[dict[str, object]]] = {} + + for row in rows: + ensure_stream_type_known(row["stream_type"]) + streams.append(render_row(row)) + + if row["event_type"] not in envelope_classes: + continue + payload = row["payload"] + kind = payload["kind"] + logbook_id = UUID(payload["logbook_id"]) + spec = resolve(kind) + entries = await spec.reader(conn, logbook_id) + logbooks.setdefault(kind, []).extend(render_row(entry) for entry in entries) + + return ExportedRecord( + streams=tuple(streams), + logbooks={kind: tuple(entries) for kind, entries in logbooks.items()}, + ) diff --git a/apps/api/src/cora/infrastructure/record_export/_hashing.py b/apps/api/src/cora/infrastructure/record_export/_hashing.py new file mode 100644 index 00000000000..4a5dea26eef --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_hashing.py @@ -0,0 +1,129 @@ +"""Hash an exported record with `cora.shared.content_hash`, no exclusions. + +Per `project_record_export_v3.md` F6: one canonicalization, `content_hash`'s +profile, for both serialization and hashing. `canonical_json` (used +elsewhere in the codebase) is a DIFFERENT recipe; mixing the two means the +committed file is not the bytes the hash covers. `cora.shared` is exactly +what `tach.toml` allows `cora.infrastructure` to depend on, so this import +is the dependency the layering rule exists to permit. + +No exclusions: every rendered stream row and every rendered logbook row +goes into the hashed body. Re-exporting the same database twice is +measured stable (`test_hashing.py`'s integration test); a regenerated +record hashing differently across genuinely different content is content +addressing working, not a defect, and cross-record comparison is the +shape layer's job, not this module's. + +Payload types follow the scheme +`application/vnd.cora.+json` (`content_hash.py`'s own +convention) and the three names `project_record_export_v3.md`'s "Naming" +section already picked: `record`, `record-streams`, `record-logbooks`. +That section calls `record+json` the manifest's type; nothing writes a +manifest yet (step 4), so there is no live collision today. Used here for +the whole-bundle hash because `{streams, logbooks}` together *is* the +record the manifest will describe -- if step 4 wants manifest.json to +carry its own distinct wrapping type when it starts writing bytes to +disk, that is a decision for whoever builds it, made with an actual +manifest body in hand rather than guessed at here. +""" + +from cora.infrastructure.record_export._dispositions import DISPOSITIONS +from cora.infrastructure.record_export._export import ExportedRecord +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + TIER2_JSONB_CLEARED_POINTERS, + TIER2_JSONB_DROPPED_COLUMNS, +) +from cora.shared.content_hash import compute_content_hash + +RECORD_PAYLOAD_TYPE = "application/vnd.cora.record+json" +STREAMS_PAYLOAD_TYPE = "application/vnd.cora.record-streams+json" +LOGBOOKS_PAYLOAD_TYPE = "application/vnd.cora.record-logbooks+json" +REDACTION_PROFILE_PAYLOAD_TYPE = "application/vnd.cora.record-redaction-profile+json" + + +def hash_streams(streams: tuple[dict[str, object], ...]) -> str: + """SHA-256 content hash over the streams tier alone, in its own order.""" + return compute_content_hash(STREAMS_PAYLOAD_TYPE, list(streams)) + + +def hash_logbooks(logbooks: dict[str, tuple[dict[str, object], ...]]) -> str: + """SHA-256 content hash over the logbooks tier alone. + + Kind keys are NFC-normalized and sort themselves via + `json.dumps(sort_keys=True)`; each kind's row order is preserved + (the registry's own order-by), never re-sorted. + """ + body = {kind: list(rows) for kind, rows in logbooks.items()} + return compute_content_hash(LOGBOOKS_PAYLOAD_TYPE, body) + + +def hash_record(record: ExportedRecord) -> str: + """SHA-256 content hash over the whole bundle, both tiers, no exclusions. + + This is THE record hash: per F2, it covers everything `export_record` + produced. Re-running the export against an unchanged database + reproduces this value exactly; any single differing byte anywhere in + either tier changes it. + """ + body = { + "streams": list(record.streams), + "logbooks": {kind: list(rows) for kind, rows in record.logbooks.items()}, + } + return compute_content_hash(RECORD_PAYLOAD_TYPE, body) + + +def hash_redaction_profile() -> str: + """SHA-256 content hash over every table that decides what a + published record discloses: tier 1's generated `DISPOSITIONS` AND + tier 2's hand-authored `TIER2_DISPOSITIONS` / + `TIER2_JSONB_CLEARED_POINTERS` / `TIER2_JSONB_DROPPED_COLUMNS`. + + This IS the redaction profile hash (H2). Step 7's security re-review + found the tier-2 tables missing from this hash: the fail-closed + switch (`redact_record`'s `expected_redaction_profile_hash` check) + was fail-closed for tier 1 only, silently blind to a tier-2 table + edit that weakened a disposition (e.g. `conduit_verdicts.reason` + `DROP` -> `KEEP`) or dropped a jsonb clearance restriction. Both + tiers must be in H2, or "the hash matches" does not mean what + `RedactionProfileMismatchError`'s docstring claims it means. + + Tuple-keyed dicts (`TIER2_JSONB_CLEARED_POINTERS` / + `TIER2_JSONB_DROPPED_COLUMNS`) are flattened to `"kind/column"` + string keys before hashing rather than relying on + `compute_content_hash`'s `str(key)` fallback for non-string Mapping + keys, which would hash a Python tuple's `repr()` instead of a + reviewable string. + + Regenerating tier 1's table via `make record-dispositions` after a + real event-model change, or hand-editing tier 2's tables, is + expected to change this value; `test_record_dispositions_drift.py` + guards tier 1's generator output specifically, and + `test_redact_tier2.py`'s live-schema drift test guards tier 2's + column coverage, but only THIS hash is what a caller's + `expected_redaction_profile_hash` actually pins. + """ + body = { + "tier1": DISPOSITIONS, + "tier2_dispositions": TIER2_DISPOSITIONS, + "tier2_jsonb_cleared_pointers": { + f"{kind}/{column}": sorted(pointers) + for (kind, column), pointers in TIER2_JSONB_CLEARED_POINTERS.items() + }, + "tier2_jsonb_dropped_columns": sorted( + f"{kind}/{column}" for kind, column in TIER2_JSONB_DROPPED_COLUMNS + ), + } + return compute_content_hash(REDACTION_PROFILE_PAYLOAD_TYPE, body) + + +__all__ = [ + "LOGBOOKS_PAYLOAD_TYPE", + "RECORD_PAYLOAD_TYPE", + "REDACTION_PROFILE_PAYLOAD_TYPE", + "STREAMS_PAYLOAD_TYPE", + "hash_logbooks", + "hash_record", + "hash_redaction_profile", + "hash_streams", +] diff --git a/apps/api/src/cora/infrastructure/record_export/_leaf_rule.py b/apps/api/src/cora/infrastructure/record_export/_leaf_rule.py new file mode 100644 index 00000000000..b8685c34ce6 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_leaf_rule.py @@ -0,0 +1,115 @@ +"""The generic recursive leaf rule. + +Shared by tier-1's `by-value` polymorphic fields (a slot that can hold a +scalar or a value object -- `_dispositions.py`'s `by-value` disposition) +and tier-2's four recursing jsonb columns (`activities.payload`, +`diagnostics.payload`, `outcomes.point`, `outcomes.measurements`). Per +`project_record_export_v3.md` F5: numeric and boolean leaves KEEP, +UUID-shaped string leaves TOKEN, every other string leaf DROPS unless +its `(column, json-pointer)` is on a caller-supplied cleared list. +Object KEYS are always published -- in practice they are field names +and they carry the structure -- only VALUES are ever redacted. + +This walker does not know, or need to know, what produced the value it +is walking (which `step_kind`, which Recipe step type, ...). That is +deliberate: a nested shape nobody enumerated a clearance for simply +drops by the same default every other unlisted field drops by, rather +than needing a special case per producer. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# Walks `Any`-typed, already-rendered JSON values; suppressed the same +# way cora.shared.content_hash._canonicalize is, for the same reason. + +import re +from typing import Any + +from cora.infrastructure.record_export._tokens import TokenMap + +OMITTED = object() +"""Sentinel: the caller must omit this key (dict) or this element (list) +entirely, not store `None` in its place.""" + +_UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def is_uuid_shaped(value: str) -> bool: + return bool(_UUID_RE.match(value)) + + +def _join_pointer(pointer: str, segment: str) -> str: + return segment if not pointer else f"{pointer}/{segment}" + + +def apply_leaf_rule( + value: Any, + *, + token_map: TokenMap, + cleared_pointers: frozenset[str] = frozenset(), + pointer: str = "", + fired_pointers: set[str] | None = None, +) -> Any: + """Recursively redact `value`. + + Returns a redacted `dict`/`list` (same shape, dropped keys/elements + omitted) for a container, a redacted scalar for a leaf, or `OMITTED` + if the top-level value itself is a leaf that must be omitted by the + caller (dict/list values are never themselves `OMITTED` -- an empty + dict/list is a valid redacted result and is returned as such). + + `fired_pointers`, when supplied, collects every `pointer` whose + value was actually kept via `cleared_pointers` (not via the + unconditional number/bool/None/UUID branches) -- the record needed + to check a cleared list ever matched anything, per F5's + unfired-clearance rejection. + """ + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, sub_value in value.items(): + child_pointer = _join_pointer(pointer, str(key)) + redacted = apply_leaf_rule( + sub_value, + token_map=token_map, + cleared_pointers=cleared_pointers, + pointer=child_pointer, + fired_pointers=fired_pointers, + ) + if redacted is not OMITTED: + result[key] = redacted + return result + + if isinstance(value, (list, tuple)): + child_pointer = _join_pointer(pointer, "*") + out: list[Any] = [] + for item in value: + redacted = apply_leaf_rule( + item, + token_map=token_map, + cleared_pointers=cleared_pointers, + pointer=child_pointer, + fired_pointers=fired_pointers, + ) + if redacted is not OMITTED: + out.append(redacted) + return out + + # Leaf values. + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + if is_uuid_shaped(value): + return token_map.token_uuid(value) + if pointer in cleared_pointers: + if fired_pointers is not None: + fired_pointers.add(pointer) + return value + return OMITTED + + # An already-rendered export body (per step 2's F6) never contains + # anything else; fail closed rather than guess at a new type's intent. + return OMITTED + + +__all__ = ["OMITTED", "apply_leaf_rule", "is_uuid_shaped"] diff --git a/apps/api/src/cora/infrastructure/record_export/_manifest.py b/apps/api/src/cora/infrastructure/record_export/_manifest.py new file mode 100644 index 00000000000..7bbdb8fa65f --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_manifest.py @@ -0,0 +1,150 @@ +"""The export manifest: git commit, watermark, both profile hashes, and +the per-kind / per-event-type / per-run facts a reader needs before +trusting the bundle. + +Per `project_record_export_build_brief.md` step 4 and +`project_record_export_v3.md` F8. "Both profile hashes" are the two +namable today: `record_hash` (H1, the whole unredacted bundle, built in +step 3) and `redaction_profile_hash` (H2, step 0's generated disposition +table -- "the table's canonical hash IS the redaction profile hash", +computed here because nothing computed it before this step existed). H3 +(the published/redacted record's hash) does not exist until step 6 +builds redaction. + +`build_manifest` is pure: every input it needs (`git_commit`, +`watermark`) is captured by the caller first and passed in, so the +function itself does no I/O and is trivial to test with synthetic +`ExportedRecord`s. +""" + +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from cora.infrastructure.record_export._export import ExportedRecord +from cora.infrastructure.record_export._hashing import hash_record, hash_redaction_profile + + +@dataclass(frozen=True, slots=True) +class Manifest: + """One export's provenance and shape, independent of its bytes on disk. + + `row_count_by_logbook_kind` and `max_schema_version_by_event_type` + are reader-facing sanity checks: a reader can recompute both from + the bundle itself and compare, catching truncation or a stale + generator without needing to trust this manifest blindly. + """ + + git_commit: str + watermark: int + record_hash: str + redaction_profile_hash: str + row_count_by_logbook_kind: dict[str, int] + max_schema_version_by_event_type: dict[str, int] + is_simulated: bool + expansion_digest_presence_by_run: dict[str, bool] + + +def capture_git_commit(*, cwd: Path | str | None = None) -> str: + """The exporting checkout's HEAD commit SHA. + + No caching, no fallback: a failure here (detached submodule, no + `.git`, corrupt repo) should stop the export rather than write a + manifest that lies about provenance. + """ + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def _require_str(value: object) -> str: + assert isinstance(value, str) + return value + + +def _payload(row: dict[str, object]) -> dict[str, object]: + payload = row["payload"] + assert isinstance(payload, dict) + return cast("dict[str, object]", payload) + + +def _row_count_by_logbook_kind(record: ExportedRecord) -> dict[str, int]: + return {kind: len(rows) for kind, rows in record.logbooks.items()} + + +def _max_schema_version_by_event_type(record: ExportedRecord) -> dict[str, int]: + versions: dict[str, int] = {} + for row in record.streams: + event_type = row["event_type"] + schema_version = row["schema_version"] + assert isinstance(event_type, str) + assert isinstance(schema_version, int) + if schema_version > versions.get(event_type, 0): + versions[event_type] = schema_version + return versions + + +def _is_simulated(record: ExportedRecord) -> bool: + """True unless an observation row explicitly says otherwise. + + Vacuously True when the export carries no observation rows at all: + nothing in the bundle contradicts "this is a simulated record". A + mixed result (some True, some False) reports as False rather than + raising -- the manifest's job is to report the fact, not gate on it. + """ + observations = record.logbooks.get("observation", ()) + return all(row["is_simulated"] is True for row in observations) + + +def _expansion_digest_presence_by_run(record: ExportedRecord) -> dict[str, bool]: + """Per F8: a run has a pinned expansion digest iff at least one of its + child Procedures was registered via `register_procedure_from_recipe` + (carries a `RecipeExpansionRecorded` on its own stream). A Procedure + registered directly, or a run recorded by observing an external + scan, has no digest to compare against; that is correct, not a gap. + """ + run_ids = { + _require_str(row["stream_id"]) for row in record.streams if row["stream_type"] == "Run" + } + + parent_run_by_procedure: dict[str, str | None] = {} + expanded_procedures: set[str] = set() + for row in record.streams: + if row["event_type"] == "ProcedureRegistered": + payload = _payload(row) + parent_run_id = payload["parent_run_id"] + parent_run_by_procedure[_require_str(payload["procedure_id"])] = ( + None if parent_run_id is None else _require_str(parent_run_id) + ) + elif row["event_type"] == "RecipeExpansionRecorded": + expanded_procedures.add(_require_str(_payload(row)["procedure_id"])) + return { + run_id: any( + parent_run_id == run_id and procedure_id in expanded_procedures + for procedure_id, parent_run_id in parent_run_by_procedure.items() + ) + for run_id in run_ids + } + + +def build_manifest(record: ExportedRecord, *, watermark: int, git_commit: str) -> Manifest: + """Assemble the manifest for one already-exported, already-rendered record.""" + return Manifest( + git_commit=git_commit, + watermark=watermark, + record_hash=hash_record(record), + redaction_profile_hash=hash_redaction_profile(), + row_count_by_logbook_kind=_row_count_by_logbook_kind(record), + max_schema_version_by_event_type=_max_schema_version_by_event_type(record), + is_simulated=_is_simulated(record), + expansion_digest_presence_by_run=_expansion_digest_presence_by_run(record), + ) + + +__all__ = ["Manifest", "build_manifest", "capture_git_commit"] diff --git a/apps/api/src/cora/infrastructure/record_export/_redact_tier1.py b/apps/api/src/cora/infrastructure/record_export/_redact_tier1.py new file mode 100644 index 00000000000..5d95da780ef --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_redact_tier1.py @@ -0,0 +1,181 @@ +"""Redact `events` rows: fixed columns plus disposition-table-driven payload. + +Per `project_record_export_build_brief.md` step 6 and +`project_record_export_v3.md` F5. + +Two DIFFERENT failure modes, easy to conflate (the brief's own step-6 +notes and its terser Rejections list read ambiguously against each +other on this point; this resolves it the way that keeps schema +evolution possible): + +- `event_type` absent from `_dispositions.DISPOSITIONS` entirely -> + ABORT (`UnknownEventTypeError`). Step 0's generator is exhaustive + over every currently-declared event class; a stream carrying a type + the table has never heard of means the table is stale relative to + the code, which is a build problem, not a per-row one. +- A payload KEY present on a row but absent from + `DISPOSITIONS[event_type]`'s own field list -> DROP (omit the key). + This is what makes schema evolution survivable: an OLDER + `schema_version`'s row can carry a field the CURRENT dataclass no + longer declares (removed in a later version), and dropping it rather + than aborting the whole export is the graceful path. This is also + exactly what "a bare str on a NEW event drops with nobody editing a + list" is really about at the OTHER end: Step 0's generator + auto-classifies every new bare `str` field as `drop:text` the moment + it is generated, so the field already has a rule (drop) with no + manual list-editing required; it is not the "missing key" case. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# Dispatches on `Any`-typed disposition values and already-rendered +# JSON payloads; suppressed the same way cora.shared.content_hash is. + +from typing import Any + +from cora.infrastructure.record_export._dispositions import DISPOSITIONS +from cora.infrastructure.record_export._leaf_rule import OMITTED, apply_leaf_rule +from cora.infrastructure.record_export._tokens import TokenMap + +_FIXED_KEEP_COLUMNS = ( + "schema_version", + "stream_type", + "event_type", + "occurred_at", + "recorded_at", +) +_FIXED_TOKEN_COLUMNS = ( + "stream_id", + "correlation_id", + "causation_id", + "event_id", + "principal_id", +) +_FIXED_DROP_COLUMNS = ("metadata", "signature", "signature_kid", "signature_version") + + +class UnknownEventTypeError(LookupError): + """`event_type` has no entry in `_dispositions.DISPOSITIONS` at all. + + Refuses loudly: the disposition table is exhaustive over every + currently-declared event class, so this means the table is stale + relative to the code, not that this one row should be skipped. + """ + + def __init__(self, event_type: str) -> None: + super().__init__( + f"event_type {event_type!r} has no entry in DISPOSITIONS; the " + "table is stale relative to the code (run `make record-dispositions`)." + ) + self.event_type = event_type + + +def _apply_field_disposition(disposition: Any, value: Any, *, token_map: TokenMap) -> Any: + if isinstance(disposition, dict): + if "[]" in disposition: + # Fixed-length heterogeneous tuple: one disposition per position. + per_position = disposition["[]"] + if not isinstance(value, (list, tuple)): + return OMITTED + return [ + _apply_field_disposition(pos_disposition, item, token_map=token_map) + for pos_disposition, item in zip(per_position, value, strict=True) + ] + # A recursed value object: apply this same per-key logic one level down. + if not isinstance(value, dict): + return OMITTED + result: dict[str, Any] = {} + for key, sub_disposition in disposition.items(): + if key not in value: + continue + redacted = _apply_field_disposition(sub_disposition, value[key], token_map=token_map) + if redacted is not OMITTED: + result[key] = redacted + return result + + if disposition == "by-value": + return apply_leaf_rule(value, token_map=token_map) + if disposition.startswith("keep:"): + return value + if disposition == "token:uuid": + # A UUID-collection field (e.g. `target_asset_ids`) carries + # `token:uuid` too, per Step 0's census: the disposition names + # the ELEMENT type, not the field's own cardinality. + if isinstance(value, (list, tuple)): + return [token_map.token_uuid(item) for item in value] + return token_map.token_uuid(value) + if disposition.startswith("drop:"): + return OMITTED + return OMITTED + + +def redact_tier1_payload( + event_type: str, payload: dict[str, Any], *, token_map: TokenMap +) -> dict[str, Any]: + """Redact one event's `payload`, iterating the STORED payload's own + keys (never the disposition table's), per F5's fail-closed property.""" + if event_type not in DISPOSITIONS: + raise UnknownEventTypeError(event_type) + field_dispositions = DISPOSITIONS[event_type] + + result: dict[str, Any] = {} + for key, value in payload.items(): + disposition = field_dispositions.get(key) + if disposition is None: + continue # known event type, unlisted field: schema-evolution DROP + redacted = _apply_field_disposition(disposition, value, token_map=token_map) + if redacted is not OMITTED: + result[key] = redacted + return result + + +class Tier1Redactor: + """Stateful per-export redactor for `events` rows. + + Must be fed rows in the SAME order `export_record` produced them + (`(transaction_id, position)`): `position`/`transaction_id` + re-indexing depends on that order, and `version` re-indexing depends + on rows for one `stream_id` arriving in stream order (already true + of the exporter's global order). + """ + + def __init__(self, token_map: TokenMap) -> None: + self._token_map = token_map + self._next_position = 1 + self._next_version_by_stream: dict[str, int] = {} + self._transaction_id_index: dict[str, int] = {} + self._next_transaction_id = 1 + + def _dense_version(self, raw_stream_id: str) -> int: + version = self._next_version_by_stream.get(raw_stream_id, 1) + self._next_version_by_stream[raw_stream_id] = version + 1 + return version + + def _dense_transaction_id(self, raw_transaction_id: object) -> int: + key = str(raw_transaction_id) + if key not in self._transaction_id_index: + self._transaction_id_index[key] = self._next_transaction_id + self._next_transaction_id += 1 + return self._transaction_id_index[key] + + def redact_row(self, row: dict[str, Any]) -> dict[str, Any]: + raw_stream_id = row["stream_id"] + redacted: dict[str, Any] = { + "position": self._next_position, + "version": self._dense_version(raw_stream_id), + "transaction_id": self._dense_transaction_id(row["transaction_id"]), + } + self._next_position += 1 + + for column in _FIXED_KEEP_COLUMNS: + redacted[column] = row[column] + for column in _FIXED_TOKEN_COLUMNS: + redacted[column] = self._token_map.token_uuid(row[column]) + # _FIXED_DROP_COLUMNS (metadata, signature*) intentionally absent. + + redacted["payload"] = redact_tier1_payload( + row["event_type"], row["payload"], token_map=self._token_map + ) + return redacted + + +__all__ = ["Tier1Redactor", "UnknownEventTypeError", "redact_tier1_payload"] diff --git a/apps/api/src/cora/infrastructure/record_export/_redact_tier2.py b/apps/api/src/cora/infrastructure/record_export/_redact_tier2.py new file mode 100644 index 00000000000..2167d52382c --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_redact_tier2.py @@ -0,0 +1,249 @@ +"""Redact `entries_*` rows: a hand-authored disposition table per kind. + +Transcribed from `project_record_export_v3.md` F5's three tables (28 +`text` columns across 8 tables, split PROVED CLOSED / JUDGED LOW RISK / +DROPPED; the 5 jsonb columns, 4 BY_VALUE + 1 dropped-whole). Unlike +tier 1, there is no generator here: F5 itself says tier 2 is small +enough (103 columns total) to hand-enumerate, and that hand-enumeration +is exactly what a fitness test elsewhere +(`test_record_export_registry_completeness.py`-style AST discovery does +not cover, since this table has no source annotation to derive from) +makes `test_redact_tier2.py` responsible for keeping honest against +`_registry.py`'s own table list. + +`BY_VALUE` reuses the exact string tier 1's locked, generated vocabulary +already uses for the same dispatch (`_dispositions.py`'s `"by-value"`): +both name the same operation (apply the generic leaf rule), and tier 2 +is the side free to conform since it is hand-authored, not generated. + +`kind` here is the Step 1 registry's kind (`"activity"`, `"verdict"`, +...), matching `_registry.EntriesTableSpec.kind`, not the raw table name. +""" + +from typing import Any + +from cora.infrastructure.record_export._leaf_rule import OMITTED, apply_leaf_rule +from cora.infrastructure.record_export._tokens import TokenMap + +KEEP = "keep" +TOKEN = "token" +DROP = "drop" +BY_VALUE = "by-value" + +# Every column of every kind's entries table. `BY_VALUE` columns are +# jsonb and are handled via TIER2_JSONB_CLEARED_POINTERS / +# TIER2_JSONB_DROPPED_COLUMNS below, never via this dict's own value. +TIER2_DISPOSITIONS: dict[str, dict[str, str]] = { + "verdict": { + "event_id": TOKEN, + "conduit_id": TOKEN, + "logbook_id": TOKEN, + "actor_id": TOKEN, + "command_name": KEEP, # judged low risk: written from code literals + "decision": KEEP, # proved closed: DB CHECK (decision IN ('Allow','Deny')) + "reason": DROP, # P0-4: builds f"Principal {principal_id} not in policy..." + "correlation_id": TOKEN, + "causation_id": TOKEN, + "occurred_at": KEEP, + "recorded_at": KEEP, + }, + "inference": { + "event_id": TOKEN, + "decision_id": TOKEN, + "logbook_id": TOKEN, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "occurred_at": KEEP, + "duration": KEEP, + "operation_name": KEEP, # judged low risk: names an operation, not a person + "provider_name": KEEP, + "request_model": KEEP, + "response_id": DROP, # correlator into an external vendor's records + "response_model": KEEP, + "request_temperature": KEEP, + "request_top_p": KEEP, + "request_max_tokens": KEEP, + "output_type": KEEP, + "finish_reasons": DROP, # text[]: array of str, drop-unless-cleared + "input_tokens": KEEP, + "output_tokens": KEEP, + "agent_id": DROP, # OTel gen_ai.agent.id; correlator, fail closed + "agent_name": DROP, # operator-authored free text + "agent_description": DROP, + "conversation_id": DROP, + "tool_name": KEEP, # judged low risk: names a tool, not a person + "tool_call_id": DROP, + "tool_type": KEEP, + "messages": BY_VALUE, # dropped whole, see TIER2_JSONB_DROPPED_COLUMNS + "recorded_at": KEEP, + "cost_usd": KEEP, + }, + "activity": { + "event_id": TOKEN, + "procedure_id": TOKEN, + "logbook_id": TOKEN, + "actor_id": TOKEN, + "command_name": KEEP, # judged low risk (3-of-8 group) + "step_kind": KEEP, # proved closed: Literal + STEP_KIND_VALUES + "payload": BY_VALUE, + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + }, + "diagnostic": { + "event_id": TOKEN, + "procedure_id": TOKEN, + "logbook_id": TOKEN, + "iteration_index": KEEP, + "model_ref": KEEP, # judged low risk: fed from advice.model_ref, "unknown" fallback + "payload": BY_VALUE, + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + }, + "outcome": { + "event_id": TOKEN, + "procedure_id": TOKEN, + "logbook_id": TOKEN, + "iteration_index": KEEP, + "point": BY_VALUE, + "measurements": BY_VALUE, + "succeeded": KEEP, + "actuation_kind": KEEP, # judged low risk: rehearsal-vs-live gate, closed by its writers + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + }, + "observation": { + "event_id": TOKEN, + "run_id": TOKEN, + "logbook_id": TOKEN, + "actor_id": TOKEN, + "command_name": KEEP, # judged low risk (3-of-8 group) + "channel_name": KEEP, # EPICS channel address, facility-fixed, already public + "value": KEEP, + "units": KEEP, # judged low risk: unvalidated, low risk, not closed + "sampling_procedure": KEEP, # proved closed: Literal + SAMPLING_PROCEDURE_VALUES + "sampled_at": KEEP, + "occurred_at": KEEP, + "correlation_id": TOKEN, + "causation_id": TOKEN, + "recorded_at": KEEP, + "is_simulated": KEEP, + }, + "heartbeat": { + "event_id": TOKEN, + "run_id": TOKEN, + "source_id": KEEP, # EPICS channel address, facility-fixed, already public + "heartbeat_at": KEEP, + "recorded_at": KEEP, + }, + "probe": { + "event_id": TOKEN, + "enclosure_id": TOKEN, + "source_kind": KEEP, # judged low risk, same standard as heartbeat.source_id + "source_id": DROP, # NOT cleared alongside its heartbeat twin, deliberately: + # pairing a reachability failure with the exact substrate address is + # closer to a security disclosure about a safety system than to science. + "reach_tier": KEEP, # proved closed: ReachTier StrEnum + "status_claimed": KEEP, # grouped with reach_tier; needs its own human pass (watch item) + "recorded_at": KEEP, + }, +} + +# (kind, column) -> the set of string-leaf json-pointers cleared inside +# that jsonb column. Absent entries default to an empty set (every +# string leaf drops). Pointers use "*" for "any list element", matching +# F5's own notation (`outcomes.measurements/*/name`). +TIER2_JSONB_CLEARED_POINTERS: dict[tuple[str, str], frozenset[str]] = { + ("activity", "payload"): frozenset({"channel", "action_name", "units"}), + ("outcome", "measurements"): frozenset({"*/name", "*/units", "*/kind", "*/quality"}), +} + +# (kind, column) pairs whose jsonb value drops WHOLE rather than recursing. +TIER2_JSONB_DROPPED_COLUMNS: frozenset[tuple[str, str]] = frozenset({("inference", "messages")}) + + +class UnfiredClearanceError(RuntimeError): + """A declared tier-2 jsonb clearance never matched a row in a kind + that WAS present in this export. + + Per F5's Rejections list: an unfired rule inside a type that was + exported is a leak-shaped gap (most likely a typo in this file's + own `TIER2_JSONB_CLEARED_POINTERS`), not a benign no-op. + """ + + def __init__(self, kind: str, column: str, pointer: str) -> None: + super().__init__( + f"declared clearance ({kind!r}, {column!r})/{pointer!r} never matched " + f"any row while redacting {kind!r} rows present in this export." + ) + self.kind = kind + self.column = column + self.pointer = pointer + + +def redact_tier2_row( + kind: str, + row: dict[str, Any], + *, + token_map: TokenMap, + fired_pointers: dict[tuple[str, str], set[str]], +) -> dict[str, Any]: + """Redact one entries row of the given registry `kind`.""" + column_dispositions = TIER2_DISPOSITIONS[kind] + result: dict[str, Any] = {} + for column, value in row.items(): + disposition = column_dispositions.get(column) + if disposition is None: + # An entries column not enumerated above. Fail closed rather + # than silently pass it through: this is this file's own + # drift-detection surface (see test_redact_tier2.py). + continue + if disposition == KEEP: + result[column] = value + elif disposition == TOKEN: + result[column] = token_map.token_uuid(value) + elif disposition == DROP: + continue + elif disposition == BY_VALUE: + if (kind, column) in TIER2_JSONB_DROPPED_COLUMNS: + continue + cleared = TIER2_JSONB_CLEARED_POINTERS.get((kind, column), frozenset()) + fired = fired_pointers.setdefault((kind, column), set()) + redacted = apply_leaf_rule( + value, token_map=token_map, cleared_pointers=cleared, fired_pointers=fired + ) + if redacted is not OMITTED: + result[column] = redacted + return result + + +def ensure_all_clearances_fired( + fired_pointers: dict[tuple[str, str], set[str]], *, kinds_present: frozenset[str] +) -> None: + """Raise `UnfiredClearanceError` for a declared clearance that never + matched, scoped to kinds actually present in this export (an unused + clearance for a kind with zero rows is not a leak -- nothing to leak).""" + for (kind, column), cleared in TIER2_JSONB_CLEARED_POINTERS.items(): + if kind not in kinds_present: + continue + fired = fired_pointers.get((kind, column), set()) + for pointer in cleared - fired: + raise UnfiredClearanceError(kind, column, pointer) + + +__all__ = [ + "TIER2_DISPOSITIONS", + "TIER2_JSONB_CLEARED_POINTERS", + "TIER2_JSONB_DROPPED_COLUMNS", + "UnfiredClearanceError", + "ensure_all_clearances_fired", + "redact_tier2_row", +] diff --git a/apps/api/src/cora/infrastructure/record_export/_redaction.py b/apps/api/src/cora/infrastructure/record_export/_redaction.py new file mode 100644 index 00000000000..6a16ccf93e6 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_redaction.py @@ -0,0 +1,102 @@ +"""Redact a whole `ExportedRecord`: the fail-closed switch plus both tiers. + +Per `project_record_export_v3.md` F5's fail-closed switch: the exporter +refuses to produce a bundle marked publishable unless a redaction +profile is supplied and its hash matches. `redact_record` computes +`hash_redaction_profile()` itself and compares against the caller- +supplied `expected_redaction_profile_hash` BEFORE touching a single row, +so a stale or substituted disposition table aborts rather than silently +redacting under the wrong rules. +""" + +from dataclasses import dataclass + +from cora.infrastructure.record_export._export import ExportedRecord +from cora.infrastructure.record_export._hashing import hash_redaction_profile +from cora.infrastructure.record_export._redact_tier1 import Tier1Redactor, UnknownEventTypeError +from cora.infrastructure.record_export._redact_tier2 import ( + ensure_all_clearances_fired, + redact_tier2_row, +) +from cora.infrastructure.record_export._tokens import TokenMap + +__all__ = [ + "RedactedRecord", + "RedactionProfileMismatchError", + "RedactionResult", + "UnknownEventTypeError", + "redact_record", +] + + +class RedactionProfileMismatchError(RuntimeError): + """`expected_redaction_profile_hash` does not match the disposition + table this checkout would actually redact with. + + Refuses rather than redacting under an unverified or stale table: + the exact failure mode the fail-closed switch exists to catch. + """ + + def __init__(self, expected: str, actual: str) -> None: + super().__init__( + f"redaction profile mismatch: expected {expected!r}, this checkout's " + f"disposition table hashes to {actual!r}. Refusing to redact." + ) + self.expected = expected + self.actual = actual + + +@dataclass(frozen=True, slots=True) +class RedactedRecord: + """The published projection of an `ExportedRecord`: same two-tier + shape, every value passed through F5's dispositions.""" + + streams: tuple[dict[str, object], ...] + logbooks: dict[str, tuple[dict[str, object], ...]] + + +@dataclass(frozen=True, slots=True) +class RedactionResult: + """`redacted_record` is safe to hash as the published record. + `token_map` is an artifact of THIS export, retained separately under + H1's obligation; it must never be shipped or fed into anything + hashed as the published record.""" + + redacted_record: RedactedRecord + token_map: TokenMap + + +def redact_record( + record: ExportedRecord, *, expected_redaction_profile_hash: str +) -> RedactionResult: + """Redact both tiers of `record` under one shared `TokenMap`. + + Raises `RedactionProfileMismatchError` before touching any row if + the hash does not match; `UnknownEventTypeError` if a stream row's + `event_type` has no entry in the disposition table at all; + `UnfiredClearanceError` if a declared tier-2 jsonb clearance never + matched a row in a kind present in this export. + """ + actual_hash = hash_redaction_profile() + if expected_redaction_profile_hash != actual_hash: + raise RedactionProfileMismatchError(expected_redaction_profile_hash, actual_hash) + + token_map = TokenMap() + + tier1 = Tier1Redactor(token_map) + redacted_streams = tuple(tier1.redact_row(row) for row in record.streams) + + fired_pointers: dict[tuple[str, str], set[str]] = {} + redacted_logbooks = { + kind: tuple( + redact_tier2_row(kind, row, token_map=token_map, fired_pointers=fired_pointers) + for row in rows + ) + for kind, rows in record.logbooks.items() + } + ensure_all_clearances_fired(fired_pointers, kinds_present=frozenset(record.logbooks)) + + return RedactionResult( + redacted_record=RedactedRecord(streams=redacted_streams, logbooks=redacted_logbooks), + token_map=token_map, + ) diff --git a/apps/api/src/cora/infrastructure/record_export/_registry.py b/apps/api/src/cora/infrastructure/record_export/_registry.py new file mode 100644 index 00000000000..028329df983 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_registry.py @@ -0,0 +1,189 @@ +"""Registry: logbook ``kind`` -> (table, order key, reader). + +Per `project_record_export_v3.md` F0/F2 and +`project_record_is_two_tier.md`. The exporter walks the event stream and, +on each ``LogbookOpened`` envelope, reads the envelope's ``kind`` (a +bare ``str``, not a closed enum) and resolves it through this ONE +registry to the entries table that holds the fine-grained doing. An +unknown ``kind`` refuses loudly rather than being skipped. + +Eight entries, not six. Six kinds come from an envelope event on the +main stream; two tables, `entries_run_feed_heartbeats` and +`entries_enclosure_permit_probes`, have no envelope at all and are +declared here explicitly, with `envelope_class` set to `None`, per +`project_record_is_two_tier.md`'s "declare or exclude, in writing" +finding. Whether the exporter actually pulls their rows into a published +bundle, or excludes them as operational telemetry, is a separate, +still-open call for the exporter step; this registry only has to make +both tables reachable and refuse to silently drop either. + +The order key lives per kind because `sampled_at` exists on only four of +the eight tables (activity, diagnostic, outcome, observation). The other +four order by `event_id` alone: CORA mints it with `UUIDv7Generator`, so +it is total and insertion-ordered without a separate timestamp column, +and `occurred_at` is never the tiebreak because it ties across a whole +append batch (one Clock read per handler call). + +The six envelope-driven tables are scoped by `logbook_id`, the join +column the envelope carries. Heartbeats and probes are not +Logbook-and-Entry instances (no `logbook_id` column at all) and are +scoped by their owning aggregate's id instead: `run_id` for heartbeats, +`enclosure_id` for probes. +""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from uuid import UUID + +import asyncpg + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# asyncpg's stubs are loose; suppress at module level, matching the other +# entries-table readers (e.g. postgres_procedure_activity_lookup.py). + +EntriesReader = Callable[[asyncpg.Connection, UUID], Awaitable[list[asyncpg.Record]]] + + +@dataclass(frozen=True, slots=True) +class EntriesTableSpec: + """One registry row: how to read one kind's entries-tier rows. + + `envelope_class` names the `*LogbookOpened` class that carries this + kind on the main stream, or `None` for the two tables with no + envelope. `scope_column` is the column `reader` filters on: the + envelope's `logbook_id` for the six logbook-backed kinds, the owning + aggregate's id for the other two. + """ + + kind: str + table: str + envelope_class: str | None + scope_column: str + order_by: tuple[str, ...] + reader: EntriesReader + + +class UnknownLogbookKindError(LookupError): + """A `kind` with no registry entry. Refuses loudly rather than skipping.""" + + def __init__(self, kind: str) -> None: + super().__init__( + f"no entries-table registry entry for logbook kind {kind!r}; " + "an unknown kind must not be skipped" + ) + self.kind = kind + + +def _make_reader(table: str, scope_column: str, order_by: tuple[str, ...]) -> EntriesReader: + # table, scope_column and order_by are all registry-declared constants + # below, never caller input, so the interpolation cannot carry + # attacker-controlled SQL. + sql = f"SELECT * FROM {table} WHERE {scope_column} = $1 ORDER BY {', '.join(order_by)}" + + async def read(conn: asyncpg.Connection, scope_id: UUID) -> list[asyncpg.Record]: + return await conn.fetch(sql, scope_id) + + return read + + +def _spec( + *, + kind: str, + table: str, + envelope_class: str | None, + scope_column: str, + order_by: tuple[str, ...], +) -> EntriesTableSpec: + return EntriesTableSpec( + kind=kind, + table=table, + envelope_class=envelope_class, + scope_column=scope_column, + order_by=order_by, + reader=_make_reader(table, scope_column, order_by), + ) + + +_ENTRIES: tuple[EntriesTableSpec, ...] = ( + _spec( + kind="verdict", + table="entries_conduit_verdicts", + envelope_class="ConduitLogbookOpened", + scope_column="logbook_id", + order_by=("event_id",), + ), + _spec( + kind="inference", + table="entries_decision_inferences", + envelope_class="DecisionLogbookOpened", + scope_column="logbook_id", + order_by=("event_id",), + ), + _spec( + kind="activity", + table="entries_operation_procedure_activities", + envelope_class="ProcedureActivitiesLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="diagnostic", + table="entries_operation_procedure_diagnostics", + envelope_class="ProcedureDiagnosticLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="outcome", + table="entries_operation_procedure_outcomes", + envelope_class="ProcedureOutcomeLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="observation", + table="entries_run_observations", + envelope_class="RunObservationLogbookOpened", + scope_column="logbook_id", + order_by=("sampled_at", "event_id"), + ), + _spec( + kind="heartbeat", + table="entries_run_feed_heartbeats", + envelope_class=None, + scope_column="run_id", + order_by=("event_id",), + ), + _spec( + kind="probe", + table="entries_enclosure_permit_probes", + envelope_class=None, + scope_column="enclosure_id", + order_by=("event_id",), + ), +) + +_REGISTRY: dict[str, EntriesTableSpec] = {spec.kind: spec for spec in _ENTRIES} + + +def resolve(kind: str) -> EntriesTableSpec: + """Look up `kind`'s table spec. + + Raises `UnknownLogbookKindError` rather than returning `None`: an + envelope carrying a kind this registry has never heard of must stop + the export, not be skipped. + """ + try: + return _REGISTRY[kind] + except KeyError: + raise UnknownLogbookKindError(kind) from None + + +def all_specs() -> tuple[EntriesTableSpec, ...]: + """Every registered spec, in declaration order.""" + return _ENTRIES + + +def registered_envelope_classes() -> frozenset[str]: + """Every `*LogbookOpened` class name named by a registry entry.""" + return frozenset(spec.envelope_class for spec in _ENTRIES if spec.envelope_class is not None) diff --git a/apps/api/src/cora/infrastructure/record_export/_render.py b/apps/api/src/cora/infrastructure/record_export/_render.py new file mode 100644 index 00000000000..5a3a1aba472 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_render.py @@ -0,0 +1,58 @@ +"""F6 rendering: the one table for turning a raw row into export bytes-ready form. + +Per `project_record_export_v3.md` F6: UUID to string, datetime to UTC +ISO-8601, bytes to hex. UTC normalization is measured, not stylistic: +the same instant at `-05:00` and `+00:00` hashes differently, and +Postgres returns `timestamptz` columns in the session's configured +offset while payload timestamps were already written as UTC strings. + +This only has to touch the OUTER `events` / `entries_*` row columns +that asyncpg hands back as typed Python objects (`uuid.UUID`, tz-aware +`datetime`, `bytes`). It must NOT recurse into `payload` / `metadata`: +every `to_payload()` in the tree already converts UUID and datetime +fields to plain strings before the row is written (verified against +`_dispositions.py`, the Step 0 generated table, whose entries for all +six `*LogbookOpened` classes show `logbook_id` as a flat `token:uuid` +field rather than a nested one), and the asyncpg pool's jsonb codec +(`cora.infrastructure.postgres.pool`) decodes straight to `dict` / `str` +/ etc, never back to `UUID` or `datetime`. Rendering jsonb contents a +second time here would be a no-op at best and silently wrong if a future +payload ever carried a real (not pre-stringified) UUID or datetime. +""" + +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from uuid import UUID + +if TYPE_CHECKING: + import asyncpg + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# asyncpg's stubs are loose; suppress at module level, matching the other +# entries-table readers. + + +def render_value(value: object) -> object: + """Render one column value for export. + + `UUID -> str`, `datetime -> UTC ISO-8601 str`, `bytes`-like -> hex + `str`. Everything else (str, int, float, bool, None, and jsonb-decoded + dict/list contents) passes through unchanged. + """ + if isinstance(value, UUID): + return str(value) + if isinstance(value, datetime): + # asyncpg returns `timestamptz` columns as timezone-aware in the + # session's configured offset; astimezone(UTC) normalizes + # regardless of what that offset was. + return value.astimezone(UTC).isoformat() + if isinstance(value, (bytes, bytearray, memoryview)): + return value.hex() + return value + + +def render_row(row: "dict[str, object] | asyncpg.Record") -> dict[str, object]: + """Render every column of one asyncpg `Record` (production) or plain + `dict` (unit tests -- `Record` isn't constructible outside asyncpg's + own protocol machinery).""" + return {key: render_value(value) for key, value in row.items()} diff --git a/apps/api/src/cora/infrastructure/record_export/_stream_types.py b/apps/api/src/cora/infrastructure/record_export/_stream_types.py new file mode 100644 index 00000000000..677b5682cf5 --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_stream_types.py @@ -0,0 +1,88 @@ +"""The closed set of `events.stream_type` values. + +No such set exists anywhere else in the code: each BC's aggregate/slice +declares its own `_STREAM_TYPE = "X"` module constant (42 distinct +literals across 211 declarations at last count), and `stream_type` on +`events` is a bare `text` column with no DB or Python enum closing it. + +The exporter needs one anyway: per `project_record_export_build_brief.md` +step 2's acceptance criteria, "an unknown `stream_type` refuses rather +than skips ... silently skipping is pass-while-differing." So this +module declares the closed set explicitly, the same shape as +`_registry.py`'s `kind` registry, and +`test_record_export_stream_types_completeness.py` AST-discovers every +`_STREAM_TYPE` literal under `src/cora` and pins it against +`KNOWN_STREAM_TYPES` in both directions so a new BC cannot silently fall +through `ensure_stream_type_known` as "skip" instead of "refuse". +""" + +KNOWN_STREAM_TYPES: frozenset[str] = frozenset( + { + "Acquisition", + "Actor", + "Agent", + "Allocation", + "Assembly", + "Asset", + "Attestation", + "Calibration", + "Campaign", + "Capability", + "Caution", + "Clearance", + "ClearanceTemplate", + "Conduit", + "Credential", + "Dataset", + "Decision", + "Distribution", + "Edition", + "Enclosure", + "Facility", + "Family", + "Fixture", + "Frame", + "LanguageModel", + "Method", + "Model", + "Mount", + "Permit", + "Plan", + "Policy", + "Practice", + "Procedure", + "Ratification", + "Recipe", + "Role", + "Run", + "Seal", + "Subject", + "Supply", + "Surface", + "Visit", + "Zone", + } +) + + +class UnknownStreamTypeError(LookupError): + """A `stream_type` with no entry in `KNOWN_STREAM_TYPES`. + + Refuses loudly rather than being silently skipped: skipping an + unrecognised stream is "pass while differing" per the build brief's + trap list, not a safe default. + """ + + def __init__(self, stream_type: str) -> None: + super().__init__( + f"unknown events.stream_type {stream_type!r}; not in " + "KNOWN_STREAM_TYPES. A new BC's stream_type must be added " + "here before its streams can be exported." + ) + self.stream_type = stream_type + + +def ensure_stream_type_known(stream_type: str) -> None: + """Raise `UnknownStreamTypeError` unless `stream_type` is declared.""" + if stream_type not in KNOWN_STREAM_TYPES: + raise UnknownStreamTypeError(stream_type) diff --git a/apps/api/src/cora/infrastructure/record_export/_tokens.py b/apps/api/src/cora/infrastructure/record_export/_tokens.py new file mode 100644 index 00000000000..ba912ecbdad --- /dev/null +++ b/apps/api/src/cora/infrastructure/record_export/_tokens.py @@ -0,0 +1,53 @@ +"""The per-export UUID surrogate map. + +Per `project_record_export_v3.md` F5: a TOKEN is a random surrogate, +never a hash. A hash is deterministic, so against a known candidate set +(a ten-person roster, a facility's proposal list) it is brute-forceable +in as many guesses as there are candidates -- the same defect as +publishing a signature beside a redacted payload. `TokenMap` mints a +fresh `uuid4()` per distinct source string the first time it is seen +and returns that same surrogate on every later lookup within the same +export, which is what preserves joins (both within tier 1 and across +the tier-1/tier-2 seam, per F5) without making the surrogate +derivable from its source. + +The map itself is an artifact of the export, not the record: F5 says it +"never ships" and sits under the same retention obligation as the +unredacted record. `surrogate_by_source` exists so a caller can persist +it separately; nothing in this package ever feeds it back into anything +that gets hashed as "the published record". +""" + +from uuid import uuid4 + + +class TokenMap: + """Mint-once, reuse-after per-export UUID surrogates. Never a hash.""" + + def __init__(self) -> None: + self._surrogates: dict[str, str] = {} + + def token_uuid(self, source: str | None) -> str | None: + """Surrogate for `source`, or `None` if `source` is `None`. + + `None` is a legitimate value for an optional UUID column + (`causation_id`, `principal_id`, ...) and must pass through + as `None` rather than being tokenized or dropped. + """ + if source is None: + return None + if source not in self._surrogates: + self._surrogates[source] = str(uuid4()) + return self._surrogates[source] + + @property + def surrogate_by_source(self) -> dict[str, str]: + """Source UUID -> surrogate, for retention under H1's obligation. + + A copy, not a live view: callers must not be able to mutate the + map this instance uses for tokenization by holding this dict. + """ + return dict(self._surrogates) + + +__all__ = ["TokenMap"] diff --git a/apps/api/tests/architecture/test_record_dispositions_drift.py b/apps/api/tests/architecture/test_record_dispositions_drift.py new file mode 100644 index 00000000000..a80b86f2078 --- /dev/null +++ b/apps/api/tests/architecture/test_record_dispositions_drift.py @@ -0,0 +1,67 @@ +"""Redaction disposition table drift test. + +Asserts that the committed +`src/cora/infrastructure/record_export/_dispositions.py` matches what +`tools/gen_record_dispositions.py` produces right now. Adding an event +type, adding a field, or changing a field's declared type fails this +test until the table is regenerated and the diff reviewed. + +That review is the point. The table decides what a published record +discloses, so a field's arrival has to be a line in a pull request +rather than a silent behaviour change. Regenerate with: + + make record-dispositions + +The generator runs in a SUBPROCESS rather than being imported. It lives +outside `src/` precisely so nothing shippable can import it, and this +test declining to import it keeps that true. It also means the test +exercises the same entry point a developer runs. + +Fail-closed is NOT provided by this test. An unlisted field drops at +export time by the rule itself; this test only catches staleness early. +Deleting it would make the table rot silently, not make the exporter +leak. + +Running the generator in a subprocess hides this test's real dependency +(every `events.py` in the tree) from pytest-tach's impact analysis, so +`pytest --tach` would skip it after an event-only change. CI runs the +suite without that flag, so it is covered today; anyone turning impact +analysis on in CI has to exempt this test. +""" + +import subprocess +import sys +from pathlib import Path + +_API_ROOT = Path(__file__).resolve().parents[2] +_GENERATOR = _API_ROOT / "tools" / "gen_record_dispositions.py" +_TABLE = _API_ROOT / "src" / "cora" / "infrastructure" / "record_export" / "_dispositions.py" + + +def test_committed_disposition_table_matches_generator() -> None: + """Committed `_dispositions.py` equals a fresh generator run.""" + before = _TABLE.read_text(encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(_GENERATOR)], + cwd=_API_ROOT, + capture_output=True, + text=True, + timeout=120, + ) + + after = _TABLE.read_text(encoding="utf-8") + if before != after: + _TABLE.write_text(before, encoding="utf-8") + + assert result.returncode == 0, ( + "The disposition generator refused to produce a table. An " + "annotation it cannot classify ABORTS the run by design, because " + "an unrecognised type is a question about the design rather than " + f"a row to skip. Run `make record-dispositions`.\n{result.stderr}" + ) + assert before == after, ( + f"Disposition table drift detected at {_TABLE.name}. Run " + "`make record-dispositions`, then review the diff: it is the " + "list of what a published record would disclose." + ) diff --git a/apps/api/tests/architecture/test_record_export_registry_completeness.py b/apps/api/tests/architecture/test_record_export_registry_completeness.py new file mode 100644 index 00000000000..aa8d1db6bc0 --- /dev/null +++ b/apps/api/tests/architecture/test_record_export_registry_completeness.py @@ -0,0 +1,82 @@ +"""Every `*LogbookOpened` class is named by the entries-tier registry. + +`cora.infrastructure.record_export._registry` resolves a logbook +envelope's `kind` to the `entries_*` table it opened. Per +`project_record_is_two_tier.md`, the envelope never names its own table, +so the registry is the one place that link is written down, and a new +`*LogbookOpened` class that forgets to register itself would silently +narrow envelope-driven traversal the same way the eighth entries table +(`entries_enclosure_permit_probes`) silently narrowed it before this +registry existed. + +This fitness function AST-discovers every `*LogbookOpened` class under +`src/cora` (git-tracked, so pre-commit's tracked-file staging sees the +same set) and pins it against `registered_envelope_classes()`, both +directions: a new envelope class with no registry entry, and a registry +entry naming a class that no longer exists (e.g. after a rename), both +fail loudly instead of rotting quietly. +""" + +import ast +from pathlib import Path + +import pytest + +from cora.infrastructure.record_export import registered_envelope_classes +from tests.architecture.conftest import tracked_python_files + + +def _find_logbook_opened_classes(tree: ast.Module) -> list[str]: + return [ + node.name + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) and node.name.endswith("LogbookOpened") + ] + + +def _discover_logbook_opened_classes() -> dict[str, Path]: + """Map every discovered `*LogbookOpened` class name to its defining file.""" + found: dict[str, Path] = {} + for path in tracked_python_files(): + if path.name != "events.py": + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + for class_name in _find_logbook_opened_classes(tree): + found[class_name] = path + return found + + +@pytest.mark.architecture +def test_every_logbook_opened_class_is_in_the_registry() -> None: + discovered = _discover_logbook_opened_classes() + registered = registered_envelope_classes() + + unregistered = set(discovered) - registered + assert not unregistered, ( + f"{sorted(unregistered)} define a *LogbookOpened envelope with no " + "entry in cora.infrastructure.record_export._registry. Envelope-" + "driven traversal silently stops covering the new table until " + "one is added. Files: " + f"{ {name: str(discovered[name]) for name in sorted(unregistered)} }" + ) + + +@pytest.mark.architecture +def test_no_registry_entry_names_a_class_that_no_longer_exists() -> None: + discovered = _discover_logbook_opened_classes() + registered = registered_envelope_classes() + + stale = registered - set(discovered) + assert not stale, ( + f"{sorted(stale)} are named by the registry's envelope_class field " + "but no *LogbookOpened class by that name exists under src/cora " + "anymore (renamed?). Update _registry.py's envelope_class." + ) + + +@pytest.mark.architecture +def test_exactly_six_logbook_opened_classes_exist() -> None: + """Pins the count so a seventh envelope class is a deliberate, reviewed + registry addition rather than a silent drift. Bump alongside a new + _registry.py entry, never on its own.""" + assert len(_discover_logbook_opened_classes()) == 6 diff --git a/apps/api/tests/architecture/test_record_export_stream_types_completeness.py b/apps/api/tests/architecture/test_record_export_stream_types_completeness.py new file mode 100644 index 00000000000..9527261724d --- /dev/null +++ b/apps/api/tests/architecture/test_record_export_stream_types_completeness.py @@ -0,0 +1,58 @@ +"""Every `_STREAM_TYPE = "X"` literal under `src/cora` is in `KNOWN_STREAM_TYPES`. + +`events.stream_type` is a bare `text` column with no DB or Python enum +closing it; each BC/slice declares its own `_STREAM_TYPE` module +constant instead (211 declarations across 42 distinct literals at last +count). `cora.infrastructure.record_export._stream_types` declares that +set explicitly so the exporter can refuse an unknown `stream_type` +rather than silently skip it. This fitness function AST-discovers every +`_STREAM_TYPE = "X"` assignment and pins it against `KNOWN_STREAM_TYPES` +both directions: a new BC that forgets to add its stream_type here would +otherwise pass every other test and still make `export_record` refuse +on the very first row of its stream. +""" + +import ast + +import pytest + +from cora.infrastructure.record_export import KNOWN_STREAM_TYPES +from tests.architecture.conftest import tracked_python_files + + +def _discover_stream_type_literals() -> set[str]: + found: set[str] = set() + for path in tracked_python_files(): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if not (isinstance(target, ast.Name) and target.id == "_STREAM_TYPE"): + continue + if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + found.add(node.value.value) + return found + + +@pytest.mark.architecture +def test_every_declared_stream_type_literal_is_known() -> None: + discovered = _discover_stream_type_literals() + unknown = discovered - KNOWN_STREAM_TYPES + assert not unknown, ( + f"{sorted(unknown)} appear as `_STREAM_TYPE = ...` literals under " + "src/cora but are not in KNOWN_STREAM_TYPES " + "(cora.infrastructure.record_export._stream_types). export_record " + "would refuse on the first row of that stream." + ) + + +@pytest.mark.architecture +def test_no_known_stream_type_is_unused() -> None: + discovered = _discover_stream_type_literals() + stale = KNOWN_STREAM_TYPES - discovered + assert not stale, ( + f"{sorted(stale)} are declared in KNOWN_STREAM_TYPES but no " + "`_STREAM_TYPE = ...` literal under src/cora uses them anymore " + "(renamed or removed BC?). Update _stream_types.py." + ) diff --git a/apps/api/tests/integration/test_record_export_exporter_postgres.py b/apps/api/tests/integration/test_record_export_exporter_postgres.py new file mode 100644 index 00000000000..bdef404e556 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_exporter_postgres.py @@ -0,0 +1,220 @@ +"""Acceptance test for Step 2 of the record exporter build brief. + +Per `project_record_export_build_brief.md` step 2: a Procedure stream +folds to `Running` with its activity rows beside it, unfolded; an +unknown `stream_type` refuses rather than skips; zero rows exported is +an error. + +The Procedure is built the same way +`test_append_activities_handler_postgres.py` does it: `ProcedureRegistered` ++ `ProcedureStarted` seeded directly into the event store (bypassing +`register_procedure`/`start_procedure`'s cross-aggregate validation, +which is not this test's concern), then the real `append_activities` +handler so `ProcedureActivitiesLogbookOpened` fires for real and rows +land in `entries_operation_procedure_activities`. + +The "folds to Running" assertion reconstructs `StoredEvent`s from +`export_record`'s own rendered `streams` output (not from a fresh +`event_store.load`), because the point of the acceptance test is that +the EXPORTED representation is complete and correct enough to fold, the +same property the standalone verifier (step 5) will depend on. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC as _UTC +from datetime import datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.ports.event_store import StoredEvent +from cora.infrastructure.record_export import ( + EmptyExportError, + UnknownStreamTypeError, + export_record, +) +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + ProcedureStatus, + event_type_name, + fold, + from_stored, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=_UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +async def _seed_running_procedure(event_store: object, procedure_id: UUID) -> None: + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await event_store.append( # type: ignore[attr-defined] + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + +def _stored_event_from_rendered_row(row: dict[str, object]) -> StoredEvent: + """Reconstruct a `StoredEvent` from one of `export_record`'s rendered + `streams` rows, undoing F6 rendering. This is exactly the + reconstruction the standalone verifier (step 5) will need to do.""" + return StoredEvent( + position=row["position"], # type: ignore[arg-type] + event_id=UUID(row["event_id"]), # type: ignore[arg-type] + stream_type=row["stream_type"], # type: ignore[arg-type] + stream_id=UUID(row["stream_id"]), # type: ignore[arg-type] + version=row["version"], # type: ignore[arg-type] + event_type=row["event_type"], # type: ignore[arg-type] + schema_version=row["schema_version"], # type: ignore[arg-type] + payload=row["payload"], # type: ignore[arg-type] + metadata=row["metadata"], # type: ignore[arg-type] + correlation_id=UUID(row["correlation_id"]), # type: ignore[arg-type] + causation_id=UUID(row["causation_id"]) if row["causation_id"] else None, # type: ignore[arg-type] + occurred_at=datetime.fromisoformat(row["occurred_at"]), # type: ignore[arg-type] + recorded_at=datetime.fromisoformat(row["recorded_at"]), # type: ignore[arg-type] + transaction_id=int(row["transaction_id"]), # type: ignore[arg-type] + principal_id=UUID(row["principal_id"]) if row["principal_id"] else None, # type: ignore[arg-type] + signature=bytes.fromhex(row["signature"]) if row["signature"] else None, # type: ignore[arg-type] + signature_kid=row["signature_kid"], # type: ignore[arg-type] + signature_version=row["signature_version"], # type: ignore[arg-type] + ) + + +async def _read_activity_rows(db_pool: asyncpg.Pool, procedure_id: UUID) -> list[asyncpg.Record]: + async with db_pool.acquire() as conn: + return await conn.fetch( + """ + SELECT event_id, procedure_id, logbook_id, step_kind, payload, + sampled_at, occurred_at, recorded_at + FROM entries_operation_procedure_activities + WHERE procedure_id = $1 + ORDER BY sampled_at, event_id + """, + procedure_id, + ) + + +@pytest.mark.integration +async def test_procedure_stream_folds_to_running_with_activity_rows_beside_it( + db_pool: asyncpg.Pool, +) -> None: + procedure_id = UUID("01900000-0000-7000-8000-0000020a0a01") + logbook_id = UUID("01900000-0000-7000-8000-0000020a0a02") + open_event_id = UUID("01900000-0000-7000-8000-0000020a0a03") + + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + await _seed_running_procedure(deps.event_store, procedure_id) + + step_store = PostgresActivityStore(db_pool) + handler = bind_append(deps, step_store=step_store) + sampled_a = datetime(2026, 5, 15, 12, 0, 1, tzinfo=_UTC) + sampled_b = datetime(2026, 5, 15, 12, 0, 2, tzinfo=_UTC) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=UUID("01900000-0000-7000-8000-0000020a0b01"), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0}, + sampled_at=sampled_a, + ), + ActivityInput( + event_id=UUID("01900000-0000-7000-8000-0000020a0b02"), + step_kind="check", + payload={"channel": "T_oven", "passed": True}, + sampled_at=sampled_b, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + result = await export_record(pg_conn) + + procedure_rows = [row for row in result.streams if row["stream_id"] == str(procedure_id)] + assert len(procedure_rows) == 3 # Registered, Started, ActivitiesLogbookOpened + procedure_rows.sort(key=lambda row: int(row["version"])) # type: ignore[arg-type] + stored = [_stored_event_from_rendered_row(row) for row in procedure_rows] + state = fold([from_stored(s) for s in stored]) + assert state is not None + assert state.status == ProcedureStatus.RUNNING + assert state.activity_logbook_id == logbook_id + + exported_activity_rows = result.logbooks["activity"] + assert len(exported_activity_rows) == 2 + live_rows = await _read_activity_rows(db_pool, procedure_id) + assert {row["event_id"] for row in exported_activity_rows} == { + str(row["event_id"]) for row in live_rows + } + by_kind = {row["step_kind"]: row for row in exported_activity_rows} + assert by_kind["setpoint"]["payload"] == {"channel": "T_oven", "target_value": 423.0} + assert by_kind["check"]["payload"] == {"channel": "T_oven", "passed": True} + # Unfolded: two distinct rows, not a summary of the logbook. + assert by_kind["setpoint"] is not by_kind["check"] + + +@pytest.mark.integration +async def test_unknown_stream_type_refuses_rather_than_skips(db_pool: asyncpg.Pool) -> None: + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + await pg_conn.execute( + """ + INSERT INTO events (event_id, stream_type, stream_id, version, + event_type, payload, correlation_id, occurred_at) + VALUES ($1, 'Widget', $2, 1, 'WidgetRegistered', '{}'::jsonb, $3, now()) + """, + uuid4(), + uuid4(), + uuid4(), + ) + with pytest.raises(UnknownStreamTypeError) as excinfo: + await export_record(pg_conn) + assert excinfo.value.stream_type == "Widget" + + +@pytest.mark.integration +async def test_zero_rows_exported_is_an_error(db_pool: asyncpg.Pool) -> None: + """A freshly migrated template is NOT empty: two seed migrations + (`20260519000000_seed_bootstrap_policy.sql`, + `20260519200000_seed_default_surfaces_and_v2_policy.sql`) insert + bootstrap events. Emptying `events` explicitly is what actually + exercises the zero-rows refusal, on this test's own disposable + per-test database.""" + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + await pg_conn.execute("DELETE FROM events") + with pytest.raises(EmptyExportError): + await export_record(pg_conn) diff --git a/apps/api/tests/integration/test_record_export_hashing_postgres.py b/apps/api/tests/integration/test_record_export_hashing_postgres.py new file mode 100644 index 00000000000..5dfa6d30e39 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_hashing_postgres.py @@ -0,0 +1,120 @@ +"""Acceptance test for Step 3 of the record exporter build brief. + +Per `project_record_export_build_brief.md` step 3: same DB exported +twice, identical hash; a flipped byte fails. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.record_export import ExportedRecord, export_record, hash_record +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + event_type_name, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +async def _seed_running_procedure_with_activity(db_pool: asyncpg.Pool, procedure_id: UUID) -> None: + logbook_id = uuid4() + open_event_id = uuid4() + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await deps.event_store.append( + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + handler = bind_append(deps, step_store=PostgresActivityStore(db_pool)) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=uuid4(), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0}, + sampled_at=_NOW, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + +@pytest.mark.integration +async def test_same_database_exported_twice_hashes_identically(db_pool: asyncpg.Pool) -> None: + await _seed_running_procedure_with_activity(db_pool, uuid4()) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + first = await export_record(pg_conn) + async with db_pool.acquire() as conn: + pg_conn = conn # type: ignore[assignment] + second = await export_record(pg_conn) + + assert hash_record(first) == hash_record(second) + + +@pytest.mark.integration +async def test_a_flipped_byte_in_the_exported_record_changes_the_hash( + db_pool: asyncpg.Pool, +) -> None: + await _seed_running_procedure_with_activity(db_pool, uuid4()) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + baseline_hash = hash_record(exported) + + tampered_first_row = dict(exported.streams[0]) + original_event_type = tampered_first_row["event_type"] + assert isinstance(original_event_type, str) and original_event_type + # Flip exactly one character of one field. + tampered_first_row["event_type"] = original_event_type[:-1] + ( + "X" if original_event_type[-1] != "X" else "Y" + ) + tampered = ExportedRecord( + streams=(tampered_first_row, *exported.streams[1:]), + logbooks=exported.logbooks, + ) + + assert hash_record(tampered) != baseline_hash diff --git a/apps/api/tests/integration/test_record_export_manifest_postgres.py b/apps/api/tests/integration/test_record_export_manifest_postgres.py new file mode 100644 index 00000000000..d562aee1363 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_manifest_postgres.py @@ -0,0 +1,105 @@ +"""Manifest built from a real `export_record` result, not synthetic fixtures. + +`build_manifest` itself is pure and unit-tested against hand-built +`ExportedRecord`s in `tests/unit/infrastructure/record_export/test_manifest.py`. +This test exists because a real rendered row's shape could diverge from +those synthetic fixtures in ways only a live export would surface (e.g. +if `event_type_name()` or `to_payload()` ever changed a key name). +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.record_export import ( + build_manifest, + capture_git_commit, + export_record, + hash_record, +) +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + event_type_name, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +@pytest.mark.integration +async def test_manifest_built_from_a_real_export(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + logbook_id = uuid4() + open_event_id = uuid4() + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await deps.event_store.append( + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + handler = bind_append(deps, step_store=PostgresActivityStore(db_pool)) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=uuid4(), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0}, + sampled_at=_NOW, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + manifest = build_manifest(exported, watermark=1, git_commit=capture_git_commit()) + + assert manifest.record_hash == hash_record(exported) + assert len(manifest.redaction_profile_hash) == 64 + assert manifest.row_count_by_logbook_kind["activity"] == 1 + assert manifest.max_schema_version_by_event_type["ProcedureRegistered"] >= 1 + # No Run stream in this fixture (parent_run_id=None, no RunStarted + # seeded), so the per-run map must be empty, not crash. + assert manifest.expansion_digest_presence_by_run == {} + # No observation rows in this fixture: vacuously simulated. + assert manifest.is_simulated is True diff --git a/apps/api/tests/integration/test_record_export_redaction_postgres.py b/apps/api/tests/integration/test_record_export_redaction_postgres.py new file mode 100644 index 00000000000..48b4cc62e50 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_redaction_postgres.py @@ -0,0 +1,249 @@ +"""Acceptance test for Step 6 of the record exporter build brief. + +Per `project_record_export_build_brief.md` step 6: a bare `str` on a NEW +event drops with nobody editing a list; an event type absent from the +disposition table aborts; no raw store identifier survives the export +(NOT "no join key" -- timestamps remain one, deliberately); a rehearsal +export contains no real principal, ASSERTED rather than assumed. + +Built the same way steps 2-4's acceptance tests were: a real Procedure +through Running with a real activity appended, via the real handlers, +then `export_record` + `redact_record` against the live database. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +import dataclasses +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.event_envelope import to_new_event +from cora.infrastructure.record_export import ( + RedactionProfileMismatchError, + UnknownEventTypeError, + export_record, + hash_redaction_profile, + redact_record, +) +from cora.infrastructure.record_export._redact_tier2 import TIER2_DISPOSITIONS +from cora.infrastructure.record_export._registry import all_specs +from cora.operation.aggregates.procedure import ( + PostgresActivityStore, + ProcedureRegistered, + ProcedureStarted, + event_type_name, + to_payload, +) +from cora.operation.features.append_activities import ActivityInput, AppendProcedureActivities +from cora.operation.features.append_activities import bind as bind_append +from tests.integration._helpers import build_postgres_deps + +_NOW = datetime(2026, 5, 15, 12, 0, 0, tzinfo=UTC) +_PRINCIPAL_ID = UUID("01900000-0000-7000-8000-000000000099") +_CORRELATION_ID = UUID("01900000-0000-7000-8000-0000000000aa") + + +def _collect_string_leaves(value: object, out: set[str]) -> None: + if isinstance(value, dict): + for sub in value.values(): + _collect_string_leaves(sub, out) + elif isinstance(value, (list, tuple)): + for item in value: + _collect_string_leaves(item, out) + elif isinstance(value, str): + out.add(value) + + +async def _seed_running_procedure_with_activity(db_pool: asyncpg.Pool, procedure_id: UUID) -> None: + logbook_id = uuid4() + open_event_id = uuid4() + deps = build_postgres_deps(db_pool, now=_NOW, ids=[logbook_id, open_event_id]) + + registered = ProcedureRegistered( + procedure_id=procedure_id, + name="Vessel-A bakeout", + kind="bakeout", + target_asset_ids=(), + parent_run_id=None, + occurred_at=_NOW, + ) + started = ProcedureStarted(procedure_id=procedure_id, occurred_at=_NOW) + for index, event in enumerate((registered, started)): + new_event = to_new_event( + event_type=event_type_name(event), + payload=to_payload(event), + occurred_at=event.occurred_at, + event_id=uuid4(), + command_name="RegisterProcedure" if index == 0 else "StartProcedure", + correlation_id=_CORRELATION_ID, + principal_id=_PRINCIPAL_ID, + ) + await deps.event_store.append( + stream_type="Procedure", + stream_id=procedure_id, + expected_version=index, + events=[new_event], + ) + + # A single setpoint-only activity would leave the activity kind's + # `action_name` and `units` clearances unfired: a real whole-database + # export accumulates many activities across many procedures over + # time and would plausibly exercise every step kind, so this mirrors + # that with one of each (same 3-entry shape as step 2's fixture) + # rather than under-representing a real export. + handler = bind_append(deps, step_store=PostgresActivityStore(db_pool)) + await handler( + AppendProcedureActivities( + procedure_id=procedure_id, + entries=( + ActivityInput( + event_id=uuid4(), + step_kind="setpoint", + payload={"channel": "T_oven", "target_value": 423.0, "units": "K"}, + sampled_at=_NOW, + ), + ActivityInput( + event_id=uuid4(), + step_kind="action", + payload={"action_name": "open_valve", "params": {"valve": "V12"}}, + sampled_at=_NOW, + ), + ActivityInput( + event_id=uuid4(), + step_kind="check", + payload={"channel": "T_oven", "passed": True}, + sampled_at=_NOW, + ), + ), + ), + principal_id=_PRINCIPAL_ID, + correlation_id=_CORRELATION_ID, + ) + + +@pytest.mark.integration +async def test_bare_str_field_drops_and_no_real_principal_survives(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + result = redact_record(exported, expected_redaction_profile_hash=hash_redaction_profile()) + redacted = result.redacted_record + + registered_rows = [ + row for row in redacted.streams if row["event_type"] == "ProcedureRegistered" + ] + assert len(registered_rows) == 1 + registered_payload = registered_rows[0]["payload"] + assert isinstance(registered_payload, dict) + # ProcedureRegistered.name and .kind are drop:text in the real, + # generated table -- no test-only disposition edited for this. + assert "name" not in registered_payload + assert "kind" not in registered_payload + + # ASSERTED, not assumed: the real principal/actor UUID used to build + # this fixture must not appear anywhere in the redacted output. + leaves: set[str] = set() + for row in redacted.streams: + _collect_string_leaves(row, leaves) + for rows in redacted.logbooks.values(): + for row in rows: + _collect_string_leaves(row, leaves) + assert str(_PRINCIPAL_ID) not in leaves + assert str(procedure_id) not in leaves + assert str(_CORRELATION_ID) not in leaves + + +@pytest.mark.integration +async def test_event_type_absent_from_disposition_table_aborts(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + tampered_streams = list(exported.streams) + tampered_streams[0] = {**tampered_streams[0], "event_type": "ThisEventTypeDoesNotExist"} + tampered = dataclasses.replace(exported, streams=tuple(tampered_streams)) + + with pytest.raises(UnknownEventTypeError) as excinfo: + redact_record(tampered, expected_redaction_profile_hash=hash_redaction_profile()) + assert excinfo.value.event_type == "ThisEventTypeDoesNotExist" + + +@pytest.mark.integration +async def test_no_raw_uuid_survives_the_redacted_export(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + raw_uuids: set[str] = set() + for row in exported.streams: + for key in ("event_id", "stream_id", "correlation_id", "causation_id", "principal_id"): + value = row.get(key) + if isinstance(value, str): + raw_uuids.add(value) + for rows in exported.logbooks.values(): + for row in rows: + for value in row.values(): + if isinstance(value, str) and len(value) == 36 and value.count("-") == 4: + raw_uuids.add(value) + + result = redact_record(exported, expected_redaction_profile_hash=hash_redaction_profile()) + + redacted_leaves: set[str] = set() + for row in result.redacted_record.streams: + _collect_string_leaves(row, redacted_leaves) + for rows in result.redacted_record.logbooks.values(): + for row in rows: + _collect_string_leaves(row, redacted_leaves) + + assert raw_uuids, "fixture produced no UUIDs to check -- test would pass vacuously" + assert raw_uuids.isdisjoint(redacted_leaves) + + +@pytest.mark.integration +async def test_wrong_redaction_profile_hash_refuses_before_redacting(db_pool: asyncpg.Pool) -> None: + procedure_id = uuid4() + await _seed_running_procedure_with_activity(db_pool, procedure_id) + + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exported = await export_record(pg_conn) + + with pytest.raises(RedactionProfileMismatchError): + redact_record(exported, expected_redaction_profile_hash="0" * 64) + + +@pytest.mark.integration +@pytest.mark.parametrize("spec", all_specs(), ids=lambda spec: spec.kind) +async def test_tier2_disposition_table_columns_match_live_schema( + db_pool: asyncpg.Pool, spec: object +) -> None: + """Drift guard: TIER2_DISPOSITIONS[kind] must enumerate every column + the live table actually has, nothing more, nothing less -- a new + migration adding a column must fail this test until this file is + updated, the same posture step 1's registry completeness test takes + for the registry itself.""" + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + rows = await pg_conn.fetch( + "SELECT column_name FROM information_schema.columns WHERE table_name = $1", + spec.table, # type: ignore[attr-defined] + ) + live_columns = {row["column_name"] for row in rows} + declared_columns = set(TIER2_DISPOSITIONS[spec.kind].keys()) # type: ignore[attr-defined] + assert declared_columns == live_columns, ( + f"kind={spec.kind!r}: TIER2_DISPOSITIONS declares {declared_columns} but " # type: ignore[attr-defined] + f"{spec.table!r} actually has {live_columns}" # type: ignore[attr-defined] + ) diff --git a/apps/api/tests/integration/test_record_export_registry_postgres.py b/apps/api/tests/integration/test_record_export_registry_postgres.py new file mode 100644 index 00000000000..4b8e8767551 --- /dev/null +++ b/apps/api/tests/integration/test_record_export_registry_postgres.py @@ -0,0 +1,75 @@ +"""Integration test: the entries-tier registry against a live schema. + +AST enumeration can prove a `*LogbookOpened` class exists; it cannot +prove the table or columns the registry names for it are real, because +renames defeat it. Three of these eight tables have already been +renamed (see `project_record_is_two_tier.md`), so this test queries +`information_schema` directly against the fully-migrated template +database and fails if `_registry.py` still points at a name from before +a rename. + +Also exercises the reader end to end: for a table with zero rows scoped +to a random id, the reader returns an empty list rather than raising, +which is the shape the exporter needs to distinguish "this stream opened +a logbook with no entries yet" (Conduit's eager-open pattern) from a +registry or SQL error. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false + +from uuid import uuid4 + +import asyncpg +import pytest + +from cora.infrastructure.record_export import EntriesTableSpec, all_specs + + +async def _columns(conn: asyncpg.Connection, table: str) -> set[str]: + rows = await conn.fetch( + "SELECT column_name FROM information_schema.columns WHERE table_name = $1", + table, + ) + return {row["column_name"] for row in rows} + + +@pytest.mark.integration +@pytest.mark.parametrize("spec", all_specs(), ids=lambda spec: spec.kind) +async def test_registered_table_exists_with_its_declared_columns( + db_pool: asyncpg.Pool, spec: EntriesTableSpec +) -> None: + async with db_pool.acquire() as conn: + # PoolConnectionProxy and Connection expose the same runtime API; + # narrowed here to match the `_columns` / EntriesReader signature, + # per the postgres_profile_store.py convention. + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + exists = await pg_conn.fetchval( + "SELECT count(*) FROM information_schema.tables WHERE table_name = $1", + spec.table, + ) + assert exists == 1, ( + f"kind={spec.kind!r} names table {spec.table!r}, which does not " + "exist in the live schema. A migration rename that forgot to " + "update _registry.py is exactly the failure mode this test " + "exists to catch." + ) + + columns = await _columns(pg_conn, spec.table) + needed = {spec.scope_column, *spec.order_by} + missing = needed - columns + assert not missing, ( + f"kind={spec.kind!r}: {spec.table!r} is missing column(s) " + f"{sorted(missing)} that _registry.py's scope_column/order_by " + "declare for it." + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("spec", all_specs(), ids=lambda spec: spec.kind) +async def test_reader_returns_empty_list_for_an_unscoped_id( + db_pool: asyncpg.Pool, spec: EntriesTableSpec +) -> None: + async with db_pool.acquire() as conn: + pg_conn: asyncpg.Connection = conn # type: ignore[assignment] + rows = await spec.reader(pg_conn, uuid4()) + assert rows == [] diff --git a/apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py b/apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py new file mode 100644 index 00000000000..da73751e88f --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_hash_redaction_profile.py @@ -0,0 +1,50 @@ +"""Unit tests for `hash_redaction_profile` (H2). + +Step 7's security re-review found tier 2's hand-authored tables +(`TIER2_DISPOSITIONS`, `TIER2_JSONB_CLEARED_POINTERS`, +`TIER2_JSONB_DROPPED_COLUMNS`) missing from H2: the hash covered only +tier 1's generated `DISPOSITIONS`, so `redact_record`'s fail-closed +switch could not detect a tier-2 table edit that weakened a +disposition. These tests pin the fix: every one of the four tables H2 +is supposed to cover must actually move the hash. +""" + +import pytest + +from cora.infrastructure.record_export import hash_redaction_profile +from cora.infrastructure.record_export._dispositions import DISPOSITIONS +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + TIER2_JSONB_CLEARED_POINTERS, +) + + +def test_changing_a_tier1_disposition_changes_the_hash(monkeypatch: pytest.MonkeyPatch) -> None: + baseline = hash_redaction_profile() + some_event_type = next(iter(DISPOSITIONS)) + some_field = next(iter(DISPOSITIONS[some_event_type])) + monkeypatch.setitem(DISPOSITIONS[some_event_type], some_field, "keep:enum:Tampered") + assert hash_redaction_profile() != baseline + + +def test_changing_a_tier2_disposition_changes_the_hash(monkeypatch: pytest.MonkeyPatch) -> None: + """The gap Step 7 found: this used to be a no-op on the hash.""" + baseline = hash_redaction_profile() + monkeypatch.setitem(TIER2_DISPOSITIONS["verdict"], "reason", "keep") + assert hash_redaction_profile() != baseline + + +def test_widening_a_tier2_jsonb_clearance_changes_the_hash( + monkeypatch: pytest.MonkeyPatch, +) -> None: + baseline = hash_redaction_profile() + monkeypatch.setitem( + TIER2_JSONB_CLEARED_POINTERS, + ("activity", "payload"), + frozenset({"channel", "action_name", "units", "an_extra_leaked_pointer"}), + ) + assert hash_redaction_profile() != baseline + + +def test_hash_redaction_profile_is_stable_across_repeated_calls() -> None: + assert hash_redaction_profile() == hash_redaction_profile() diff --git a/apps/api/tests/unit/infrastructure/record_export/test_hashing.py b/apps/api/tests/unit/infrastructure/record_export/test_hashing.py new file mode 100644 index 00000000000..3deaf669e63 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_hashing.py @@ -0,0 +1,109 @@ +"""Unit tests for hashing an exported record. + +Per `project_record_export_v3.md` F2: no exclusions, stable across +re-exports of the same content, sensitive to any single differing byte. +These tests exercise that purely as a function of Python data (no DB +needed); the DB-backed "same database exported twice" acceptance test +lives in `tests/integration/test_record_export_hashing_postgres.py`. +""" + +from cora.infrastructure.record_export import ( + ExportedRecord, + hash_logbooks, + hash_record, + hash_streams, +) + +_STREAM_ROW: dict[str, object] = { + "event_id": "12345678-1234-5678-1234-567812345678", + "event_type": "ProcedureRegistered", + "version": 1, +} +_ACTIVITY_ROW: dict[str, object] = { + "event_id": "aaaaaaaa-1234-5678-1234-567812345678", + "step_kind": "setpoint", + "payload": {"channel": "T_oven", "target_value": 423.0}, +} + + +def _record() -> ExportedRecord: + return ExportedRecord( + streams=(_STREAM_ROW,), + logbooks={"activity": (_ACTIVITY_ROW,)}, + ) + + +def test_hash_record_is_stable_across_independently_built_equal_records() -> None: + """Two structurally identical ExportedRecords (fresh dict/tuple copies, + not the same objects) must hash identically -- this is what "the same + database exported twice" reduces to at the data level.""" + first = ExportedRecord( + streams=(dict(_STREAM_ROW),), logbooks={"activity": (dict(_ACTIVITY_ROW),)} + ) + second = ExportedRecord( + streams=(dict(_STREAM_ROW),), logbooks={"activity": (dict(_ACTIVITY_ROW),)} + ) + assert hash_record(first) == hash_record(second) + + +def test_hash_record_changes_on_a_single_field_change_anywhere() -> None: + baseline = hash_record(_record()) + + changed_stream = ExportedRecord( + streams=({**_STREAM_ROW, "version": 2},), + logbooks={"activity": (_ACTIVITY_ROW,)}, + ) + assert hash_record(changed_stream) != baseline + + changed_logbook_row = dict(_ACTIVITY_ROW) + changed_logbook_row["payload"] = {"channel": "T_oven", "target_value": 424.0} + changed_logbook = ExportedRecord( + streams=(_STREAM_ROW,), + logbooks={"activity": (changed_logbook_row,)}, + ) + assert hash_record(changed_logbook) != baseline + + +def test_hash_record_is_sensitive_to_stream_row_order() -> None: + """Stream order is significant (it is the replay order); reordering + two rows must change the hash even though the SET of rows is the + same.""" + row_a = {**_STREAM_ROW, "version": 1} + row_b = {**_STREAM_ROW, "version": 2, "event_id": "bbbbbbbb-1234-5678-1234-567812345678"} + forward = ExportedRecord(streams=(row_a, row_b), logbooks={}) + backward = ExportedRecord(streams=(row_b, row_a), logbooks={}) + assert hash_record(forward) != hash_record(backward) + + +def test_hash_record_is_insensitive_to_logbook_kind_key_order() -> None: + """Kind keys sort themselves via json.dumps(sort_keys=True); insertion + order into the `logbooks` dict must not affect the hash.""" + forward = ExportedRecord( + streams=(), + logbooks={"activity": (_ACTIVITY_ROW,), "outcome": ({"a": 1},)}, + ) + backward = ExportedRecord( + streams=(), + logbooks={"outcome": ({"a": 1},), "activity": (_ACTIVITY_ROW,)}, + ) + assert hash_record(forward) == hash_record(backward) + + +def test_hash_record_covers_both_tiers_not_just_one() -> None: + """A record hash must differ from hashing either tier alone -- proof + that hash_record is not accidentally ignoring one of the two.""" + record = _record() + assert hash_record(record) != hash_streams(record.streams) + assert hash_record(record) != hash_logbooks(record.logbooks) + + +def test_empty_record_hashes_deterministically() -> None: + empty = ExportedRecord(streams=(), logbooks={}) + assert hash_record(empty) == hash_record(ExportedRecord(streams=(), logbooks={})) + + +def test_hash_streams_and_hash_logbooks_are_pinned_to_distinct_payload_types() -> None: + """Same body, different payload_type, must hash differently: this is + the whole point of binding payload_type into the PAE wrap.""" + record = _record() + assert hash_streams(record.streams) != hash_logbooks({"x": record.streams}) diff --git a/apps/api/tests/unit/infrastructure/record_export/test_manifest.py b/apps/api/tests/unit/infrastructure/record_export/test_manifest.py new file mode 100644 index 00000000000..334864a5b6a --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_manifest.py @@ -0,0 +1,139 @@ +"""Unit tests for the export manifest. + +`build_manifest` is pure (no I/O): every input is passed in, so these +tests construct synthetic `ExportedRecord`s directly rather than going +through a live database. The DB-backed acceptance path (via a real +Procedure + Run) lives in +`tests/integration/test_record_export_manifest_postgres.py`. +""" + +import re + +from cora.infrastructure.record_export import ( + ExportedRecord, + build_manifest, + capture_git_commit, + hash_record, + hash_redaction_profile, +) + +_RUN_A = "01900000-0000-7000-8000-0000000000a1" +_RUN_B = "01900000-0000-7000-8000-0000000000a2" +_PROC_EXPANDED = "01900000-0000-7000-8000-0000000000b1" +_PROC_DIRECT = "01900000-0000-7000-8000-0000000000b2" +_PROC_NO_RUN = "01900000-0000-7000-8000-0000000000b3" + + +def _stream_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "stream_type": "Procedure", + "stream_id": _PROC_EXPANDED, + "event_type": "ProcedureStarted", + "schema_version": 1, + "payload": {}, + } + row.update(overrides) + return row + + +def _procedure_registered( + procedure_id: str, parent_run_id: str | None, *, schema_version: int = 1 +) -> dict[str, object]: + return _stream_row( + stream_id=procedure_id, + event_type="ProcedureRegistered", + schema_version=schema_version, + payload={"procedure_id": procedure_id, "parent_run_id": parent_run_id}, + ) + + +def _recipe_expansion_recorded(procedure_id: str) -> dict[str, object]: + return _stream_row( + stream_id=procedure_id, + event_type="RecipeExpansionRecorded", + payload={"procedure_id": procedure_id}, + ) + + +def _run_started(run_id: str) -> dict[str, object]: + return _stream_row(stream_type="Run", stream_id=run_id, event_type="RunStarted", payload={}) + + +def _record() -> ExportedRecord: + streams = ( + _run_started(_RUN_A), + _run_started(_RUN_B), + _procedure_registered(_PROC_EXPANDED, _RUN_A), + _recipe_expansion_recorded(_PROC_EXPANDED), + _procedure_registered(_PROC_DIRECT, _RUN_B), + _procedure_registered(_PROC_NO_RUN, None), + _procedure_registered("01900000-0000-7000-8000-0000000000b9", None, schema_version=2), + ) + logbooks: dict[str, tuple[dict[str, object], ...]] = { + "activity": ({"step_kind": "setpoint"}, {"step_kind": "check"}), + "observation": ({"is_simulated": True}, {"is_simulated": True}), + } + return ExportedRecord(streams=streams, logbooks=logbooks) + + +def test_logbook_row_counts_match_each_kinds_length() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert manifest.row_count_by_logbook_kind == {"activity": 2, "observation": 2} + + +def test_max_schema_version_takes_the_max_per_event_type() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + # Two ProcedureRegistered rows in the fixture: schema_version 1 (the + # three _procedure_registered() calls) and 2 (the last row). + assert manifest.max_schema_version_by_event_type["ProcedureRegistered"] == 2 + assert manifest.max_schema_version_by_event_type["RunStarted"] == 1 + + +def test_is_simulated_true_when_every_observation_says_so() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert manifest.is_simulated is True + + +def test_is_simulated_false_on_a_single_dissenting_observation() -> None: + record = ExportedRecord( + streams=(), + logbooks={"observation": ({"is_simulated": True}, {"is_simulated": False})}, + ) + manifest = build_manifest(record, watermark=100, git_commit="deadbeef") + assert manifest.is_simulated is False + + +def test_is_simulated_vacuously_true_with_no_observations() -> None: + record = ExportedRecord(streams=(), logbooks={}) + manifest = build_manifest(record, watermark=100, git_commit="deadbeef") + assert manifest.is_simulated is True + + +def test_expansion_digest_present_only_for_the_run_whose_child_was_expanded() -> None: + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert manifest.expansion_digest_presence_by_run == {_RUN_A: True, _RUN_B: False} + + +def test_expansion_digest_ignores_procedures_with_no_parent_run() -> None: + """_PROC_NO_RUN has parent_run_id=None; it must not create a phantom + run entry or affect either real run's result.""" + manifest = build_manifest(_record(), watermark=100, git_commit="deadbeef") + assert set(manifest.expansion_digest_presence_by_run) == {_RUN_A, _RUN_B} + + +def test_manifest_hashes_match_calling_the_hash_functions_directly() -> None: + record = _record() + manifest = build_manifest(record, watermark=100, git_commit="deadbeef") + assert manifest.record_hash == hash_record(record) + assert manifest.redaction_profile_hash == hash_redaction_profile() + + +def test_manifest_carries_the_watermark_and_commit_verbatim() -> None: + manifest = build_manifest(_record(), watermark=4242, git_commit="cafef00d") + assert manifest.watermark == 4242 + assert manifest.git_commit == "cafef00d" + + +def test_capture_git_commit_returns_a_full_sha() -> None: + commit = capture_git_commit() + assert re.fullmatch(r"[0-9a-f]{40}", commit) diff --git a/apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py new file mode 100644 index 00000000000..20e06b8ff35 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier1.py @@ -0,0 +1,163 @@ +"""Unit tests for tier-1 (`events`) redaction. + +Uses `AgentDefined`'s real, generated disposition entry (not a +hand-tuned test-only one) for the payload-shape tests, so "a bare str +drops with nobody editing a list" is demonstrated against the actual +committed table. +""" + +from uuid import uuid4 + +import pytest + +from cora.infrastructure.record_export import TokenMap, UnknownEventTypeError, redact_tier1_payload +from cora.infrastructure.record_export._redact_tier1 import Tier1Redactor + +_AGENT_ID = "01900000-0000-7000-8000-0000000000c1" +_PROMPT_TEMPLATE_ID = "01900000-0000-7000-8000-0000000000c2" + + +def _agent_defined_payload() -> dict[str, object]: + return { + "agent_id": _AGENT_ID, + "canonical_uri": "https://internal/agents/foo", + "capabilities": "read,write", + "daily_token_cap": 1000, + "description": "an agent that does things", + "kind": "assistant", + "model_ref": {"model": "claude-x", "provider": "anthropic", "snapshot_pin": "2026-01-01"}, + "monthly_usd_cap": 50.0, + "name": "Foo Agent", + "occurred_at": "2026-05-15T12:00:00+00:00", + "prompt_template_id": _PROMPT_TEMPLATE_ID, + "tools": "search,browse", + "version": "v1", + } + + +def test_bare_str_drop_text_field_drops_with_no_list_edited() -> None: + """AgentDefined.name is drop:text in the real generated table.""" + redacted = redact_tier1_payload("AgentDefined", _agent_defined_payload(), token_map=TokenMap()) + assert "name" not in redacted + assert "canonical_uri" not in redacted + assert "kind" not in redacted + assert "tools" not in redacted + assert "version" not in redacted + + +def test_keep_number_and_keep_time_fields_survive_unchanged() -> None: + payload = _agent_defined_payload() + redacted = redact_tier1_payload("AgentDefined", payload, token_map=TokenMap()) + assert redacted["daily_token_cap"] == 1000 + assert redacted["monthly_usd_cap"] == 50.0 + assert redacted["occurred_at"] == "2026-05-15T12:00:00+00:00" + + +def test_token_uuid_fields_become_a_distinct_surrogate() -> None: + token_map = TokenMap() + redacted = redact_tier1_payload("AgentDefined", _agent_defined_payload(), token_map=token_map) + assert redacted["agent_id"] == token_map.token_uuid(_AGENT_ID) + assert redacted["agent_id"] != _AGENT_ID + assert redacted["prompt_template_id"] != _PROMPT_TEMPLATE_ID + + +def test_recursed_value_object_drops_its_own_drop_text_subfields() -> None: + """model_ref is a recursed VO whose 3 fields are all drop:text.""" + redacted = redact_tier1_payload("AgentDefined", _agent_defined_payload(), token_map=TokenMap()) + assert "model_ref" not in redacted or redacted["model_ref"] == {} + + +def test_a_payload_key_absent_from_a_known_events_field_list_drops() -> None: + """Schema evolution: an older schema_version's row can carry a field + the current dataclass no longer declares. Must drop, not abort -- + aborting would make every export containing legacy-schema rows fail.""" + payload = _agent_defined_payload() + payload["a_field_removed_in_a_later_schema_version"] = "still in an old row" + redacted = redact_tier1_payload("AgentDefined", payload, token_map=TokenMap()) + assert "a_field_removed_in_a_later_schema_version" not in redacted + # The known fields are still processed normally. + assert redacted["daily_token_cap"] == 1000 + + +def test_unknown_event_type_aborts() -> None: + with pytest.raises(UnknownEventTypeError) as excinfo: + redact_tier1_payload("TotallyMadeUpEventType", {}, token_map=TokenMap()) + assert excinfo.value.event_type == "TotallyMadeUpEventType" + + +def _stream_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "position": 999, + "version": 999, + "transaction_id": "12345", + "schema_version": 1, + "stream_type": "Agent", + "event_type": "AgentDefined", + "occurred_at": "2026-05-15T12:00:00+00:00", + "recorded_at": "2026-05-15T12:00:01+00:00", + "stream_id": str(uuid4()), + "correlation_id": str(uuid4()), + "causation_id": None, + "event_id": str(uuid4()), + "principal_id": str(uuid4()), + "metadata": {"some": "metadata"}, + "signature": "deadbeef", + "signature_kid": "key-1", + "signature_version": "v1", + "payload": _agent_defined_payload(), + } + row.update(overrides) + return row + + +def test_metadata_and_signature_columns_are_absent_from_the_redacted_row() -> None: + redacted = Tier1Redactor(TokenMap()).redact_row(_stream_row()) + assert "metadata" not in redacted + assert "signature" not in redacted + assert "signature_kid" not in redacted + assert "signature_version" not in redacted + + +def test_position_is_dense_from_one_across_calls() -> None: + redactor = Tier1Redactor(TokenMap()) + first = redactor.redact_row(_stream_row(position=4400)) + second = redactor.redact_row(_stream_row(position=9100)) + assert first["position"] == 1 + assert second["position"] == 2 + + +def test_version_is_dense_per_stream_and_independent_across_streams() -> None: + redactor = Tier1Redactor(TokenMap()) + stream_a = str(uuid4()) + stream_b = str(uuid4()) + a1 = redactor.redact_row(_stream_row(stream_id=stream_a, version=1)) + b1 = redactor.redact_row(_stream_row(stream_id=stream_b, version=1)) + a2 = redactor.redact_row(_stream_row(stream_id=stream_a, version=2)) + assert a1["version"] == 1 + assert b1["version"] == 1 + assert a2["version"] == 2 + + +def test_transaction_id_is_a_small_monotone_int_not_the_raw_value() -> None: + redactor = Tier1Redactor(TokenMap()) + first = redactor.redact_row(_stream_row(transaction_id="99999999")) + second_same_tx = redactor.redact_row(_stream_row(transaction_id="99999999")) + third_new_tx = redactor.redact_row(_stream_row(transaction_id="100000000")) + assert first["transaction_id"] == 1 + assert second_same_tx["transaction_id"] == 1 # same raw tx -> same dense id + assert third_new_tx["transaction_id"] == 2 + assert third_new_tx["transaction_id"] != 100000000 + + +def test_stream_id_correlation_id_and_event_id_are_tokened() -> None: + token_map = TokenMap() + raw_stream_id = str(uuid4()) + row = _stream_row(stream_id=raw_stream_id) + redacted = Tier1Redactor(token_map).redact_row(row) + assert redacted["stream_id"] == token_map.token_uuid(raw_stream_id) + assert redacted["stream_id"] != raw_stream_id + + +def test_causation_id_none_stays_none() -> None: + redacted = Tier1Redactor(TokenMap()).redact_row(_stream_row(causation_id=None)) + assert redacted["causation_id"] is None diff --git a/apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py new file mode 100644 index 00000000000..3829d3c2ef2 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_redact_tier2.py @@ -0,0 +1,153 @@ +"""Unit tests for tier-2 (`entries_*`) redaction: the hand-authored +per-kind disposition table, jsonb recursion, and the unfired-clearance +check.""" + +from uuid import uuid4 + +import pytest + +from cora.infrastructure.record_export import TokenMap +from cora.infrastructure.record_export._redact_tier2 import ( + TIER2_DISPOSITIONS, + UnfiredClearanceError, + ensure_all_clearances_fired, + redact_tier2_row, +) + + +def test_proved_closed_text_column_survives() -> None: + """conduit_verdicts.decision: DB CHECK (decision IN ('Allow','Deny')).""" + row = {"event_id": str(uuid4()), "decision": "Allow"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert redacted["decision"] == "Allow" + + +def test_judged_low_risk_text_column_survives() -> None: + row = {"event_id": str(uuid4()), "command_name": "AppendActivities"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert redacted["command_name"] == "AppendActivities" + + +def test_dropped_text_column_is_omitted() -> None: + """conduit_verdicts.reason: P0-4, builds free text that can republish + a tokened UUID.""" + row = {"event_id": str(uuid4()), "reason": "Principal ... not in policy"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert "reason" not in redacted + + +def test_uuid_column_is_tokened() -> None: + token_map = TokenMap() + raw = str(uuid4()) + row = {"event_id": raw} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=token_map, fired_pointers=fired) + assert redacted["event_id"] == token_map.token_uuid(raw) + assert redacted["event_id"] != raw + + +def test_a_column_absent_from_the_disposition_table_is_omitted() -> None: + """Fail closed: an entries column this file never enumerated drops, + same posture as tier 1's missing-key rule.""" + row = {"event_id": str(uuid4()), "a_column_nobody_listed": "surprise"} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("verdict", row, token_map=TokenMap(), fired_pointers=fired) + assert "a_column_nobody_listed" not in redacted + + +def test_activities_payload_keeps_cleared_string_leaves_and_drops_others() -> None: + row = { + "event_id": str(uuid4()), + "payload": { + "channel": "T_oven", + "target_value": 423.0, + "units": "K", + "action_name": "open_valve", + "an_uncleared_free_text_field": "should drop", + }, + } + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("activity", row, token_map=TokenMap(), fired_pointers=fired) + payload = redacted["payload"] + assert payload["channel"] == "T_oven" + assert payload["units"] == "K" + assert payload["action_name"] == "open_valve" + assert payload["target_value"] == 423.0 + assert "an_uncleared_free_text_field" not in payload + + +def test_activities_payload_tokens_a_uuid_shaped_string_leaf() -> None: + token_map = TokenMap() + raw = str(uuid4()) + row = {"event_id": str(uuid4()), "payload": {"asset_id": raw}} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("activity", row, token_map=token_map, fired_pointers=fired) + assert redacted["payload"]["asset_id"] == token_map.token_uuid(raw) + + +def test_outcomes_measurements_keeps_cleared_pointers_and_drops_quality_detail() -> None: + row = { + "event_id": str(uuid4()), + "measurements": [ + { + "name": "flux", + "value": 1.23, + "kind": "Scalar", + "quality": "Good", + "quality_detail": "opaque forensic string", + "units": "cps", + } + ], + } + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("outcome", row, token_map=TokenMap(), fired_pointers=fired) + measurement = redacted["measurements"][0] + assert measurement["name"] == "flux" + assert measurement["units"] == "cps" + assert measurement["kind"] == "Scalar" + assert measurement["quality"] == "Good" + assert measurement["value"] == 1.23 + assert "quality_detail" not in measurement + + +def test_decision_inferences_messages_drops_whole() -> None: + row = {"event_id": str(uuid4()), "messages": [{"role": "user", "content": "secret prompt"}]} + fired: dict[tuple[str, str], set[str]] = {} + redacted = redact_tier2_row("inference", row, token_map=TokenMap(), fired_pointers=fired) + assert "messages" not in redacted + + +@pytest.mark.parametrize("kind", sorted(TIER2_DISPOSITIONS)) +def test_every_declared_kind_has_at_least_one_uuid_scope_column(kind: str) -> None: + """Sanity check on the hand-authored table itself: every kind has at + least one `token` column (its logbook_id/run_id/enclosure_id scope + column at minimum).""" + assert "token" in TIER2_DISPOSITIONS[kind].values() + + +def test_unfired_clearance_raises_when_a_declared_pointer_never_matched() -> None: + # No channel/action_name/units in this payload. + row = {"event_id": str(uuid4()), "payload": {"target_value": 423.0}} + fired: dict[tuple[str, str], set[str]] = {} + redact_tier2_row("activity", row, token_map=TokenMap(), fired_pointers=fired) + with pytest.raises(UnfiredClearanceError): + ensure_all_clearances_fired(fired, kinds_present=frozenset({"activity"})) + + +def test_unfired_clearance_does_not_raise_for_a_kind_not_present() -> None: + """An unused clearance for a kind with zero rows in this export is + not a leak -- nothing exported to leak.""" + ensure_all_clearances_fired({}, kinds_present=frozenset({"verdict"})) + + +def test_all_declared_clearances_fire_when_every_pointer_is_exercised() -> None: + row = { + "event_id": str(uuid4()), + "payload": {"channel": "T_oven", "action_name": "open_valve", "units": "K"}, + } + fired: dict[tuple[str, str], set[str]] = {} + redact_tier2_row("activity", row, token_map=TokenMap(), fired_pointers=fired) + ensure_all_clearances_fired(fired, kinds_present=frozenset({"activity"})) # must not raise diff --git a/apps/api/tests/unit/infrastructure/record_export/test_registry.py b/apps/api/tests/unit/infrastructure/record_export/test_registry.py new file mode 100644 index 00000000000..6a84a185a3d --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_registry.py @@ -0,0 +1,79 @@ +"""Unit tests for the entries-tier registry: `kind -> (table, order key, reader)`. + +See `cora.infrastructure.record_export._registry` for the design: eight +entries, six resolved from a `*LogbookOpened` envelope's `kind` and two +(`heartbeat`, `probe`) declared explicitly because they have no envelope. +""" + +import pytest + +from cora.infrastructure.record_export import ( + UnknownLogbookKindError, + all_specs, + registered_envelope_classes, + resolve, +) + +_ENVELOPE_DRIVEN_KINDS = ( + "verdict", + "inference", + "activity", + "diagnostic", + "outcome", + "observation", +) +_DECLARED_KINDS = ("heartbeat", "probe") + + +def test_registry_has_eight_entries_not_six() -> None: + assert len(all_specs()) == 8 + + +@pytest.mark.parametrize("kind", [*_ENVELOPE_DRIVEN_KINDS, *_DECLARED_KINDS]) +def test_resolve_finds_every_registered_kind(kind: str) -> None: + spec = resolve(kind) + assert spec.kind == kind + + +@pytest.mark.parametrize("kind", _ENVELOPE_DRIVEN_KINDS) +def test_envelope_driven_kinds_carry_their_envelope_class(kind: str) -> None: + spec = resolve(kind) + assert spec.envelope_class is not None + assert spec.envelope_class.endswith("LogbookOpened") + assert spec.scope_column == "logbook_id" + + +@pytest.mark.parametrize("kind", _DECLARED_KINDS) +def test_declared_kinds_have_no_envelope_class(kind: str) -> None: + spec = resolve(kind) + assert spec.envelope_class is None + assert spec.scope_column != "logbook_id" + + +def test_registered_envelope_classes_has_exactly_the_six_logbook_kinds() -> None: + assert registered_envelope_classes() == { + "ConduitLogbookOpened", + "DecisionLogbookOpened", + "ProcedureActivitiesLogbookOpened", + "ProcedureDiagnosticLogbookOpened", + "ProcedureOutcomeLogbookOpened", + "RunObservationLogbookOpened", + } + + +def test_unknown_kind_refuses_loudly_instead_of_returning_none() -> None: + with pytest.raises(UnknownLogbookKindError) as excinfo: + resolve("steps") # the pre-rename name; must not silently resurrect it + assert excinfo.value.kind == "steps" + + +def test_order_by_uses_sampled_at_on_exactly_the_four_tables_that_have_it() -> None: + with_sampled_at = {spec.kind for spec in all_specs() if spec.order_by[0] == "sampled_at"} + assert with_sampled_at == {"activity", "diagnostic", "outcome", "observation"} + without_sampled_at = {spec.kind for spec in all_specs() if spec.order_by == ("event_id",)} + assert without_sampled_at == {"verdict", "inference", "heartbeat", "probe"} + + +def test_table_names_are_unique_across_the_registry() -> None: + tables = [spec.table for spec in all_specs()] + assert len(tables) == len(set(tables)) diff --git a/apps/api/tests/unit/infrastructure/record_export/test_render.py b/apps/api/tests/unit/infrastructure/record_export/test_render.py new file mode 100644 index 00000000000..f6c35bd9e6d --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_render.py @@ -0,0 +1,73 @@ +"""Unit tests for F6 rendering: `render_value` / `render_row`. + +See `cora.infrastructure.record_export._render` for why this only +touches typed outer-row columns and must not recurse into jsonb. +""" + +from datetime import UTC, datetime, timedelta, timezone +from uuid import UUID + +from cora.infrastructure.record_export import render_row, render_value + +_SOME_UUID = UUID("12345678-1234-5678-1234-567812345678") + + +def test_uuid_renders_as_its_string_form() -> None: + assert render_value(_SOME_UUID) == "12345678-1234-5678-1234-567812345678" + + +def test_utc_datetime_renders_as_iso8601() -> None: + dt = datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC) + assert render_value(dt) == "2026-05-31T12:00:00+00:00" + + +def test_session_local_offset_datetime_normalizes_to_utc() -> None: + """The same instant at -05:00 must render identically to +00:00: this + is the measured hazard F6 exists to close, not a stylistic choice.""" + minus_five = datetime(2026, 5, 31, 7, 0, 0, tzinfo=timezone(timedelta(hours=-5))) + utc = datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC) + assert render_value(minus_five) == render_value(utc) == "2026-05-31T12:00:00+00:00" + + +def test_bytes_renders_as_hex() -> None: + assert render_value(b"\x00\xff\xab") == "00ffab" + + +def test_bytearray_and_memoryview_render_as_hex() -> None: + assert render_value(bytearray(b"\x01\x02")) == "0102" + assert render_value(memoryview(b"\x01\x02")) == "0102" + + +def test_plain_primitives_pass_through_unchanged() -> None: + assert render_value("channel-a") == "channel-a" + assert render_value(42) == 42 + assert render_value(3.14) == 3.14 + assert render_value(True) is True + assert render_value(None) is None + + +def test_jsonb_decoded_dict_and_list_pass_through_unrendered() -> None: + """payload/metadata already arrive as JSON-safe primitives (every + to_payload() pre-converts UUID/datetime to strings before the row is + written); render_value must not try to walk into them.""" + payload = {"logbook_id": "12345678-1234-5678-1234-567812345678", "kind": "activity"} + assert render_value(payload) is payload + assert render_value([1, "a", None]) == [1, "a", None] + + +def test_render_row_applies_render_value_to_every_column() -> None: + row: dict[str, object] = { + "event_id": _SOME_UUID, + "occurred_at": datetime(2026, 5, 31, 12, 0, 0, tzinfo=UTC), + "signature": b"\xde\xad", + "event_type": "ProcedureRegistered", + "payload": {"a": 1}, + } + rendered = render_row(row) + assert rendered == { + "event_id": "12345678-1234-5678-1234-567812345678", + "occurred_at": "2026-05-31T12:00:00+00:00", + "signature": "dead", + "event_type": "ProcedureRegistered", + "payload": {"a": 1}, + } diff --git a/apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py b/apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py new file mode 100644 index 00000000000..269157bd7ec --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_standalone_verifier.py @@ -0,0 +1,190 @@ +"""Tests for the standalone, zero-cora-import verifier at +`scripts/verify_record_hash.py`. + +Per `project_record_export_build_brief.md` step 5's acceptance: a unit +test cross-checking it byte-for-byte against `cora.shared.content_hash` +over a corpus including NFC and float cases; a subprocess test that +fails on a flipped byte. + +The script is loaded via `importlib` for the byte-for-byte cross-check +(same dynamic-import bridge `tests/unit/deployments/test_beamline_descriptor.py` +uses for other `scripts/` modules, since `scripts/` is not on the +type-checker's or the `cora` package's path) and invoked as a genuine +subprocess for the CLI / flipped-byte tests, so at least one test proves +the file runs as an actual standalone OS process, not just an +importable module. +""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from cora.shared.content_hash import compute_content_hash as cora_compute_content_hash + +if TYPE_CHECKING: + from types import ModuleType + +_REPO_ROOT = Path(__file__).resolve().parents[6] +_SCRIPT = _REPO_ROOT / "scripts" / "verify_record_hash.py" +_PAYLOAD_TYPE = "application/vnd.cora.record-test+json" + +# Precomposed 'e-acute' (single codepoint U+00E9) vs decomposed ('e' +# U+0065 + combining acute accent U+0301). Both display as "e" with an +# accent but are different byte sequences until NFC-normalized. +_PRECOMPOSED_E_ACUTE = "café" +_DECOMPOSED_E_ACUTE = "café" + + +def _load_verifier() -> ModuleType: + spec = importlib.util.spec_from_file_location("verify_record_hash", _SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load verify_record_hash from {_SCRIPT}") + module = importlib.util.module_from_spec(spec) + sys.modules["verify_record_hash"] = module + spec.loader.exec_module(module) + return module + + +_verifier = _load_verifier() + +_CORPUS: list[Any] = [ + {"a": 1, "b": "text"}, + {"target_value": 423.0, "tolerance": 3.14159, "ramp_rate": 0.0}, + {"name": _PRECOMPOSED_E_ACUTE}, + {"name": _DECOMPOSED_E_ACUTE}, + {"flag": True, "missing": None}, + {"items": [{"k": "v"}, {"k2": [1, 2, 3]}], "nested": {"a": {"b": {"c": None}}}}, + {"turkish": "dotless ı vs dotted i İ", "emoji": "\U0001f52c"}, # noqa: RUF001 + [1, "two", 3.0, None, True, {"five": 5}], + {}, + [], + "bare string body", + 42, +] + + +def test_script_has_zero_cora_imports() -> None: + source = _SCRIPT.read_text(encoding="utf-8") + for line in source.splitlines(): + stripped = line.strip() + assert not stripped.startswith("import cora"), ( + f"{_SCRIPT} imports cora ({stripped!r}); the whole point of this " + "file is running on a machine with no CORA installed." + ) + assert not stripped.startswith("from cora"), ( + f"{_SCRIPT} imports cora ({stripped!r}); the whole point of this " + "file is running on a machine with no CORA installed." + ) + + +@pytest.mark.parametrize("body", _CORPUS, ids=range(len(_CORPUS))) +def test_compute_content_hash_matches_cora_byte_for_byte(body: Any) -> None: + assert _verifier.compute_content_hash(_PAYLOAD_TYPE, body) == cora_compute_content_hash( + _PAYLOAD_TYPE, body + ) + + +def test_composed_and_decomposed_nfc_forms_hash_identically() -> None: + assert _PRECOMPOSED_E_ACUTE != _DECOMPOSED_E_ACUTE # different bytes pre-normalization + composed = {"name": _PRECOMPOSED_E_ACUTE} + decomposed = {"name": _DECOMPOSED_E_ACUTE} + assert _verifier.compute_content_hash( + _PAYLOAD_TYPE, composed + ) == _verifier.compute_content_hash(_PAYLOAD_TYPE, decomposed) + + +def test_cli_hash_subcommand_prints_the_matching_digest(tmp_path: Path) -> None: + body = {"a": 1, "b": _PRECOMPOSED_E_ACUTE} + body_file = tmp_path / "body.json" + body_file.write_text(json.dumps(body), encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(_SCRIPT), "hash", "--payload-type", _PAYLOAD_TYPE, str(body_file)], + capture_output=True, + text=True, + check=True, + ) + + expected = cora_compute_content_hash(_PAYLOAD_TYPE, body) + assert result.stdout.strip() == expected + + +def test_cli_verify_subcommand_exits_zero_on_a_match(tmp_path: Path) -> None: + body = {"a": 1, "b": _PRECOMPOSED_E_ACUTE} + body_file = tmp_path / "body.json" + body_file.write_text(json.dumps(body), encoding="utf-8") + expected = cora_compute_content_hash(_PAYLOAD_TYPE, body) + + result = subprocess.run( + [ + sys.executable, + str(_SCRIPT), + "verify", + "--payload-type", + _PAYLOAD_TYPE, + "--expected-hash", + expected, + str(body_file), + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "OK" in result.stdout + + +def test_cli_verify_subcommand_fails_on_a_flipped_byte(tmp_path: Path) -> None: + """The acceptance test named explicitly: a subprocess run against a + tampered file must FAIL (nonzero exit), proving the recomputed hash + is sensitive to the tamper rather than silently passing.""" + body = {"a": 1, "b": _PRECOMPOSED_E_ACUTE, "target_value": 423.0} + body_file = tmp_path / "body.json" + body_file.write_text(json.dumps(body), encoding="utf-8") + expected = cora_compute_content_hash(_PAYLOAD_TYPE, body) + + def _verify() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(_SCRIPT), + "verify", + "--payload-type", + _PAYLOAD_TYPE, + "--expected-hash", + expected, + str(body_file), + ], + capture_output=True, + text=True, + ) + + before = _verify() + assert before.returncode == 0 + + # Flip exactly one character in the file's bytes on disk. + original_text = body_file.read_text(encoding="utf-8") + tampered_text = original_text.replace("423.0", "424.0") + assert tampered_text != original_text + body_file.write_text(tampered_text, encoding="utf-8") + + after = _verify() + assert after.returncode == 1 + assert "MISMATCH" in after.stderr + + +def test_cli_reports_a_clean_error_on_unreadable_input(tmp_path: Path) -> None: + missing = tmp_path / "does_not_exist.json" + result = subprocess.run( + [sys.executable, str(_SCRIPT), "hash", "--payload-type", _PAYLOAD_TYPE, str(missing)], + capture_output=True, + text=True, + ) + assert result.returncode == 2 diff --git a/apps/api/tests/unit/infrastructure/record_export/test_stream_types.py b/apps/api/tests/unit/infrastructure/record_export/test_stream_types.py new file mode 100644 index 00000000000..5b2567bca72 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_stream_types.py @@ -0,0 +1,25 @@ +"""Unit tests for the closed `stream_type` set.""" + +import pytest + +from cora.infrastructure.record_export import ( + KNOWN_STREAM_TYPES, + UnknownStreamTypeError, + ensure_stream_type_known, +) + + +@pytest.mark.parametrize("stream_type", sorted(KNOWN_STREAM_TYPES)) +def test_ensure_stream_type_known_accepts_every_declared_stream_type(stream_type: str) -> None: + ensure_stream_type_known(stream_type) # must not raise + + +def test_ensure_stream_type_known_refuses_an_undeclared_stream_type() -> None: + with pytest.raises(UnknownStreamTypeError) as excinfo: + ensure_stream_type_known("Widget") + assert excinfo.value.stream_type == "Widget" + + +def test_known_stream_types_has_no_empty_or_duplicate_entries() -> None: + assert "" not in KNOWN_STREAM_TYPES + assert len(KNOWN_STREAM_TYPES) == len({s.strip() for s in KNOWN_STREAM_TYPES}) diff --git a/apps/api/tests/unit/infrastructure/record_export/test_tokens.py b/apps/api/tests/unit/infrastructure/record_export/test_tokens.py new file mode 100644 index 00000000000..b756b7aa379 --- /dev/null +++ b/apps/api/tests/unit/infrastructure/record_export/test_tokens.py @@ -0,0 +1,52 @@ +"""Unit tests for the per-export UUID surrogate map.""" + +from cora.infrastructure.record_export import TokenMap + +_SOURCE_A = "01900000-0000-7000-8000-0000000000a1" +_SOURCE_B = "01900000-0000-7000-8000-0000000000a2" + + +def test_none_passes_through_as_none() -> None: + assert TokenMap().token_uuid(None) is None + + +def test_same_source_returns_the_same_surrogate_within_one_map() -> None: + token_map = TokenMap() + first = token_map.token_uuid(_SOURCE_A) + second = token_map.token_uuid(_SOURCE_A) + assert first == second + + +def test_distinct_sources_get_distinct_surrogates() -> None: + token_map = TokenMap() + assert token_map.token_uuid(_SOURCE_A) != token_map.token_uuid(_SOURCE_B) + + +def test_surrogate_is_never_equal_to_its_source() -> None: + token_map = TokenMap() + assert token_map.token_uuid(_SOURCE_A) != _SOURCE_A + + +def test_surrogate_is_not_derivable_from_its_source() -> None: + """Two independent TokenMaps produce DIFFERENT surrogates for the + SAME source: a hash would be deterministic (same input, same + output, every time) and therefore brute-forceable against a known + candidate roster. A random mint is not.""" + first_map = TokenMap() + second_map = TokenMap() + assert first_map.token_uuid(_SOURCE_A) != second_map.token_uuid(_SOURCE_A) + + +def test_surrogate_by_source_reflects_every_source_tokenized_so_far() -> None: + token_map = TokenMap() + a = token_map.token_uuid(_SOURCE_A) + b = token_map.token_uuid(_SOURCE_B) + assert token_map.surrogate_by_source == {_SOURCE_A: a, _SOURCE_B: b} + + +def test_surrogate_by_source_is_a_copy_not_a_live_view() -> None: + token_map = TokenMap() + token_map.token_uuid(_SOURCE_A) + snapshot = token_map.surrogate_by_source + snapshot["injected"] = "not-real" + assert "injected" not in token_map.surrogate_by_source diff --git a/apps/api/tools/gen_record_dispositions.py b/apps/api/tools/gen_record_dispositions.py new file mode 100644 index 00000000000..fe7e2c0d6c9 --- /dev/null +++ b/apps/api/tools/gen_record_dispositions.py @@ -0,0 +1,316 @@ +"""Generate the record-export redaction disposition table. + +The record exporter decides what to publish from a field's DECLARED TYPE +(see the F5 section of `project_record_export_v3.md`). Answering "is this +annotation a StrEnum, a str alias, or a value object" needs the defining +module imported, and the exporter lives at `cora.infrastructure`, where +`tach.toml` allows `cora.shared` and nothing else. So the question is +answered HERE, at build time, by a tool that may import every bounded +context, and the answer is committed as inert data the exporter reads. + +This mirrors `make openapi-snapshot`: a generated artifact in the tree, +a drift test that fails until it is regenerated, and a diff a reviewer +can read. Living OUTSIDE `src/` is deliberate and structural, not +cosmetic: nothing the application ships can import this module, so the +exporter cannot acquire the BC dependency the design forbids it. + +Run it with: + + make record-dispositions + +Resolution happens on real type objects rather than on annotation +strings, which is the point. A plain alias (`ActorId = UUID`) collapses +to its target for free; only `NewType` and genuine value objects need +handling. An annotation this tool cannot classify ABORTS the run: an +unrecognized type is a question about the design, not a row to skip. +""" + +from __future__ import annotations + +import dataclasses +import enum +import importlib +import inspect +import json +import subprocess +import sys +import types +import typing +from collections.abc import Iterable, Mapping, MutableMapping, Sequence +from datetime import datetime +from pathlib import Path +from typing import Any, Literal, NewType, Union, get_args, get_origin +from uuid import UUID + +_API_ROOT = Path(__file__).resolve().parents[1] +_SRC = _API_ROOT / "src" +_OUT = _SRC / "cora" / "infrastructure" / "record_export" / "_dispositions.py" + +KEEP_ENUM = "keep:enum" +KEEP_NUMBER = "keep:number" +KEEP_TIME = "keep:time" +TOKEN_UUID = "token:uuid" +DROP_TEXT = "drop:text" +DROP_OPAQUE = "drop:opaque" +BY_VALUE = "by-value" + +_SCALAR_KEEP: Mapping[type, str] = { + bool: KEEP_NUMBER, + int: KEEP_NUMBER, + float: KEEP_NUMBER, + datetime: KEEP_TIME, +} + +_CONTAINER_ORIGINS = (tuple, frozenset, set, list) + + +class UnclassifiedAnnotationError(Exception): + """An annotation no rule in F5 covers. Aborts the run by design.""" + + def __init__(self, event: str, field: str, annotation: object) -> None: + super().__init__( + f"{event}.{field}: cannot classify {annotation!r}. " + "Add a rule to gen_record_dispositions.py, or change the " + "declared type. Do NOT silently drop it." + ) + + +def _union_variants(annotation: Any) -> list[Any] | None: + """Non-None members of a union, or None if this is not a union.""" + origin = get_origin(annotation) + if origin is not Union and origin is not types.UnionType: + return None + return [a for a in get_args(annotation) if a is not type(None)] + + +def _merge( + variants: Sequence[str | dict[str, Any]], event: str, field: str, ann: Any +) -> str | dict[str, Any]: + """Fold the dispositions of a union's variants into one. + + Scalars must agree. Value objects are MERGED, because a discriminated + union stores whichever arm's keys, so the published rule has to cover + every arm. A key two arms disagree on is a real ambiguity and aborts. + """ + if all(isinstance(v, str) for v in variants): + if len({typing.cast("str", v) for v in variants}) == 1: + return typing.cast("str", variants[0]) + return BY_VALUE + if not all(isinstance(v, dict) for v in variants): + # A slot spanning scalars AND value objects (a Recipe setpoint is + # a number, a channel string, or a binding reference) cannot be + # decided statically. Defer to the same leaf rule the tier-2 jsonb + # columns already use, which is fail-closed on strings. + return BY_VALUE + merged: dict[str, Any] = {} + for variant in variants: + for key, disposition in typing.cast("dict[str, Any]", variant).items(): + if key in merged and merged[key] != disposition: + raise UnclassifiedAnnotationError(event, field, ann) + merged[key] = disposition + return merged + + +def _is_value_object(annotation: Any) -> bool: + """True for a frozen dataclass, which has declared fields to recurse into.""" + if not inspect.isclass(annotation) or not dataclasses.is_dataclass(annotation): + return False + params: Any = getattr(annotation, "__dataclass_params__", None) + return bool(params is not None and params.frozen) + + +def _classify(annotation: Any, event: str, field: str) -> str | dict[str, Any]: + """Map one resolved annotation to a disposition, per the F5 table.""" + if isinstance(annotation, typing.TypeAliasType): + return _classify(annotation.__value__, event, field) + + variants = _union_variants(annotation) + if variants is not None: + if len(variants) == 1: + return _classify(variants[0], event, field) + return _merge([_classify(v, event, field) for v in variants], event, field, annotation) + + if annotation is Any: + return DROP_OPAQUE + + if isinstance(annotation, NewType): + return _classify(annotation.__supertype__, event, field) + + if get_origin(annotation) is Literal: + args = get_args(annotation) + if all(isinstance(a, str) for a in args): + return KEEP_ENUM + if all(isinstance(a, bool | int | float) for a in args): + return KEEP_NUMBER + raise UnclassifiedAnnotationError(event, field, annotation) + + if inspect.isclass(annotation): + if issubclass(annotation, enum.Enum): + # Name the enum. The value set is what a human signs off, so + # the disposition has to say WHICH set, and swapping one enum + # for another must show up as drift rather than as no change. + return f"{KEEP_ENUM}:{annotation.__name__}" + if annotation is UUID: + return TOKEN_UUID + if annotation is str: + return DROP_TEXT + for scalar, disposition in _SCALAR_KEEP.items(): + if annotation is scalar: + return disposition + if _is_value_object(annotation): + return _resolve_fields(annotation) + + origin = get_origin(annotation) + if origin in _CONTAINER_ORIGINS: + raw = get_args(annotation) + args = [a for a in raw if a is not Ellipsis] + if not args: + raise UnclassifiedAnnotationError(event, field, annotation) + inner = [_classify(a, event, field) for a in args] + if origin is tuple and Ellipsis not in raw and len(inner) > 1: + # A fixed-length heterogeneous tuple is a positional record, + # not a collection: `tuple[str, float]` is (name, value) and + # its two slots get different answers. Emit one per position. + return {"[]": inner} + if any(candidate != inner[0] for candidate in inner[1:]): + raise UnclassifiedAnnotationError(event, field, annotation) + return inner[0] + if origin in (dict, Mapping, MutableMapping): + return DROP_OPAQUE + + raise UnclassifiedAnnotationError(event, field, annotation) + + +def _resolve_fields(cls: type) -> dict[str, Any]: + """Disposition per field of one dataclass, recursing into value objects.""" + hints = typing.get_type_hints(cls) + out: dict[str, Any] = {} + for spec in dataclasses.fields(cls): + out[spec.name] = _classify(hints[spec.name], cls.__name__, spec.name) + return out + + +def _event_modules() -> list[str]: + """Dotted names of every `events.py` under the cora package.""" + names: list[str] = [] + for path in sorted((_SRC / "cora").rglob("events.py")): + names.append(".".join(path.relative_to(_SRC).with_suffix("").parts)) + return names + + +def _event_classes(module_name: str) -> Iterable[type]: + """Frozen dataclasses DEFINED in this module, deduplicated by identity. + + A single-member union alias (`AcquisitionEvent = AcquisitionRecorded`) + binds one class under two module-level names, so `getmembers` yields + it twice. Dedupe on identity, not on name. + """ + module = importlib.import_module(module_name) + seen: set[int] = set() + for _, obj in inspect.getmembers(module, inspect.isclass): + if obj.__module__ != module_name or not _is_value_object(obj): + continue + if id(obj) in seen: + continue + seen.add(id(obj)) + yield obj + + +def build_table(survey: bool = False) -> tuple[dict[str, dict[str, Any]], list[str]]: + """Disposition per (event type, field) across every bounded context. + + With `survey`, collect every unclassified annotation instead of + aborting on the first. The generator still refuses to write a table + while any remain; the flag exists so the tail can be read in one + pass rather than one exception at a time. + """ + table: dict[str, dict[str, Any]] = {} + unclassified: list[str] = [] + for module_name in _event_modules(): + for cls in _event_classes(module_name): + if cls.__name__ in table: + raise RuntimeError( + f"Duplicate event class name {cls.__name__!r}; the table is " + "keyed on the bare name because that is what `events.event_type` " + "stores. Rename one, or key on the qualified name." + ) + if not survey: + table[cls.__name__] = _resolve_fields(cls) + continue + try: + table[cls.__name__] = _resolve_fields(cls) + except UnclassifiedAnnotationError as exc: + unclassified.append(str(exc).split(".", 1)[0] + ": " + str(exc)) + return dict(sorted(table.items())), unclassified + + +def render(table: Mapping[str, Mapping[str, Any]]) -> str: + """Render the table as a committed Python module.""" + body = json.dumps(table, indent=4, sort_keys=True) + return f'''"""Generated redaction dispositions. DO NOT EDIT BY HAND. + +Regenerate with `make record-dispositions`; +`tests/architecture/test_record_dispositions_drift.py` fails until you do. + +One entry per event type, one disposition per declared field, resolved +from the field's real type by `tools/gen_record_dispositions.py`. The +vocabulary: + + keep:enum: closed value set, provably reviewable. The enum is + NAMED because a human signs off the value set, and + swapping one enum for another must read as drift. + keep:number int / float / bool + keep:time datetime + token:uuid replaced with a per-export random surrogate + drop:text free text, no finite range, dropped by default + drop:opaque a dict with no declared keys, nothing to allowlist + by-value the slot is polymorphic across scalars and objects, + so no static answer exists. Apply the tier-2 leaf + rule at export time: numbers and booleans keep, + UUID-shaped strings token, other strings drop. + +A nested mapping is a value object recursed into. A mapping whose sole +key is `[]` is a fixed-length heterogeneous tuple, and its value lists +one disposition per position. + +Redaction iterates the STORED payload's keys and looks each up here. A +key absent from its event's entry is dropped; an event type absent from +this table aborts the export. The canonical hash of this mapping is the +redaction profile hash recorded in the export manifest. +""" + +from typing import Any + +DISPOSITIONS: dict[str, dict[str, Any]] = {body} +''' + + +def main() -> int: + survey = "--survey" in sys.argv + table, unclassified = build_table(survey=survey) + if unclassified: + print(f"{len(unclassified)} unclassified annotations:", file=sys.stderr) + for line in unclassified: + print(f" {line}", file=sys.stderr) + return 1 + _OUT.parent.mkdir(parents=True, exist_ok=True) + _OUT.write_text(render(table), encoding="utf-8") + # Format in place so the committed artifact is what `make lint` + # expects. Without this the generator and the formatter disagree and + # the drift test can never be green at the same time as lint. + formatted = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--quiet", str(_OUT)], + capture_output=True, + text=True, + check=False, + ) + if formatted.returncode != 0: + print(formatted.stderr, file=sys.stderr) + return 1 + fields = sum(len(v) for v in table.values()) + print(f"{len(table)} event types, {fields} fields -> {_OUT.relative_to(_API_ROOT)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify_record_hash.py b/scripts/verify_record_hash.py new file mode 100644 index 00000000000..a77fe83404a --- /dev/null +++ b/scripts/verify_record_hash.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Standalone content-hash verifier for a CORA record export. + +Per `project_record_export_build_brief.md` step 5 and +`project_record_export_v3.md` F4: "sha256 and a JSON reader, no CORA" was +the earlier (falsified) claim -- `compute_content_hash` is SHA-256 over +DSSE-PAE-wrapped, NFC-normalized, sorted-key JSON, so a standalone +checker has to reimplement that canonicalization, not just hash raw +bytes. This file is that reimplementation: stdlib only (`json`, +`hashlib`, `unicodedata`, `argparse`), zero imports of `cora` or any +third-party package, so it runs with any Python 3.13 interpreter on a +machine that has never installed CORA. + +This is deliberately NOT the full `cora.shared.content_hash` port. That +module also canonicalizes dataclasses, Pydantic models, and sets/ +frozensets, because it hashes live domain objects elsewhere in CORA. A +record export never contains any of those: `render_row` (step 2) +already reduces every column to a JSON primitive before it reaches +`hash_record` (step 3), and the disposition table `hash_redaction_profile` +hashes (step 4) is itself a plain nested dict/str structure. So this +file's `_canonicalize` only has to handle what actually appears in an +exported body: `str`, `dict`, `list`/`tuple`, and JSON scalars passed +through unchanged. `tests/unit/infrastructure/record_export/ +test_standalone_verifier.py` cross-checks this file byte-for-byte +against `cora.shared.content_hash.compute_content_hash` to keep the two +recipes from drifting apart. + +Usage: + python3 verify_record_hash.py hash --payload-type TYPE body.json + python3 verify_record_hash.py verify --payload-type TYPE --expected-hash HASH body.json + +Exit codes: 0 success (hash printed, or verify matched); 1 verify +mismatch; 2 the input file could not be read or parsed as JSON. +""" + +# pyright: reportUnknownMemberType=false, reportUnknownVariableType=false, reportUnknownArgumentType=false +# `_canonicalize` deliberately takes `Any`, mirroring +# cora.shared.content_hash._canonicalize; suppressed the same way there. + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import unicodedata +from pathlib import Path +from typing import Any + + +def _canonicalize(value: Any) -> Any: + """Recursively normalize a value into a JSON-stable Python structure. + + Mirrors `cora.shared.content_hash._canonicalize`'s str/Mapping/list + branches exactly (NFC normalization of strings and dict keys, list + recursion); omits the dataclass/Pydantic/set branches, which never + apply to an already-rendered export body (see module docstring). + """ + if isinstance(value, str): + return unicodedata.normalize("NFC", value) + if isinstance(value, dict): + return {unicodedata.normalize("NFC", str(k)): _canonicalize(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_canonicalize(item) for item in value] + return value + + +def canonical_body_bytes(body: Any) -> bytes: + """Produce canonical UTF-8 JSON bytes, byte-for-byte identical to + `cora.shared.content_hash.canonical_body_bytes` for any body an + export can actually contain.""" + canonical = _canonicalize(body) + return json.dumps( + canonical, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def pae_bytes(payload_type: str, body: bytes) -> bytes: + """DSSE Pre-Authentication Encoding, identical recipe to + `cora.shared.content_hash.pae_bytes`. LEN is BYTE length, not + character length: matters for a non-ASCII `payload_type`.""" + payload_type_bytes = payload_type.encode("utf-8") + return b"DSSEv1 %d %b %d %b" % ( + len(payload_type_bytes), + payload_type_bytes, + len(body), + body, + ) + + +def compute_content_hash(payload_type: str, body: Any) -> str: + """SHA-256 content hash, 64-char lowercase hex. Identical pipeline to + `cora.shared.content_hash.compute_content_hash`.""" + body_bytes = canonical_body_bytes(body) + pae = pae_bytes(payload_type, body_bytes) + return hashlib.sha256(pae).hexdigest() + + +def _load_body(path: Path) -> Any: + return json.loads(path.read_text(encoding="utf-8")) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Standalone content-hash verifier for a CORA record export." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + hash_parser = subparsers.add_parser("hash", help="Print the content hash of a JSON body.") + hash_parser.add_argument("--payload-type", required=True) + hash_parser.add_argument("body_file", type=Path) + + verify_parser = subparsers.add_parser( + "verify", help="Verify a JSON body's hash matches an expected value." + ) + verify_parser.add_argument("--payload-type", required=True) + verify_parser.add_argument("--expected-hash", required=True) + verify_parser.add_argument("body_file", type=Path) + + args = parser.parse_args(argv) + + try: + body = _load_body(args.body_file) + except (OSError, json.JSONDecodeError) as exc: + print(f"cannot read {args.body_file} as JSON: {exc}", file=sys.stderr) + return 2 + + digest = compute_content_hash(args.payload_type, body) + + if args.command == "hash": + print(digest) + return 0 + + if digest == args.expected_hash: + print(f"OK {digest}") + return 0 + print(f"MISMATCH: expected {args.expected_hash}, computed {digest}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())