diff --git a/docs/evidence/issue-62/README.md b/docs/evidence/issue-62/README.md index 75a45748..1785d386 100644 --- a/docs/evidence/issue-62/README.md +++ b/docs/evidence/issue-62/README.md @@ -22,8 +22,15 @@ payloads are redacted. Gateway filtering beyond the observed prefix. The exact-version Desktop core and Code Mode app-server controls pass. -The source snapshot is OpenAI Codex commit -9e552e9d15ba52bed7077d5357f3e18e330f8f38. At that revision, the dynamic +The retained Desktop capture is historical: it was captured on 2026-07-12 +with Codex CLI `0.144.0-alpha.4`, source commit +`9e552e9d15ba52bed7077d5357f3e18e330f8f38`. It must not be relabeled as the +later CLI 0.146.0 release. The separate +`codex-0.146-source-contract.json` records the 0.146.0 source contract (tag +`rust-v0.146.0`, attested commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`, +and exact binary hash) with `capture_status=not_observed` and +`qualification_status=unqualified`. At that +historical revision, the dynamic tool protocol defines optional deferLoading; the dynamic handler maps true to Deferred and missing or false to Direct. ToolExposure keeps Direct, DirectModelOnly, Deferred, and Hidden distinct. Tool search is planned only @@ -90,6 +97,10 @@ session, task, turn, call, item, request, or response identifiers. The bounded audit establishes these additional facts without a restart, reconnect, configuration write, or production-handler change: +Its candidate provenance is explicitly `capture_status=not_observed` and is +bound to the same 0.146 source contract; the retained historical capture +metadata remains recorded separately rather than being promoted to 0.146. + - Forty-three retained Sol transport rows resolve to three actual model-visible planner surfaces. The largest retained surface includes the base functions, collaboration namespace, goal functions, image generation, @@ -123,6 +134,7 @@ Run the sanitizer with explicit bounded inputs and observation cutoffs: ```powershell python scripts/audit_issue_62_runtime_artifacts.py ` + --source-contract docs/evidence/issue-62/codex-0.146-source-contract.json ` --codex-log-db ` --gateway-db ` --model gpt-5.6-sol ` @@ -146,18 +158,28 @@ records one per-scope disposition for every taxonomy item the Codex CLI exposes over the core Responses contract and the explicitly-deferred advanced capabilities. -The artifact is bound to CLI floor `0.145.0` and to the candidate identity -derived from the existing sanitized artifacts (`cli_version=0.144.0-alpha.4`, -source commit `9e552e9d15ba52bed7077d5357f3e18e330f8f38`, official Responses -route). Because the captured CLI is below the floor, the generated -`qualification.ready_for_beta1` is `false` and the candidate is explicitly -marked `legacy_below_floor`; this evidence cannot be used as the beta.1 -candidate. The generator rejects an explicitly supplied CLI/source value that -does not match the trace, binds route/provider/model fields across trace and -wire fixtures (including pre/post models, catalog binding, and route profile), -and records a canonical-LF SHA-256 manifest for all three input artifacts. It -never fabricates a capability disposition for a gate the artifacts do not -qualify. +The artifact is bound to CLI floor `0.146.0` and to the candidate identity +from the unobserved 0.146 source contract (`cli_version=0.146.0`, source +commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`, candidate revision +`accab8ff6eb4d6ebd93cda84585fb5f6cb89da82`, official Responses route). The +historical trace and wire fixture remain explicitly bound as 0.144.0 evidence. +The audit carries the 0.146 source-contract provenance with +`capture_status=not_observed` and nested historical-capture metadata; it is not +a 0.146 runtime capture. The candidate is version-eligible, but +`qualification.ready_for_beta2` remains +`false`: planner completeness, clean current-binding cold start, +independently fingerprinted full pre/post request and response bodies, +non-streaming/terminal/error/hosted/unknown controls, and wire replay evidence +are still incomplete. This is evidence for #62's downstream gates, not a #65 +qualification or capability unlock. The generator rejects an explicitly +supplied CLI/source value that does not match the trace, binds route/provider/ +model fields across trace and wire fixtures (including pre/post models, catalog +binding, and route profile), and records a canonical-LF SHA-256 manifest for +all four input artifacts (including the source contract). The audit sanitizer +must receive that source-contract path so reruns retain the exact +`capture_status=not_observed` 0.146 provenance and nested historical 0.144 +capture metadata. It never fabricates a capability disposition for a gate the +artifacts do not qualify. The qualification also has a separate `wire_identity_replay` gate. A full request/response fingerprint is not treated as replay proof by itself: a @@ -219,7 +241,7 @@ reconciliation invokes the same drift check, then independently checks the input fingerprints, rejects duplicate scopes, and requires each core scope to point at its declared evidence path. A zero `unclassified_core_items` count therefore describes vocabulary validity only; -`qualification.ready_for_beta1` is the separate completion gate. That gate also +`qualification.ready_for_beta2` is the separate completion gate. That gate also consumes planner completeness, current-binding cold-start, full-wire fingerprinting, non-streaming, and identity-replay statuses; item dispositions alone cannot make an incomplete evidence set ready. @@ -260,7 +282,7 @@ Evidence may use `cli_source_commit: null` with not acceptable. If the npm provenance attestation has been independently verified, its exact SLSA resolved-dependency commit may instead be recorded with status `published` (for `0.146.0`, the attested release commit is -`e363b08c9175ac1cbe5893615dd2cb9ddf95043b2`). +`e363b08c9175ac1cbe5893615dd2cb9ddf95043b`). ## Isolated live-evidence sidecar lane diff --git a/docs/evidence/issue-62/codex-0.146-source-contract.json b/docs/evidence/issue-62/codex-0.146-source-contract.json new file mode 100644 index 00000000..4f5626a3 --- /dev/null +++ b/docs/evidence/issue-62/codex-0.146-source-contract.json @@ -0,0 +1,179 @@ +{ + "schema_version": 1, + "fixture_kind": "codex_cli_source_contract", + "capture_status": "not_observed", + "qualification_status": "unqualified", + "captured_at": null, + "provenance": { + "cli_version": "0.146.0", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", + "cli_source_commit_status": "published_attested", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" + }, + "runtime_wire_surface": { + "source": "Codex CLI 0.146.0 ToolSpec and ResponseItem source contract", + "declaration_family_order": [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind" + ], + "declaration_families": [ + {"family": "plain_function", "runtime_type": "function", "wire_declaration_type": "function", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "preserve declaration and inverse call/result/history IDs"}, + {"family": "custom_freeform", "runtime_type": "custom", "wire_declaration_type": "custom", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "preserve declaration and inverse call/result/history IDs"}, + {"family": "namespace", "runtime_type": "namespace", "wire_declaration_type": "namespace", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "preserve declaration and inverse call/result/history IDs"}, + {"family": "client_executed_tool_discovery", "runtime_type": "tool_search", "wire_declaration_type": "tool_search", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "discovery request/result stays client-executed"}, + {"family": "selected_provider_hosted", "runtime_type": "web_search", "wire_declaration_type": "web_search", "observed": false, "observation": "not_observed_selected_provider_control_required", "executor": "selected_provider", "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly"}, + {"family": "unknown_future_kind", "runtime_type": "unknown", "wire_declaration_type": "", "observed": false, "observation": "opaque_sentinel_only", "executor": "unknown", "loss_boundary": "retain tag and opaque payload; do not normalize"} + ], + "request_shape": { + "protocol": "responses", + "streaming_fields": [ + "model", + "instructions", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "store", + "stream", + "stream_options", + "include", + "service_tier", + "prompt_cache_key", + "text", + "client_metadata" + ], + "representative": { + "model": "gpt-5.6-sol", + "input": "", + "tools": "", + "tool_choice": "auto", + "parallel_tool_calls": false, + "stream": true, + "store": false + }, + "non_streaming_control": { + "stream": false, + "response_body": "", + "captured": false, + "status": "unqualified" + } + }, + "response_shape": { + "response_item_types": [ + "message", + "reasoning", + "function_call", + "function_call_output", + "custom_tool_call", + "custom_tool_call_output", + "tool_search_call", + "tool_search_output", + "web_search_call", + "local_shell_call", + "compaction", + "context_compaction", + "unknown" + ], + "stream_event_order": [ + "response.created", + "response.in_progress", + "response.output_item.added", + "response.output_text.delta", + "response.function_call_arguments.delta", + "response.custom_tool_call_input.delta", + "response.function_call_arguments.done", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.completed" + ], + "terminal_events": [ + "response.completed", + "response.incomplete", + "response.failed" + ], + "error_shape": { + "event": "response.failed", + "response": {"id": "response_error_001", "status": "failed", "error": ""}, + "classification": "unqualified" + } + }, + "declaration_family_examples": { + "plain_function": { + "declaration": {"type": "function", "name": "function_alpha", "parameters": ""}, + "call": {"type": "function_call", "item_id": "item_call_function_001", "call_id": "call_function_001", "arguments": ""}, + "result": {"type": "function_call_output", "item_id": "item_output_function_001", "call_id": "call_function_001", "output": ""}, + "history": {"call_id": "call_function_001", "call_item_id": "item_call_function_001", "output_item_id": "item_output_function_001"}, + "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "arguments_done": "response.function_call_arguments.done", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "response.function_call_arguments.delta", "response.function_call_arguments.done", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + "custom_freeform": { + "declaration": {"type": "custom", "name": "custom_alpha", "format": ""}, + "call": {"type": "custom_tool_call", "item_id": "item_call_custom_001", "call_id": "call_custom_001", "input": ""}, + "result": {"type": "custom_tool_call_output", "item_id": "item_output_custom_001", "call_id": "call_custom_001", "output": ""}, + "history": {"call_id": "call_custom_001", "call_item_id": "item_call_custom_001", "output_item_id": "item_output_custom_001"}, + "streaming": {"added": "response.output_item.added", "delta": "response.custom_tool_call_input.delta", "input_done": "response.custom_tool_call_input.done", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "response.custom_tool_call_input.delta", "response.custom_tool_call_input.done", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + "namespace": { + "declaration": {"type": "namespace", "name": "namespace_alpha", "tools": [{"type": "function", "name": "function_beta", "parameters": ""}]}, + "call": {"type": "function_call", "item_id": "item_call_namespace_001", "call_id": "call_namespace_001", "namespace": "namespace_alpha", "arguments": ""}, + "result": {"type": "function_call_output", "item_id": "item_output_namespace_001", "call_id": "call_namespace_001", "output": ""}, + "history": {"call_id": "call_namespace_001", "call_item_id": "item_call_namespace_001", "output_item_id": "item_output_namespace_001", "namespace": "namespace_alpha"}, + "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "arguments_done": "response.function_call_arguments.done", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "response.function_call_arguments.delta", "response.function_call_arguments.done", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + "client_executed_tool_discovery": { + "declaration": {"type": "tool_search", "execution": "client", "parameters": ""}, + "call": {"type": "tool_search_call", "item_id": "item_call_search_001", "call_id": "call_search_001", "execution": "client", "arguments": ""}, + "result": {"type": "tool_search_output", "item_id": "item_output_search_001", "call_id": "call_search_001", "execution": "client", "tools": ""}, + "history": {"call_id": "call_search_001", "call_item_id": "item_call_search_001", "output_item_id": "item_output_search_001", "executor": "codex_client"}, + "streaming": {"added": null, "delta": null, "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "discovery request/result stays client-executed" + }, + "selected_provider_hosted": { + "declaration": {"type": "web_search", "executor": "selected_provider", "provider_scope": "selected_provider_only"}, + "call": {"type": "web_search_call", "item_id": "item_call_hosted_001", "status": "", "action": ""}, + "result": {"type": "web_search_call", "item_id": "item_output_hosted_001", "status": "", "provider_scope": "selected_provider_only"}, + "history": {"call_id": null, "call_item_id": "item_call_hosted_001", "output_item_id": "item_output_hosted_001", "executor": "selected_provider", "cross_provider_proxy": "forbidden"}, + "streaming": {"added": "response.output_item.added", "delta": "", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "observed": false, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden" + }, + "unknown_future_kind": { + "declaration": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "call": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "result": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "history": {"call_id": null, "call_item_id": null, "output_item_id": null, "loss_rule": "retain opaque sentinel"}, + "streaming": {"added": "unknown.future_event", "delta": "unknown.future_delta", "done": "unknown.future_done", "terminal": "response.completed", "event_order": ["unknown.future_event", "unknown.future_delta", "unknown.future_done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "retain tag and opaque payload; do not normalize", + "observed": false, + "status": "opaque_sentinel_only" + } + } + } +} diff --git a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json index 77a22173..2d6469d4 100644 --- a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json +++ b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json @@ -25,6 +25,7 @@ "provenance": { "cli_version": "0.144.0-alpha.4", "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "captured_at": "2026-07-12T14:57:55+08:00", "capture_id": "sanitized-current-side-capture" }, "exposure_state_tags": ["Direct", "DirectModelOnly", "Deferred", "Hidden", "hosted-only", "host-unavailable"], diff --git a/docs/evidence/issue-62/read-only-gate-audit.json b/docs/evidence/issue-62/read-only-gate-audit.json index bbde3b56..a4c9e8ef 100644 --- a/docs/evidence/issue-62/read-only-gate-audit.json +++ b/docs/evidence/issue-62/read-only-gate-audit.json @@ -1,5 +1,19 @@ { "capture_kind": "sanitized_bounded_read_only_audit", + "provenance": { + "capture_status": "not_observed", + "cli_version": "0.146.0", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", + "cli_source_commit_status": "published_attested", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", + "historical_capture": { + "captured_at": "2026-07-12T14:57:55+08:00", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38" + } + }, "gate_classification": { "choice_controls": "observed", "clean_cold_start_current_binding": "live_control_required", @@ -349,6 +363,62 @@ } ] }, + "declaration_families": [ + { + "family": "plain_function", + "runtime_type": "function", + "wire_declaration_type": "function", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "custom_freeform", + "runtime_type": "custom", + "wire_declaration_type": "custom", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "namespace", + "runtime_type": "namespace", + "wire_declaration_type": "namespace", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "client_executed_tool_discovery", + "runtime_type": "tool_search", + "wire_declaration_type": "tool_search", + "observed": true, + "observation": "observed_client_execution", + "executor": "codex_client", + "loss_boundary": "discovery request/result stays client-executed" + }, + { + "family": "selected_provider_hosted", + "runtime_type": "web_search", + "wire_declaration_type": "web_search", + "observed": false, + "observation": "not_observed_selected_provider_only", + "executor": "selected_provider", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly" + }, + { + "family": "unknown_future_kind", + "runtime_type": "unknown", + "wire_declaration_type": "", + "observed": false, + "observation": "opaque_sentinel_only", + "executor": "unknown", + "loss_boundary": "retain tag and opaque payload; do not normalize" + } + ], "top_level_field_presence": [ "client_metadata", "include", diff --git a/docs/evidence/issue-62/runtime-wire-inventory.json b/docs/evidence/issue-62/runtime-wire-inventory.json index 03dfe230..af3e99d3 100644 --- a/docs/evidence/issue-62/runtime-wire-inventory.json +++ b/docs/evidence/issue-62/runtime-wire-inventory.json @@ -1,21 +1,397 @@ { "artifact_kind": "runtime_wire_inventory", "candidate_identity": { + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", "catalog_binding": "official Codex catalog entry for openai/gpt-5.6-sol", "catalog_model_entry_id": "gpt-5.6-sol", "catalog_snapshot_sha256": "307a09f22b0c827ae77192a4beaf7059efcf8a698ec4f252aa4ba4787f8d1876", - "cli_version": "0.144.0-alpha.4", - "codex_source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "cli_source_commit_status": "published_attested", + "cli_source_tag": "rust-v0.146.0", + "cli_version": "0.146.0", + "codex_source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "configured_provider_id": "custom", - "evidence_manifest_sha256": "9dfb5f48f52f27603d4d27066d0ed8a2e300600bcdbdff9a24f594a9892e5c22", + "evidence_manifest_sha256": "685858e1affdb073df17467f53ab3de538c0c7f2d930b01c10797a94963c6619", "inbound_format": "responses", "model": "gpt-5.6-sol", "route_behavior_profile": "official_codex_app_http_passthrough", "route_upstream": "official", - "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "upstream_format": "responses" }, - "cli_version_floor": "0.145.0", + "cli_version_floor": "0.146.0", + "declaration_families": [ + { + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", + "executor": "codex_client", + "family": "plain_function", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "observation": "not_observed_source_contract_only", + "observed": false, + "optional_rule": "preserve_when_supported_else_omit", + "representative": { + "call": { + "arguments": "", + "call_id": "call_function_001", + "item_id": "item_call_function_001", + "type": "function_call" + }, + "declaration": { + "name": "function_alpha", + "parameters": "", + "type": "function" + }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, + "history": { + "call_id": "call_function_001", + "call_item_id": "item_call_function_001", + "output_item_id": "item_output_function_001" + }, + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "result": { + "call_id": "call_function_001", + "item_id": "item_output_function_001", + "output": "", + "type": "function_call_output" + }, + "streaming": { + "added": "response.output_item.added", + "arguments_done": "response.function_call_arguments.done", + "delta": "response.function_call_arguments.delta", + "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed" + ], + "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" + } + }, + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "runtime_type": "function", + "selected_protocol_disposition": "native", + "wire_declaration_type": "function" + }, + { + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.custom_freeform", + "executor": "codex_client", + "family": "custom_freeform", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "observation": "not_observed_source_contract_only", + "observed": false, + "optional_rule": "preserve_when_supported_else_omit", + "representative": { + "call": { + "call_id": "call_custom_001", + "input": "", + "item_id": "item_call_custom_001", + "type": "custom_tool_call" + }, + "declaration": { + "format": "", + "name": "custom_alpha", + "type": "custom" + }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, + "history": { + "call_id": "call_custom_001", + "call_item_id": "item_call_custom_001", + "output_item_id": "item_output_custom_001" + }, + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "result": { + "call_id": "call_custom_001", + "item_id": "item_output_custom_001", + "output": "", + "type": "custom_tool_call_output" + }, + "streaming": { + "added": "response.output_item.added", + "delta": "response.custom_tool_call_input.delta", + "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "response.custom_tool_call_input.delta", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.completed" + ], + "input_done": "response.custom_tool_call_input.done", + "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" + } + }, + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "runtime_type": "custom", + "selected_protocol_disposition": "native", + "wire_declaration_type": "custom" + }, + { + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.namespace", + "executor": "codex_client", + "family": "namespace", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "observation": "not_observed_source_contract_only", + "observed": false, + "optional_rule": "preserve_when_supported_else_omit", + "representative": { + "call": { + "arguments": "", + "call_id": "call_namespace_001", + "item_id": "item_call_namespace_001", + "namespace": "namespace_alpha", + "type": "function_call" + }, + "declaration": { + "name": "namespace_alpha", + "tools": [ + { + "name": "function_beta", + "parameters": "", + "type": "function" + } + ], + "type": "namespace" + }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, + "history": { + "call_id": "call_namespace_001", + "call_item_id": "item_call_namespace_001", + "namespace": "namespace_alpha", + "output_item_id": "item_output_namespace_001" + }, + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "result": { + "call_id": "call_namespace_001", + "item_id": "item_output_namespace_001", + "output": "", + "type": "function_call_output" + }, + "streaming": { + "added": "response.output_item.added", + "arguments_done": "response.function_call_arguments.done", + "delta": "response.function_call_arguments.delta", + "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed" + ], + "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" + } + }, + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "runtime_type": "namespace", + "selected_protocol_disposition": "native", + "wire_declaration_type": "namespace" + }, + { + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.client_executed_tool_discovery", + "executor": "codex_client", + "family": "client_executed_tool_discovery", + "loss_boundary": "discovery request/result stays client-executed", + "observation": "not_observed_source_contract_only", + "observed": false, + "optional_rule": "preserve_client_execution_else_omit", + "representative": { + "call": { + "arguments": "", + "call_id": "call_search_001", + "execution": "client", + "item_id": "item_call_search_001", + "type": "tool_search_call" + }, + "declaration": { + "execution": "client", + "parameters": "", + "type": "tool_search" + }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, + "history": { + "call_id": "call_search_001", + "call_item_id": "item_call_search_001", + "executor": "codex_client", + "output_item_id": "item_output_search_001" + }, + "loss_boundary": "discovery request/result stays client-executed", + "result": { + "call_id": "call_search_001", + "execution": "client", + "item_id": "item_output_search_001", + "tools": "", + "type": "tool_search_output" + }, + "streaming": { + "added": null, + "delta": null, + "done": "response.output_item.done", + "event_order": [ + "response.output_item.done", + "response.completed" + ], + "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" + } + }, + "required_rule": "preserve_client_execution_else_required-but-unavailable", + "runtime_type": "tool_search", + "selected_protocol_disposition": "native", + "wire_declaration_type": "tool_search" + }, + { + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.selected_provider_hosted", + "executor": "selected_provider", + "family": "selected_provider_hosted", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "observation": "not_observed_selected_provider_control_required", + "observed": false, + "optional_rule": "omit_if_selected_provider_unsupported", + "representative": { + "call": { + "action": "", + "item_id": "item_call_hosted_001", + "status": "", + "type": "web_search_call" + }, + "cross_provider_proxy": "forbidden", + "declaration": { + "executor": "selected_provider", + "provider_scope": "selected_provider_only", + "type": "web_search" + }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, + "history": { + "call_id": null, + "call_item_id": "item_call_hosted_001", + "cross_provider_proxy": "forbidden", + "executor": "selected_provider", + "output_item_id": "item_output_hosted_001" + }, + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "observed": false, + "provider_scope": "selected_provider_only", + "result": { + "item_id": "item_output_hosted_001", + "provider_scope": "selected_provider_only", + "status": "", + "type": "web_search_call" + }, + "status": "selected_provider_control_required", + "streaming": { + "added": "response.output_item.added", + "delta": "", + "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "", + "response.output_item.done", + "response.completed" + ], + "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" + } + }, + "required_rule": "required-but-unavailable_if_selected_provider_unsupported", + "runtime_type": "web_search", + "selected_protocol_disposition": "native_if_selected_provider_supports", + "wire_declaration_type": "web_search" + }, + { + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.unknown_future_kind", + "executor": "unknown", + "family": "unknown_future_kind", + "loss_boundary": "retain tag and opaque payload; do not normalize", + "observation": "opaque_sentinel_only", + "observed": false, + "optional_rule": "omit_and_emit_sanitized_diagnostic", + "representative": { + "call": { + "opaque_payload": "", + "tag": "unknown", + "type": "unknown" + }, + "declaration": { + "opaque_payload": "", + "tag": "unknown", + "type": "unknown" + }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, + "history": { + "call_id": null, + "call_item_id": null, + "loss_rule": "retain opaque sentinel", + "output_item_id": null + }, + "loss_boundary": "retain tag and opaque payload; do not normalize", + "observed": false, + "result": { + "opaque_payload": "", + "tag": "unknown", + "type": "unknown" + }, + "status": "opaque_sentinel_only", + "streaming": { + "added": "unknown.future_event", + "delta": "unknown.future_delta", + "done": "unknown.future_done", + "event_order": [ + "unknown.future_event", + "unknown.future_delta", + "unknown.future_done", + "response.completed" + ], + "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" + } + }, + "required_rule": "required-but-unavailable", + "runtime_type": "unknown", + "selected_protocol_disposition": "omit", + "wire_declaration_type": "" + } + ], "disposition_vocabulary": [ "preserved", "reversibly_adapted", @@ -26,7 +402,11 @@ "evidence_binding": { "audit": { "file": "read-only-gate-audit.json", - "sha256": "a27f27622f89436f9530e2a7ca31f4b1b48872a31bbced8426302682f8516fa0" + "sha256": "775efbc763dfae1c5d6a61532d9394e5cd181b2e6803bf2e42f6c886a0a964e3" + }, + "source_contract": { + "file": "codex-0.146-source-contract.json", + "sha256": "6f38b8b07b98c6f28edd7418b63449242ee41d6396694392a70c0d7fb2b70f2c" }, "trace": { "file": "current-codexhub-thread-tool-surface.json", @@ -34,7 +414,7 @@ }, "wire_fixture": { "file": "codexhub-runtime-wire-fixture.json", - "sha256": "3a8654854539799b526bc013c50bc9f16c40c09319cff8da716b75e6deecbc3a" + "sha256": "1e3ecfc51f59de7f21bee8a7a0b743bba26d35fb3dc9eaca5f7b4ea3dd57705f" } }, "evidence_sources": { @@ -237,8 +617,8 @@ "terminal_events", "unknown_tagged_sentinels" ], - "candidate_version_eligible": false, - "candidate_version_status": "legacy_below_floor", + "candidate_version_eligible": true, + "candidate_version_status": "eligible", "evidence_gates": { "clean_cold_start_current_binding": "not_run", "complete_model_visible_plan": "partial", @@ -253,7 +633,7 @@ "terminal_events": "not_captured", "wire_identity_replay": "not_captured" }, - "ready_for_beta1": false + "ready_for_beta2": false }, "schema_version": 1 } diff --git a/scripts/audit_issue_62_runtime_artifacts.py b/scripts/audit_issue_62_runtime_artifacts.py index 1962f0ee..afea3521 100644 --- a/scripts/audit_issue_62_runtime_artifacts.py +++ b/scripts/audit_issue_62_runtime_artifacts.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import importlib.util import json import re import sqlite3 @@ -15,6 +16,22 @@ TRANSPORT_TARGET = "codex_http_client::transport" +DEFAULT_SOURCE_CONTRACT = Path( + "docs/evidence/issue-62/codex-0.146-source-contract.json" +) +SOURCE_CONTRACT_PROVENANCE = { + "cli_version": "0.146.0", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", + "cli_source_commit_status": "published_attested", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", +} +HISTORICAL_CAPTURE = { + "captured_at": "2026-07-12T14:57:55+08:00", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", +} POST_BODY_PATTERN = re.compile( r": POST to (?Phttps?://.+?): (?P\{.*\})\s*$", re.DOTALL, @@ -37,6 +54,24 @@ "web_search_call", } VALID_TOOL_CHOICE_STRINGS = {"auto", "none", "required"} +# Structural declaration families emitted by Codex CLI 0.146's Responses +# ``ToolSpec`` surface. These labels describe shape/execution only; they are +# deliberately not model or Provider qualification records. +DECLARATION_FAMILIES = ( + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", +) +_TOOL_TYPE_TO_FAMILY = { + "function": "plain_function", + "custom": "custom_freeform", + "namespace": "namespace", + "tool_search": "client_executed_tool_discovery", + "web_search": "selected_provider_hosted", +} ROUTE_FIELDS = ( "upstream", "route_mode", @@ -137,6 +172,237 @@ def _is_proven_tool_choice(value: Any) -> bool: ) +def _validate_source_contract_with_inventory(source_contract: dict[str, Any]) -> dict[str, Any]: + """Apply the canonical source-contract validator used by inventory replay. + + The sanitizer and inventory are independent entry points, but they must + reject the same unobserved source-contract mutations. Load the sibling + generator by path so this module remains directly executable and importable + from tests without relying on ``scripts`` being a Python package. + """ + + validator_path = Path(__file__).with_name("build_issue_62_runtime_inventory.py") + spec = importlib.util.spec_from_file_location( + "_issue_62_runtime_inventory_contract_validator", validator_path + ) + if spec is None or spec.loader is None: + raise ValueError("unable to load canonical source-contract validator") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + validator = getattr(module, "_validate_source_contract", None) + if not callable(validator): + raise ValueError("canonical source-contract validator is unavailable") + return validator(source_contract) + + +def _source_contract_provenance(path: Path) -> dict[str, Any]: + """Load and validate the unobserved 0.146 source-contract identity.""" + + try: + source_contract = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"unable to read source contract: {path}") from exc + if not isinstance(source_contract, dict): + raise ValueError("source contract must be a JSON object") + try: + _validate_source_contract_with_inventory(source_contract) + except ValueError as exc: + raise ValueError(str(exc)) from exc + if set(source_contract) != { + "schema_version", + "fixture_kind", + "capture_status", + "qualification_status", + "captured_at", + "provenance", + "runtime_wire_surface", + }: + raise ValueError("source contract has unknown top-level fields") + if ( + source_contract.get("schema_version") != 1 + or source_contract.get("fixture_kind") != "codex_cli_source_contract" + or source_contract.get("capture_status") != "not_observed" + or source_contract.get("qualification_status") != "unqualified" + or "captured_at" not in source_contract + or source_contract.get("captured_at") is not None + ): + raise ValueError( + "source contract must remain not_observed, unqualified, and uncaptured" + ) + provenance = source_contract.get("provenance") + if not isinstance(provenance, dict) or set(provenance) != set(SOURCE_CONTRACT_PROVENANCE) or any( + provenance.get(field) != value + for field, value in SOURCE_CONTRACT_PROVENANCE.items() + ): + raise ValueError("source contract provenance is invalid or has unknown fields") + runtime_surface = source_contract.get("runtime_wire_surface") + if not isinstance(runtime_surface, dict): + raise ValueError("source contract runtime surface is missing") + if set(runtime_surface) != { + "source", + "declaration_family_order", + "declaration_families", + "request_shape", + "response_shape", + "declaration_family_examples", + }: + raise ValueError("source contract runtime_wire_surface has unknown fields") + if runtime_surface.get("declaration_family_order") != list(DECLARATION_FAMILIES): + raise ValueError("source contract declaration family order is invalid") + declaration_families = runtime_surface.get("declaration_families") + if not isinstance(declaration_families, list) or len(declaration_families) != len( + DECLARATION_FAMILIES + ): + raise ValueError("source contract declaration families are invalid") + expected_observations = { + "plain_function": "not_observed_source_contract_only", + "custom_freeform": "not_observed_source_contract_only", + "namespace": "not_observed_source_contract_only", + "client_executed_tool_discovery": "not_observed_source_contract_only", + "selected_provider_hosted": "not_observed_selected_provider_control_required", + "unknown_future_kind": "opaque_sentinel_only", + } + for family, entry in zip(DECLARATION_FAMILIES, declaration_families): + if not isinstance(entry, dict) or entry.get("family") != family: + raise ValueError("source contract declaration family identity is invalid") + if set(entry) != { + "family", + "runtime_type", + "wire_declaration_type", + "observed", + "observation", + "executor", + "loss_boundary", + }: + raise ValueError("source contract declaration family has unknown fields") + if entry.get("observed") is not False: + raise ValueError("source contract cannot claim an observed declaration family") + if entry.get("observation") != expected_observations[family]: + raise ValueError("source contract declaration observation is invalid") + request_shape = runtime_surface.get("request_shape") + if not isinstance(request_shape, dict) or set(request_shape) != { + "protocol", + "streaming_fields", + "representative", + "non_streaming_control", + }: + raise ValueError("source contract request_shape has unknown fields") + if not isinstance(request_shape.get("representative"), dict) or set( + request_shape["representative"] + ) != { + "model", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "stream", + "store", + }: + raise ValueError("source contract request_shape.representative has unknown fields") + non_streaming_control = ( + request_shape.get("non_streaming_control") + if isinstance(request_shape, dict) + else None + ) + if not isinstance(non_streaming_control, dict) or set(non_streaming_control) != { + "stream", + "response_body", + "captured", + "status", + }: + raise ValueError("source contract request_shape has unknown fields") + if not isinstance(non_streaming_control, dict) or non_streaming_control.get( + "captured" + ) is not False or non_streaming_control.get("status") != "unqualified": + raise ValueError("source contract cannot claim a captured non-streaming response") + examples = runtime_surface.get("declaration_family_examples") + if not isinstance(examples, dict): + raise ValueError("source contract declaration examples are missing") + if set(examples) != set(DECLARATION_FAMILIES): + raise ValueError("source contract declaration examples have unknown fields") + for family in DECLARATION_FAMILIES: + example = examples.get(family) + if not isinstance(example, dict): + raise ValueError(f"source contract example is missing for {family}") + expected_example_fields = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } + if family == "selected_provider_hosted": + expected_example_fields |= { + "observed", + "status", + "provider_scope", + "cross_provider_proxy", + } + elif family == "unknown_future_kind": + expected_example_fields |= {"observed", "status"} + if set(example) != expected_example_fields: + raise ValueError(f"source contract {family} has unknown example fields") + if ( + not isinstance(example.get("terminal"), dict) + or example["terminal"].get("event") != "response.completed" + or example["terminal"].get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"source contract terminal status is invalid for {family}") + if ( + not isinstance(example.get("error"), dict) + or example["error"].get("event") != "response.failed" + or example["error"].get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"source contract error status is invalid for {family}") + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"source contract loss boundary is invalid for {family}") + response_shape = runtime_surface.get("response_shape") + if not isinstance(response_shape, dict) or set(response_shape) != { + "response_item_types", + "stream_event_order", + "terminal_events", + "error_shape", + }: + raise ValueError("source contract response_shape has unknown fields") + error_shape = response_shape.get("error_shape") if isinstance(response_shape, dict) else None + if not isinstance(error_shape, dict) or set(error_shape) != { + "event", + "response", + "classification", + }: + raise ValueError("source contract response_shape.error_shape has unknown fields") + if not isinstance(error_shape.get("response"), dict) or set( + error_shape["response"] + ) != {"id", "status", "error"}: + raise ValueError( + "source contract response_shape.error_shape.response has unknown fields" + ) + if not isinstance(error_shape, dict) or error_shape.get("classification") != "unqualified": + raise ValueError("source contract response error status is invalid") + + def _captured_true(value: Any) -> bool: + if isinstance(value, dict): + return any( + (key in {"captured", "observed"} and child is True) + or _captured_true(child) + for key, child in value.items() + ) + if isinstance(value, list): + return any(_captured_true(child) for child in value) + return False + + if _captured_true(runtime_surface): + raise ValueError("source contract contains a captured response claim") + return { + "capture_status": source_contract["capture_status"], + **SOURCE_CONTRACT_PROVENANCE, + "historical_capture": dict(HISTORICAL_CAPTURE), + } + + def _sanitize_tool(tool: dict[str, Any]) -> dict[str, Any]: tool_type = tool.get("type") if isinstance(tool.get("type"), str) else None name = tool.get("name") if isinstance(tool.get("name"), str) else None @@ -160,6 +426,73 @@ def _sanitize_tool(tool: dict[str, Any]) -> dict[str, Any]: return sanitized +def _declaration_families(tool_surfaces: list[list[dict[str, Any]]]) -> list[dict[str, Any]]: + """Summarize observed declaration families without retaining tool content. + + The selected-provider hosted family and the unknown future sentinel are + included even when the bounded request rows do not observe them. This is + intentional: #62 inventories the runtime contract and makes the evidence + boundary explicit rather than silently dropping a family. + """ + + observed_types = { + str(tool.get("type")) + for surface in tool_surfaces + for tool in surface + if isinstance(tool, dict) and isinstance(tool.get("type"), str) + } + families: list[dict[str, Any]] = [] + for family in DECLARATION_FAMILIES: + runtime_type = next( + ( + tool_type + for tool_type, mapped_family in _TOOL_TYPE_TO_FAMILY.items() + if mapped_family == family + ), + None, + ) + if family == "unknown_future_kind": + families.append( + { + "family": family, + "runtime_type": "unknown", + "wire_declaration_type": "", + "observed": False, + "observation": "opaque_sentinel_only", + "executor": "unknown", + "loss_boundary": "retain tag and opaque payload; do not normalize", + } + ) + continue + observed = runtime_type in observed_types + if family == "selected_provider_hosted": + executor = "selected_provider" + observation = "not_observed_selected_provider_only" + loss_boundary = ( + "optional unsupported hosted capability is omitted; required capability fails visibly" + ) + elif family == "client_executed_tool_discovery": + executor = "codex_client" + observation = "observed_client_execution" if observed else "not_observed" + loss_boundary = "discovery request/result stays client-executed" + else: + executor = "codex_client" + observation = "observed" if observed else "not_observed" + loss_boundary = "preserve declaration and inverse call/result/history IDs" + families.append( + { + "family": family, + "runtime_type": runtime_type, + "wire_declaration_type": runtime_type, + "observed": observed, + "observation": observation, + "executor": executor, + "loss_boundary": loss_boundary, + } + ) + return families + + def _sanitize_request_plan(payload: dict[str, Any]) -> dict[str, Any]: item_types: Counter[str] = Counter() tool_surface: list[dict[str, Any]] = [] @@ -285,6 +618,7 @@ def _codex_request_evidence( variants.append(variant) unclassified = sorted(set(observed_item_types) - KNOWN_INPUT_ITEM_TYPES) + declaration_families = _declaration_families(list(tool_surfaces.values())) return { "current_request_endpoint_classes": dict(sorted(current_endpoint_classes.items())), "model_visible_request_plan": { @@ -292,6 +626,7 @@ def _codex_request_evidence( "observed_input_item_type_counts": dict(sorted(observed_item_types.items())), "plan_variants": variants, "tool_surfaces": tool_surfaces, + "declaration_families": declaration_families, "top_level_field_presence": sorted(top_level_fields), "transport_log_rows": transport_log_rows, "unclassified_item_types": unclassified, @@ -523,7 +858,9 @@ def audit_artifacts( config_written_at: str, catalog_written_at: str, snapshot_ended_at: str, + source_contract: Path = DEFAULT_SOURCE_CONTRACT, ) -> dict[str, Any]: + provenance = _source_contract_provenance(source_contract) codex = _codex_request_evidence( codex_log_db, model=model, @@ -574,6 +911,7 @@ def audit_artifacts( return { "capture_kind": "sanitized_bounded_read_only_audit", + "provenance": provenance, "gate_classification": { "choice_controls": "observed" if choice_observed else "unclassified", "clean_cold_start_current_binding": "met" @@ -628,6 +966,7 @@ def audit_artifacts( def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-contract", type=Path, default=DEFAULT_SOURCE_CONTRACT) parser.add_argument("--codex-log-db", type=Path, required=True) parser.add_argument("--gateway-db", type=Path, required=True) parser.add_argument("--model", required=True) @@ -650,6 +989,7 @@ def main() -> int: config_written_at=args.config_written_at, catalog_written_at=args.catalog_written_at, snapshot_ended_at=args.snapshot_ended_at, + source_contract=args.source_contract, ) print(json.dumps(audit, ensure_ascii=True, indent=2, sort_keys=True)) return 0 diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index a6653ccd..6995eb2c 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -36,10 +36,63 @@ SCHEMA_VERSION = 1 ARTIFACT_KIND = "runtime_wire_inventory" -DEFAULT_CLI_FLOOR = "0.145.0" +DEFAULT_CLI_FLOOR = "0.146.0" SUPPORTED_CLI_FLOOR = DEFAULT_CLI_FLOOR DEFAULT_CANDIDATE_CLI_VERSION = None DEFAULT_CANDIDATE_SOURCE_COMMIT = None +CLI_SOURCE_TAG = "rust-v0.146.0" +CLI_SOURCE_COMMIT_STATUS = "published_attested" +CLI_BINARY_SHA256 = "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb" +CLI_SOURCE_COMMIT = "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" +CANDIDATE_REVISION = "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" +HISTORICAL_CLI_VERSION = "0.144.0-alpha.4" +HISTORICAL_SOURCE_COMMIT = "9e552e9d15ba52bed7077d5357f3e18e330f8f38" +HISTORICAL_CAPTURED_AT = "2026-07-12T14:57:55+08:00" +DEFAULT_SOURCE_CONTRACT = Path( + "docs/evidence/issue-62/codex-0.146-source-contract.json" +) +STRUCTURAL_FAMILIES = ( + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", +) + +REQUEST_SHAPE_PROTOCOL = "responses" +REQUEST_SHAPE_STREAMING_FIELDS = [ + "model", + "instructions", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "store", + "stream", + "stream_options", + "include", + "service_tier", + "prompt_cache_key", + "text", + "client_metadata", +] +REQUEST_SHAPE_REPRESENTATIVE = { + "model": "gpt-5.6-sol", + "input": "", + "tools": "", + "tool_choice": "auto", + "parallel_tool_calls": False, + "stream": True, + "store": False, +} +REQUEST_SHAPE_NON_STREAMING_CONTROL = { + "stream": False, + "response_body": "", + "captured": False, + "status": "unqualified", +} ALLOWED_DISPOSITIONS = ( "preserved", @@ -269,8 +322,209 @@ "catalog_model_entry_id", "route_behavior_profile", "evidence_manifest_sha256", + "candidate_revision", + "cli_binary_sha256", + "cli_source_commit_status", + "cli_source_tag", ) +# This artifact is deliberately bound to the current Issue #62 evidence +# package. A standalone reconciliation has no evidence root to reopen, so +# it must still reject a hand-edited candidate or pointer that merely has the +# right type/shape but no longer names the retained package. +EXPECTED_CANDIDATE_VALUES = { + "cli_version": DEFAULT_CLI_FLOOR, + "source_commit": CLI_SOURCE_COMMIT, + "codex_source_commit": CLI_SOURCE_COMMIT, + "route_upstream": "official", + "inbound_format": "responses", + "upstream_format": "responses", + "configured_provider_id": "custom", + "model": "gpt-5.6-sol", + "catalog_binding": "official Codex catalog entry for openai/gpt-5.6-sol", + "catalog_model_entry_id": "gpt-5.6-sol", + "route_behavior_profile": "official_codex_app_http_passthrough", + "candidate_revision": CANDIDATE_REVISION, + "cli_binary_sha256": CLI_BINARY_SHA256, + "cli_source_commit_status": CLI_SOURCE_COMMIT_STATUS, + "cli_source_tag": CLI_SOURCE_TAG, +} + +EXPECTED_EVIDENCE_BINDING_FILES = { + "source_contract": "codex-0.146-source-contract.json", + "trace": "current-codexhub-thread-tool-surface.json", + "wire_fixture": "codexhub-runtime-wire-fixture.json", + "audit": "read-only-gate-audit.json", +} +INVENTORY_TOP_LEVEL_FIELDS = frozenset( + { + "schema_version", + "artifact_kind", + "cli_version_floor", + "candidate_identity", + "declaration_families", + "qualification", + "disposition_vocabulary", + "items", + "identity_control", + "feeds", + "evidence_sources", + "evidence_binding", + } +) +INVENTORY_CANDIDATE_FIELDS = frozenset( + { + "candidate_revision", + "catalog_binding", + "catalog_model_entry_id", + "catalog_snapshot_sha256", + "cli_binary_sha256", + "cli_source_commit_status", + "cli_source_tag", + "cli_version", + "codex_source_commit", + "configured_provider_id", + "evidence_manifest_sha256", + "inbound_format", + "model", + "route_behavior_profile", + "route_upstream", + "source_commit", + "upstream_format", + } +) +INVENTORY_FAMILY_FIELDS = frozenset( + { + "family", + "runtime_type", + "wire_declaration_type", + "observed", + "observation", + "executor", + "loss_boundary", + "selected_protocol_disposition", + "optional_rule", + "required_rule", + "evidence_source", + "representative", + } +) +INVENTORY_ITEM_FIELDS = frozenset({"scope", "disposition", "evidence_source"}) +INVENTORY_QUALIFICATION_FIELDS = frozenset( + { + "candidate_version_status", + "candidate_version_eligible", + "blocking_scopes", + "evidence_gates", + "blocking_gates", + "ready_for_beta2", + } +) +INVENTORY_EVIDENCE_GATES = frozenset( + { + "complete_model_visible_plan", + "clean_cold_start_current_binding", + "full_pre_post_request_response", + "full_request_fingerprint", + "full_response_fingerprint", + "sse_identity", + "terminal_events", + "error_events", + "non_streaming", + "non_streaming_fixture", + "identity_replay", + "wire_identity_replay", + } +) +INVENTORY_IDENTITY_CONTROL_FIELDS = frozenset( + { + "fail_closed", + "replay_cases", + "unclassified_core_items", + "unclassified_scopes", + "unknown_tagged_source_count", + } +) +INVENTORY_EVIDENCE_SOURCES_FIELDS = frozenset({"trace", "wire_fixture", "audit"}) +INVENTORY_FEEDS_FIELDS = frozenset({"capability_gate", "chat_conversion_matrix"}) +INVENTORY_EVIDENCE_BINDING_ENTRY_FIELDS = frozenset({"file", "sha256"}) +EXPECTED_EVIDENCE_SOURCES = { + "trace": "current-codexhub-thread-tool-surface.json", + "wire_fixture": "codexhub-runtime-wire-fixture.json", + "audit": "read-only-gate-audit.json", +} +EXPECTED_FEEDS = {"capability_gate": "#249", "chat_conversion_matrix": "#66"} + +# ``reconcile_inventory`` is also used as a standalone replay check. In that +# mode there is deliberately no evidence root to reopen, so hashes, manifests, +# and unobserved qualification fields must be compared with the retained +# package identity rather than recalculated from the candidate's own fields. +# The evidence-root path below validates the currently retained, unobserved +# source-contract package and does not use these retained-only assertions. A +# later observed source contract requires a deliberate versioned schema update; +# this branch does not claim to admit that transition. +RETAINED_EVIDENCE_BINDING = { + "source_contract": { + "file": "codex-0.146-source-contract.json", + "sha256": "6f38b8b07b98c6f28edd7418b63449242ee41d6396694392a70c0d7fb2b70f2c", + }, + "trace": { + "file": "current-codexhub-thread-tool-surface.json", + "sha256": "a4367f96fc2aceaa419577ee43f91870fb3fcca4cc95644bc3347cbdd4206ee3", + }, + "wire_fixture": { + "file": "codexhub-runtime-wire-fixture.json", + "sha256": "1e3ecfc51f59de7f21bee8a7a0b743bba26d35fb3dc9eaca5f7b4ea3dd57705f", + }, + "audit": { + "file": "read-only-gate-audit.json", + "sha256": "775efbc763dfae1c5d6a61532d9394e5cd181b2e6803bf2e42f6c886a0a964e3", + }, +} +RETAINED_EVIDENCE_MANIFEST_SHA256 = ( + "685858e1affdb073df17467f53ab3de538c0c7f2d930b01c10797a94963c6619" +) +RETAINED_UNOBSERVED_EVIDENCE_GATES = { + "clean_cold_start_current_binding": "not_run", + "complete_model_visible_plan": "partial", + "error_events": "not_captured", + "full_pre_post_request_response": "live_control_required", + "full_request_fingerprint": "not_captured", + "full_response_fingerprint": "not_captured", + "identity_replay": "partial", + "non_streaming": "live_control_required", + "non_streaming_fixture": "not_captured", + "sse_identity": "not_captured", + "terminal_events": "not_captured", + "wire_identity_replay": "not_captured", +} +RETAINED_UNOBSERVED_BLOCKING_GATES = [ + "clean_cold_start_current_binding", + "complete_model_visible_plan", + "error_events", + "full_pre_post_request_response", + "full_request_fingerprint", + "full_response_fingerprint", + "identity_replay", + "non_streaming", + "non_streaming_fixture", + "sse_identity", + "terminal_events", + "wire_identity_replay", +] +RETAINED_UNOBSERVED_BLOCKING_SCOPES = [ + "choice_controls", + "core_function_replay", + "core_sse_errors", + "core_sse_terminal_events", + "core_text_non_streaming", + "default_runtime_fields", + "errors", + "hosted_only_declarations", + "terminal_events", + "unknown_tagged_sentinels", +] + def _sha256_file(path: Path) -> str: # Evidence is JSON text; hash its canonical LF representation so a @@ -332,7 +586,9 @@ def _validate_supported_floor(value: str) -> str: return value -def _candidate_identity_mismatches(candidate_identity: Any) -> list[str]: +def _candidate_identity_mismatches( + candidate_identity: Any, *, enforce_retained: bool = False +) -> list[str]: """Validate candidate metadata that is self-contained in the inventory. Reconciliation is also used for in-memory replay controls where no evidence @@ -387,7 +643,32 @@ def _candidate_identity_mismatches(candidate_identity: Any) -> list[str]: if not isinstance(value, str) or not value.strip(): mismatches.append(f"candidate_identity.{field} is missing or blank") - for field in ("catalog_snapshot_sha256", "evidence_manifest_sha256"): + candidate_revision = candidate_identity.get("candidate_revision") + if not isinstance(candidate_revision, str) or not re.fullmatch( + r"[0-9a-f]{40}", candidate_revision + ): + mismatches.append( + "candidate_identity.candidate_revision is not a lowercase 40-character SHA-1" + ) + source_status = candidate_identity.get("cli_source_commit_status") + if source_status not in {"published_attested", "not_published_by_registry"}: + mismatches.append("candidate_identity.cli_source_commit_status is invalid") + source_tag = candidate_identity.get("cli_source_tag") + if not isinstance(source_tag, str) or not source_tag.strip(): + mismatches.append("candidate_identity.cli_source_tag is missing or blank") + + if enforce_retained: + for field, expected in EXPECTED_CANDIDATE_VALUES.items(): + if candidate_identity.get(field) != expected: + mismatches.append( + f"candidate_identity.{field} does not match the retained Issue #62 candidate" + ) + + for field in ( + "catalog_snapshot_sha256", + "evidence_manifest_sha256", + "cli_binary_sha256", + ): value = candidate_identity.get(field) if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): mismatches.append( @@ -396,6 +677,100 @@ def _candidate_identity_mismatches(candidate_identity: Any) -> list[str]: return mismatches +def _standalone_retained_mismatches( + *, + inventory: dict[str, Any], + candidate_identity: Any, + qualification: Any, +) -> list[str]: + """Reject self-consistent edits when retained evidence cannot be reopened. + + A no-root replay has no trustworthy input from which to recompute a hash, + gate, or readiness value. Compare those fields with the immutable retained + package identity instead. Bound-evidence reconciliation supplies an + evidence root and intentionally takes the dynamic path below. The + source-contract validator remains versioned and unobserved-only; an + observed source contract needs a separate schema transition rather than + silently bypassing these retained checks. + """ + + mismatches: list[str] = [] + evidence_binding = inventory.get("evidence_binding") + if isinstance(evidence_binding, dict): + for name, expected in RETAINED_EVIDENCE_BINDING.items(): + actual = evidence_binding.get(name) + if actual != expected: + mismatches.append( + f"mutation: evidence_binding.{name} does not match the retained evidence artifact" + ) + if isinstance(candidate_identity, dict) and candidate_identity.get( + "evidence_manifest_sha256" + ) != RETAINED_EVIDENCE_MANIFEST_SHA256: + mismatches.append( + "mutation: candidate_identity.evidence_manifest_sha256 does not match the retained evidence manifest" + ) + + if isinstance(qualification, dict): + if qualification.get("evidence_gates") != RETAINED_UNOBSERVED_EVIDENCE_GATES: + mismatches.append( + "mutation: qualification.evidence_gates does not match the retained unobserved evidence" + ) + if qualification.get("blocking_gates") != RETAINED_UNOBSERVED_BLOCKING_GATES: + mismatches.append( + "mutation: qualification.blocking_gates does not match the retained unobserved evidence" + ) + if qualification.get("blocking_scopes") != RETAINED_UNOBSERVED_BLOCKING_SCOPES: + mismatches.append( + "mutation: qualification.blocking_scopes does not match the retained unobserved evidence" + ) + if qualification.get("ready_for_beta2") is not False: + mismatches.append( + "mutation: qualification.ready_for_beta2 does not match the retained unobserved evidence" + ) + + identity_control = inventory.get("identity_control") + if isinstance(identity_control, dict): + if identity_control.get("unknown_tagged_source_count") != 2: + mismatches.append( + "mutation: identity_control.unknown_tagged_source_count does not match the retained wire evidence" + ) + if identity_control.get("unclassified_core_items") != 0: + mismatches.append( + "mutation: identity_control.unclassified_core_items does not match the retained evidence" + ) + if identity_control.get("unclassified_scopes") != []: + mismatches.append( + "mutation: identity_control.unclassified_scopes does not match the retained evidence" + ) + + families = inventory.get("declaration_families") + if isinstance(families, list): + expected_observations = { + family: STRUCTURAL_FAMILY_SCHEMAS[family]["observation"] + for family in STRUCTURAL_FAMILIES + } + for index, family in enumerate(STRUCTURAL_FAMILIES): + if index >= len(families) or not isinstance(families[index], dict): + continue + entry = families[index] + representative = entry.get("representative") + if entry.get("observed") is not False or entry.get("observation") != expected_observations[family]: + mismatches.append( + f"mutation: declaration_families[{index}] does not match the retained unobserved source contract" + ) + if not isinstance(representative, dict): + continue + expected_statuses = {"terminal": "not_observed", "error": "unqualified"} + for section, expected in expected_statuses.items(): + actual_section = representative.get(section) + if not isinstance(actual_section, dict) or actual_section.get("classification") != expected: + mismatches.append( + f"mutation: declaration_families[{index}].representative.{section} classification does not match the retained unobserved source contract" + ) + + return mismatches + + def _evidence_source_allowed(scope: str, evidence_source: Any) -> bool: expected = SCOPE_EVIDENCE_SOURCES.get(scope) if expected is None: @@ -495,6 +870,23 @@ def _count_unknown_tags(value: Any) -> int: return 0 +def _count_response_unknown_tags(wire: dict[str, Any]) -> int: + """Count unknown-tag sentinels in the two response modes only. + + Structural family examples intentionally carry their own opaque unknown + sentinel. That declaration inventory is separate from the response-tag + evidence used by the identity control, so it must not inflate this count. + """ + + response = wire.get("response", {}) + return _count_unknown_tags( + { + "streaming": response.get("streaming", {}), + "non_streaming": response.get("non_streaming", {}), + } + ) + + def _unknown_tag_mode_counts(wire: dict[str, Any]) -> tuple[int, int]: streaming = _streaming_events(wire) non_streaming = wire.get("response", {}).get("non_streaming", {}).get( @@ -515,6 +907,1053 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: return [event for event in events if isinstance(event, dict)] +_STRUCTURAL_RULES = { + "plain_function": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_when_supported_else_omit", + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "executor": "codex_client", + }, + "custom_freeform": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_when_supported_else_omit", + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "executor": "codex_client", + }, + "namespace": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_when_supported_else_omit", + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "executor": "codex_client", + }, + "client_executed_tool_discovery": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_client_execution_else_omit", + "required_rule": "preserve_client_execution_else_required-but-unavailable", + "executor": "codex_client", + }, + "selected_provider_hosted": { + "selected_protocol_disposition": "native_if_selected_provider_supports", + "optional_rule": "omit_if_selected_provider_unsupported", + "required_rule": "required-but-unavailable_if_selected_provider_unsupported", + "executor": "selected_provider", + }, + "unknown_future_kind": { + "selected_protocol_disposition": "omit", + "optional_rule": "omit_and_emit_sanitized_diagnostic", + "required_rule": "required-but-unavailable", + "executor": "unknown", + }, +} + +# The source contract is intentionally closed over the six declaration +# families. Keep the wire/runtime names and identity-bearing sections +# canonical so a regenerated inventory cannot turn an arbitrary fixture value +# into a qualified family claim. +STRUCTURAL_FAMILY_SCHEMAS = { + "plain_function": { + "runtime_type": "function", + "wire_declaration_type": "function", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "declaration": {"type": "function", "required": ("name", "parameters")}, + "call": { + "type": "function_call", + "required": ("item_id", "call_id", "arguments"), + }, + "result": { + "type": "function_call_output", + "required": ("item_id", "call_id", "output"), + }, + "history": {"required": ("call_id", "call_item_id", "output_item_id")}, + }, + "custom_freeform": { + "runtime_type": "custom", + "wire_declaration_type": "custom", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "declaration": {"type": "custom", "required": ("name", "format")}, + "call": { + "type": "custom_tool_call", + "required": ("item_id", "call_id", "input"), + }, + "result": { + "type": "custom_tool_call_output", + "required": ("item_id", "call_id", "output"), + }, + "history": {"required": ("call_id", "call_item_id", "output_item_id")}, + }, + "namespace": { + "runtime_type": "namespace", + "wire_declaration_type": "namespace", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "declaration": {"type": "namespace", "required": ("name", "tools")}, + "call": { + "type": "function_call", + "required": ("item_id", "call_id", "namespace", "arguments"), + }, + "result": { + "type": "function_call_output", + "required": ("item_id", "call_id", "output"), + }, + "history": { + "required": ("call_id", "call_item_id", "output_item_id", "namespace") + }, + }, + "client_executed_tool_discovery": { + "runtime_type": "tool_search", + "wire_declaration_type": "tool_search", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "discovery request/result stays client-executed", + "declaration": { + "type": "tool_search", + "required": ("execution", "parameters"), + "equals": {"execution": "client"}, + }, + "call": { + "type": "tool_search_call", + "required": ("item_id", "call_id", "execution", "arguments"), + "equals": {"execution": "client"}, + }, + "result": { + "type": "tool_search_output", + "required": ("item_id", "call_id", "execution", "tools"), + "equals": {"execution": "client"}, + }, + "history": { + "required": ("call_id", "call_item_id", "output_item_id", "executor"), + "equals": {"executor": "codex_client"}, + }, + }, + "selected_provider_hosted": { + "runtime_type": "web_search", + "wire_declaration_type": "web_search", + "executor": "selected_provider", + "observation": "not_observed_selected_provider_control_required", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "declaration": { + "type": "web_search", + "required": ("executor", "provider_scope"), + "equals": { + "executor": "selected_provider", + "provider_scope": "selected_provider_only", + }, + }, + "call": { + "type": "web_search_call", + "required": ("item_id", "status", "action"), + }, + "result": { + "type": "web_search_call", + "required": ("item_id", "status", "provider_scope"), + "equals": {"provider_scope": "selected_provider_only"}, + }, + "history": { + "required": ( + "call_id", + "call_item_id", + "output_item_id", + "executor", + "cross_provider_proxy", + ), + "equals": { + "executor": "selected_provider", + "cross_provider_proxy": "forbidden", + }, + "nullable": ("call_id",), + }, + }, + "unknown_future_kind": { + "runtime_type": "unknown", + "wire_declaration_type": "", + "executor": "unknown", + "observation": "opaque_sentinel_only", + "loss_boundary": "retain tag and opaque payload; do not normalize", + "declaration": { + "type": "unknown", + "required": ("tag", "opaque_payload"), + "equals": {"tag": "unknown"}, + }, + "call": { + "type": "unknown", + "required": ("tag", "opaque_payload"), + "equals": {"tag": "unknown"}, + }, + "result": { + "type": "unknown", + "required": ("tag", "opaque_payload"), + "equals": {"tag": "unknown"}, + }, + "history": { + "required": ("call_id", "call_item_id", "output_item_id", "loss_rule"), + "nullable": ("call_id", "call_item_id", "output_item_id"), + "equals": { + "call_id": None, + "call_item_id": None, + "output_item_id": None, + "loss_rule": "retain opaque sentinel", + }, + }, + }, +} +STRUCTURAL_EXAMPLE_BASE_FIELDS = frozenset( + { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } +) +STRUCTURAL_EXAMPLE_STATUS_FIELDS = { + "selected_provider_hosted": { + "observed": False, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden", + }, + "unknown_future_kind": { + "observed": False, + "status": "opaque_sentinel_only", + }, +} +STRUCTURAL_REDACTED_FIELDS = frozenset( + { + "parameters", + "arguments", + "input", + "output", + "format", + "action", + "status", + "tools", + "opaque_payload", + } +) +STRUCTURAL_RESPONSE_ITEM_TYPES = [ + "message", + "reasoning", + "function_call", + "function_call_output", + "custom_tool_call", + "custom_tool_call_output", + "tool_search_call", + "tool_search_output", + "web_search_call", + "local_shell_call", + "compaction", + "context_compaction", + "unknown", +] +STRUCTURAL_RESPONSE_STREAM_EVENT_ORDER = [ + "response.created", + "response.in_progress", + "response.output_item.added", + "response.output_text.delta", + "response.function_call_arguments.delta", + "response.custom_tool_call_input.delta", + "response.function_call_arguments.done", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.completed", +] +STRUCTURAL_RESPONSE_TERMINAL_EVENTS = [ + "response.completed", + "response.incomplete", + "response.failed", +] +STRUCTURAL_NONEMPTY_STRING_FIELDS = frozenset( + { + "name", + "namespace", + "executor", + "execution", + "provider_scope", + "cross_provider_proxy", + "tag", + "loss_rule", + "item_id", + "call_id", + "call_item_id", + "output_item_id", + } +) + + +def _require_exact_fields( + value: dict[str, Any], expected: set[str] | frozenset[str], *, context: str +) -> None: + actual = set(value) + if actual != set(expected): + missing = sorted(set(expected) - actual) + unknown = sorted(actual - set(expected)) + raise ValueError( + f"{context} has unknown or missing fields " + f"(missing={missing!r}, unknown={unknown!r})" + ) + + +def _exact_inventory_fields_mismatch( + value: Any, + expected: set[str] | frozenset[str], + *, + context: str, + optional: set[str] | frozenset[str] = frozenset(), +) -> str | None: + """Return a fail-closed mismatch for an inventory object with extra keys.""" + + if not isinstance(value, dict): + return f"{context} must be an object" + actual = set(value) + if set(expected) <= actual and not (actual - set(expected) - set(optional)): + return None + missing = sorted(set(expected) - actual) + unknown = sorted(actual - set(expected) - set(optional)) + return f"{context} has unknown or missing fields (missing={missing!r}, unknown={unknown!r})" + + +def _validate_structural_section( + family_name: str, section_name: str, section: dict[str, Any], *, context: str +) -> None: + section_schema = STRUCTURAL_FAMILY_SCHEMAS[family_name][section_name] + expected_fields = set(section_schema.get("required", ())) + if section_schema.get("type") is not None: + expected_fields.add("type") + _require_exact_fields(section, expected_fields, context=context) + nullable = set(section_schema.get("nullable", ())) + equals = section_schema.get("equals", {}) + for key in expected_fields: + value = section[key] + if key in equals: + if value != equals[key]: + raise ValueError( + f"{context}.{key} does not match the canonical family schema" + ) + continue + if key == "type": + if value != section_schema["type"]: + raise ValueError( + f"{context}.type does not match the canonical family schema" + ) + continue + if value is None: + if key not in nullable: + raise ValueError(f"{context}.{key} cannot be null") + continue + if family_name == "namespace" and section_name == "declaration" and key == "tools": + continue + if key in STRUCTURAL_REDACTED_FIELDS: + if value != "": + raise ValueError(f"{context}.{key} must remain the redacted sentinel") + continue + if key in STRUCTURAL_NONEMPTY_STRING_FIELDS and ( + not isinstance(value, str) or not value + ): + raise ValueError(f"{context}.{key} must be a non-empty string") + if not isinstance(value, str): + raise ValueError(f"{context}.{key} must be a string") + + if family_name == "namespace" and section_name == "declaration": + tools = section["tools"] + if not isinstance(tools, list) or not tools: + raise ValueError(f"{context}.tools must contain a function") + for index, nested in enumerate(tools): + nested_context = f"{context}.tools[{index}]" + if not isinstance(nested, dict): + raise ValueError(f"{nested_context} must be an object") + _require_exact_fields( + nested, + {"type", "name", "parameters"}, + context=nested_context, + ) + if nested.get("type") != "function": + raise ValueError(f"{nested_context}.type must be function") + if not isinstance(nested.get("name"), str) or not nested["name"]: + raise ValueError(f"{nested_context}.name must be a non-empty string") + if nested.get("parameters") != "": + raise ValueError( + f"{nested_context}.parameters must remain the redacted sentinel" + ) + + +def _validate_structural_terminal_error( + example: dict[str, Any], *, context: str +) -> None: + terminal = example.get("terminal") + if not isinstance(terminal, dict): + raise ValueError(f"{context}.terminal must be an object") + _require_exact_fields(terminal, {"event", "classification"}, context=f"{context}.terminal") + if terminal.get("event") != "response.completed": + raise ValueError(f"{context}.terminal.event is invalid") + if terminal.get("classification") not in {"not_observed", "unqualified"}: + raise ValueError(f"{context}.terminal.classification is invalid") + + error = example.get("error") + if not isinstance(error, dict): + raise ValueError(f"{context}.error must be an object") + _require_exact_fields(error, {"event", "classification"}, context=f"{context}.error") + if error.get("event") != "response.failed": + raise ValueError(f"{context}.error.event is invalid") + if error.get("classification") not in {"not_observed", "unqualified"}: + raise ValueError(f"{context}.error.classification is invalid") + + +def _validate_response_shape_schema(response_shape: Any, *, context: str) -> None: + if not isinstance(response_shape, dict): + raise ValueError(f"{context} must be an object") + _require_exact_fields( + response_shape, + {"response_item_types", "stream_event_order", "terminal_events", "error_shape"}, + context=context, + ) + if response_shape.get("response_item_types") != STRUCTURAL_RESPONSE_ITEM_TYPES: + raise ValueError(f"{context}.response_item_types does not match the canonical list") + if response_shape.get("stream_event_order") != STRUCTURAL_RESPONSE_STREAM_EVENT_ORDER: + raise ValueError(f"{context}.stream_event_order does not match the canonical list") + if response_shape.get("terminal_events") != STRUCTURAL_RESPONSE_TERMINAL_EVENTS: + raise ValueError(f"{context}.terminal_events does not match the canonical list") + error_shape = response_shape.get("error_shape") + if not isinstance(error_shape, dict): + raise ValueError(f"{context}.error_shape must be an object") + _require_exact_fields( + error_shape, + {"event", "response", "classification"}, + context=f"{context}.error_shape", + ) + if error_shape.get("event") != "response.failed": + raise ValueError(f"{context}.error_shape.event is invalid") + if error_shape.get("classification") != "unqualified": + raise ValueError(f"{context}.error_shape.classification is invalid") + error_response = error_shape.get("response") + if not isinstance(error_response, dict): + raise ValueError(f"{context}.error_shape.response must be an object") + _require_exact_fields( + error_response, + {"id", "status", "error"}, + context=f"{context}.error_shape.response", + ) + if error_response.get("id") != "response_error_001": + raise ValueError(f"{context}.error_shape.response.id is invalid") + if error_response.get("status") != "failed": + raise ValueError(f"{context}.error_shape.response.status is invalid") + if error_response.get("error") != "": + raise ValueError(f"{context}.error_shape.response.error must remain redacted") + + +def _validate_structural_family_schema( + family_name: str, + family: dict[str, Any], + example: dict[str, Any], + *, + context: str, +) -> None: + schema = STRUCTURAL_FAMILY_SCHEMAS.get(family_name) + if schema is None: + raise ValueError(f"{context} has an unknown family schema") + for field in ("runtime_type", "wire_declaration_type", "executor", "observation", "loss_boundary"): + if family.get(field) != schema[field]: + raise ValueError(f"{context}.{field} does not match the canonical family schema") + if not isinstance(example, dict): + raise ValueError(f"{context} must be an object") + expected_example_fields = STRUCTURAL_EXAMPLE_BASE_FIELDS | set( + STRUCTURAL_EXAMPLE_STATUS_FIELDS.get(family_name, {}) + ) + _require_exact_fields(example, expected_example_fields, context=context) + for field, expected in STRUCTURAL_EXAMPLE_STATUS_FIELDS.get(family_name, {}).items(): + if example.get(field) != expected: + raise ValueError(f"{context}.{field} does not match the canonical status") + for section_name in ("declaration", "call", "result", "history"): + section = example.get(section_name) + if not isinstance(section, dict): + raise ValueError(f"{context}.{section_name} must be an object") + _validate_structural_section( + family_name, + section_name, + section, + context=f"{context}.{section_name}", + ) + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"{context}.loss_boundary is missing or blank") + if example["loss_boundary"] != schema["loss_boundary"]: + raise ValueError(f"{context}.loss_boundary does not match the canonical family schema") + _validate_structural_terminal_error(example, context=context) + if family_name == "namespace": + namespace = example["declaration"].get("name") + if ( + example["call"].get("namespace") != namespace + or example["history"].get("namespace") != namespace + ): + raise ValueError(f"{context} namespace ownership does not reconcile") + if family_name in { + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + }: + if ( + example["call"].get("item_id") != example["history"].get("call_item_id") + or example["result"].get("item_id") + != example["history"].get("output_item_id") + ): + raise ValueError(f"{context} call/result item IDs do not reconcile") + _validate_structural_stream_example( + family_name, + example.get("streaming"), + context=context, + ) + +STRUCTURAL_EVIDENCE_SOURCES = { + "plain_function": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", + "custom_freeform": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.custom_freeform", + "namespace": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.namespace", + "client_executed_tool_discovery": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.client_executed_tool_discovery", + "selected_provider_hosted": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.selected_provider_hosted", + "unknown_future_kind": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.unknown_future_kind", +} + +STRUCTURAL_STREAM_DONE_EVENTS = { + "plain_function": ("arguments_done", "response.function_call_arguments.done"), + "custom_freeform": ("input_done", "response.custom_tool_call_input.done"), + "namespace": ("arguments_done", "response.function_call_arguments.done"), + "unknown_future_kind": ("done", "unknown.future_done"), +} + +STRUCTURAL_STREAM_SCHEMAS = { + "plain_function": { + "added": "response.output_item.added", + "delta": "response.function_call_arguments.delta", + "done_field": "arguments_done", + "done": "response.function_call_arguments.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed", + ], + }, + "custom_freeform": { + "added": "response.output_item.added", + "delta": "response.custom_tool_call_input.delta", + "done_field": "input_done", + "done": "response.custom_tool_call_input.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "response.custom_tool_call_input.delta", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.completed", + ], + }, + "namespace": { + "added": "response.output_item.added", + "delta": "response.function_call_arguments.delta", + "done_field": "arguments_done", + "done": "response.function_call_arguments.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed", + ], + }, + "client_executed_tool_discovery": { + "added": None, + "delta": None, + "done_field": "done", + "done": "response.output_item.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": ["response.output_item.done", "response.completed"], + }, + "selected_provider_hosted": { + "added": "response.output_item.added", + "delta": "", + "done_field": "done", + "done": "response.output_item.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "", + "response.output_item.done", + "response.completed", + ], + }, + "unknown_future_kind": { + "added": "unknown.future_event", + "delta": "unknown.future_delta", + "done_field": "done", + "done": "unknown.future_done", + "item_done": "unknown.future_done", + "terminal": "response.completed", + "event_order": [ + "unknown.future_event", + "unknown.future_delta", + "unknown.future_done", + "response.completed", + ], + }, +} + + +def _validate_structural_stream_example( + family_name: str, + streaming: Any, + *, + context: str, +) -> None: + schema = STRUCTURAL_STREAM_SCHEMAS.get(family_name) + if schema is None: + raise ValueError(f"{context} has an unknown stream schema") + if not isinstance(streaming, dict): + raise ValueError(f"{context}.streaming must be an object") + expected_fields = { + "added", + "delta", + "done", + "terminal", + "event_order", + schema["done_field"], + } + _require_exact_fields(streaming, expected_fields, context=f"{context}.streaming") + for field in ("added", "delta", "terminal"): + if streaming.get(field) != schema[field]: + raise ValueError(f"{context}.streaming.{field} does not match the canonical SSE schema") + done_field = schema["done_field"] + if streaming.get(done_field) != schema["done"]: + raise ValueError(f"{context}.streaming.{done_field} does not match the canonical SSE schema") + if streaming.get("done") != schema["item_done"]: + raise ValueError(f"{context}.streaming.done does not match the canonical SSE schema") + if streaming.get("event_order") != schema["event_order"]: + raise ValueError(f"{context}.streaming.event_order does not match the canonical SSE schema") + + +def _validate_structural_stream_contract( + *, wire: dict[str, Any], examples: dict[str, Any] +) -> None: + """Require family SSE examples to retain done-before-terminal ordering.""" + + response_shape = wire.get("runtime_wire_surface", {}).get("response_shape", {}) + stream_event_order = response_shape.get("stream_event_order") + if not isinstance(stream_event_order, list) or not all( + isinstance(event, str) and event for event in stream_event_order + ): + raise ValueError("wire response_shape.stream_event_order must be a non-empty string array") + if "response.completed" not in stream_event_order: + raise ValueError("wire response_shape.stream_event_order is missing response.completed") + completed_index = stream_event_order.index("response.completed") + output_done_index = ( + stream_event_order.index("response.output_item.done") + if "response.output_item.done" in stream_event_order + else None + ) + for family_name, (_, event) in STRUCTURAL_STREAM_DONE_EVENTS.items(): + if event.startswith("unknown."): + continue + if event not in stream_event_order: + raise ValueError( + f"wire response_shape.stream_event_order is missing {event} for {family_name}" + ) + event_index = stream_event_order.index(event) + if event_index >= completed_index or ( + output_done_index is not None and event_index >= output_done_index + ): + raise ValueError(f"wire stream event {event} must precede terminal/item completion") + + for family_name in STRUCTURAL_FAMILIES: + example = examples[family_name] + streaming = example.get("streaming") + _validate_structural_stream_example( + family_name, + streaming, + context=f"wire runtime family {family_name}", + ) + if not isinstance(streaming, dict): + raise ValueError(f"wire runtime streaming example is malformed for {family_name}") + event_order = streaming.get("event_order") + if not isinstance(event_order, list) or not all( + isinstance(event, str) and event for event in event_order + ): + raise ValueError(f"wire runtime streaming event_order is invalid for {family_name}") + if streaming.get("terminal") != "response.completed" or event_order[-1] != "response.completed": + raise ValueError(f"wire runtime streaming terminal ordering is invalid for {family_name}") + if family_name != "unknown_future_kind" and ( + "response.output_item.done" not in event_order + or event_order.index("response.output_item.done") >= len(event_order) - 1 + ): + raise ValueError(f"wire runtime streaming item completion is invalid for {family_name}") + done_spec = STRUCTURAL_STREAM_DONE_EVENTS.get(family_name) + if done_spec is not None: + field, event = done_spec + if streaming.get(field) != event or event not in event_order: + raise ValueError( + f"wire runtime streaming family done event is missing for {family_name}" + ) + if event_order.index(event) >= event_order.index("response.completed"): + raise ValueError(f"wire runtime streaming family done event follows terminal for {family_name}") + if family_name == "client_executed_tool_discovery": + if event_order != ["response.output_item.done", "response.completed"]: + raise ValueError("tool_search_call SSE must contain only item completion and terminal events") + if streaming.get("added") is not None or streaming.get("delta") is not None: + raise ValueError("tool_search_call SSE must not claim a text delta or item-added event") + + +def _validate_structural_evidence_pointers( + *, + source_contract: dict[str, Any], + wire: dict[str, Any], + audit: dict[str, Any], + declaration_families: list[dict[str, Any]], +) -> None: + """Require every declaration-family evidence source to resolve in its fixture.""" + + documents = { + "codex-0.146-source-contract.json": source_contract, + "codexhub-runtime-wire-fixture.json": wire, + "read-only-gate-audit.json": audit, + } + for entry in declaration_families: + family = entry.get("family") + source = entry.get("evidence_source") + expected = STRUCTURAL_EVIDENCE_SOURCES.get(family) + if expected is None or source != expected: + raise ValueError( + f"declaration-family evidence source is invalid for {family!r}: {source!r}" + ) + filename, pointer_text = source.split("#", 1) + document = documents.get(filename) + if document is None: + raise ValueError(f"declaration-family evidence fixture is unknown: {filename}") + try: + _resolve_fixture_pointer(document, tuple(pointer_text.split("."))) + except KeyError as exc: + raise ValueError( + f"declaration-family evidence pointer for {family} is missing: " + f"{filename}#{exc.args[0]}" + ) from exc + + +def _build_structural_inventory( + *, + source_contract: dict[str, Any], + trace: dict[str, Any], + wire: dict[str, Any], + audit: dict[str, Any], +) -> list[dict[str, Any]]: + """Join the unobserved 0.146 source contract with sanitized examples.""" + + families = source_contract.get("runtime_wire_surface", {}).get( + "declaration_families", [] + ) + if not isinstance(families, list): + raise ValueError("source contract declaration_families must be an array") + by_family: dict[str, dict[str, Any]] = {} + for family in families: + if not isinstance(family, dict) or not isinstance(family.get("family"), str): + raise ValueError("runtime planner declaration family is malformed") + name = family["family"] + if name in by_family: + raise ValueError(f"duplicate runtime planner declaration family: {name}") + by_family[name] = family + if set(by_family) != set(STRUCTURAL_FAMILIES): + missing = sorted(set(STRUCTURAL_FAMILIES) - set(by_family)) + extra = sorted(set(by_family) - set(STRUCTURAL_FAMILIES)) + raise ValueError( + "runtime planner declaration families are incomplete: " + f"missing={missing!r} extra={extra!r}" + ) + + examples = source_contract.get("runtime_wire_surface", {}).get( + "declaration_family_examples", {} + ) + if not isinstance(examples, dict): + raise ValueError("wire runtime declaration_family_examples must be an object") + output: list[dict[str, Any]] = [] + for family_name in STRUCTURAL_FAMILIES: + family = by_family[family_name] + example = examples.get(family_name) + if not isinstance(example, dict): + raise ValueError(f"wire runtime example missing for {family_name}") + required_example_parts = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } + if not required_example_parts.issubset(example): + missing = sorted(required_example_parts - set(example)) + raise ValueError( + f"wire runtime example for {family_name} is missing {missing!r}" + ) + _validate_structural_family_schema( + family_name, + family, + example, + context=f"runtime planner family {family_name}", + ) + if not isinstance(example.get("terminal"), dict) or example["terminal"].get( + "classification" + ) not in {"not_observed", "unqualified"}: + raise ValueError(f"runtime terminal classification is invalid for {family_name}") + if not isinstance(example.get("error"), dict) or example["error"].get( + "classification" + ) not in {"not_observed", "unqualified"}: + raise ValueError(f"runtime error classification is invalid for {family_name}") + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"runtime loss boundary is missing for {family_name}") + rule = _STRUCTURAL_RULES[family_name] + if family.get("executor") != rule["executor"]: + raise ValueError( + f"runtime planner executor for {family_name} contradicts the CLI contract" + ) + if family_name == "selected_provider_hosted": + if example.get("provider_scope") != "selected_provider_only": + raise ValueError("hosted declaration is not bound to the selected Provider") + if example.get("cross_provider_proxy") != "forbidden": + raise ValueError("hosted declaration permits a cross-Provider proxy") + if family_name in { + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + }: + call = example["call"] + result = example["result"] + history = example["history"] + if not all(isinstance(part, dict) for part in (call, result, history)): + raise ValueError(f"runtime identity sections are malformed for {family_name}") + if ( + call.get("item_id") != history.get("call_item_id") + or result.get("item_id") != history.get("output_item_id") + or call.get("call_id") != result.get("call_id") + or call.get("call_id") != history.get("call_id") + ): + raise ValueError(f"runtime call/result/history IDs do not reconcile for {family_name}") + output.append( + { + "family": family_name, + "runtime_type": family.get("runtime_type"), + "wire_declaration_type": family.get("wire_declaration_type"), + "observed": bool(family.get("observed")), + "observation": family.get("observation"), + "executor": rule["executor"], + "selected_protocol_disposition": rule["selected_protocol_disposition"], + "optional_rule": rule["optional_rule"], + "required_rule": rule["required_rule"], + "loss_boundary": family.get("loss_boundary"), + "evidence_source": STRUCTURAL_EVIDENCE_SOURCES[family_name], + "representative": example, + } + ) + _validate_structural_stream_contract(wire=source_contract, examples=examples) + _validate_structural_evidence_pointers( + source_contract=source_contract, + wire=wire, + audit=audit, + declaration_families=output, + ) + return output + + +def _structural_inventory_mismatches( + value: Any, *, require_unobserved: bool = False +) -> list[str]: + """Validate the stable shape of the emitted declaration-family inventory.""" + + if not isinstance(value, list): + return ["declaration_families must be an array"] + mismatches: list[str] = [] + if len(value) != len(STRUCTURAL_FAMILIES): + mismatches.append( + "declaration_families must contain exactly the six known families" + ) + required_example_parts = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } + for index, family_name in enumerate(STRUCTURAL_FAMILIES): + if index >= len(value): + break + entry = value[index] + prefix = f"declaration_families[{index}]" + if not isinstance(entry, dict): + mismatches.append(f"{prefix} must be an object") + continue + extra_fields = _exact_inventory_fields_mismatch( + entry, INVENTORY_FAMILY_FIELDS, context=prefix + ) + if extra_fields: + mismatches.append(extra_fields) + if entry.get("family") != family_name: + mismatches.append( + f"{prefix}.family must be {family_name!r} in the canonical order" + ) + continue + rule = _STRUCTURAL_RULES[family_name] + if entry.get("executor") != rule["executor"]: + mismatches.append(f"{prefix}.executor does not match the family rule") + for field in ("runtime_type", "wire_declaration_type", "observation", "loss_boundary"): + if not isinstance(entry.get(field), str) or not entry[field]: + mismatches.append(f"{prefix}.{field} is missing or blank") + if not isinstance(entry.get("observed"), bool): + mismatches.append(f"{prefix}.observed must be boolean") + elif require_unobserved and entry.get("observed") is not False: + mismatches.append( + f"{prefix}.observed must remain false for the unobserved 0.146 source contract" + ) + for field in ("selected_protocol_disposition", "optional_rule", "required_rule"): + if entry.get(field) != rule[field]: + mismatches.append(f"{prefix}.{field} does not match the family rule") + expected_source = STRUCTURAL_EVIDENCE_SOURCES[family_name] + if entry.get("evidence_source") != expected_source: + mismatches.append(f"{prefix}.evidence_source does not match the family source") + representative = entry.get("representative") + if not isinstance(representative, dict): + mismatches.append(f"{prefix}.representative must be an object") + continue + if not required_example_parts.issubset(representative): + mismatches.append(f"{prefix}.representative is missing a wire example section") + else: + try: + _validate_structural_family_schema( + family_name, + entry, + representative, + context=prefix, + ) + except ValueError as exc: + mismatches.append(str(exc)) + terminal = representative.get("terminal") + if not isinstance(terminal, dict): + mismatches.append(f"{prefix}.representative.terminal must be an object") + else: + if terminal.get("event") != "response.completed": + mismatches.append( + f"{prefix}.representative.terminal.event must be response.completed" + ) + if terminal.get("classification") not in {"not_observed", "unqualified"}: + mismatches.append( + f"{prefix}.representative.terminal.classification is invalid" + ) + error = representative.get("error") + if not isinstance(error, dict): + mismatches.append(f"{prefix}.representative.error must be an object") + else: + if error.get("event") != "response.failed": + mismatches.append( + f"{prefix}.representative.error.event must be response.failed" + ) + if error.get("classification") not in {"not_observed", "unqualified"}: + mismatches.append( + f"{prefix}.representative.error.classification is invalid" + ) + if not isinstance(representative.get("loss_boundary"), str) or not representative[ + "loss_boundary" + ]: + mismatches.append(f"{prefix}.representative.loss_boundary is missing or blank") + elif representative["loss_boundary"] != entry.get("loss_boundary"): + mismatches.append( + f"{prefix}.representative.loss_boundary does not match the family boundary" + ) + streaming = representative.get("streaming") + if not isinstance(streaming, dict): + mismatches.append(f"{prefix}.representative.streaming must be an object") + else: + event_order = streaming.get("event_order") + if not isinstance(event_order, list) or not event_order: + mismatches.append(f"{prefix}.representative.streaming.event_order is invalid") + elif streaming.get("terminal") != "response.completed" or event_order[-1] != "response.completed": + mismatches.append( + f"{prefix}.representative.streaming terminal ordering is invalid" + ) + else: + done_spec = STRUCTURAL_STREAM_DONE_EVENTS.get(family_name) + if family_name != "unknown_future_kind" and ( + "response.output_item.done" not in event_order + or event_order.index("response.output_item.done") >= len(event_order) - 1 + ): + mismatches.append( + f"{prefix}.representative.streaming item completion is invalid" + ) + if done_spec is not None: + field, event = done_spec + if streaming.get(field) != event or event not in event_order: + mismatches.append( + f"{prefix}.representative.streaming family done event is missing" + ) + elif event_order.index(event) >= event_order.index("response.completed"): + mismatches.append( + f"{prefix}.representative.streaming family done event follows terminal" + ) + if family_name == "client_executed_tool_discovery": + if event_order != ["response.output_item.done", "response.completed"]: + mismatches.append( + f"{prefix}.representative.streaming tool-search order is invalid" + ) + if streaming.get("added") is not None or streaming.get("delta") is not None: + mismatches.append( + f"{prefix}.representative.streaming tool-search claims text events" + ) + if family_name in { + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + }: + call = representative.get("call") + result = representative.get("result") + history = representative.get("history") + if not all(isinstance(part, dict) for part in (call, result, history)): + mismatches.append(f"{prefix}.representative identity sections are malformed") + elif ( + call.get("item_id") != history.get("call_item_id") + or result.get("item_id") != history.get("output_item_id") + or call.get("call_id") != result.get("call_id") + or call.get("call_id") != history.get("call_id") + ): + mismatches.append(f"{prefix}.representative call/result/history IDs do not reconcile") + if family_name == "selected_provider_hosted": + if representative.get("provider_scope") != "selected_provider_only": + mismatches.append(f"{prefix}.representative is not selected-provider scoped") + if representative.get("cross_provider_proxy") != "forbidden": + mismatches.append(f"{prefix}.representative permits a cross-provider proxy") + if family_name == "unknown_future_kind": + declaration = representative.get("declaration", {}) + if not isinstance(declaration, dict) or declaration.get("tag") != "unknown": + mismatches.append(f"{prefix}.representative does not retain the unknown sentinel") + return mismatches + + def _has_terminal_event(wire: dict[str, Any]) -> bool: return any(event.get("event") == "response.completed" for event in _streaming_events(wire)) @@ -778,7 +2217,7 @@ def _classify_live_control_items( full_wire_gate = gate_classification.get("full_pre_post_request_response") non_direct_gate = gate_classification.get("non_direct_states") non_streaming_captured = _real_non_streaming_captured(wire) - unknown_tag_count = _count_unknown_tags(wire) + unknown_tag_count = _count_response_unknown_tags(wire) unknown_stream_count, unknown_non_stream_count = _unknown_tag_mode_counts(wire) choice_captured = ( @@ -1028,30 +2467,289 @@ def _build_qualification( "blocking_scopes": blocking_scopes, "evidence_gates": evidence_gates, "blocking_gates": blocking_gates, - "ready_for_beta1": candidate_version_eligible + "ready_for_beta2": candidate_version_eligible and not blocking_scopes and not blocking_gates, } +def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any]: + expected_top_level_fields = { + "schema_version", + "fixture_kind", + "capture_status", + "qualification_status", + "captured_at", + "provenance", + "runtime_wire_surface", + } + if set(source_contract) != expected_top_level_fields: + raise ValueError("Codex 0.146 source contract has unknown top-level fields") + if ( + source_contract.get("schema_version") != 1 + or source_contract.get("fixture_kind") != "codex_cli_source_contract" + or source_contract.get("capture_status") != "not_observed" + or source_contract.get("qualification_status") != "unqualified" + or "captured_at" not in source_contract + or source_contract.get("captured_at") is not None + ): + raise ValueError("Codex 0.146 source contract must remain not_observed and unqualified") + provenance = source_contract.get("provenance", {}) + expected_provenance_fields = { + "cli_version", + "source_commit", + "cli_source_tag", + "cli_source_commit_status", + "cli_binary_sha256", + "candidate_revision", + } + if not isinstance(provenance, dict) or set(provenance) != expected_provenance_fields: + raise ValueError("Codex 0.146 source contract provenance has unknown fields") + expected = { + "cli_version": DEFAULT_CLI_FLOOR, + "source_commit": CLI_SOURCE_COMMIT, + "cli_source_tag": CLI_SOURCE_TAG, + "cli_source_commit_status": CLI_SOURCE_COMMIT_STATUS, + "cli_binary_sha256": CLI_BINARY_SHA256, + "candidate_revision": CANDIDATE_REVISION, + } + if any(provenance.get(field) != value for field, value in expected.items()): + raise ValueError("Codex 0.146 source contract provenance is invalid") + runtime_surface = source_contract.get("runtime_wire_surface") + if not isinstance(runtime_surface, dict): + raise ValueError("Codex 0.146 source contract runtime surface is missing") + expected_runtime_surface_fields = { + "source", + "declaration_family_order", + "declaration_families", + "request_shape", + "response_shape", + "declaration_family_examples", + } + if set(runtime_surface) != expected_runtime_surface_fields: + raise ValueError( + "Codex 0.146 source contract runtime_wire_surface has unknown fields" + ) + if runtime_surface.get("declaration_family_order") != list(STRUCTURAL_FAMILIES): + raise ValueError("Codex 0.146 source contract declaration family order is invalid") + declaration_families = runtime_surface.get("declaration_families") + if not isinstance(declaration_families, list) or len(declaration_families) != len( + STRUCTURAL_FAMILIES + ): + raise ValueError("Codex 0.146 source contract declaration families are invalid") + declaration_families_by_name: dict[str, dict[str, Any]] = {} + expected_observations = { + "plain_function": "not_observed_source_contract_only", + "custom_freeform": "not_observed_source_contract_only", + "namespace": "not_observed_source_contract_only", + "client_executed_tool_discovery": "not_observed_source_contract_only", + "selected_provider_hosted": "not_observed_selected_provider_control_required", + "unknown_future_kind": "opaque_sentinel_only", + } + for family, expected_family in zip(STRUCTURAL_FAMILIES, declaration_families): + if not isinstance(expected_family, dict) or expected_family.get("family") != family: + raise ValueError("Codex 0.146 source contract declaration family identity is invalid") + if set(expected_family) != { + "family", + "runtime_type", + "wire_declaration_type", + "observed", + "observation", + "executor", + "loss_boundary", + }: + raise ValueError( + "Codex 0.146 source contract declaration family has unknown fields" + ) + declaration_families_by_name[family] = expected_family + if expected_family.get("observed") is not False: + raise ValueError( + "Codex 0.146 source contract cannot claim an observed declaration family" + ) + if expected_family.get("observation") != expected_observations[family]: + raise ValueError( + "Codex 0.146 source contract declaration observation is invalid" + ) + request_shape = runtime_surface.get("request_shape") + expected_request_shape_fields = { + "protocol", + "streaming_fields", + "representative", + "non_streaming_control", + } + if not isinstance(request_shape, dict) or set(request_shape) != expected_request_shape_fields: + raise ValueError( + "Codex 0.146 source contract request_shape has unknown fields" + ) + if request_shape.get("protocol") != REQUEST_SHAPE_PROTOCOL: + raise ValueError("Codex 0.146 source contract request_shape.protocol is invalid") + if request_shape.get("streaming_fields") != REQUEST_SHAPE_STREAMING_FIELDS: + raise ValueError( + "Codex 0.146 source contract request_shape.streaming_fields is invalid" + ) + expected_request_representative_fields = { + "model", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "stream", + "store", + } + if ( + not isinstance(request_shape.get("representative"), dict) + or set(request_shape["representative"]) != expected_request_representative_fields + ): + raise ValueError( + "Codex 0.146 source contract request_shape.representative has unknown fields" + ) + if request_shape["representative"] != REQUEST_SHAPE_REPRESENTATIVE: + raise ValueError( + "Codex 0.146 source contract request_shape.representative values are invalid" + ) + expected_non_streaming_control_fields = { + "stream", + "response_body", + "captured", + "status", + } + non_streaming_control = ( + request_shape.get("non_streaming_control") + if isinstance(request_shape, dict) + else None + ) + if ( + not isinstance(non_streaming_control, dict) + or set(non_streaming_control) != expected_non_streaming_control_fields + ): + raise ValueError( + "Codex 0.146 source contract request_shape.non_streaming_control has unknown fields" + ) + if non_streaming_control != REQUEST_SHAPE_NON_STREAMING_CONTROL: + raise ValueError( + "Codex 0.146 source contract cannot claim a captured non-streaming response; " + "request_shape.non_streaming_control values are invalid" + ) + examples = runtime_surface.get("declaration_family_examples") + if not isinstance(examples, dict): + raise ValueError("Codex 0.146 source contract declaration examples are missing") + if set(examples) != set(STRUCTURAL_FAMILIES): + raise ValueError( + "Codex 0.146 source contract declaration_family_examples has unknown fields" + ) + for family in STRUCTURAL_FAMILIES: + example = examples.get(family) + if not isinstance(example, dict): + raise ValueError(f"Codex 0.146 source contract example is missing for {family}") + expected_status_fields = { + "selected_provider_hosted": { + "observed": False, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden", + }, + "unknown_future_kind": { + "observed": False, + "status": "opaque_sentinel_only", + }, + }.get(family, {}) + expected_example_fields = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + *expected_status_fields, + } + if set(example) != expected_example_fields: + raise ValueError( + f"Codex 0.146 source contract {family} has unknown example fields" + ) + for field, expected_value in expected_status_fields.items(): + if example.get(field) != expected_value: + raise ValueError( + f"Codex 0.146 source contract {family} {field} status is invalid" + ) + if not expected_status_fields and any( + field in example for field in ("observed", "status") + ): + raise ValueError( + f"Codex 0.146 source contract {family} has an unknown status field" + ) + _validate_structural_family_schema( + family, + declaration_families_by_name[family], + example, + context=f"Codex 0.146 source contract family {family}", + ) + terminal = example.get("terminal") + error = example.get("error") + if ( + not isinstance(terminal, dict) + or terminal.get("event") != "response.completed" + or terminal.get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"Codex 0.146 source contract terminal status is invalid for {family}") + if ( + not isinstance(error, dict) + or error.get("event") != "response.failed" + or error.get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"Codex 0.146 source contract error status is invalid for {family}") + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"Codex 0.146 source contract loss boundary is invalid for {family}") + response_shape = runtime_surface.get("response_shape") + _validate_response_shape_schema( + response_shape, + context="Codex 0.146 source contract response_shape", + ) + + def _captured_true(value: Any) -> bool: + if isinstance(value, dict): + return any( + (key in {"captured", "observed"} and child is True) + or _captured_true(child) + for key, child in value.items() + ) + if isinstance(value, list): + return any(_captured_true(child) for child in value) + return False + + if _captured_true(runtime_surface): + raise ValueError("Codex 0.146 source contract contains a captured response claim") + return provenance + + def _validate_candidate_binding( *, + source_contract_data: dict[str, Any], trace_data: dict[str, Any], wire_data: dict[str, Any], + audit_data: dict[str, Any], cli_version_floor: str, candidate_cli_version: str | None, candidate_source_commit: str | None, ) -> tuple[dict[str, Any], str]: _validate_supported_floor(cli_version_floor) + contract_provenance = _validate_source_contract(source_contract_data) source = trace_data.get("source", {}) planner_gates = trace_data.get("planner_gates", {}) trace_cli_version = source.get("cli_version") trace_source_commit = planner_gates.get("source_commit") trace_capture_id = source.get("capture_id") - if not trace_cli_version or not trace_source_commit or not trace_capture_id: + trace_captured_at = trace_data.get("captured_at") + if ( + not trace_cli_version + or not trace_source_commit + or not trace_capture_id + or not trace_captured_at + ): raise ValueError( - "trace evidence is missing source.cli_version, source.capture_id, " - "or planner_gates.source_commit" + "historical trace evidence is missing source.cli_version, " + "source.capture_id, planner_gates.source_commit, or captured_at" ) if not isinstance(trace_cli_version, str): raise ValueError("trace source.cli_version must be a string") @@ -1059,29 +2757,31 @@ def _validate_candidate_binding( _version_key(trace_cli_version) except ValueError as exc: raise ValueError("trace source.cli_version is malformed") from exc - if not isinstance(trace_source_commit, str) or not re.fullmatch( - r"[0-9a-f]{40}", trace_source_commit + if ( + trace_cli_version != HISTORICAL_CLI_VERSION + or trace_source_commit != HISTORICAL_SOURCE_COMMIT + or trace_captured_at != HISTORICAL_CAPTURED_AT ): raise ValueError( - "trace planner_gates.source_commit must be a lowercase 40-character SHA-1" + "trace evidence must retain the historical Codex 0.144.0-alpha.4 provenance" ) if candidate_cli_version is None: - candidate_cli_version = trace_cli_version - elif candidate_cli_version != trace_cli_version: + candidate_cli_version = contract_provenance["cli_version"] + elif candidate_cli_version != contract_provenance["cli_version"]: raise ValueError( - "candidate CLI version does not match trace evidence: " - f"requested={candidate_cli_version!r} observed={trace_cli_version!r}" + "candidate CLI version does not match the source contract: " + f"requested={candidate_cli_version!r} observed={contract_provenance['cli_version']!r}" ) if not isinstance(candidate_cli_version, str): raise ValueError("candidate CLI version must be a string") if candidate_source_commit is None: - candidate_source_commit = trace_source_commit - elif candidate_source_commit != trace_source_commit: + candidate_source_commit = contract_provenance["source_commit"] + elif candidate_source_commit != contract_provenance["source_commit"]: raise ValueError( - "candidate source commit does not match trace evidence: " - f"requested={candidate_source_commit!r} observed={trace_source_commit!r}" + "candidate source commit does not match the source contract: " + f"requested={candidate_source_commit!r} observed={contract_provenance['source_commit']!r}" ) if not isinstance(candidate_source_commit, str) or not re.fullmatch( @@ -1138,8 +2838,26 @@ def _validate_candidate_binding( wire_provenance.get("cli_version") != trace_cli_version or wire_provenance.get("source_commit") != trace_source_commit or wire_provenance.get("capture_id") != trace_capture_id + or wire_provenance.get("captured_at") != trace_captured_at + ): + raise ValueError("historical wire provenance is not bound to the historical trace") + audit_provenance = audit_data.get("provenance", {}) + if ( + audit_provenance.get("capture_status") != "not_observed" + or any(audit_provenance.get(field) != value for field, value in contract_provenance.items()) ): - raise ValueError("wire provenance is not bound to the trace candidate") + raise ValueError("read-only audit provenance is not bound to the 0.146 source contract") + historical_capture = audit_provenance.get("historical_capture") + if ( + not isinstance(historical_capture, dict) + or historical_capture.get("captured_at") != trace_captured_at + or historical_capture.get("cli_version") != trace_cli_version + or historical_capture.get("source_commit") != trace_source_commit + or historical_capture.get("captured_at") != wire_provenance.get("captured_at") + or historical_capture.get("cli_version") != wire_provenance.get("cli_version") + or historical_capture.get("source_commit") != wire_provenance.get("source_commit") + ): + raise ValueError("read-only audit historical capture provenance is not bound") catalog_snapshot = ( trace_data.get("planner_gates", {}) .get("catalog_source", {}) @@ -1175,12 +2893,17 @@ def _validate_candidate_binding( "catalog_snapshot_sha256": wire_route["catalog_snapshot_sha256"], "catalog_model_entry_id": wire_route["catalog_model_entry_id"], "route_behavior_profile": trace_route.get("behavior_profile"), + "candidate_revision": contract_provenance["candidate_revision"], + "cli_binary_sha256": contract_provenance["cli_binary_sha256"], + "cli_source_commit_status": contract_provenance["cli_source_commit_status"], + "cli_source_tag": contract_provenance["cli_source_tag"], } return identity, status def build_inventory( *, + source_contract: Path = DEFAULT_SOURCE_CONTRACT, trace: Path, wire_fixture: Path, audit: Path, @@ -1192,6 +2915,7 @@ def build_inventory( trace_data = _load_json(trace) wire_data = _load_json(wire_fixture) audit_data = _load_json(audit) + source_contract_data = _load_json(source_contract) wire_fixture_sha256 = _sha256_file(wire_fixture) if ( trace_data.get("schema_version") != 4 @@ -1201,6 +2925,7 @@ def build_inventory( or audit_data.get("capture_kind") != "sanitized_bounded_read_only_audit" ): raise ValueError("Issue #62 evidence schema identity is invalid") + _validate_source_contract(source_contract_data) _validate_core_evidence_pointers(wire=wire_data, audit=audit_data) items: list[dict[str, Any]] = [] @@ -1218,7 +2943,7 @@ def build_inventory( raise ValueError(f"duplicate scope: {item['scope']}") seen.add(item["scope"]) - unknown_tagged_source_count = _count_unknown_tags(wire_data) + unknown_tagged_source_count = _count_response_unknown_tags(wire_data) unknown_stream_count, unknown_non_stream_count = _unknown_tag_mode_counts(wire_data) if unknown_tagged_source_count == 0: raise ValueError("wire evidence contains no unknown-tag sentinel to classify") @@ -1229,9 +2954,17 @@ def build_inventory( identity_control = _build_identity_control( items, unknown_tagged_source_count=unknown_tagged_source_count ) + structural_families = _build_structural_inventory( + source_contract=source_contract_data, + trace=trace_data, + wire=wire_data, + audit=audit_data, + ) candidate_identity, candidate_version_status = _validate_candidate_binding( + source_contract_data=source_contract_data, trace_data=trace_data, wire_data=wire_data, + audit_data=audit_data, cli_version_floor=cli_version_floor, candidate_cli_version=candidate_cli_version, candidate_source_commit=candidate_source_commit, @@ -1243,6 +2976,10 @@ def build_inventory( "sha256": wire_fixture_sha256, }, "audit": {"file": audit.name, "sha256": _sha256_file(audit)}, + "source_contract": { + "file": source_contract.name, + "sha256": _sha256_file(source_contract), + }, } candidate_identity["evidence_manifest_sha256"] = _evidence_manifest_sha256( evidence_binding @@ -1261,6 +2998,7 @@ def build_inventory( "artifact_kind": ARTIFACT_KIND, "cli_version_floor": cli_version_floor, "candidate_identity": candidate_identity, + "declaration_families": structural_families, "qualification": qualification, "disposition_vocabulary": list(ALLOWED_DISPOSITIONS), "items": items, @@ -1322,12 +3060,27 @@ def reconcile_inventory( """Reconcile an inventory (possibly mutated) against the identity contract.""" mismatches: list[str] = [] + if not isinstance(inventory, dict): + return {"reconciled": False, "mismatches": ["inventory must be an object"]} + top_level_fields = _exact_inventory_fields_mismatch( + inventory, INVENTORY_TOP_LEVEL_FIELDS, context="inventory" + ) + if top_level_fields: + mismatches.append(top_level_fields) + if inventory.get("schema_version") != SCHEMA_VERSION: mismatches.append("inventory schema_version is invalid") if inventory.get("artifact_kind") != ARTIFACT_KIND: mismatches.append("inventory artifact_kind is invalid") if inventory.get("disposition_vocabulary") != list(ALLOWED_DISPOSITIONS): mismatches.append("inventory disposition_vocabulary is invalid") + mismatches.extend( + f"mutation: {message}" + for message in _structural_inventory_mismatches( + inventory.get("declaration_families"), + require_unobserved=evidence_root is None, + ) + ) items = inventory.get("items", []) if not isinstance(items, list): @@ -1338,6 +3091,16 @@ def reconcile_inventory( if not isinstance(item, dict): mismatches.append("item must be an object") continue + item_fields = _exact_inventory_fields_mismatch( + item, + INVENTORY_ITEM_FIELDS, + context="inventory item", + optional={"notes"}, + ) + if item_fields: + mismatches.append(item_fields) + if "notes" in item and not isinstance(item.get("notes"), str): + mismatches.append("inventory item notes must be a string") scope = item.get("scope") if not scope: mismatches.append("item missing scope") @@ -1386,7 +3149,16 @@ def reconcile_inventory( mismatches.append(f"deletion: missing scopes {missing}") candidate_identity = inventory.get("candidate_identity", {}) - mismatches.extend(_candidate_identity_mismatches(candidate_identity)) + candidate_fields = _exact_inventory_fields_mismatch( + candidate_identity, INVENTORY_CANDIDATE_FIELDS, context="candidate_identity" + ) + if candidate_fields: + mismatches.append(candidate_fields) + mismatches.extend( + _candidate_identity_mismatches( + candidate_identity, enforce_retained=evidence_root is None + ) + ) if not isinstance(candidate_identity, dict): candidate_identity = {} for field in REQUIRED_CANDIDATE_FIELDS: @@ -1394,6 +3166,11 @@ def reconcile_inventory( mismatches.append(f"loss: candidate_identity.{field} is missing") qualification = inventory.get("qualification", {}) + qualification_fields = _exact_inventory_fields_mismatch( + qualification, INVENTORY_QUALIFICATION_FIELDS, context="qualification" + ) + if qualification_fields: + mismatches.append(qualification_fields) if not isinstance(qualification, dict): mismatches.append("qualification must be an object") qualification = {} @@ -1435,22 +3212,14 @@ def reconcile_inventory( ) evidence_gates = qualification.get("evidence_gates", {}) - expected_gate_keys = { - "complete_model_visible_plan", - "clean_cold_start_current_binding", - "full_pre_post_request_response", - "full_request_fingerprint", - "full_response_fingerprint", - "sse_identity", - "terminal_events", - "error_events", - "non_streaming", - "non_streaming_fixture", - "identity_replay", - "wire_identity_replay", - } - if set(evidence_gates) != expected_gate_keys: + if not isinstance(evidence_gates, dict) or set(evidence_gates) != set( + INVENTORY_EVIDENCE_GATES + ): mismatches.append("qualification.evidence_gates has an unexpected key set") + if "ready_for_beta1" in qualification: + mismatches.append( + "qualification.ready_for_beta1 is stale; use ready_for_beta2" + ) accepted_gate_statuses = { "complete_model_visible_plan": {"complete"}, "clean_cold_start_current_binding": {"complete", "pass"}, @@ -1479,33 +3248,88 @@ def reconcile_inventory( and not actual_blocking_scopes and not actual_blocking_gates ) - if qualification.get("ready_for_beta1") is not expected_ready: + if qualification.get("ready_for_beta2") is not expected_ready: + mismatches.append( + "qualification.ready_for_beta2 is inconsistent with candidate eligibility and blockers" + ) + if evidence_root is None and qualification.get("ready_for_beta2") is True: mismatches.append( - "qualification.ready_for_beta1 is inconsistent with candidate eligibility and blockers" + "qualification.ready_for_beta2 cannot be asserted without bound evidence" + ) + if evidence_root is None and not actual_blocking_gates: + mismatches.append( + "qualification.blocking_gates cannot be empty without bound evidence" ) evidence_binding = inventory.get("evidence_binding", {}) if not isinstance(evidence_binding, dict): mismatches.append("evidence_binding must be an object") evidence_binding = {} - for name in ("trace", "wire_fixture", "audit"): + elif set(evidence_binding) != set(EXPECTED_EVIDENCE_BINDING_FILES): + mismatches.append("evidence_binding has an unexpected key set") + for name in ("source_contract", "trace", "wire_fixture", "audit"): entry = evidence_binding.get(name, {}) + entry_fields = _exact_inventory_fields_mismatch( + entry, + INVENTORY_EVIDENCE_BINDING_ENTRY_FIELDS, + context=f"evidence_binding.{name}", + ) + if entry_fields: + mismatches.append(entry_fields) if not isinstance(entry, dict) or not entry.get("file") or not re.fullmatch( r"[0-9a-f]{64}", str(entry.get("sha256", "")) ): mismatches.append(f"loss: evidence_binding.{name} is missing or malformed") + elif entry.get("file") != EXPECTED_EVIDENCE_BINDING_FILES[name]: + mismatches.append( + f"mutation: evidence_binding.{name}.file does not name the retained fixture" + ) + + evidence_sources = inventory.get("evidence_sources") + evidence_sources_fields = _exact_inventory_fields_mismatch( + evidence_sources, INVENTORY_EVIDENCE_SOURCES_FIELDS, context="evidence_sources" + ) + if evidence_sources_fields: + mismatches.append(evidence_sources_fields) + if isinstance(evidence_sources, dict): + for name, expected in EXPECTED_EVIDENCE_SOURCES.items(): + if evidence_sources.get(name) != expected: + mismatches.append( + f"mutation: evidence_sources.{name} does not match the retained fixture" + ) + feeds = inventory.get("feeds") + feeds_fields = _exact_inventory_fields_mismatch( + feeds, INVENTORY_FEEDS_FIELDS, context="feeds" + ) + if feeds_fields: + mismatches.append(feeds_fields) + if isinstance(feeds, dict): + for name, expected in EXPECTED_FEEDS.items(): + if feeds.get(name) != expected: + mismatches.append( + f"mutation: feeds.{name} does not match the retained issue feed" + ) if evidence_binding and all( isinstance(evidence_binding.get(name), dict) - for name in ("trace", "wire_fixture", "audit") + for name in ("source_contract", "trace", "wire_fixture", "audit") ): manifest = _evidence_manifest_sha256(evidence_binding) if candidate_identity.get("evidence_manifest_sha256") != manifest: mismatches.append("loss: candidate_identity.evidence_manifest_sha256 is stale") + if evidence_root is None: + mismatches.extend( + _standalone_retained_mismatches( + inventory=inventory, + candidate_identity=candidate_identity, + qualification=qualification, + ) + ) + if evidence_root is not None and not mismatches: bound_paths: dict[str, Path] = {} - for name in ("trace", "wire_fixture", "audit"): + for name in ("source_contract", "trace", "wire_fixture", "audit"): entry = evidence_binding[name] relative_name = str(entry["file"]) relative_path = Path(relative_name) @@ -1525,7 +3349,11 @@ def reconcile_inventory( trace_data = _load_json(bound_paths["trace"]) wire_data = _load_json(bound_paths["wire_fixture"]) audit_data = _load_json(bound_paths["audit"]) + source_contract_data = _load_json(bound_paths["source_contract"]) if ( + source_contract_data.get("schema_version") != 1 + or source_contract_data.get("fixture_kind") != "codex_cli_source_contract" + or trace_data.get("schema_version") != 4 or wire_data.get("schema_version") != 1 or wire_data.get("fixture_kind") != "sanitized_artifact_backed_replay" @@ -1535,14 +3363,23 @@ def reconcile_inventory( mismatches.append("mutation: bound evidence schema identity is invalid") return {"reconciled": False, "mismatches": mismatches} try: + _validate_source_contract(source_contract_data) + _validate_structural_evidence_pointers( + source_contract=source_contract_data, + wire=wire_data, + audit=audit_data, + declaration_families=inventory.get("declaration_families", []), + ) _validate_core_evidence_pointers(wire=wire_data, audit=audit_data) except ValueError as exc: mismatches.append(f"mutation: bound evidence pointer validation failed: {exc}") return {"reconciled": False, "mismatches": mismatches} try: expected_identity, expected_status_from_evidence = _validate_candidate_binding( + source_contract_data=source_contract_data, trace_data=trace_data, wire_data=wire_data, + audit_data=audit_data, cli_version_floor=str(inventory.get("cli_version_floor", "")), candidate_cli_version=candidate_identity.get("cli_version"), candidate_source_commit=candidate_identity.get("source_commit"), @@ -1571,7 +3408,7 @@ def reconcile_inventory( "evidence_gates", "blocking_gates", "blocking_scopes", - "ready_for_beta1", + "ready_for_beta2", ): if qualification.get(field) != expected_qualification[field]: mismatches.append( @@ -1579,6 +3416,7 @@ def reconcile_inventory( ) try: generated_inventory = build_inventory( + source_contract=bound_paths["source_contract"], trace=bound_paths["trace"], wire_fixture=bound_paths["wire_fixture"], audit=bound_paths["audit"], @@ -1597,6 +3435,13 @@ def reconcile_inventory( ) identity_control = inventory.get("identity_control", {}) + identity_control_fields = _exact_inventory_fields_mismatch( + identity_control, + INVENTORY_IDENTITY_CONTROL_FIELDS, + context="identity_control", + ) + if identity_control_fields: + mismatches.append(identity_control_fields) if not isinstance(identity_control, dict): mismatches.append("identity_control must be an object") identity_control = {} @@ -1634,7 +3479,7 @@ def reconcile_inventory( wire_entry = evidence_binding.get("wire_fixture", {}) wire_path = evidence_root / str(wire_entry.get("file", "")) if wire_path.is_file(): - actual_unknown_tagged_source_count = _count_unknown_tags( + actual_unknown_tagged_source_count = _count_response_unknown_tags( _load_json(wire_path) ) if unknown_tagged_source_count != actual_unknown_tagged_source_count: @@ -1648,6 +3493,11 @@ def reconcile_inventory( def _build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source-contract", + type=Path, + default=DEFAULT_SOURCE_CONTRACT, + ) parser.add_argument( "--trace", type=Path, @@ -1689,6 +3539,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: def main() -> int: args = _build_arg_parser().parse_args() inventory = build_inventory( + source_contract=args.source_contract, trace=args.trace, wire_fixture=args.wire_fixture, audit=args.audit, diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 8741d2ab..34b04e74 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -1,4 +1,5 @@ param( + [string]$SourceContractPath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\codex-0.146-source-contract.json'), [string]$TracePath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\current-codexhub-thread-tool-surface.json'), [string]$WireFixturePath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\codexhub-runtime-wire-fixture.json'), [string]$AuditPath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\read-only-gate-audit.json'), @@ -11,12 +12,13 @@ param( $ErrorActionPreference = 'Stop' -foreach ($path in @($TracePath, $WireFixturePath, $AuditPath, $InventoryPath)) { +foreach ($path in @($SourceContractPath, $TracePath, $WireFixturePath, $AuditPath, $InventoryPath)) { if (-not (Test-Path -LiteralPath $path)) { throw "Evidence file not found: $path" } } +$sourceContract = Get-Content -Raw -LiteralPath $SourceContractPath | ConvertFrom-Json $trace = Get-Content -Raw -LiteralPath $TracePath | ConvertFrom-Json $wire = Get-Content -Raw -LiteralPath $WireFixturePath | ConvertFrom-Json $audit = Get-Content -Raw -LiteralPath $AuditPath | ConvertFrom-Json @@ -28,6 +30,331 @@ function Add-Mismatch { $script:mismatches.Add($Message) } +$expectedFamilySchemas = @( + @{ Name = 'plain_function'; RuntimeType = 'function'; WireType = 'function'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'preserve declaration and inverse call/result/history IDs'; DeclarationType = 'function'; CallType = 'function_call'; ResultType = 'function_call_output'; DeclarationRequired = @('name', 'parameters'); CallRequired = @('item_id', 'call_id', 'arguments'); ResultRequired = @('item_id', 'call_id', 'output'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id') }, + @{ Name = 'custom_freeform'; RuntimeType = 'custom'; WireType = 'custom'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'preserve declaration and inverse call/result/history IDs'; DeclarationType = 'custom'; CallType = 'custom_tool_call'; ResultType = 'custom_tool_call_output'; DeclarationRequired = @('name', 'format'); CallRequired = @('item_id', 'call_id', 'input'); ResultRequired = @('item_id', 'call_id', 'output'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id') }, + @{ Name = 'namespace'; RuntimeType = 'namespace'; WireType = 'namespace'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'preserve declaration and inverse call/result/history IDs'; DeclarationType = 'namespace'; CallType = 'function_call'; ResultType = 'function_call_output'; DeclarationRequired = @('name', 'tools'); CallRequired = @('item_id', 'call_id', 'namespace', 'arguments'); ResultRequired = @('item_id', 'call_id', 'output'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'namespace') }, + @{ Name = 'client_executed_tool_discovery'; RuntimeType = 'tool_search'; WireType = 'tool_search'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'discovery request/result stays client-executed'; DeclarationType = 'tool_search'; CallType = 'tool_search_call'; ResultType = 'tool_search_output'; DeclarationRequired = @('execution', 'parameters'); CallRequired = @('item_id', 'call_id', 'execution', 'arguments'); ResultRequired = @('item_id', 'call_id', 'execution', 'tools'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'executor') }, + @{ Name = 'selected_provider_hosted'; RuntimeType = 'web_search'; WireType = 'web_search'; Executor = 'selected_provider'; Observation = 'not_observed_selected_provider_control_required'; LossBoundary = 'optional unsupported hosted capability is omitted; required capability fails visibly'; DeclarationType = 'web_search'; CallType = 'web_search_call'; ResultType = 'web_search_call'; DeclarationRequired = @('executor', 'provider_scope'); CallRequired = @('item_id', 'status', 'action'); ResultRequired = @('item_id', 'status', 'provider_scope'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'executor', 'cross_provider_proxy') }, + @{ Name = 'unknown_future_kind'; RuntimeType = 'unknown'; WireType = ''; Executor = 'unknown'; Observation = 'opaque_sentinel_only'; LossBoundary = 'retain tag and opaque payload; do not normalize'; DeclarationType = 'unknown'; CallType = 'unknown'; ResultType = 'unknown'; DeclarationRequired = @('tag', 'opaque_payload'); CallRequired = @('tag', 'opaque_payload'); ResultRequired = @('tag', 'opaque_payload'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'loss_rule') } +) +$sourceContractSchemaValid = $true +$sourceContractFamilies = @($sourceContract.runtime_wire_surface.declaration_families) +$sourceContractExamples = $sourceContract.runtime_wire_surface.declaration_family_examples + +function Test-ExactPropertySet { + param( + [object]$Value, + [string[]]$Expected + ) + + if ($null -eq $Value) { + return $false + } + $actual = @($Value.PSObject.Properties.Name) + foreach ($name in $Expected) { + if ($actual -notcontains $name) { + return $false + } + } + foreach ($name in $actual) { + if ($Expected -notcontains $name) { + return $false + } + } + return $true +} + +if (-not (Test-ExactPropertySet -Value $sourceContract -Expected @( + 'schema_version', 'fixture_kind', 'capture_status', + 'qualification_status', 'captured_at', 'provenance', + 'runtime_wire_surface' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.provenance -Expected @( + 'cli_version', 'source_commit', 'cli_source_tag', + 'cli_source_commit_status', 'cli_binary_sha256', 'candidate_revision' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface -Expected @( + 'source', 'declaration_family_order', 'declaration_families', + 'request_shape', 'response_shape', 'declaration_family_examples' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.request_shape -Expected @( + 'protocol', 'streaming_fields', 'representative', 'non_streaming_control' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.request_shape.representative -Expected @( + 'model', 'input', 'tools', 'tool_choice', 'parallel_tool_calls', 'stream', 'store' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.request_shape.non_streaming_control -Expected @( + 'stream', 'response_body', 'captured', 'status' + ))) { + $sourceContractSchemaValid = $false +} +$expectedRequestStreamingFields = @( + 'model', 'instructions', 'input', 'tools', 'tool_choice', + 'parallel_tool_calls', 'reasoning', 'store', 'stream', 'stream_options', + 'include', 'service_tier', 'prompt_cache_key', 'text', 'client_metadata' +) +if ( + $sourceContract.runtime_wire_surface.request_shape.protocol -ne 'responses' -or + ($sourceContract.runtime_wire_surface.request_shape.streaming_fields -join '|') -ne ($expectedRequestStreamingFields -join '|') +) { + $sourceContractSchemaValid = $false +} +if ( + $sourceContract.runtime_wire_surface.request_shape.representative.model -ne 'gpt-5.6-sol' -or + $sourceContract.runtime_wire_surface.request_shape.representative.input -ne '' -or + $sourceContract.runtime_wire_surface.request_shape.representative.tools -ne '' -or + $sourceContract.runtime_wire_surface.request_shape.representative.tool_choice -ne 'auto' -or + $sourceContract.runtime_wire_surface.request_shape.representative.parallel_tool_calls -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.representative.stream -ne $true -or + $sourceContract.runtime_wire_surface.request_shape.representative.store -ne $false +) { + $sourceContractSchemaValid = $false +} +if ( + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.stream -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.response_body -ne '' -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.captured -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.status -ne 'unqualified' +) { + $sourceContractSchemaValid = $false +} +$expectedResponseItemTypes = @( + 'message', 'reasoning', 'function_call', 'function_call_output', + 'custom_tool_call', 'custom_tool_call_output', 'tool_search_call', + 'tool_search_output', 'web_search_call', 'local_shell_call', + 'compaction', 'context_compaction', 'unknown' +) +$expectedResponseStreamEventOrder = @( + 'response.created', 'response.in_progress', 'response.output_item.added', + 'response.output_text.delta', 'response.function_call_arguments.delta', + 'response.custom_tool_call_input.delta', 'response.function_call_arguments.done', + 'response.custom_tool_call_input.done', 'response.output_item.done', + 'response.reasoning_summary_part.added', 'response.reasoning_summary_text.delta', + 'response.reasoning_summary_text.done', 'response.completed' +) +$expectedResponseTerminalEvents = @('response.completed', 'response.incomplete', 'response.failed') +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape -Expected @( + 'response_item_types', 'stream_event_order', 'terminal_events', 'error_shape' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape.error_shape -Expected @( + 'event', 'response', 'classification' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape.error_shape.response -Expected @( + 'id', 'status', 'error' + ))) { + $sourceContractSchemaValid = $false +} +if (($sourceContract.runtime_wire_surface.response_shape.response_item_types -join '|') -ne ($expectedResponseItemTypes -join '|')) { + $sourceContractSchemaValid = $false +} +if (($sourceContract.runtime_wire_surface.response_shape.stream_event_order -join '|') -ne ($expectedResponseStreamEventOrder -join '|')) { + $sourceContractSchemaValid = $false +} +if (($sourceContract.runtime_wire_surface.response_shape.terminal_events -join '|') -ne ($expectedResponseTerminalEvents -join '|')) { + $sourceContractSchemaValid = $false +} +if ( + $sourceContract.runtime_wire_surface.response_shape.error_shape.event -ne 'response.failed' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.classification -ne 'unqualified' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.response.id -ne 'response_error_001' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.response.status -ne 'failed' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.response.error -ne '' +) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContractExamples -Expected @( + 'plain_function', 'custom_freeform', 'namespace', + 'client_executed_tool_discovery', 'selected_provider_hosted', + 'unknown_future_kind' + ))) { + $sourceContractSchemaValid = $false +} +$nullableByFamily = @{ + selected_provider_hosted = @{ history = @('call_id') } + unknown_future_kind = @{ history = @('call_id', 'call_item_id', 'output_item_id') } +} +$nonEmptyStringFields = @('name', 'namespace', 'executor', 'execution', 'provider_scope', 'cross_provider_proxy', 'tag', 'loss_rule', 'item_id', 'call_id', 'call_item_id', 'output_item_id') +$expectedSseSchemas = @{ + plain_function = @{ Added = 'response.output_item.added'; Delta = 'response.function_call_arguments.delta'; DoneField = 'arguments_done'; Done = 'response.function_call_arguments.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', 'response.function_call_arguments.delta', 'response.function_call_arguments.done', 'response.output_item.done', 'response.completed') } + custom_freeform = @{ Added = 'response.output_item.added'; Delta = 'response.custom_tool_call_input.delta'; DoneField = 'input_done'; Done = 'response.custom_tool_call_input.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', 'response.custom_tool_call_input.delta', 'response.custom_tool_call_input.done', 'response.output_item.done', 'response.completed') } + namespace = @{ Added = 'response.output_item.added'; Delta = 'response.function_call_arguments.delta'; DoneField = 'arguments_done'; Done = 'response.function_call_arguments.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', 'response.function_call_arguments.delta', 'response.function_call_arguments.done', 'response.output_item.done', 'response.completed') } + client_executed_tool_discovery = @{ Added = $null; Delta = $null; DoneField = 'done'; Done = 'response.output_item.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.done', 'response.completed') } + selected_provider_hosted = @{ Added = 'response.output_item.added'; Delta = ''; DoneField = 'done'; Done = 'response.output_item.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', '', 'response.output_item.done', 'response.completed') } + unknown_future_kind = @{ Added = 'unknown.future_event'; Delta = 'unknown.future_delta'; DoneField = 'done'; Done = 'unknown.future_done'; ItemDone = 'unknown.future_done'; Terminal = 'response.completed'; Order = @('unknown.future_event', 'unknown.future_delta', 'unknown.future_done', 'response.completed') } +} +$redactedStructuralFields = @('parameters', 'arguments', 'input', 'output', 'format', 'action', 'status', 'tools', 'opaque_payload') +$nonEmptyStructuralFields = @('name', 'namespace', 'executor', 'execution', 'provider_scope', 'cross_provider_proxy', 'tag', 'loss_rule', 'item_id', 'call_id', 'call_item_id', 'output_item_id') +foreach ($expected in $expectedFamilySchemas) { + $family = @($sourceContractFamilies | Where-Object { $_.family -eq $expected.Name }) | Select-Object -First 1 + $exampleProperty = if ($null -ne $sourceContractExamples) { $sourceContractExamples.PSObject.Properties[$expected.Name] } else { $null } + $example = if ($null -ne $exampleProperty) { $exampleProperty.Value } else { $null } + if ($null -eq $family -or $null -eq $example -or + $family.runtime_type -ne $expected.RuntimeType -or + $family.wire_declaration_type -ne $expected.WireType -or + $family.executor -ne $expected.Executor -or + $family.observation -ne $expected.Observation -or + $family.loss_boundary -ne $expected.LossBoundary) { + $sourceContractSchemaValid = $false + continue + } + if (-not (Test-ExactPropertySet -Value $family -Expected @( + 'family', 'runtime_type', 'wire_declaration_type', 'observed', + 'observation', 'executor', 'loss_boundary' + ))) { + $sourceContractSchemaValid = $false + } + $expectedExampleFields = @( + 'declaration', 'call', 'result', 'history', 'streaming', + 'terminal', 'error', 'loss_boundary' + ) + if ($expected.Name -eq 'selected_provider_hosted') { + $expectedExampleFields += @('observed', 'status', 'provider_scope', 'cross_provider_proxy') + } elseif ($expected.Name -eq 'unknown_future_kind') { + $expectedExampleFields += @('observed', 'status') + } + if (-not (Test-ExactPropertySet -Value $example -Expected $expectedExampleFields)) { + $sourceContractSchemaValid = $false + } + if ($expected.Name -eq 'selected_provider_hosted' -and + ($example.observed -ne $false -or + $example.status -ne 'selected_provider_control_required' -or + $example.provider_scope -ne 'selected_provider_only' -or + $example.cross_provider_proxy -ne 'forbidden')) { + $sourceContractSchemaValid = $false + } elseif ($expected.Name -eq 'unknown_future_kind' -and + ($example.observed -ne $false -or $example.status -ne 'opaque_sentinel_only')) { + $sourceContractSchemaValid = $false + } + $sections = @( + @{ Name = 'declaration'; Type = $expected.DeclarationType; Required = $expected.DeclarationRequired }, + @{ Name = 'call'; Type = $expected.CallType; Required = $expected.CallRequired }, + @{ Name = 'result'; Type = $expected.ResultType; Required = $expected.ResultRequired }, + @{ Name = 'history'; Type = $null; Required = $expected.HistoryRequired } + ) + foreach ($sectionSpec in $sections) { + $sectionProperty = $example.PSObject.Properties[$sectionSpec.Name] + $section = if ($null -ne $sectionProperty) { $sectionProperty.Value } else { $null } + if ($null -eq $section -or ($null -ne $sectionSpec.Type -and $section.type -ne $sectionSpec.Type)) { + $sourceContractSchemaValid = $false + continue + } + $expectedSectionFields = @($sectionSpec.Required) + if ($null -ne $sectionSpec.Type) { + $expectedSectionFields += 'type' + } + if (-not (Test-ExactPropertySet -Value $section -Expected $expectedSectionFields)) { + $sourceContractSchemaValid = $false + } + foreach ($required in $sectionSpec.Required) { + $requiredProperty = $section.PSObject.Properties[$required] + $nullableFields = @() + if ($nullableByFamily.ContainsKey($expected.Name) -and $nullableByFamily[$expected.Name].ContainsKey($sectionSpec.Name)) { + $nullableFields = @($nullableByFamily[$expected.Name][$sectionSpec.Name]) + } + if ($null -eq $requiredProperty -or + ($required -in $nonEmptyStringFields -and + $required -notin $nullableFields -and + ($null -eq $requiredProperty.Value -or + $requiredProperty.Value -isnot [string] -or + [string]::IsNullOrEmpty($requiredProperty.Value)))) { + $sourceContractSchemaValid = $false + } + } + foreach ($property in $section.PSObject.Properties) { + if ($property.Name -eq 'type' -or $property.Name -in $nonEmptyStructuralFields) { + continue + } + if ($property.Name -eq 'tools' -and $expected.Name -eq 'namespace' -and $sectionSpec.Name -eq 'declaration') { + if ($property.Value -isnot [array] -or @($property.Value).Count -eq 0) { + $sourceContractSchemaValid = $false + continue + } + foreach ($nested in @($property.Value)) { + if (-not (Test-ExactPropertySet -Value $nested -Expected @('type', 'name', 'parameters')) -or + $nested.type -ne 'function' -or + $nested.name -isnot [string] -or [string]::IsNullOrEmpty($nested.name) -or + $nested.parameters -ne '') { + $sourceContractSchemaValid = $false + } + } + continue + } + if ($property.Name -in $redactedStructuralFields -and $property.Value -ne '') { + $sourceContractSchemaValid = $false + } elseif ($property.Value -isnot [string]) { + $sourceContractSchemaValid = $false + } + } + } + if ($expected.Name -eq 'client_executed_tool_discovery' -and + ($example.declaration.execution -ne 'client' -or + $example.call.execution -ne 'client' -or + $example.result.execution -ne 'client' -or + $example.history.executor -ne 'codex_client')) { + $sourceContractSchemaValid = $false + } + if ($expected.Name -eq 'selected_provider_hosted' -and + ($example.declaration.executor -ne 'selected_provider' -or + $example.declaration.provider_scope -ne 'selected_provider_only' -or + $example.result.provider_scope -ne 'selected_provider_only' -or + $example.history.executor -ne 'selected_provider' -or + $example.history.cross_provider_proxy -ne 'forbidden')) { + $sourceContractSchemaValid = $false + } + if ($expected.Name -eq 'unknown_future_kind' -and + ($example.declaration.tag -ne 'unknown' -or + $example.call.tag -ne 'unknown' -or + $example.result.tag -ne 'unknown' -or + $example.history.call_id -ne $null -or + $example.history.call_item_id -ne $null -or + $example.history.output_item_id -ne $null -or + $example.history.loss_rule -ne 'retain opaque sentinel')) { + $sourceContractSchemaValid = $false + } + if (-not (Test-ExactPropertySet -Value $example.terminal -Expected @('event', 'classification')) -or + $example.terminal.event -ne 'response.completed' -or + $example.terminal.classification -notin @('not_observed', 'unqualified') -or + -not (Test-ExactPropertySet -Value $example.error -Expected @('event', 'classification')) -or + $example.error.event -ne 'response.failed' -or + $example.error.classification -notin @('not_observed', 'unqualified') -or + $example.loss_boundary -isnot [string] -or [string]::IsNullOrEmpty($example.loss_boundary)) { + $sourceContractSchemaValid = $false + } + $sseSchema = $expectedSseSchemas[$expected.Name] + $streamProperty = $example.PSObject.Properties['streaming'] + $stream = if ($null -ne $streamProperty) { $streamProperty.Value } else { $null } + $doneProperty = if ($null -ne $stream) { $stream.PSObject.Properties[$sseSchema.DoneField] } else { $null } + $expectedStreamFields = @('added', 'delta', 'done', 'terminal', 'event_order', $sseSchema.DoneField) | Select-Object -Unique + if (-not (Test-ExactPropertySet -Value $stream -Expected $expectedStreamFields)) { + $sourceContractSchemaValid = $false + } + if ($null -eq $stream -or + $stream.added -ne $sseSchema.Added -or + $stream.delta -ne $sseSchema.Delta -or + $stream.terminal -ne $sseSchema.Terminal -or + $stream.done -ne $sseSchema.ItemDone -or + $null -eq $doneProperty -or + $doneProperty.Value -ne $sseSchema.Done -or + (($stream.event_order -join '|') -ne ($sseSchema.Order -join '|'))) { + $sourceContractSchemaValid = $false + } +} + # Rebuild the inventory from the bound evidence and compare it with the # committed artifact. The PowerShell checks below remain an independent # reconciliation, while this call catches stale generated fields/notes that @@ -39,6 +366,7 @@ if ($null -eq $python) { } else { try { $generatorOutput = & $python.Source $inventoryGenerator ` + --source-contract $SourceContractPath ` --trace $TracePath ` --wire-fixture $WireFixturePath ` --audit $AuditPath ` @@ -405,7 +733,11 @@ foreach ($link in $callLinks) { $streamUnknown = @($wire.response.streaming.events | Where-Object { $_.tag -eq 'unknown' }) $nonStreamingUnknown = @($wire.response.non_streaming.response_items | Where-Object { $_.tag -eq 'unknown' }) -$unknownTaggedSourceCount = Get-UnknownTaggedSourceCount -Value $wire +$responseUnknownSource = [PSCustomObject]@{ + streaming = $wire.response.streaming + non_streaming = $wire.response.non_streaming +} +$unknownTaggedSourceCount = Get-UnknownTaggedSourceCount -Value $responseUnknownSource if ( $inventory.identity_control.unknown_tagged_source_count -le 0 -or $inventory.identity_control.unknown_tagged_source_count -ne $unknownTaggedSourceCount @@ -619,7 +951,7 @@ foreach ($scope in $allRequiredScopes) { if ( $inventory.artifact_kind -ne 'runtime_wire_inventory' -or $inventory.schema_version -ne 1 -or - $inventory.cli_version_floor -ne '0.145.0' + $inventory.cli_version_floor -ne '0.146.0' ) { Add-Mismatch 'inventory artifact identity or CLI version floor is invalid' } @@ -655,16 +987,70 @@ foreach ($field in @('catalog_snapshot_sha256','evidence_manifest_sha256')) { Add-Mismatch "inventory candidate $field is not a lowercase 64-character SHA-256" } } +if ([string]$inventoryCandidate.candidate_revision -notmatch '^[0-9a-f]{40}$') { + Add-Mismatch 'inventory candidate revision is not a lowercase 40-character SHA-1' +} +if ([string]$inventoryCandidate.cli_binary_sha256 -notmatch '^[0-9a-f]{64}$') { + Add-Mismatch 'inventory candidate CLI binary SHA-256 is invalid' +} +if ([string]$inventoryCandidate.cli_binary_sha256 -ne 'bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb') { + Add-Mismatch 'inventory candidate CLI binary SHA-256 is not the retained Codex 0.146.0 binary' +} +if ([string]$inventoryCandidate.cli_source_commit_status -notin @('published_attested','not_published_by_registry')) { + Add-Mismatch 'inventory candidate CLI source commit status is invalid' +} +if ([string]::IsNullOrWhiteSpace([string]$inventoryCandidate.cli_source_tag)) { + Add-Mismatch 'inventory candidate CLI source tag is missing or blank' +} if ( + $sourceContract.schema_version -ne 1 -or + $sourceContract.fixture_kind -ne 'codex_cli_source_contract' -or + $sourceContract.capture_status -ne 'not_observed' -or + $sourceContract.qualification_status -ne 'unqualified' -or + $null -ne $sourceContract.captured_at -or + $sourceContract.provenance.cli_version -ne '0.146.0' -or + $sourceContract.provenance.source_commit -ne 'e363b08c9175ac1cbe5893615dd2cb9ddf95043b' -or + $sourceContract.provenance.cli_source_tag -ne 'rust-v0.146.0' -or + $sourceContract.provenance.cli_source_commit_status -ne 'published_attested' -or + $sourceContract.provenance.cli_binary_sha256 -ne 'bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb' -or + $sourceContract.provenance.candidate_revision -ne 'accab8ff6eb4d6ebd93cda84585fb5f6cb89da82' -or + -not $sourceContractSchemaValid -or + $sourceContract.runtime_wire_surface.declaration_families.Count -ne 6 -or + ($sourceContract.runtime_wire_surface.declaration_families | Where-Object { $_.observed -ne $false }).Count -ne 0 -or + ($sourceContract.runtime_wire_surface.declaration_families | Where-Object { $_.observation -notin @('not_observed_source_contract_only', 'not_observed_selected_provider_control_required', 'opaque_sentinel_only') }).Count -ne 0 -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.captured -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.status -ne 'unqualified' -or [string]::IsNullOrWhiteSpace([string]$trace.source.capture_id) -or - [string]::IsNullOrWhiteSpace([string]$wire.provenance.capture_id) + [string]::IsNullOrWhiteSpace([string]$wire.provenance.capture_id) -or + $trace.captured_at -ne '2026-07-12T14:57:55+08:00' -or + $wire.provenance.captured_at -ne '2026-07-12T14:57:55+08:00' ) { - Add-Mismatch 'candidate trace/wire capture identity is missing' + Add-Mismatch 'source contract or historical trace/wire provenance is invalid' } if ( - $inventoryCandidate.cli_version -ne $trace.source.cli_version -or - $inventoryCandidate.source_commit -ne $trace.planner_gates.source_commit -or - $inventoryCandidate.codex_source_commit -ne $trace.planner_gates.source_commit -or + $trace.source.cli_version -ne '0.144.0-alpha.4' -or + $trace.planner_gates.source_commit -ne '9e552e9d15ba52bed7077d5357f3e18e330f8f38' -or + $trace.captured_at -ne '2026-07-12T14:57:55+08:00' -or + $wire.provenance.cli_version -ne $trace.source.cli_version -or + $wire.provenance.source_commit -ne $trace.planner_gates.source_commit -or + $wire.provenance.captured_at -ne $trace.captured_at -or + $inventoryCandidate.cli_version -ne $sourceContract.provenance.cli_version -or + $inventoryCandidate.source_commit -ne $sourceContract.provenance.source_commit -or + $inventoryCandidate.codex_source_commit -ne $sourceContract.provenance.source_commit -or + $inventoryCandidate.candidate_revision -ne $sourceContract.provenance.candidate_revision -or + $inventoryCandidate.cli_binary_sha256 -ne $sourceContract.provenance.cli_binary_sha256 -or + $inventoryCandidate.cli_source_commit_status -ne $sourceContract.provenance.cli_source_commit_status -or + $inventoryCandidate.cli_source_tag -ne $sourceContract.provenance.cli_source_tag -or + $audit.provenance.capture_status -ne 'not_observed' -or + $audit.provenance.cli_version -ne $sourceContract.provenance.cli_version -or + $audit.provenance.source_commit -ne $sourceContract.provenance.source_commit -or + $audit.provenance.candidate_revision -ne $sourceContract.provenance.candidate_revision -or + $audit.provenance.cli_binary_sha256 -ne $sourceContract.provenance.cli_binary_sha256 -or + $audit.provenance.cli_source_commit_status -ne $sourceContract.provenance.cli_source_commit_status -or + $audit.provenance.cli_source_tag -ne $sourceContract.provenance.cli_source_tag -or + $audit.provenance.historical_capture.captured_at -ne $trace.captured_at -or + $audit.provenance.historical_capture.cli_version -ne $trace.source.cli_version -or + $audit.provenance.historical_capture.source_commit -ne $trace.planner_gates.source_commit -or $inventoryCandidate.route_upstream -ne $wire.route.upstream_route -or $inventoryCandidate.inbound_format -ne $wire.route.inbound_format -or $inventoryCandidate.upstream_format -ne $wire.route.upstream_format -or @@ -687,8 +1073,6 @@ if ( $wire.route.catalog_snapshot_sha256 -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.sha256 -or $wire.route.catalog_model_entry_id -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.model_entry_id -or $wire.route.catalog_model_supports_search_tool -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.model_entry_supports_search_tool -or - $wire.provenance.cli_version -ne $trace.source.cli_version -or - $wire.provenance.source_commit -ne $trace.planner_gates.source_commit -or $wire.provenance.capture_id -ne $trace.source.capture_id -or $wire.pre_gateway.model -ne $trace.source.model -or $wire.post_gateway.model -ne $trace.source.model @@ -696,6 +1080,7 @@ if ( Add-Mismatch 'inventory candidate identity does not bind to the exact trace and wire candidate route' } $evidenceBindings = @{ + source_contract = $SourceContractPath trace = $TracePath wire_fixture = $WireFixturePath audit = $AuditPath @@ -712,7 +1097,7 @@ foreach ($name in $evidenceBindings.Keys) { Add-Mismatch "inventory evidence binding $name hash does not match the input artifact" } } -$manifestParts = foreach ($name in @('audit','trace','wire_fixture')) { +$manifestParts = foreach ($name in @('audit','source_contract','trace','wire_fixture')) { $binding = $inventory.evidence_binding.$name '{0}:{1}:{2}' -f $name, $binding.file, $binding.sha256 } @@ -816,6 +1201,9 @@ $qualification = $inventory.qualification if (-not $qualification -or $qualification.candidate_version_status -notin @('eligible','legacy_below_floor')) { Add-Mismatch 'inventory qualification candidate version status is invalid' } +if ($qualification.PSObject.Properties.Name -contains 'ready_for_beta1') { + Add-Mismatch 'inventory qualification ready_for_beta1 is stale; use ready_for_beta2' +} $blockingScopes = @( $coreEvidence.Keys + $requiredLiveControlScopes + $requiredChoiceScope | Sort-Object -Unique @@ -889,8 +1277,8 @@ if ((($qualification.blocking_gates | Sort-Object) -join '|') -ne (($observedBlo Add-Mismatch 'inventory qualification blocking_gates does not match trace/audit' } $expectedReady = $expectedCandidateEligible -and $observedBlockingScopes.Count -eq 0 -and $observedBlockingGates.Count -eq 0 -if ([bool]$qualification.ready_for_beta1 -ne $expectedReady) { - Add-Mismatch 'inventory qualification ready_for_beta1 is inconsistent with evidence blockers' +if ([bool]$qualification.ready_for_beta2 -ne $expectedReady) { + Add-Mismatch 'inventory qualification ready_for_beta2 is inconsistent with evidence blockers' } $advancedScopes = @('code_mode','tool_search','collaboration_v2','chat_conversion') foreach ($scope in $advancedScopes) { diff --git a/tests/test_issue_62_runtime_audit.py b/tests/test_issue_62_runtime_audit.py index 1e4839b8..84044777 100644 --- a/tests/test_issue_62_runtime_audit.py +++ b/tests/test_issue_62_runtime_audit.py @@ -10,6 +10,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "audit_issue_62_runtime_artifacts.py" AUDIT = ROOT / "docs" / "evidence" / "issue-62" / "read-only-gate-audit.json" +SOURCE_CONTRACT = ROOT / "docs" / "evidence" / "issue-62" / "codex-0.146-source-contract.json" def load_audit_module(): @@ -27,6 +28,7 @@ def run_audit( gateway_db: Path, *, config_written_at: str = "1970-01-01T00:06:00Z", + source_contract: Path = SOURCE_CONTRACT, ): return module.audit_artifacts( codex_log_db=codex_db, @@ -37,6 +39,7 @@ def run_audit( config_written_at=config_written_at, catalog_written_at="1970-01-01T00:02:00Z", snapshot_ended_at="1970-01-01T00:10:00Z", + source_contract=source_contract, ) @@ -282,6 +285,14 @@ def test_audit_reports_only_sanitized_schema_and_gate_facts(tmp_path: Path) -> N "type": "tool_search", }, ] + assert [entry["family"] for entry in planner["declaration_families"]] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] gateway = audit["gateway_identity_route"] assert gateway["request_starts"] == 1 @@ -385,7 +396,17 @@ def test_audit_surfaces_unclassified_items_and_prefix_mismatch(tmp_path: Path) - def test_committed_audit_preserves_the_bounded_fact_and_sanitization_boundary() -> None: audit = json.loads(AUDIT.read_text(encoding="utf-8")) - + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + + assert audit["provenance"]["capture_status"] == "not_observed" + assert audit["provenance"]["capture_status"] == source_contract["capture_status"] + for field, value in source_contract["provenance"].items(): + assert audit["provenance"][field] == value + assert audit["provenance"]["historical_capture"] == { + "captured_at": "2026-07-12T14:57:55+08:00", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + } assert audit["gateway_identity_route"]["request_starts"] == 525 assert audit["gateway_identity_route"]["prefix_equal"] == 525 assert audit["gateway_identity_route"]["prefix_mismatch"] == 0 @@ -406,7 +427,194 @@ def test_committed_audit_preserves_the_bounded_fact_and_sanitization_boundary() assert "https://" not in serialized assert ".codex" not in serialized.lower() assert not re.search(r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}", serialized) - assert not re.search(r'(? None: + module = load_audit_module() + codex_db = tmp_path / "codex.sqlite" + gateway_db = tmp_path / "gateway.sqlite" + create_codex_log_db(codex_db) + create_gateway_db(gateway_db) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["provenance"]["source_commit"] = "0" * 40 + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="source contract provenance"): + run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) + + +def test_audit_rejects_captured_source_contract_claim(tmp_path: Path) -> None: + module = load_audit_module() + codex_db = tmp_path / "codex.sqlite" + gateway_db = tmp_path / "gateway.sqlite" + create_codex_log_db(codex_db) + create_gateway_db(gateway_db) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["request_shape"]["non_streaming_control"][ + "captured" + ] = True + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="captured non-streaming"): + run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) + + +@pytest.mark.parametrize( + "mutation", + [ + "protocol", + "streaming_fields", + "representative_model", + "representative_input", + "representative_tools", + "representative_tool_choice", + "representative_parallel_tool_calls", + "representative_stream", + "representative_store", + "non_streaming_stream", + "non_streaming_response_body", + ], +) +def test_audit_rejects_source_contract_request_shape_value_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_audit_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + request_shape = source_contract["runtime_wire_surface"]["request_shape"] + if mutation == "protocol": + request_shape["protocol"] = "chat_completions" + elif mutation == "streaming_fields": + request_shape["streaming_fields"] = ["model"] + elif mutation.startswith("representative_"): + field = mutation.removeprefix("representative_") + values = { + "model": "gpt-5.5", + "input": "not-redacted", + "tools": [], + "tool_choice": "required", + "parallel_tool_calls": True, + "stream": False, + "store": True, + } + request_shape["representative"][field] = values[field] + elif mutation == "non_streaming_stream": + request_shape["non_streaming_control"]["stream"] = True + else: + request_shape["non_streaming_control"]["response_body"] = "captured-body" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="request_shape"): + module._source_contract_provenance(source_contract_path) + + +@pytest.mark.parametrize( + "mutation", + [ + "family_runtime_type", + "family_wire_type", + "family_executor", + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "namespace_tool", + "response_item_type", + ], +) +def test_audit_rejects_nested_source_contract_schema_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_audit_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + surface = source_contract["runtime_wire_surface"] + examples = surface["declaration_family_examples"] + if mutation == "family_runtime_type": + surface["declaration_families"][0]["runtime_type"] = "bogus" + elif mutation == "family_wire_type": + surface["declaration_families"][0]["wire_declaration_type"] = "bogus" + elif mutation == "family_executor": + surface["declaration_families"][0]["executor"] = "bogus" + elif mutation == "declaration": + examples["plain_function"]["declaration"]["type"] = "bogus" + elif mutation == "call": + examples["plain_function"]["call"]["type"] = "bogus" + elif mutation == "result": + examples["plain_function"]["result"]["type"] = "bogus" + elif mutation == "history": + examples["plain_function"]["history"]["future_field"] = "must fail" + elif mutation == "streaming": + examples["plain_function"]["streaming"]["future_field"] = "must fail" + elif mutation == "terminal": + examples["plain_function"]["terminal"]["event"] = "bogus" + elif mutation == "error": + examples["plain_function"]["error"]["event"] = "bogus" + elif mutation == "namespace_tool": + examples["namespace"]["declaration"]["tools"][0]["future_field"] = "must fail" + else: + surface["response_shape"]["response_item_types"].append("future_item") + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError): + module._source_contract_provenance(source_contract_path) + + +@pytest.mark.parametrize( + "mutation", + ["top_level", "runtime_wire_surface", "request_shape", "response_shape"], +) +def test_audit_rejects_unknown_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + module = load_audit_module() + codex_db = tmp_path / "codex.sqlite" + gateway_db = tmp_path / "gateway.sqlite" + create_codex_log_db(codex_db) + create_gateway_db(gateway_db) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation == "top_level": + source_contract["future_field"] = "must not be accepted" + elif mutation == "runtime_wire_surface": + source_contract["runtime_wire_surface"]["future_field"] = "must not be accepted" + elif mutation == "request_shape": + source_contract["runtime_wire_surface"]["request_shape"][ + "future_field" + ] = "must not be accepted" + else: + source_contract["runtime_wire_surface"]["response_shape"][ + "future_field" + ] = "must not be accepted" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unknown"): + run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) def test_audit_detects_generic_response_body_fingerprint_fields(tmp_path: Path) -> None: diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index e4fc2538..b85bfee7 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -8,6 +8,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "build_issue_62_runtime_inventory.py" INVENTORY = ROOT / "docs" / "evidence" / "issue-62" / "runtime-wire-inventory.json" +SOURCE_CONTRACT = ROOT / "docs" / "evidence" / "issue-62" / "codex-0.146-source-contract.json" TRACE = ROOT / "docs" / "evidence" / "issue-62" / "current-codexhub-thread-tool-surface.json" WIRE_FIXTURE = ROOT / "docs" / "evidence" / "issue-62" / "codexhub-runtime-wire-fixture.json" AUDIT = ROOT / "docs" / "evidence" / "issue-62" / "read-only-gate-audit.json" @@ -35,11 +36,15 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N assert inventory["schema_version"] == 1 assert inventory["artifact_kind"] == "runtime_wire_inventory" - assert inventory["cli_version_floor"] == "0.145.0" + assert inventory["cli_version_floor"] == "0.146.0" candidate = inventory["candidate_identity"] - assert candidate["cli_version"] == "0.144.0-alpha.4" - assert candidate["source_commit"] == "9e552e9d15ba52bed7077d5357f3e18e330f8f38" + assert candidate["cli_version"] == "0.146.0" + assert candidate["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" assert candidate["codex_source_commit"] == candidate["source_commit"] + assert candidate["candidate_revision"] == "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" + assert candidate["cli_binary_sha256"] == "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb" + assert candidate["cli_source_commit_status"] == "published_attested" + assert candidate["cli_source_tag"] == "rust-v0.146.0" assert candidate["route_upstream"] == "official" assert candidate["inbound_format"] == "responses" assert candidate["upstream_format"] == "responses" @@ -48,9 +53,9 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N assert candidate["catalog_model_entry_id"] == "gpt-5.6-sol" assert candidate["route_behavior_profile"] == "official_codex_app_http_passthrough" assert len(candidate["evidence_manifest_sha256"]) == 64 - assert inventory["qualification"]["candidate_version_status"] == "legacy_below_floor" - assert inventory["qualification"]["candidate_version_eligible"] is False - assert inventory["qualification"]["ready_for_beta1"] is False + assert inventory["qualification"]["candidate_version_status"] == "eligible" + assert inventory["qualification"]["candidate_version_eligible"] is True + assert inventory["qualification"]["ready_for_beta2"] is False assert inventory["identity_control"]["unknown_tagged_source_count"] == 2 assert inventory["qualification"]["blocking_gates"] == [ "clean_cold_start_current_binding", @@ -82,6 +87,8 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N } assert inventory["evidence_binding"]["trace"]["file"] == TRACE.name assert len(inventory["evidence_binding"]["trace"]["sha256"]) == 64 + assert inventory["evidence_binding"]["source_contract"]["file"] == SOURCE_CONTRACT.name + assert len(inventory["evidence_binding"]["source_contract"]["sha256"]) == 64 def test_inventory_uses_only_final_capability_disposition_vocabulary() -> None: @@ -129,6 +136,62 @@ def test_inventory_covers_every_required_taxonomy_scope() -> None: assert not missing, f"missing required scopes: {sorted(missing)}" +def test_inventory_records_all_structural_declaration_families() -> None: + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + + families = inventory["declaration_families"] + assert [entry["family"] for entry in families] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] + by_family = {entry["family"]: entry for entry in families} + assert by_family["client_executed_tool_discovery"]["executor"] == "codex_client" + assert by_family["selected_provider_hosted"]["executor"] == "selected_provider" + assert by_family["selected_provider_hosted"]["representative"]["cross_provider_proxy"] == "forbidden" + assert by_family["unknown_future_kind"]["selected_protocol_disposition"] == "omit" + assert by_family["plain_function"]["representative"]["streaming"]["arguments_done"] == "response.function_call_arguments.done" + assert by_family["custom_freeform"]["representative"]["streaming"]["input_done"] == "response.custom_tool_call_input.done" + assert by_family["client_executed_tool_discovery"]["representative"]["streaming"]["event_order"] == [ + "response.output_item.done", + "response.completed", + ] + for entry in families: + representative = entry["representative"] + assert representative["terminal"]["classification"] in {"not_observed", "unqualified"} + assert representative["error"]["classification"] in {"not_observed", "unqualified"} + assert representative["loss_boundary"] + + +def test_declaration_family_evidence_sources_resolve_to_bound_fixtures() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + + module._validate_structural_evidence_pointers( + source_contract=json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")), + wire=wire, + audit=audit, + declaration_families=inventory["declaration_families"], + ) + + dangling = json.loads(json.dumps(inventory["declaration_families"])) + dangling[-1]["evidence_source"] = ( + "codex-0.146-source-contract.json#runtime_wire_surface.unknown_future" + ) + with pytest.raises(ValueError, match="evidence source is invalid"): + module._validate_structural_evidence_pointers( + source_contract=json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")), + wire=wire, + audit=audit, + declaration_families=dangling, + ) + + def test_every_item_carries_an_allowed_disposition() -> None: inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) @@ -196,7 +259,7 @@ def test_inventory_reports_zero_unclassified_core_items() -> None: assert inventory["identity_control"]["unclassified_core_items"] == 0 assert inventory["identity_control"]["unclassified_scopes"] == [] - assert inventory["qualification"]["ready_for_beta1"] is False + assert inventory["qualification"]["ready_for_beta2"] is False assert "core_function_replay" in inventory["qualification"]["blocking_scopes"] @@ -249,19 +312,19 @@ def test_build_inventory_reads_existing_artifacts_and_clamps_live_gates() -> Non trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - cli_version_floor="0.145.0", - candidate_cli_version="0.144.0-alpha.4", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + cli_version_floor="0.146.0", + candidate_cli_version="0.146.0", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) assert inventory["schema_version"] == 1 - assert inventory["cli_version_floor"] == "0.145.0" - assert inventory["candidate_identity"]["cli_version"] == "0.144.0-alpha.4" + assert inventory["cli_version_floor"] == "0.146.0" + assert inventory["candidate_identity"]["cli_version"] == "0.146.0" scopes = {entry["scope"] for entry in inventory["items"]} assert "core_text_streaming" in scopes assert "code_mode" in scopes assert inventory["identity_control"]["unclassified_core_items"] == 0 - assert inventory["qualification"]["candidate_version_status"] == "legacy_below_floor" + assert inventory["qualification"]["candidate_version_status"] == "eligible" def test_build_inventory_rejects_candidate_metadata_drift() -> None: @@ -273,7 +336,7 @@ def test_build_inventory_rejects_candidate_metadata_drift() -> None: wire_fixture=WIRE_FIXTURE, audit=AUDIT, candidate_cli_version="0.145.0", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) with pytest.raises(ValueError, match="candidate source commit"): @@ -281,7 +344,7 @@ def test_build_inventory_rejects_candidate_metadata_drift() -> None: trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - candidate_cli_version="0.144.0-alpha.4", + candidate_cli_version="0.146.0", candidate_source_commit="0" * 40, ) @@ -294,9 +357,9 @@ def test_build_inventory_rejects_a_floor_other_than_supported_candidate_floor() trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - cli_version_floor="0.144.0", - candidate_cli_version="0.144.0-alpha.4", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + cli_version_floor="0.145.0", + candidate_cli_version="0.146.0", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) @@ -309,7 +372,7 @@ def test_build_inventory_rejects_malformed_candidate_provenance() -> None: wire_fixture=WIRE_FIXTURE, audit=AUDIT, candidate_cli_version="not-a-version", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) with pytest.raises(ValueError, match="candidate source commit"): @@ -317,7 +380,7 @@ def test_build_inventory_rejects_malformed_candidate_provenance() -> None: trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - candidate_cli_version="0.144.0-alpha.4", + candidate_cli_version="0.146.0", candidate_source_commit="A" * 40, ) @@ -337,6 +400,563 @@ def test_build_inventory_rejects_missing_capture_provenance(tmp_path: Path) -> N ) +def test_build_inventory_rejects_audit_candidate_provenance_drift(tmp_path: Path) -> None: + module = load_inventory_module() + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + audit["provenance"]["source_commit"] = "0" * 40 + audit_path = tmp_path / AUDIT.name + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="audit provenance"): + module.build_inventory( + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=audit_path, + ) + + +def test_build_inventory_rejects_audit_historical_capture_drift(tmp_path: Path) -> None: + module = load_inventory_module() + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + audit["provenance"]["historical_capture"]["captured_at"] = "2026-07-13T14:57:55+08:00" + audit_path = tmp_path / AUDIT.name + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="historical capture"): + module.build_inventory( + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=audit_path, + ) + + +def test_build_inventory_rejects_observed_source_contract_family(tmp_path: Path) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["declaration_families"][0]["observed"] = True + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="cannot claim an observed"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "family, field, value", + [ + ("selected_provider_hosted", "status", "captured"), + ("unknown_future_kind", "status", "captured"), + ("plain_function", "status", "future_status"), + ("plain_function", "unknown_field", "future_status"), + ], +) +def test_build_inventory_rejects_source_contract_status_or_unknown_fields( + tmp_path: Path, family: str, field: str, value: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["declaration_family_examples"][family][ + field + ] = value + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises( + ValueError, match="(?:status is invalid|unknown status field|unknown example fields)" + ): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +def test_build_inventory_rejects_captured_source_contract_control(tmp_path: Path) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["request_shape"]["non_streaming_control"][ + "captured" + ] = True + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="captured non-streaming"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "protocol", + "streaming_fields", + "representative_model", + "representative_input", + "representative_tools", + "representative_tool_choice", + "representative_parallel_tool_calls", + "representative_stream", + "representative_store", + "non_streaming_stream", + "non_streaming_response_body", + ], +) +def test_build_inventory_rejects_source_contract_request_shape_value_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + request_shape = source_contract["runtime_wire_surface"]["request_shape"] + if mutation == "protocol": + request_shape["protocol"] = "chat_completions" + elif mutation == "streaming_fields": + request_shape["streaming_fields"] = ["model"] + elif mutation.startswith("representative_"): + field = mutation.removeprefix("representative_") + values = { + "model": "gpt-5.5", + "input": "not-redacted", + "tools": [], + "tool_choice": "required", + "parallel_tool_calls": True, + "stream": False, + "store": True, + } + request_shape["representative"][field] = values[field] + elif mutation == "non_streaming_stream": + request_shape["non_streaming_control"]["stream"] = True + else: + request_shape["non_streaming_control"]["response_body"] = "captured-body" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="request_shape"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "top_level", + "runtime_wire_surface", + "request_shape", + "response_shape", + ], +) +def test_build_inventory_rejects_unknown_regenerated_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation == "top_level": + source_contract["future_field"] = "must not be accepted" + elif mutation == "runtime_wire_surface": + source_contract["runtime_wire_surface"]["future_field"] = "must not be accepted" + elif mutation == "request_shape": + source_contract["runtime_wire_surface"]["request_shape"][ + "future_field" + ] = "must not be accepted" + else: + source_contract["runtime_wire_surface"]["response_shape"][ + "future_field" + ] = "must not be accepted" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="unknown"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "runtime_type", + "wire_declaration_type", + "declaration_type", + "call_type", + "result_type", + ], +) +def test_build_inventory_rejects_source_contract_schema_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation in {"runtime_type", "wire_declaration_type"}: + source_contract["runtime_wire_surface"]["declaration_families"][0][mutation] = "bogus" + else: + source_contract["runtime_wire_surface"]["declaration_family_examples"]["plain_function"][ + {"declaration_type": "declaration", "call_type": "call", "result_type": "result"}[mutation] + ]["type"] = "web_search_call" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="canonical family schema"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "plain_call_item_id_empty", + "plain_history_call_id_empty", + "namespace_owner_empty", + "plain_added_event", + "custom_delta_event", + "namespace_done_event", + "tool_search_added_event", + "unknown_event_label", + ], +) +def test_build_inventory_rejects_source_contract_identity_and_sse_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + examples = source_contract["runtime_wire_surface"]["declaration_family_examples"] + if mutation == "plain_call_item_id_empty": + examples["plain_function"]["call"]["item_id"] = "" + elif mutation == "plain_history_call_id_empty": + examples["plain_function"]["history"]["call_id"] = "" + elif mutation == "namespace_owner_empty": + examples["namespace"]["call"]["namespace"] = "" + elif mutation == "plain_added_event": + examples["plain_function"]["streaming"]["added"] = "bogus.event" + elif mutation == "custom_delta_event": + examples["custom_freeform"]["streaming"]["delta"] = "bogus.event" + elif mutation == "namespace_done_event": + examples["namespace"]["streaming"]["arguments_done"] = "bogus.event" + elif mutation == "tool_search_added_event": + examples["client_executed_tool_discovery"]["streaming"]["added"] = ( + "response.output_item.added" + ) + elif mutation == "unknown_event_label": + examples["unknown_future_kind"]["streaming"]["event_order"][0] = "bogus.event" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="(?:non-empty string|canonical SSE)"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + [ + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "namespace_tool", + ], +) +def test_build_inventory_rejects_unknown_nested_family_fields( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + examples = source_contract["runtime_wire_surface"]["declaration_family_examples"] + if mutation == "declaration": + examples["plain_function"]["declaration"]["future_field"] = "must fail" + elif mutation == "call": + examples["plain_function"]["call"]["future_field"] = "must fail" + elif mutation == "result": + examples["custom_freeform"]["result"]["future_field"] = "must fail" + elif mutation == "history": + examples["client_executed_tool_discovery"]["history"]["future_field"] = "must fail" + elif mutation == "streaming": + examples["plain_function"]["streaming"]["future_field"] = "must fail" + elif mutation == "terminal": + examples["plain_function"]["terminal"]["future_field"] = "must fail" + elif mutation == "error": + examples["plain_function"]["error"]["future_field"] = "must fail" + else: + examples["namespace"]["declaration"]["tools"][0]["future_field"] = "must fail" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="unknown or missing fields"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + ["redacted_value", "hosted_status", "response_item_type"], +) +def test_build_inventory_rejects_nested_family_value_extensions( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + surface = source_contract["runtime_wire_surface"] + examples = surface["declaration_family_examples"] + if mutation == "redacted_value": + examples["plain_function"]["call"]["arguments"] = "future-arguments" + elif mutation == "hosted_status": + examples["selected_provider_hosted"]["status"] = "captured" + else: + surface["response_shape"]["response_item_types"].append("future_item") + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises( + ValueError, + match="(?:redacted sentinel|canonical status|canonical list|status is invalid)", + ): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["declaration_families"][0]["representative"]["call"]["type"] = "web_search_call" + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("canonical family schema" in mismatch for mismatch in report["mismatches"]) + + id_mutation = json.loads(INVENTORY.read_text(encoding="utf-8")) + id_mutation["declaration_families"][0]["representative"]["call"]["item_id"] = "" + id_report = module.reconcile_inventory(id_mutation) + assert id_report["reconciled"] is False + assert any("non-empty string" in mismatch for mismatch in id_report["mismatches"]) + + +@pytest.mark.parametrize("mutation", ["representative_future_field", "representative_status"]) +def test_inventory_reconcile_rejects_nested_representative_edits(mutation: str) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + representative = inventory["declaration_families"][0]["representative"] + if mutation == "representative_future_field": + representative["future_field"] = "must fail" + else: + representative["status"] = "captured" + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert report["mismatches"] + assert any("unknown or missing fields" in mismatch for mismatch in report["mismatches"]) + + +@pytest.mark.parametrize( + "mutation", + [ + "top_level", + "candidate_identity", + "family", + "item", + "qualification", + "identity_control", + "evidence_sources", + "feeds", + "evidence_binding_entry", + ], +) +def test_inventory_reconcile_standalone_rejects_unknown_container_fields( + mutation: str, +) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + if mutation == "top_level": + inventory["future_field"] = "must fail" + elif mutation == "candidate_identity": + inventory["candidate_identity"]["future_field"] = "must fail" + elif mutation == "family": + inventory["declaration_families"][0]["future_field"] = "must fail" + elif mutation == "item": + inventory["items"][0]["future_field"] = "must fail" + elif mutation == "qualification": + inventory["qualification"]["future_field"] = "must fail" + elif mutation == "identity_control": + inventory["identity_control"]["future_field"] = "must fail" + elif mutation == "evidence_sources": + inventory["evidence_sources"]["future_field"] = "must fail" + elif mutation == "feeds": + inventory["feeds"]["future_field"] = "must fail" + else: + inventory["evidence_binding"]["trace"]["future_field"] = "must fail" + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("unknown or missing fields" in mismatch for mismatch in report["mismatches"]) + + +def test_inventory_reconcile_rejects_observed_family_without_evidence_root() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["declaration_families"][0]["observed"] = True + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("observed must remain false" in mismatch for mismatch in report["mismatches"]) + + +def test_structural_reconcile_allows_observed_family_when_bound_evidence_is_authoritative() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["declaration_families"][0]["observed"] = True + + assert not any( + "observed must remain false" + in mismatch + for mismatch in module._structural_inventory_mismatches( + inventory["declaration_families"], require_unobserved=False + ) + ) + + +@pytest.mark.parametrize( + "field, value", + [ + ("source_commit", "0" * 40), + ("route_upstream", "custom"), + ("model", "gpt-5.5"), + ], +) +def test_inventory_reconcile_rejects_candidate_binding_mutation_without_evidence_root( + field: str, value: str +) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["candidate_identity"][field] = value + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained Issue #62 candidate" in mismatch for mismatch in report["mismatches"]) + + +def test_inventory_reconcile_rejects_evidence_pointer_and_self_reported_readiness_mutation() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["evidence_binding"]["trace"]["file"] = "evil.json" + inventory["qualification"]["evidence_gates"] = { + gate: "complete" for gate in inventory["qualification"]["evidence_gates"] + } + inventory["qualification"]["blocking_gates"] = [] + inventory["qualification"]["ready_for_beta2"] = True + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained fixture" in mismatch for mismatch in report["mismatches"]) + assert any("cannot be asserted without bound evidence" in mismatch for mismatch in report["mismatches"]) + assert any("blocking_gates cannot be empty" in mismatch for mismatch in report["mismatches"]) + + +def test_inventory_reconcile_standalone_rejects_self_consistent_hash_manifest_edit() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["evidence_binding"]["trace"]["sha256"] = "0" * 64 + inventory["candidate_identity"]["evidence_manifest_sha256"] = module._evidence_manifest_sha256( + inventory["evidence_binding"] + ) + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained evidence artifact" in mismatch for mismatch in report["mismatches"]) + assert any("retained evidence manifest" in mismatch for mismatch in report["mismatches"]) + + +@pytest.mark.parametrize("mutation", ["gate", "terminal", "readiness"]) +def test_inventory_reconcile_standalone_rejects_unobserved_status_edits( + mutation: str, +) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + if mutation == "gate": + inventory["qualification"]["evidence_gates"]["terminal_events"] = "met" + elif mutation == "terminal": + inventory["declaration_families"][0]["representative"]["terminal"][ + "classification" + ] = "unqualified" + else: + inventory["qualification"]["ready_for_beta2"] = True + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained unobserved" in mismatch for mismatch in report["mismatches"]) + + +def test_inventory_reconcile_rejects_legacy_beta1_readiness_key() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["qualification"]["ready_for_beta1"] = inventory["qualification"].pop( + "ready_for_beta2" + ) + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("ready_for_beta1 is stale" in mismatch for mismatch in report["mismatches"]) + + def test_build_inventory_rejects_unbound_response_identity_pointer(tmp_path: Path) -> None: module = load_inventory_module() wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) @@ -385,7 +1005,7 @@ def test_captured_sse_status_does_not_satisfy_independent_sse_gate() -> None: ) assert qualification["evidence_gates"]["sse_identity"] == "captured" assert "sse_identity" in qualification["blocking_gates"] - assert qualification["ready_for_beta1"] is False + assert qualification["ready_for_beta2"] is False def test_committed_inventory_matches_generator_output() -> None: @@ -429,6 +1049,12 @@ def test_inventory_reconcile_rejects_duplicate_and_wrong_core_evidence() -> None assert report["reconciled"] is False assert any("unknown scope" in mismatch for mismatch in report["mismatches"]) + structural_mutation = json.loads(json.dumps(base)) + structural_mutation["declaration_families"][4]["representative"]["cross_provider_proxy"] = "allowed" + report = module.reconcile_inventory(structural_mutation) + assert report["reconciled"] is False + assert any("declaration_families" in mismatch for mismatch in report["mismatches"]) + def test_inventory_reconcile_rejects_provenance_contradictions_without_evidence_root() -> None: module = load_inventory_module() @@ -528,7 +1154,7 @@ def test_inventory_reconcile_binds_input_hashes_and_candidate_gates() -> None: ) tampered_status = json.loads(json.dumps(base)) - tampered_status["qualification"]["candidate_version_status"] = "eligible" + tampered_status["qualification"]["candidate_version_status"] = "legacy_below_floor" report = module.reconcile_inventory(tampered_status) assert report["reconciled"] is False assert any("CLI floor" in mismatch for mismatch in report["mismatches"]) @@ -633,7 +1259,7 @@ def test_qualification_accepts_audit_met_status_when_all_gates_are_complete() -> }, ) assert qualification["blocking_gates"] == [] - assert qualification["ready_for_beta1"] is True + assert qualification["ready_for_beta2"] is True @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 412ffa4d..b6cd837b 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -9,6 +9,7 @@ ROOT = Path(__file__).resolve().parents[1] TRACE = ROOT / "docs" / "evidence" / "issue-62" / "current-codexhub-thread-tool-surface.json" +SOURCE_CONTRACT = ROOT / "docs" / "evidence" / "issue-62" / "codex-0.146-source-contract.json" WIRE_FIXTURE = ROOT / "docs" / "evidence" / "issue-62" / "codexhub-runtime-wire-fixture.json" AUDIT = ROOT / "docs" / "evidence" / "issue-62" / "read-only-gate-audit.json" REPLAY_SCRIPT = ROOT / "scripts" / "check-codex-thread-tool-surface.ps1" @@ -57,6 +58,9 @@ def test_trace_covers_dynamic_tool_exposure_and_sanitizes_session_identity() -> assert trace["schema_version"] == 4 assert "session_id" not in trace["source"] + assert trace["captured_at"] == "2026-07-12T14:57:55+08:00" + assert trace["source"]["cli_version"] == "0.144.0-alpha.4" + assert trace["planner_gates"]["source_commit"] == "9e552e9d15ba52bed7077d5357f3e18e330f8f38" assert len(trace["registered_codex_app_tools"]) == 15 required = set(trace["required_thread_tools"]) assert required <= set(trace["registered_codex_app_tools"]) @@ -103,11 +107,28 @@ def test_trace_covers_dynamic_tool_exposure_and_sanitizes_session_identity() -> assert snapshot["model_entry_supports_search_tool"] is True assert len(snapshot["sha256"]) == 64 + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + assert source_contract["capture_status"] == "not_observed" + assert source_contract["qualification_status"] == "unqualified" + assert source_contract["provenance"]["cli_version"] == "0.146.0" + assert source_contract["provenance"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" + families = source_contract["runtime_wire_surface"]["declaration_families"] + assert [family["family"] for family in families] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] + def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) assert wire["route"]["upstream_route"] == "official" + assert wire["provenance"]["cli_version"] == "0.144.0-alpha.4" + assert wire["provenance"]["source_commit"] == "9e552e9d15ba52bed7077d5357f3e18e330f8f38" assert "no full request or response body fingerprint" in wire["evidence_limit"][ "transport_observation" ] @@ -140,6 +161,19 @@ def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: item.get("tag") == "unknown" for item in wire["response"]["non_streaming"]["response_items"] ) + assert json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8"))["runtime_wire_surface"]["declaration_family_order"] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] + hosted = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8"))["runtime_wire_surface"]["declaration_family_examples"][ + "selected_provider_hosted" + ] + assert hosted["provider_scope"] == "selected_provider_only" + assert hosted["cross_provider_proxy"] == "forbidden" def test_identity_replay_passes() -> None: @@ -257,8 +291,10 @@ def test_powershell_accepts_pass_non_direct_state_like_python(tmp_path: Path) -> audit["gate_classification"]["non_direct_states"] = "pass" trace_path = tmp_path / TRACE.name wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name shutil.copyfile(TRACE, trace_path) shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(SOURCE_CONTRACT, source_contract_path) audit_path = tmp_path / "read-only-gate-audit.json" audit_path.write_text( json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8" @@ -282,6 +318,8 @@ def test_powershell_accepts_pass_non_direct_state_like_python(tmp_path: Path) -> "Bypass", "-File", str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), "-TracePath", str(trace_path), "-WireFixturePath", @@ -300,6 +338,324 @@ def test_powershell_accepts_pass_non_direct_state_like_python(tmp_path: Path) -> assert result.returncode == 0, result.stderr +def test_powershell_rejects_historical_capture_provenance_drift(tmp_path: Path) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(SOURCE_CONTRACT, source_contract_path) + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + audit["provenance"]["historical_capture"]["captured_at"] = "2026-07-13T14:57:55+08:00" + audit_path.write_text( + json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + +def test_powershell_rejects_source_contract_family_schema_drift(tmp_path: Path) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["declaration_family_examples"]["plain_function"][ + "call" + ]["type"] = "web_search_call" + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + +@pytest.mark.parametrize( + "mutation", + ["top_level", "runtime_wire_surface", "request_shape", "response_shape"], +) +def test_powershell_rejects_unknown_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation == "top_level": + source_contract["future_field"] = "must not be accepted" + elif mutation == "runtime_wire_surface": + source_contract["runtime_wire_surface"]["future_field"] = "must not be accepted" + elif mutation == "request_shape": + source_contract["runtime_wire_surface"]["request_shape"][ + "future_field" + ] = "must not be accepted" + else: + source_contract["runtime_wire_surface"]["response_shape"][ + "future_field" + ] = "must not be accepted" + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + +@pytest.mark.parametrize( + "mutation", + [ + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "namespace_tool", + "response_item_type", + ], +) +def test_powershell_rejects_unknown_nested_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + examples = source_contract["runtime_wire_surface"]["declaration_family_examples"] + if mutation == "declaration": + examples["plain_function"]["declaration"]["future_field"] = "must fail" + elif mutation == "call": + examples["plain_function"]["call"]["future_field"] = "must fail" + elif mutation == "result": + examples["custom_freeform"]["result"]["future_field"] = "must fail" + elif mutation == "history": + examples["client_executed_tool_discovery"]["history"]["future_field"] = "must fail" + elif mutation == "streaming": + examples["plain_function"]["streaming"]["future_field"] = "must fail" + elif mutation == "terminal": + examples["plain_function"]["terminal"]["future_field"] = "must fail" + elif mutation == "error": + examples["plain_function"]["error"]["future_field"] = "must fail" + elif mutation == "namespace_tool": + examples["namespace"]["declaration"]["tools"][0]["future_field"] = "must fail" + else: + source_contract["runtime_wire_surface"]["response_shape"]["response_item_types"].append( + "future_item" + ) + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + +@pytest.mark.parametrize( + "mutation", + [ + "protocol", + "streaming_fields", + "representative_model", + "representative_input", + "representative_tools", + "representative_tool_choice", + "representative_parallel_tool_calls", + "representative_stream", + "representative_store", + "non_streaming_stream", + "non_streaming_response_body", + ], +) +def test_powershell_rejects_source_contract_request_shape_value_mutations( + tmp_path: Path, mutation: str +) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + request_shape = source_contract["runtime_wire_surface"]["request_shape"] + if mutation == "protocol": + request_shape["protocol"] = "chat_completions" + elif mutation == "streaming_fields": + request_shape["streaming_fields"] = ["model"] + elif mutation.startswith("representative_"): + field = mutation.removeprefix("representative_") + values = { + "model": "gpt-5.5", + "input": "not-redacted", + "tools": [], + "tool_choice": "required", + "parallel_tool_calls": True, + "stream": False, + "store": True, + } + request_shape["representative"][field] = values[field] + elif mutation == "non_streaming_stream": + request_shape["non_streaming_control"]["stream"] = True + else: + request_shape["non_streaming_control"]["response_body"] = "captured-body" + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) def test_negative_inventory_replays_fail_visibly(case: str) -> None: result = run_inventory_replay(case)