From 059968db0a74556abf4773eb5a7d570dbdd6a30b Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 16:24:11 +0800 Subject: [PATCH 01/13] feat: bind issue 62 inventory to Codex 0.146 --- docs/evidence/issue-62/README.md | 34 +- .../codexhub-runtime-wire-fixture.json | 144 ++++++++- .../current-codexhub-thread-tool-surface.json | 64 +++- .../issue-62/read-only-gate-audit.json | 56 ++++ .../issue-62/runtime-wire-inventory.json | 302 +++++++++++++++++- scripts/audit_issue_62_runtime_artifacts.py | 87 +++++ scripts/build_issue_62_runtime_inventory.py | 294 ++++++++++++++++- scripts/check-codex-thread-tool-surface.ps1 | 31 +- tests/test_issue_62_runtime_audit.py | 8 + tests/test_issue_62_runtime_inventory.py | 67 ++-- tests/test_issue_62_runtime_trace.py | 31 ++ 11 files changed, 1061 insertions(+), 57 deletions(-) diff --git a/docs/evidence/issue-62/README.md b/docs/evidence/issue-62/README.md index 75a45748..9bf9ec40 100644 --- a/docs/evidence/issue-62/README.md +++ b/docs/evidence/issue-62/README.md @@ -22,8 +22,9 @@ payloads are redacted. Gateway filtering beyond the observed prefix. The exact-version Desktop core and Code Mode app-server controls pass. -The source snapshot is OpenAI Codex commit -9e552e9d15ba52bed7077d5357f3e18e330f8f38. At that revision, the dynamic +The source snapshot is OpenAI Codex CLI `0.146.0`, tag `rust-v0.146.0`, +attested to commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`. At that +revision, the dynamic tool protocol defines optional deferLoading; the dynamic handler maps true to Deferred and missing or false to Direct. ToolExposure keeps Direct, DirectModelOnly, Deferred, and Hidden distinct. Tool search is planned only @@ -146,18 +147,21 @@ records one per-scope disposition for every taxonomy item the Codex CLI exposes over the core Responses contract and the explicitly-deferred advanced capabilities. -The artifact is bound to CLI floor `0.145.0` and to the candidate identity -derived from the existing sanitized artifacts (`cli_version=0.144.0-alpha.4`, -source commit `9e552e9d15ba52bed7077d5357f3e18e330f8f38`, official Responses -route). Because the captured CLI is below the floor, the generated -`qualification.ready_for_beta1` is `false` and the candidate is explicitly -marked `legacy_below_floor`; this evidence cannot be used as the beta.1 -candidate. The generator rejects an explicitly supplied CLI/source value that -does not match the trace, binds route/provider/model fields across trace and -wire fixtures (including pre/post models, catalog binding, and route profile), -and records a canonical-LF SHA-256 manifest for all three input artifacts. It -never fabricates a capability disposition for a gate the artifacts do not -qualify. +The artifact is bound to CLI floor `0.146.0` and to the candidate identity +derived from the existing sanitized artifacts (`cli_version=0.146.0`, source +commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`, candidate revision +`accab8ff6eb4d6ebd93cda84585fb5f6cb89da82`, official Responses route). The +candidate is now version-eligible, but `qualification.ready_for_beta1` remains +`false`: planner completeness, clean current-binding cold start, +independently fingerprinted full pre/post request and response bodies, +non-streaming/terminal/error/hosted/unknown controls, and wire replay evidence +are still incomplete. This is evidence for #62's downstream gates, not a #65 +qualification or capability unlock. The generator rejects an explicitly +supplied CLI/source value that does not match the trace, binds route/provider/ +model fields across trace and wire fixtures (including pre/post models, catalog +binding, and route profile), and records a canonical-LF SHA-256 manifest for +all three input artifacts. It never fabricates a capability disposition for a +gate the artifacts do not qualify. The qualification also has a separate `wire_identity_replay` gate. A full request/response fingerprint is not treated as replay proof by itself: a @@ -260,7 +264,7 @@ Evidence may use `cli_source_commit: null` with not acceptable. If the npm provenance attestation has been independently verified, its exact SLSA resolved-dependency commit may instead be recorded with status `published` (for `0.146.0`, the attested release commit is -`e363b08c9175ac1cbe5893615dd2cb9ddf95043b2`). +`e363b08c9175ac1cbe5893615dd2cb9ddf95043b`). ## Isolated live-evidence sidecar lane diff --git a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json index 77a22173..9b059970 100644 --- a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json +++ b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json @@ -23,10 +23,150 @@ "classification_basis": "upstream_route and catalog_binding; never configured_provider_id alone" }, "provenance": { - "cli_version": "0.144.0-alpha.4", - "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "cli_version": "0.146.0", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "cli_source_commit_status": "published_attested", + "cli_source_tag": "rust-v0.146.0", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "capture_id": "sanitized-current-side-capture" }, + "runtime_wire_surface": { + "source": "Codex CLI 0.146.0 ToolSpec and ResponseItem contracts", + "declaration_family_order": [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind" + ], + "request_shape": { + "protocol": "responses", + "streaming_fields": [ + "model", + "instructions", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "store", + "stream", + "stream_options", + "include", + "service_tier", + "prompt_cache_key", + "text", + "client_metadata" + ], + "representative": { + "model": "gpt-5.6-sol", + "input": "", + "tools": "", + "tool_choice": "auto", + "parallel_tool_calls": false, + "stream": true, + "store": false + }, + "non_streaming_control": { + "stream": false, + "response_body": "", + "captured": false, + "status": "live_control_required" + } + }, + "response_shape": { + "response_item_types": [ + "message", + "reasoning", + "function_call", + "function_call_output", + "custom_tool_call", + "custom_tool_call_output", + "tool_search_call", + "tool_search_output", + "web_search_call", + "local_shell_call", + "compaction", + "context_compaction", + "unknown" + ], + "stream_event_order": [ + "response.created", + "response.in_progress", + "response.output_item.added", + "response.output_text.delta", + "response.function_call_arguments.delta", + "response.custom_tool_call_input.delta", + "response.output_item.done", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.completed" + ], + "terminal_events": [ + "response.completed", + "response.incomplete", + "response.failed" + ], + "error_shape": { + "event": "response.failed", + "response": {"id": "response_error_001", "status": "failed", "error": ""}, + "classification": "bounded_error_only" + } + }, + "declaration_family_examples": { + "plain_function": { + "declaration": {"type": "function", "name": "function_alpha", "parameters": ""}, + "call": {"type": "function_call", "item_id": "item_call_function_001", "call_id": "call_function_001", "arguments": ""}, + "result": {"type": "function_call_output", "item_id": "item_output_function_001", "call_id": "call_function_001", "output": ""}, + "history": {"call_id": "call_function_001", "call_item_id": "item_call_function_001", "output_item_id": "item_output_function_001"}, + "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "done": "response.output_item.done", "terminal": "response.completed"} + }, + "custom_freeform": { + "declaration": {"type": "custom", "name": "custom_alpha", "format": ""}, + "call": {"type": "custom_tool_call", "item_id": "item_call_custom_001", "call_id": "call_custom_001", "input": ""}, + "result": {"type": "custom_tool_call_output", "item_id": "item_output_custom_001", "call_id": "call_custom_001", "output": ""}, + "history": {"call_id": "call_custom_001", "call_item_id": "item_call_custom_001", "output_item_id": "item_output_custom_001"}, + "streaming": {"added": "response.output_item.added", "delta": "response.custom_tool_call_input.delta", "done": "response.output_item.done", "terminal": "response.completed"} + }, + "namespace": { + "declaration": {"type": "namespace", "name": "namespace_alpha", "tools": [{"type": "function", "name": "function_beta", "parameters": ""}]}, + "call": {"type": "function_call", "item_id": "item_call_namespace_001", "call_id": "call_namespace_001", "namespace": "namespace_alpha", "arguments": ""}, + "result": {"type": "function_call_output", "item_id": "item_output_namespace_001", "call_id": "call_namespace_001", "output": ""}, + "history": {"call_id": "call_namespace_001", "call_item_id": "item_call_namespace_001", "output_item_id": "item_output_namespace_001", "namespace": "namespace_alpha"}, + "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "done": "response.output_item.done", "terminal": "response.completed"} + }, + "client_executed_tool_discovery": { + "declaration": {"type": "tool_search", "execution": "client", "parameters": ""}, + "call": {"type": "tool_search_call", "item_id": "item_call_search_001", "call_id": "call_search_001", "execution": "client", "arguments": ""}, + "result": {"type": "tool_search_output", "item_id": "item_output_search_001", "call_id": "call_search_001", "execution": "client", "tools": ""}, + "history": {"call_id": "call_search_001", "call_item_id": "item_call_search_001", "output_item_id": "item_output_search_001", "executor": "codex_client"}, + "streaming": {"added": "response.output_item.added", "delta": "response.output_text.delta", "done": "response.output_item.done", "terminal": "response.completed"} + }, + "selected_provider_hosted": { + "declaration": {"type": "web_search", "executor": "selected_provider", "provider_scope": "selected_provider_only"}, + "call": {"type": "web_search_call", "item_id": "item_call_hosted_001", "status": "", "action": ""}, + "result": {"type": "web_search_call", "item_id": "item_output_hosted_001", "status": "", "provider_scope": "selected_provider_only"}, + "history": {"call_id": null, "call_item_id": "item_call_hosted_001", "output_item_id": "item_output_hosted_001", "executor": "selected_provider", "cross_provider_proxy": "forbidden"}, + "streaming": {"added": "response.output_item.added", "delta": "", "done": "response.output_item.done", "terminal": "response.completed"}, + "observed": false, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden" + }, + "unknown_future_kind": { + "declaration": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "call": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "result": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "history": {"call_id": null, "call_item_id": null, "output_item_id": null, "loss_rule": "retain opaque sentinel"}, + "streaming": {"added": "unknown.future_event", "delta": "unknown.future_delta", "done": "unknown.future_done", "terminal": "response.completed"}, + "observed": false, + "status": "opaque_sentinel_only" + } + } + }, "exposure_state_tags": ["Direct", "DirectModelOnly", "Deferred", "Hidden", "hosted-only", "host-unavailable"], "pre_gateway": { "request_id": "request_001", diff --git a/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json b/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json index fcfedaff..60ff68df 100644 --- a/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json +++ b/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json @@ -4,7 +4,10 @@ "source": { "capture_id": "sanitized-current-side-capture", "originator": "Codex Desktop", - "cli_version": "0.144.0-alpha.4", + "cli_version": "0.146.0", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "cli_source_commit_status": "published_attested", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", "configured_provider_id": "custom", "model": "gpt-5.6-sol", "reasoning_effort": "xhigh", @@ -87,7 +90,8 @@ {"state": "host-unavailable", "plane": "host-binding", "runtime_observed": false, "evidence": "tagged reconciliation sentinel only; not inferred as a ToolExposure enum variant"} ], "planner_gates": { - "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", "predicate": "model_info.supports_search_tool && provider.capabilities.namespace_tools", "catalog_source": { "post_switch_catalog_basename": "codexhub-model-catalog.json", @@ -115,6 +119,62 @@ "codex_app_direct_tools": ["navigate_to_codex_page", "read_thread_terminal", "load_workspace_dependencies"], "codex_app_deferred_tools_discoverable_through_tool_search": ["automation_update", "fork_thread", "handoff_thread", "get_handoff_status", "list_projects", "create_thread", "list_threads", "read_thread", "send_message_to_thread", "set_thread_pinned", "set_thread_archived", "set_thread_title"] }, + "declaration_families": [ + { + "family": "plain_function", + "runtime_type": "function", + "wire_declaration_type": "function", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "custom_freeform", + "runtime_type": "custom", + "wire_declaration_type": "custom", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "namespace", + "runtime_type": "namespace", + "wire_declaration_type": "namespace", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "client_executed_tool_discovery", + "runtime_type": "tool_search", + "wire_declaration_type": "tool_search", + "observed": true, + "observation": "observed_client_execution", + "executor": "codex_client", + "loss_boundary": "discovery request/result stays client-executed" + }, + { + "family": "selected_provider_hosted", + "runtime_type": "web_search", + "wire_declaration_type": "web_search", + "observed": false, + "observation": "not_observed_selected_provider_only", + "executor": "selected_provider", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly" + }, + { + "family": "unknown_future_kind", + "runtime_type": "unknown", + "wire_declaration_type": "", + "observed": false, + "observation": "opaque_sentinel_only", + "executor": "unknown", + "loss_boundary": "retain tag and opaque payload; do not normalize" + } + ], "runtime_gate_evaluation_for_this_trace": "not_independently_captured" }, "independent_controls": [ diff --git a/docs/evidence/issue-62/read-only-gate-audit.json b/docs/evidence/issue-62/read-only-gate-audit.json index bbde3b56..d17337d7 100644 --- a/docs/evidence/issue-62/read-only-gate-audit.json +++ b/docs/evidence/issue-62/read-only-gate-audit.json @@ -349,6 +349,62 @@ } ] }, + "declaration_families": [ + { + "family": "plain_function", + "runtime_type": "function", + "wire_declaration_type": "function", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "custom_freeform", + "runtime_type": "custom", + "wire_declaration_type": "custom", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "namespace", + "runtime_type": "namespace", + "wire_declaration_type": "namespace", + "observed": true, + "observation": "observed", + "executor": "codex_client", + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + { + "family": "client_executed_tool_discovery", + "runtime_type": "tool_search", + "wire_declaration_type": "tool_search", + "observed": true, + "observation": "observed_client_execution", + "executor": "codex_client", + "loss_boundary": "discovery request/result stays client-executed" + }, + { + "family": "selected_provider_hosted", + "runtime_type": "web_search", + "wire_declaration_type": "web_search", + "observed": false, + "observation": "not_observed_selected_provider_only", + "executor": "selected_provider", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly" + }, + { + "family": "unknown_future_kind", + "runtime_type": "unknown", + "wire_declaration_type": "", + "observed": false, + "observation": "opaque_sentinel_only", + "executor": "unknown", + "loss_boundary": "retain tag and opaque payload; do not normalize" + } + ], "top_level_field_presence": [ "client_metadata", "include", diff --git a/docs/evidence/issue-62/runtime-wire-inventory.json b/docs/evidence/issue-62/runtime-wire-inventory.json index 03dfe230..b8e69c99 100644 --- a/docs/evidence/issue-62/runtime-wire-inventory.json +++ b/docs/evidence/issue-62/runtime-wire-inventory.json @@ -1,21 +1,303 @@ { "artifact_kind": "runtime_wire_inventory", "candidate_identity": { + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", "catalog_binding": "official Codex catalog entry for openai/gpt-5.6-sol", "catalog_model_entry_id": "gpt-5.6-sol", "catalog_snapshot_sha256": "307a09f22b0c827ae77192a4beaf7059efcf8a698ec4f252aa4ba4787f8d1876", - "cli_version": "0.144.0-alpha.4", - "codex_source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "cli_source_commit_status": "published_attested", + "cli_source_tag": "rust-v0.146.0", + "cli_version": "0.146.0", + "codex_source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "configured_provider_id": "custom", - "evidence_manifest_sha256": "9dfb5f48f52f27603d4d27066d0ed8a2e300600bcdbdff9a24f594a9892e5c22", + "evidence_manifest_sha256": "1e7d07eed427926b69b7bf7d8113f9b3913e02b16be7ffbf17ee64ca58e3d94f", "inbound_format": "responses", "model": "gpt-5.6-sol", "route_behavior_profile": "official_codex_app_http_passthrough", "route_upstream": "official", - "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "upstream_format": "responses" }, - "cli_version_floor": "0.145.0", + "cli_version_floor": "0.146.0", + "declaration_families": [ + { + "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "executor": "codex_client", + "family": "plain_function", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "observation": "observed", + "observed": true, + "optional_rule": "preserve_when_supported_else_omit", + "representative": { + "call": { + "arguments": "", + "call_id": "call_function_001", + "item_id": "item_call_function_001", + "type": "function_call" + }, + "declaration": { + "name": "function_alpha", + "parameters": "", + "type": "function" + }, + "history": { + "call_id": "call_function_001", + "call_item_id": "item_call_function_001", + "output_item_id": "item_output_function_001" + }, + "result": { + "call_id": "call_function_001", + "item_id": "item_output_function_001", + "output": "", + "type": "function_call_output" + }, + "streaming": { + "added": "response.output_item.added", + "delta": "response.function_call_arguments.delta", + "done": "response.output_item.done", + "terminal": "response.completed" + } + }, + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "runtime_type": "function", + "selected_protocol_disposition": "native", + "wire_declaration_type": "function" + }, + { + "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "executor": "codex_client", + "family": "custom_freeform", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "observation": "observed", + "observed": true, + "optional_rule": "preserve_when_supported_else_omit", + "representative": { + "call": { + "call_id": "call_custom_001", + "input": "", + "item_id": "item_call_custom_001", + "type": "custom_tool_call" + }, + "declaration": { + "format": "", + "name": "custom_alpha", + "type": "custom" + }, + "history": { + "call_id": "call_custom_001", + "call_item_id": "item_call_custom_001", + "output_item_id": "item_output_custom_001" + }, + "result": { + "call_id": "call_custom_001", + "item_id": "item_output_custom_001", + "output": "", + "type": "custom_tool_call_output" + }, + "streaming": { + "added": "response.output_item.added", + "delta": "response.custom_tool_call_input.delta", + "done": "response.output_item.done", + "terminal": "response.completed" + } + }, + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "runtime_type": "custom", + "selected_protocol_disposition": "native", + "wire_declaration_type": "custom" + }, + { + "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "executor": "codex_client", + "family": "namespace", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "observation": "observed", + "observed": true, + "optional_rule": "preserve_when_supported_else_omit", + "representative": { + "call": { + "arguments": "", + "call_id": "call_namespace_001", + "item_id": "item_call_namespace_001", + "namespace": "namespace_alpha", + "type": "function_call" + }, + "declaration": { + "name": "namespace_alpha", + "tools": [ + { + "name": "function_beta", + "parameters": "", + "type": "function" + } + ], + "type": "namespace" + }, + "history": { + "call_id": "call_namespace_001", + "call_item_id": "item_call_namespace_001", + "namespace": "namespace_alpha", + "output_item_id": "item_output_namespace_001" + }, + "result": { + "call_id": "call_namespace_001", + "item_id": "item_output_namespace_001", + "output": "", + "type": "function_call_output" + }, + "streaming": { + "added": "response.output_item.added", + "delta": "response.function_call_arguments.delta", + "done": "response.output_item.done", + "terminal": "response.completed" + } + }, + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "runtime_type": "namespace", + "selected_protocol_disposition": "native", + "wire_declaration_type": "namespace" + }, + { + "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "executor": "codex_client", + "family": "client_executed_tool_discovery", + "loss_boundary": "discovery request/result stays client-executed", + "observation": "observed_client_execution", + "observed": true, + "optional_rule": "preserve_client_execution_else_omit", + "representative": { + "call": { + "arguments": "", + "call_id": "call_search_001", + "execution": "client", + "item_id": "item_call_search_001", + "type": "tool_search_call" + }, + "declaration": { + "execution": "client", + "parameters": "", + "type": "tool_search" + }, + "history": { + "call_id": "call_search_001", + "call_item_id": "item_call_search_001", + "executor": "codex_client", + "output_item_id": "item_output_search_001" + }, + "result": { + "call_id": "call_search_001", + "execution": "client", + "item_id": "item_output_search_001", + "tools": "", + "type": "tool_search_output" + }, + "streaming": { + "added": "response.output_item.added", + "delta": "response.output_text.delta", + "done": "response.output_item.done", + "terminal": "response.completed" + } + }, + "required_rule": "preserve_client_execution_else_required-but-unavailable", + "runtime_type": "tool_search", + "selected_protocol_disposition": "native", + "wire_declaration_type": "tool_search" + }, + { + "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "executor": "selected_provider", + "family": "selected_provider_hosted", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "observation": "not_observed_selected_provider_only", + "observed": false, + "optional_rule": "omit_if_selected_provider_unsupported", + "representative": { + "call": { + "action": "", + "item_id": "item_call_hosted_001", + "status": "", + "type": "web_search_call" + }, + "cross_provider_proxy": "forbidden", + "declaration": { + "executor": "selected_provider", + "provider_scope": "selected_provider_only", + "type": "web_search" + }, + "history": { + "call_id": null, + "call_item_id": "item_call_hosted_001", + "cross_provider_proxy": "forbidden", + "executor": "selected_provider", + "output_item_id": "item_output_hosted_001" + }, + "observed": false, + "provider_scope": "selected_provider_only", + "result": { + "item_id": "item_output_hosted_001", + "provider_scope": "selected_provider_only", + "status": "", + "type": "web_search_call" + }, + "status": "selected_provider_control_required", + "streaming": { + "added": "response.output_item.added", + "delta": "", + "done": "response.output_item.done", + "terminal": "response.completed" + } + }, + "required_rule": "required-but-unavailable_if_selected_provider_unsupported", + "runtime_type": "web_search", + "selected_protocol_disposition": "native_if_selected_provider_supports", + "wire_declaration_type": "web_search" + }, + { + "evidence_source": "codexhub-runtime-wire-fixture.json#runtime_wire_surface.unknown_future", + "executor": "unknown", + "family": "unknown_future_kind", + "loss_boundary": "retain tag and opaque payload; do not normalize", + "observation": "opaque_sentinel_only", + "observed": false, + "optional_rule": "omit_and_emit_sanitized_diagnostic", + "representative": { + "call": { + "opaque_payload": "", + "tag": "unknown", + "type": "unknown" + }, + "declaration": { + "opaque_payload": "", + "tag": "unknown", + "type": "unknown" + }, + "history": { + "call_id": null, + "call_item_id": null, + "loss_rule": "retain opaque sentinel", + "output_item_id": null + }, + "observed": false, + "result": { + "opaque_payload": "", + "tag": "unknown", + "type": "unknown" + }, + "status": "opaque_sentinel_only", + "streaming": { + "added": "unknown.future_event", + "delta": "unknown.future_delta", + "done": "unknown.future_done", + "terminal": "response.completed" + } + }, + "required_rule": "required-but-unavailable", + "runtime_type": "unknown", + "selected_protocol_disposition": "omit", + "wire_declaration_type": "" + } + ], "disposition_vocabulary": [ "preserved", "reversibly_adapted", @@ -26,15 +308,15 @@ "evidence_binding": { "audit": { "file": "read-only-gate-audit.json", - "sha256": "a27f27622f89436f9530e2a7ca31f4b1b48872a31bbced8426302682f8516fa0" + "sha256": "eeed7c0ad927d2abf0a10f695530bc38242fafa0ec59ae5898c0bfb307bc83f0" }, "trace": { "file": "current-codexhub-thread-tool-surface.json", - "sha256": "a4367f96fc2aceaa419577ee43f91870fb3fcca4cc95644bc3347cbdd4206ee3" + "sha256": "47bdc0b56a879a734779aeb66aedce6c18f024578219a77548077a25a2976856" }, "wire_fixture": { "file": "codexhub-runtime-wire-fixture.json", - "sha256": "3a8654854539799b526bc013c50bc9f16c40c09319cff8da716b75e6deecbc3a" + "sha256": "106cfe30d5933d758fb52f8ff015d578c7c3211107f8c490c3fd1ebd9224d8ce" } }, "evidence_sources": { @@ -237,8 +519,8 @@ "terminal_events", "unknown_tagged_sentinels" ], - "candidate_version_eligible": false, - "candidate_version_status": "legacy_below_floor", + "candidate_version_eligible": true, + "candidate_version_status": "eligible", "evidence_gates": { "clean_cold_start_current_binding": "not_run", "complete_model_visible_plan": "partial", diff --git a/scripts/audit_issue_62_runtime_artifacts.py b/scripts/audit_issue_62_runtime_artifacts.py index 1962f0ee..f78112cb 100644 --- a/scripts/audit_issue_62_runtime_artifacts.py +++ b/scripts/audit_issue_62_runtime_artifacts.py @@ -37,6 +37,24 @@ "web_search_call", } VALID_TOOL_CHOICE_STRINGS = {"auto", "none", "required"} +# Structural declaration families emitted by Codex CLI 0.146's Responses +# ``ToolSpec`` surface. These labels describe shape/execution only; they are +# deliberately not model or Provider qualification records. +DECLARATION_FAMILIES = ( + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", +) +_TOOL_TYPE_TO_FAMILY = { + "function": "plain_function", + "custom": "custom_freeform", + "namespace": "namespace", + "tool_search": "client_executed_tool_discovery", + "web_search": "selected_provider_hosted", +} ROUTE_FIELDS = ( "upstream", "route_mode", @@ -160,6 +178,73 @@ def _sanitize_tool(tool: dict[str, Any]) -> dict[str, Any]: return sanitized +def _declaration_families(tool_surfaces: list[list[dict[str, Any]]]) -> list[dict[str, Any]]: + """Summarize observed declaration families without retaining tool content. + + The selected-provider hosted family and the unknown future sentinel are + included even when the bounded request rows do not observe them. This is + intentional: #62 inventories the runtime contract and makes the evidence + boundary explicit rather than silently dropping a family. + """ + + observed_types = { + str(tool.get("type")) + for surface in tool_surfaces + for tool in surface + if isinstance(tool, dict) and isinstance(tool.get("type"), str) + } + families: list[dict[str, Any]] = [] + for family in DECLARATION_FAMILIES: + runtime_type = next( + ( + tool_type + for tool_type, mapped_family in _TOOL_TYPE_TO_FAMILY.items() + if mapped_family == family + ), + None, + ) + if family == "unknown_future_kind": + families.append( + { + "family": family, + "runtime_type": "unknown", + "wire_declaration_type": "", + "observed": False, + "observation": "opaque_sentinel_only", + "executor": "unknown", + "loss_boundary": "retain tag and opaque payload; do not normalize", + } + ) + continue + observed = runtime_type in observed_types + if family == "selected_provider_hosted": + executor = "selected_provider" + observation = "not_observed_selected_provider_only" + loss_boundary = ( + "optional unsupported hosted capability is omitted; required capability fails visibly" + ) + elif family == "client_executed_tool_discovery": + executor = "codex_client" + observation = "observed_client_execution" if observed else "not_observed" + loss_boundary = "discovery request/result stays client-executed" + else: + executor = "codex_client" + observation = "observed" if observed else "not_observed" + loss_boundary = "preserve declaration and inverse call/result/history IDs" + families.append( + { + "family": family, + "runtime_type": runtime_type, + "wire_declaration_type": runtime_type, + "observed": observed, + "observation": observation, + "executor": executor, + "loss_boundary": loss_boundary, + } + ) + return families + + def _sanitize_request_plan(payload: dict[str, Any]) -> dict[str, Any]: item_types: Counter[str] = Counter() tool_surface: list[dict[str, Any]] = [] @@ -285,6 +370,7 @@ def _codex_request_evidence( variants.append(variant) unclassified = sorted(set(observed_item_types) - KNOWN_INPUT_ITEM_TYPES) + declaration_families = _declaration_families(list(tool_surfaces.values())) return { "current_request_endpoint_classes": dict(sorted(current_endpoint_classes.items())), "model_visible_request_plan": { @@ -292,6 +378,7 @@ def _codex_request_evidence( "observed_input_item_type_counts": dict(sorted(observed_item_types.items())), "plan_variants": variants, "tool_surfaces": tool_surfaces, + "declaration_families": declaration_families, "top_level_field_presence": sorted(top_level_fields), "transport_log_rows": transport_log_rows, "unclassified_item_types": unclassified, diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index a6653ccd..9642a73f 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -36,10 +36,21 @@ SCHEMA_VERSION = 1 ARTIFACT_KIND = "runtime_wire_inventory" -DEFAULT_CLI_FLOOR = "0.145.0" +DEFAULT_CLI_FLOOR = "0.146.0" SUPPORTED_CLI_FLOOR = DEFAULT_CLI_FLOOR DEFAULT_CANDIDATE_CLI_VERSION = None DEFAULT_CANDIDATE_SOURCE_COMMIT = None +CLI_SOURCE_TAG = "rust-v0.146.0" +CLI_SOURCE_COMMIT_STATUS = "published_attested" +CLI_BINARY_SHA256 = "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb" +STRUCTURAL_FAMILIES = ( + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", +) ALLOWED_DISPOSITIONS = ( "preserved", @@ -269,6 +280,10 @@ "catalog_model_entry_id", "route_behavior_profile", "evidence_manifest_sha256", + "candidate_revision", + "cli_binary_sha256", + "cli_source_commit_status", + "cli_source_tag", ) @@ -387,7 +402,25 @@ def _candidate_identity_mismatches(candidate_identity: Any) -> list[str]: if not isinstance(value, str) or not value.strip(): mismatches.append(f"candidate_identity.{field} is missing or blank") - for field in ("catalog_snapshot_sha256", "evidence_manifest_sha256"): + candidate_revision = candidate_identity.get("candidate_revision") + if not isinstance(candidate_revision, str) or not re.fullmatch( + r"[0-9a-f]{40}", candidate_revision + ): + mismatches.append( + "candidate_identity.candidate_revision is not a lowercase 40-character SHA-1" + ) + source_status = candidate_identity.get("cli_source_commit_status") + if source_status not in {"published_attested", "not_published_by_registry"}: + mismatches.append("candidate_identity.cli_source_commit_status is invalid") + source_tag = candidate_identity.get("cli_source_tag") + if not isinstance(source_tag, str) or not source_tag.strip(): + mismatches.append("candidate_identity.cli_source_tag is missing or blank") + + for field in ( + "catalog_snapshot_sha256", + "evidence_manifest_sha256", + "cli_binary_sha256", + ): value = candidate_identity.get(field) if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): mismatches.append( @@ -495,6 +528,23 @@ def _count_unknown_tags(value: Any) -> int: return 0 +def _count_response_unknown_tags(wire: dict[str, Any]) -> int: + """Count unknown-tag sentinels in the two response modes only. + + Structural family examples intentionally carry their own opaque unknown + sentinel. That declaration inventory is separate from the response-tag + evidence used by the identity control, so it must not inflate this count. + """ + + response = wire.get("response", {}) + return _count_unknown_tags( + { + "streaming": response.get("streaming", {}), + "non_streaming": response.get("non_streaming", {}), + } + ) + + def _unknown_tag_mode_counts(wire: dict[str, Any]) -> tuple[int, int]: streaming = _streaming_events(wire) non_streaming = wire.get("response", {}).get("non_streaming", {}).get( @@ -515,6 +565,186 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: return [event for event in events if isinstance(event, dict)] +_STRUCTURAL_RULES = { + "plain_function": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_when_supported_else_omit", + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "executor": "codex_client", + }, + "custom_freeform": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_when_supported_else_omit", + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "executor": "codex_client", + }, + "namespace": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_when_supported_else_omit", + "required_rule": "preserve_when_supported_else_required-but-unavailable", + "executor": "codex_client", + }, + "client_executed_tool_discovery": { + "selected_protocol_disposition": "native", + "optional_rule": "preserve_client_execution_else_omit", + "required_rule": "preserve_client_execution_else_required-but-unavailable", + "executor": "codex_client", + }, + "selected_provider_hosted": { + "selected_protocol_disposition": "native_if_selected_provider_supports", + "optional_rule": "omit_if_selected_provider_unsupported", + "required_rule": "required-but-unavailable_if_selected_provider_unsupported", + "executor": "selected_provider", + }, + "unknown_future_kind": { + "selected_protocol_disposition": "omit", + "optional_rule": "omit_and_emit_sanitized_diagnostic", + "required_rule": "required-but-unavailable", + "executor": "unknown", + }, +} + + +def _build_structural_inventory( + *, trace: dict[str, Any], wire: dict[str, Any], audit: dict[str, Any] +) -> list[dict[str, Any]]: + """Join exact 0.146 planner family records with sanitized wire examples.""" + + audit_families = ( + audit.get("model_visible_request_plan", {}).get("declaration_families", []) + ) + trace_families = trace.get("planner_gates", {}).get("declaration_families", []) + if audit_families and trace_families and audit_families != trace_families: + raise ValueError("runtime planner declaration families contradict across trace and audit") + families = audit_families or trace_families + if not isinstance(families, list): + raise ValueError("runtime planner declaration_families must be an array") + by_family: dict[str, dict[str, Any]] = {} + for family in families: + if not isinstance(family, dict) or not isinstance(family.get("family"), str): + raise ValueError("runtime planner declaration family is malformed") + name = family["family"] + if name in by_family: + raise ValueError(f"duplicate runtime planner declaration family: {name}") + by_family[name] = family + if set(by_family) != set(STRUCTURAL_FAMILIES): + missing = sorted(set(STRUCTURAL_FAMILIES) - set(by_family)) + extra = sorted(set(by_family) - set(STRUCTURAL_FAMILIES)) + raise ValueError( + "runtime planner declaration families are incomplete: " + f"missing={missing!r} extra={extra!r}" + ) + + examples = ( + wire.get("runtime_wire_surface", {}).get("declaration_family_examples", {}) + ) + if not isinstance(examples, dict): + raise ValueError("wire runtime declaration_family_examples must be an object") + output: list[dict[str, Any]] = [] + for family_name in STRUCTURAL_FAMILIES: + family = by_family[family_name] + example = examples.get(family_name) + if not isinstance(example, dict): + raise ValueError(f"wire runtime example missing for {family_name}") + required_example_parts = {"declaration", "call", "result", "history", "streaming"} + if not required_example_parts.issubset(example): + missing = sorted(required_example_parts - set(example)) + raise ValueError( + f"wire runtime example for {family_name} is missing {missing!r}" + ) + rule = _STRUCTURAL_RULES[family_name] + if family.get("executor") != rule["executor"]: + raise ValueError( + f"runtime planner executor for {family_name} contradicts the CLI contract" + ) + if family_name == "selected_provider_hosted": + if example.get("provider_scope") != "selected_provider_only": + raise ValueError("hosted declaration is not bound to the selected Provider") + if example.get("cross_provider_proxy") != "forbidden": + raise ValueError("hosted declaration permits a cross-Provider proxy") + output.append( + { + "family": family_name, + "runtime_type": family.get("runtime_type"), + "wire_declaration_type": family.get("wire_declaration_type"), + "observed": bool(family.get("observed")), + "observation": family.get("observation"), + "executor": rule["executor"], + "selected_protocol_disposition": rule["selected_protocol_disposition"], + "optional_rule": rule["optional_rule"], + "required_rule": rule["required_rule"], + "loss_boundary": family.get("loss_boundary"), + "evidence_source": ( + "read-only-gate-audit.json#model_visible_request_plan.declaration_families" + if family_name != "unknown_future_kind" + else "codexhub-runtime-wire-fixture.json#runtime_wire_surface.unknown_future" + ), + "representative": example, + } + ) + return output + + +def _structural_inventory_mismatches(value: Any) -> list[str]: + """Validate the stable shape of the emitted declaration-family inventory.""" + + if not isinstance(value, list): + return ["declaration_families must be an array"] + mismatches: list[str] = [] + if len(value) != len(STRUCTURAL_FAMILIES): + mismatches.append( + "declaration_families must contain exactly the six known families" + ) + required_example_parts = {"declaration", "call", "result", "history", "streaming"} + for index, family_name in enumerate(STRUCTURAL_FAMILIES): + if index >= len(value): + break + entry = value[index] + prefix = f"declaration_families[{index}]" + if not isinstance(entry, dict): + mismatches.append(f"{prefix} must be an object") + continue + if entry.get("family") != family_name: + mismatches.append( + f"{prefix}.family must be {family_name!r} in the canonical order" + ) + continue + rule = _STRUCTURAL_RULES[family_name] + if entry.get("executor") != rule["executor"]: + mismatches.append(f"{prefix}.executor does not match the family rule") + for field in ("runtime_type", "wire_declaration_type", "observation", "loss_boundary"): + if not isinstance(entry.get(field), str) or not entry[field]: + mismatches.append(f"{prefix}.{field} is missing or blank") + if not isinstance(entry.get("observed"), bool): + mismatches.append(f"{prefix}.observed must be boolean") + for field in ("selected_protocol_disposition", "optional_rule", "required_rule"): + if entry.get(field) != rule[field]: + mismatches.append(f"{prefix}.{field} does not match the family rule") + expected_source = ( + "read-only-gate-audit.json#model_visible_request_plan.declaration_families" + if family_name != "unknown_future_kind" + else "codexhub-runtime-wire-fixture.json#runtime_wire_surface.unknown_future" + ) + if entry.get("evidence_source") != expected_source: + mismatches.append(f"{prefix}.evidence_source does not match the family source") + representative = entry.get("representative") + if not isinstance(representative, dict): + mismatches.append(f"{prefix}.representative must be an object") + continue + if not required_example_parts.issubset(representative): + mismatches.append(f"{prefix}.representative is missing a wire example section") + if family_name == "selected_provider_hosted": + if representative.get("provider_scope") != "selected_provider_only": + mismatches.append(f"{prefix}.representative is not selected-provider scoped") + if representative.get("cross_provider_proxy") != "forbidden": + mismatches.append(f"{prefix}.representative permits a cross-provider proxy") + if family_name == "unknown_future_kind": + declaration = representative.get("declaration", {}) + if not isinstance(declaration, dict) or declaration.get("tag") != "unknown": + mismatches.append(f"{prefix}.representative does not retain the unknown sentinel") + return mismatches + + def _has_terminal_event(wire: dict[str, Any]) -> bool: return any(event.get("event") == "response.completed" for event in _streaming_events(wire)) @@ -778,7 +1008,7 @@ def _classify_live_control_items( full_wire_gate = gate_classification.get("full_pre_post_request_response") non_direct_gate = gate_classification.get("non_direct_states") non_streaming_captured = _real_non_streaming_captured(wire) - unknown_tag_count = _count_unknown_tags(wire) + unknown_tag_count = _count_response_unknown_tags(wire) unknown_stream_count, unknown_non_stream_count = _unknown_tag_mode_counts(wire) choice_captured = ( @@ -1048,10 +1278,22 @@ def _validate_candidate_binding( trace_cli_version = source.get("cli_version") trace_source_commit = planner_gates.get("source_commit") trace_capture_id = source.get("capture_id") - if not trace_cli_version or not trace_source_commit or not trace_capture_id: + trace_candidate_revision = source.get("candidate_revision") + trace_binary_sha256 = source.get("cli_binary_sha256") + trace_source_status = source.get("cli_source_commit_status") + trace_source_tag = planner_gates.get("cli_source_tag") + if ( + not trace_cli_version + or not trace_source_commit + or not trace_capture_id + or not trace_candidate_revision + or not trace_binary_sha256 + or not trace_source_status + or not trace_source_tag + ): raise ValueError( "trace evidence is missing source.cli_version, source.capture_id, " - "or planner_gates.source_commit" + "source candidate provenance, or planner_gates.source_commit" ) if not isinstance(trace_cli_version, str): raise ValueError("trace source.cli_version must be a string") @@ -1065,6 +1307,24 @@ def _validate_candidate_binding( raise ValueError( "trace planner_gates.source_commit must be a lowercase 40-character SHA-1" ) + if not isinstance(trace_candidate_revision, str) or not re.fullmatch( + r"[0-9a-f]{40}", trace_candidate_revision + ): + raise ValueError("trace source.candidate_revision must be a lowercase 40-character SHA-1") + if not isinstance(trace_binary_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", trace_binary_sha256 + ): + raise ValueError("trace source.cli_binary_sha256 must be a lowercase 64-character SHA-256") + if trace_binary_sha256 != CLI_BINARY_SHA256: + raise ValueError( + "trace source.cli_binary_sha256 does not match Codex CLI 0.146.0" + ) + if trace_source_status not in {"published_attested", "not_published_by_registry"}: + raise ValueError("trace source.cli_source_commit_status is invalid") + if trace_source_tag != CLI_SOURCE_TAG: + raise ValueError( + f"trace planner_gates.cli_source_tag must bind to {CLI_SOURCE_TAG}" + ) if candidate_cli_version is None: candidate_cli_version = trace_cli_version @@ -1138,6 +1398,10 @@ def _validate_candidate_binding( wire_provenance.get("cli_version") != trace_cli_version or wire_provenance.get("source_commit") != trace_source_commit or wire_provenance.get("capture_id") != trace_capture_id + or wire_provenance.get("candidate_revision") != trace_candidate_revision + or wire_provenance.get("cli_binary_sha256") != trace_binary_sha256 + or wire_provenance.get("cli_source_commit_status") != trace_source_status + or wire_provenance.get("cli_source_tag") != trace_source_tag ): raise ValueError("wire provenance is not bound to the trace candidate") catalog_snapshot = ( @@ -1175,6 +1439,10 @@ def _validate_candidate_binding( "catalog_snapshot_sha256": wire_route["catalog_snapshot_sha256"], "catalog_model_entry_id": wire_route["catalog_model_entry_id"], "route_behavior_profile": trace_route.get("behavior_profile"), + "candidate_revision": trace_candidate_revision, + "cli_binary_sha256": trace_binary_sha256, + "cli_source_commit_status": trace_source_status, + "cli_source_tag": trace_source_tag, } return identity, status @@ -1218,7 +1486,7 @@ def build_inventory( raise ValueError(f"duplicate scope: {item['scope']}") seen.add(item["scope"]) - unknown_tagged_source_count = _count_unknown_tags(wire_data) + unknown_tagged_source_count = _count_response_unknown_tags(wire_data) unknown_stream_count, unknown_non_stream_count = _unknown_tag_mode_counts(wire_data) if unknown_tagged_source_count == 0: raise ValueError("wire evidence contains no unknown-tag sentinel to classify") @@ -1229,6 +1497,11 @@ def build_inventory( identity_control = _build_identity_control( items, unknown_tagged_source_count=unknown_tagged_source_count ) + structural_families = _build_structural_inventory( + trace=trace_data, + wire=wire_data, + audit=audit_data, + ) candidate_identity, candidate_version_status = _validate_candidate_binding( trace_data=trace_data, wire_data=wire_data, @@ -1261,6 +1534,7 @@ def build_inventory( "artifact_kind": ARTIFACT_KIND, "cli_version_floor": cli_version_floor, "candidate_identity": candidate_identity, + "declaration_families": structural_families, "qualification": qualification, "disposition_vocabulary": list(ALLOWED_DISPOSITIONS), "items": items, @@ -1328,6 +1602,12 @@ def reconcile_inventory( mismatches.append("inventory artifact_kind is invalid") if inventory.get("disposition_vocabulary") != list(ALLOWED_DISPOSITIONS): mismatches.append("inventory disposition_vocabulary is invalid") + mismatches.extend( + f"mutation: {message}" + for message in _structural_inventory_mismatches( + inventory.get("declaration_families") + ) + ) items = inventory.get("items", []) if not isinstance(items, list): @@ -1634,7 +1914,7 @@ def reconcile_inventory( wire_entry = evidence_binding.get("wire_fixture", {}) wire_path = evidence_root / str(wire_entry.get("file", "")) if wire_path.is_file(): - actual_unknown_tagged_source_count = _count_unknown_tags( + actual_unknown_tagged_source_count = _count_response_unknown_tags( _load_json(wire_path) ) if unknown_tagged_source_count != actual_unknown_tagged_source_count: diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 8741d2ab..3e1edb16 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -405,7 +405,11 @@ foreach ($link in $callLinks) { $streamUnknown = @($wire.response.streaming.events | Where-Object { $_.tag -eq 'unknown' }) $nonStreamingUnknown = @($wire.response.non_streaming.response_items | Where-Object { $_.tag -eq 'unknown' }) -$unknownTaggedSourceCount = Get-UnknownTaggedSourceCount -Value $wire +$responseUnknownSource = [PSCustomObject]@{ + streaming = $wire.response.streaming + non_streaming = $wire.response.non_streaming +} +$unknownTaggedSourceCount = Get-UnknownTaggedSourceCount -Value $responseUnknownSource if ( $inventory.identity_control.unknown_tagged_source_count -le 0 -or $inventory.identity_control.unknown_tagged_source_count -ne $unknownTaggedSourceCount @@ -619,7 +623,7 @@ foreach ($scope in $allRequiredScopes) { if ( $inventory.artifact_kind -ne 'runtime_wire_inventory' -or $inventory.schema_version -ne 1 -or - $inventory.cli_version_floor -ne '0.145.0' + $inventory.cli_version_floor -ne '0.146.0' ) { Add-Mismatch 'inventory artifact identity or CLI version floor is invalid' } @@ -655,6 +659,21 @@ foreach ($field in @('catalog_snapshot_sha256','evidence_manifest_sha256')) { Add-Mismatch "inventory candidate $field is not a lowercase 64-character SHA-256" } } +if ([string]$inventoryCandidate.candidate_revision -notmatch '^[0-9a-f]{40}$') { + Add-Mismatch 'inventory candidate revision is not a lowercase 40-character SHA-1' +} +if ([string]$inventoryCandidate.cli_binary_sha256 -notmatch '^[0-9a-f]{64}$') { + Add-Mismatch 'inventory candidate CLI binary SHA-256 is invalid' +} +if ([string]$inventoryCandidate.cli_binary_sha256 -ne 'bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb') { + Add-Mismatch 'inventory candidate CLI binary SHA-256 is not the retained Codex 0.146.0 binary' +} +if ([string]$inventoryCandidate.cli_source_commit_status -notin @('published_attested','not_published_by_registry')) { + Add-Mismatch 'inventory candidate CLI source commit status is invalid' +} +if ([string]::IsNullOrWhiteSpace([string]$inventoryCandidate.cli_source_tag)) { + Add-Mismatch 'inventory candidate CLI source tag is missing or blank' +} if ( [string]::IsNullOrWhiteSpace([string]$trace.source.capture_id) -or [string]::IsNullOrWhiteSpace([string]$wire.provenance.capture_id) @@ -665,6 +684,10 @@ if ( $inventoryCandidate.cli_version -ne $trace.source.cli_version -or $inventoryCandidate.source_commit -ne $trace.planner_gates.source_commit -or $inventoryCandidate.codex_source_commit -ne $trace.planner_gates.source_commit -or + $inventoryCandidate.candidate_revision -ne $trace.source.candidate_revision -or + $inventoryCandidate.cli_binary_sha256 -ne $trace.source.cli_binary_sha256 -or + $inventoryCandidate.cli_source_commit_status -ne $trace.source.cli_source_commit_status -or + $inventoryCandidate.cli_source_tag -ne $trace.planner_gates.cli_source_tag -or $inventoryCandidate.route_upstream -ne $wire.route.upstream_route -or $inventoryCandidate.inbound_format -ne $wire.route.inbound_format -or $inventoryCandidate.upstream_format -ne $wire.route.upstream_format -or @@ -689,6 +712,10 @@ if ( $wire.route.catalog_model_supports_search_tool -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.model_entry_supports_search_tool -or $wire.provenance.cli_version -ne $trace.source.cli_version -or $wire.provenance.source_commit -ne $trace.planner_gates.source_commit -or + $wire.provenance.candidate_revision -ne $trace.source.candidate_revision -or + $wire.provenance.cli_binary_sha256 -ne $trace.source.cli_binary_sha256 -or + $wire.provenance.cli_source_commit_status -ne $trace.source.cli_source_commit_status -or + $wire.provenance.cli_source_tag -ne $trace.planner_gates.cli_source_tag -or $wire.provenance.capture_id -ne $trace.source.capture_id -or $wire.pre_gateway.model -ne $trace.source.model -or $wire.post_gateway.model -ne $trace.source.model diff --git a/tests/test_issue_62_runtime_audit.py b/tests/test_issue_62_runtime_audit.py index 1e4839b8..aee5125a 100644 --- a/tests/test_issue_62_runtime_audit.py +++ b/tests/test_issue_62_runtime_audit.py @@ -282,6 +282,14 @@ def test_audit_reports_only_sanitized_schema_and_gate_facts(tmp_path: Path) -> N "type": "tool_search", }, ] + assert [entry["family"] for entry in planner["declaration_families"]] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] gateway = audit["gateway_identity_route"] assert gateway["request_starts"] == 1 diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index e4fc2538..774e431a 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -35,11 +35,15 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N assert inventory["schema_version"] == 1 assert inventory["artifact_kind"] == "runtime_wire_inventory" - assert inventory["cli_version_floor"] == "0.145.0" + assert inventory["cli_version_floor"] == "0.146.0" candidate = inventory["candidate_identity"] - assert candidate["cli_version"] == "0.144.0-alpha.4" - assert candidate["source_commit"] == "9e552e9d15ba52bed7077d5357f3e18e330f8f38" + assert candidate["cli_version"] == "0.146.0" + assert candidate["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" assert candidate["codex_source_commit"] == candidate["source_commit"] + assert candidate["candidate_revision"] == "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" + assert candidate["cli_binary_sha256"] == "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb" + assert candidate["cli_source_commit_status"] == "published_attested" + assert candidate["cli_source_tag"] == "rust-v0.146.0" assert candidate["route_upstream"] == "official" assert candidate["inbound_format"] == "responses" assert candidate["upstream_format"] == "responses" @@ -48,8 +52,8 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N assert candidate["catalog_model_entry_id"] == "gpt-5.6-sol" assert candidate["route_behavior_profile"] == "official_codex_app_http_passthrough" assert len(candidate["evidence_manifest_sha256"]) == 64 - assert inventory["qualification"]["candidate_version_status"] == "legacy_below_floor" - assert inventory["qualification"]["candidate_version_eligible"] is False + assert inventory["qualification"]["candidate_version_status"] == "eligible" + assert inventory["qualification"]["candidate_version_eligible"] is True assert inventory["qualification"]["ready_for_beta1"] is False assert inventory["identity_control"]["unknown_tagged_source_count"] == 2 assert inventory["qualification"]["blocking_gates"] == [ @@ -129,6 +133,25 @@ def test_inventory_covers_every_required_taxonomy_scope() -> None: assert not missing, f"missing required scopes: {sorted(missing)}" +def test_inventory_records_all_structural_declaration_families() -> None: + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + + families = inventory["declaration_families"] + assert [entry["family"] for entry in families] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] + by_family = {entry["family"]: entry for entry in families} + assert by_family["client_executed_tool_discovery"]["executor"] == "codex_client" + assert by_family["selected_provider_hosted"]["executor"] == "selected_provider" + assert by_family["selected_provider_hosted"]["representative"]["cross_provider_proxy"] == "forbidden" + assert by_family["unknown_future_kind"]["selected_protocol_disposition"] == "omit" + + def test_every_item_carries_an_allowed_disposition() -> None: inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) @@ -249,19 +272,19 @@ def test_build_inventory_reads_existing_artifacts_and_clamps_live_gates() -> Non trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - cli_version_floor="0.145.0", - candidate_cli_version="0.144.0-alpha.4", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + cli_version_floor="0.146.0", + candidate_cli_version="0.146.0", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) assert inventory["schema_version"] == 1 - assert inventory["cli_version_floor"] == "0.145.0" - assert inventory["candidate_identity"]["cli_version"] == "0.144.0-alpha.4" + assert inventory["cli_version_floor"] == "0.146.0" + assert inventory["candidate_identity"]["cli_version"] == "0.146.0" scopes = {entry["scope"] for entry in inventory["items"]} assert "core_text_streaming" in scopes assert "code_mode" in scopes assert inventory["identity_control"]["unclassified_core_items"] == 0 - assert inventory["qualification"]["candidate_version_status"] == "legacy_below_floor" + assert inventory["qualification"]["candidate_version_status"] == "eligible" def test_build_inventory_rejects_candidate_metadata_drift() -> None: @@ -273,7 +296,7 @@ def test_build_inventory_rejects_candidate_metadata_drift() -> None: wire_fixture=WIRE_FIXTURE, audit=AUDIT, candidate_cli_version="0.145.0", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) with pytest.raises(ValueError, match="candidate source commit"): @@ -281,7 +304,7 @@ def test_build_inventory_rejects_candidate_metadata_drift() -> None: trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - candidate_cli_version="0.144.0-alpha.4", + candidate_cli_version="0.146.0", candidate_source_commit="0" * 40, ) @@ -294,9 +317,9 @@ def test_build_inventory_rejects_a_floor_other_than_supported_candidate_floor() trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - cli_version_floor="0.144.0", - candidate_cli_version="0.144.0-alpha.4", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + cli_version_floor="0.145.0", + candidate_cli_version="0.146.0", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) @@ -309,7 +332,7 @@ def test_build_inventory_rejects_malformed_candidate_provenance() -> None: wire_fixture=WIRE_FIXTURE, audit=AUDIT, candidate_cli_version="not-a-version", - candidate_source_commit="9e552e9d15ba52bed7077d5357f3e18e330f8f38", + candidate_source_commit="e363b08c9175ac1cbe5893615dd2cb9ddf95043b", ) with pytest.raises(ValueError, match="candidate source commit"): @@ -317,7 +340,7 @@ def test_build_inventory_rejects_malformed_candidate_provenance() -> None: trace=TRACE, wire_fixture=WIRE_FIXTURE, audit=AUDIT, - candidate_cli_version="0.144.0-alpha.4", + candidate_cli_version="0.146.0", candidate_source_commit="A" * 40, ) @@ -429,6 +452,12 @@ def test_inventory_reconcile_rejects_duplicate_and_wrong_core_evidence() -> None assert report["reconciled"] is False assert any("unknown scope" in mismatch for mismatch in report["mismatches"]) + structural_mutation = json.loads(json.dumps(base)) + structural_mutation["declaration_families"][4]["representative"]["cross_provider_proxy"] = "allowed" + report = module.reconcile_inventory(structural_mutation) + assert report["reconciled"] is False + assert any("declaration_families" in mismatch for mismatch in report["mismatches"]) + def test_inventory_reconcile_rejects_provenance_contradictions_without_evidence_root() -> None: module = load_inventory_module() @@ -528,7 +557,7 @@ def test_inventory_reconcile_binds_input_hashes_and_candidate_gates() -> None: ) tampered_status = json.loads(json.dumps(base)) - tampered_status["qualification"]["candidate_version_status"] = "eligible" + tampered_status["qualification"]["candidate_version_status"] = "legacy_below_floor" report = module.reconcile_inventory(tampered_status) assert report["reconciled"] is False assert any("CLI floor" in mismatch for mismatch in report["mismatches"]) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 412ffa4d..5eabab58 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -57,6 +57,11 @@ def test_trace_covers_dynamic_tool_exposure_and_sanitizes_session_identity() -> assert trace["schema_version"] == 4 assert "session_id" not in trace["source"] + assert trace["source"]["cli_version"] == "0.146.0" + assert trace["source"]["candidate_revision"] == "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" + assert trace["source"]["cli_source_commit_status"] == "published_attested" + assert trace["planner_gates"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" + assert trace["planner_gates"]["cli_source_tag"] == "rust-v0.146.0" assert len(trace["registered_codex_app_tools"]) == 15 required = set(trace["required_thread_tools"]) assert required <= set(trace["registered_codex_app_tools"]) @@ -103,11 +108,24 @@ def test_trace_covers_dynamic_tool_exposure_and_sanitizes_session_identity() -> assert snapshot["model_entry_supports_search_tool"] is True assert len(snapshot["sha256"]) == 64 + families = trace["planner_gates"]["declaration_families"] + assert [family["family"] for family in families] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] + def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) assert wire["route"]["upstream_route"] == "official" + assert wire["provenance"]["cli_version"] == "0.146.0" + assert wire["provenance"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" + assert wire["provenance"]["candidate_revision"] == "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" assert "no full request or response body fingerprint" in wire["evidence_limit"][ "transport_observation" ] @@ -140,6 +158,19 @@ def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: item.get("tag") == "unknown" for item in wire["response"]["non_streaming"]["response_items"] ) + assert wire["runtime_wire_surface"]["declaration_family_order"] == [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind", + ] + hosted = wire["runtime_wire_surface"]["declaration_family_examples"][ + "selected_provider_hosted" + ] + assert hosted["provider_scope"] == "selected_provider_only" + assert hosted["cross_provider_proxy"] == "forbidden" def test_identity_replay_passes() -> None: From f8248126b4782e0970ddb54166f24946d8fa96d7 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 17:27:38 +0800 Subject: [PATCH 02/13] fix(issue-62): separate historical capture from source contract --- docs/evidence/issue-62/README.md | 26 +- .../issue-62/codex-0.146-source-contract.json | 179 +++++++ .../codexhub-runtime-wire-fixture.json | 144 +----- .../current-codexhub-thread-tool-surface.json | 64 +-- .../issue-62/read-only-gate-audit.json | 14 + .../issue-62/runtime-wire-inventory.json | 140 +++++- scripts/build_issue_62_runtime_inventory.py | 443 ++++++++++++++---- scripts/check-codex-thread-tool-surface.ps1 | 52 +- tests/test_issue_62_runtime_audit.py | 12 +- tests/test_issue_62_runtime_inventory.py | 55 +++ tests/test_issue_62_runtime_trace.py | 29 +- 11 files changed, 818 insertions(+), 340 deletions(-) create mode 100644 docs/evidence/issue-62/codex-0.146-source-contract.json diff --git a/docs/evidence/issue-62/README.md b/docs/evidence/issue-62/README.md index 9bf9ec40..ed2596cf 100644 --- a/docs/evidence/issue-62/README.md +++ b/docs/evidence/issue-62/README.md @@ -22,9 +22,15 @@ payloads are redacted. Gateway filtering beyond the observed prefix. The exact-version Desktop core and Code Mode app-server controls pass. -The source snapshot is OpenAI Codex CLI `0.146.0`, tag `rust-v0.146.0`, -attested to commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`. At that -revision, the dynamic +The retained Desktop capture is historical: it was captured on 2026-07-12 +with Codex CLI `0.144.0-alpha.4`, source commit +`9e552e9d15ba52bed7077d5357f3e18e330f8f38`. It must not be relabeled as the +later CLI 0.146.0 release. The separate +`codex-0.146-source-contract.json` records the 0.146.0 source contract (tag +`rust-v0.146.0`, attested commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`, +and exact binary hash) with `capture_status=not_observed` and +`qualification_status=unqualified`. At that +historical revision, the dynamic tool protocol defines optional deferLoading; the dynamic handler maps true to Deferred and missing or false to Direct. ToolExposure keeps Direct, DirectModelOnly, Deferred, and Hidden distinct. Tool search is planned only @@ -91,6 +97,10 @@ session, task, turn, call, item, request, or response identifiers. The bounded audit establishes these additional facts without a restart, reconnect, configuration write, or production-handler change: +Its candidate provenance is explicitly `capture_status=not_observed` and is +bound to the same 0.146 source contract; the retained historical capture +metadata remains recorded separately rather than being promoted to 0.146. + - Forty-three retained Sol transport rows resolve to three actual model-visible planner surfaces. The largest retained surface includes the base functions, collaboration namespace, goal functions, image generation, @@ -148,10 +158,12 @@ exposes over the core Responses contract and the explicitly-deferred advanced capabilities. The artifact is bound to CLI floor `0.146.0` and to the candidate identity -derived from the existing sanitized artifacts (`cli_version=0.146.0`, source +from the unobserved 0.146 source contract (`cli_version=0.146.0`, source commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`, candidate revision `accab8ff6eb4d6ebd93cda84585fb5f6cb89da82`, official Responses route). The -candidate is now version-eligible, but `qualification.ready_for_beta1` remains +historical trace, wire fixture, and audit remain explicitly bound as +0.144.0/historical evidence; they do not become 0.146 runtime captures. The +candidate is version-eligible, but `qualification.ready_for_beta1` remains `false`: planner completeness, clean current-binding cold start, independently fingerprinted full pre/post request and response bodies, non-streaming/terminal/error/hosted/unknown controls, and wire replay evidence @@ -160,8 +172,8 @@ qualification or capability unlock. The generator rejects an explicitly supplied CLI/source value that does not match the trace, binds route/provider/ model fields across trace and wire fixtures (including pre/post models, catalog binding, and route profile), and records a canonical-LF SHA-256 manifest for -all three input artifacts. It never fabricates a capability disposition for a -gate the artifacts do not qualify. +all four input artifacts (including the source contract). It never fabricates a +capability disposition for a gate the artifacts do not qualify. The qualification also has a separate `wire_identity_replay` gate. A full request/response fingerprint is not treated as replay proof by itself: a diff --git a/docs/evidence/issue-62/codex-0.146-source-contract.json b/docs/evidence/issue-62/codex-0.146-source-contract.json new file mode 100644 index 00000000..4f5626a3 --- /dev/null +++ b/docs/evidence/issue-62/codex-0.146-source-contract.json @@ -0,0 +1,179 @@ +{ + "schema_version": 1, + "fixture_kind": "codex_cli_source_contract", + "capture_status": "not_observed", + "qualification_status": "unqualified", + "captured_at": null, + "provenance": { + "cli_version": "0.146.0", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", + "cli_source_commit_status": "published_attested", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" + }, + "runtime_wire_surface": { + "source": "Codex CLI 0.146.0 ToolSpec and ResponseItem source contract", + "declaration_family_order": [ + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + "unknown_future_kind" + ], + "declaration_families": [ + {"family": "plain_function", "runtime_type": "function", "wire_declaration_type": "function", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "preserve declaration and inverse call/result/history IDs"}, + {"family": "custom_freeform", "runtime_type": "custom", "wire_declaration_type": "custom", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "preserve declaration and inverse call/result/history IDs"}, + {"family": "namespace", "runtime_type": "namespace", "wire_declaration_type": "namespace", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "preserve declaration and inverse call/result/history IDs"}, + {"family": "client_executed_tool_discovery", "runtime_type": "tool_search", "wire_declaration_type": "tool_search", "observed": false, "observation": "not_observed_source_contract_only", "executor": "codex_client", "loss_boundary": "discovery request/result stays client-executed"}, + {"family": "selected_provider_hosted", "runtime_type": "web_search", "wire_declaration_type": "web_search", "observed": false, "observation": "not_observed_selected_provider_control_required", "executor": "selected_provider", "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly"}, + {"family": "unknown_future_kind", "runtime_type": "unknown", "wire_declaration_type": "", "observed": false, "observation": "opaque_sentinel_only", "executor": "unknown", "loss_boundary": "retain tag and opaque payload; do not normalize"} + ], + "request_shape": { + "protocol": "responses", + "streaming_fields": [ + "model", + "instructions", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "store", + "stream", + "stream_options", + "include", + "service_tier", + "prompt_cache_key", + "text", + "client_metadata" + ], + "representative": { + "model": "gpt-5.6-sol", + "input": "", + "tools": "", + "tool_choice": "auto", + "parallel_tool_calls": false, + "stream": true, + "store": false + }, + "non_streaming_control": { + "stream": false, + "response_body": "", + "captured": false, + "status": "unqualified" + } + }, + "response_shape": { + "response_item_types": [ + "message", + "reasoning", + "function_call", + "function_call_output", + "custom_tool_call", + "custom_tool_call_output", + "tool_search_call", + "tool_search_output", + "web_search_call", + "local_shell_call", + "compaction", + "context_compaction", + "unknown" + ], + "stream_event_order": [ + "response.created", + "response.in_progress", + "response.output_item.added", + "response.output_text.delta", + "response.function_call_arguments.delta", + "response.custom_tool_call_input.delta", + "response.function_call_arguments.done", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.completed" + ], + "terminal_events": [ + "response.completed", + "response.incomplete", + "response.failed" + ], + "error_shape": { + "event": "response.failed", + "response": {"id": "response_error_001", "status": "failed", "error": ""}, + "classification": "unqualified" + } + }, + "declaration_family_examples": { + "plain_function": { + "declaration": {"type": "function", "name": "function_alpha", "parameters": ""}, + "call": {"type": "function_call", "item_id": "item_call_function_001", "call_id": "call_function_001", "arguments": ""}, + "result": {"type": "function_call_output", "item_id": "item_output_function_001", "call_id": "call_function_001", "output": ""}, + "history": {"call_id": "call_function_001", "call_item_id": "item_call_function_001", "output_item_id": "item_output_function_001"}, + "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "arguments_done": "response.function_call_arguments.done", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "response.function_call_arguments.delta", "response.function_call_arguments.done", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + "custom_freeform": { + "declaration": {"type": "custom", "name": "custom_alpha", "format": ""}, + "call": {"type": "custom_tool_call", "item_id": "item_call_custom_001", "call_id": "call_custom_001", "input": ""}, + "result": {"type": "custom_tool_call_output", "item_id": "item_output_custom_001", "call_id": "call_custom_001", "output": ""}, + "history": {"call_id": "call_custom_001", "call_item_id": "item_call_custom_001", "output_item_id": "item_output_custom_001"}, + "streaming": {"added": "response.output_item.added", "delta": "response.custom_tool_call_input.delta", "input_done": "response.custom_tool_call_input.done", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "response.custom_tool_call_input.delta", "response.custom_tool_call_input.done", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + "namespace": { + "declaration": {"type": "namespace", "name": "namespace_alpha", "tools": [{"type": "function", "name": "function_beta", "parameters": ""}]}, + "call": {"type": "function_call", "item_id": "item_call_namespace_001", "call_id": "call_namespace_001", "namespace": "namespace_alpha", "arguments": ""}, + "result": {"type": "function_call_output", "item_id": "item_output_namespace_001", "call_id": "call_namespace_001", "output": ""}, + "history": {"call_id": "call_namespace_001", "call_item_id": "item_call_namespace_001", "output_item_id": "item_output_namespace_001", "namespace": "namespace_alpha"}, + "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "arguments_done": "response.function_call_arguments.done", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "response.function_call_arguments.delta", "response.function_call_arguments.done", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "preserve declaration and inverse call/result/history IDs" + }, + "client_executed_tool_discovery": { + "declaration": {"type": "tool_search", "execution": "client", "parameters": ""}, + "call": {"type": "tool_search_call", "item_id": "item_call_search_001", "call_id": "call_search_001", "execution": "client", "arguments": ""}, + "result": {"type": "tool_search_output", "item_id": "item_output_search_001", "call_id": "call_search_001", "execution": "client", "tools": ""}, + "history": {"call_id": "call_search_001", "call_item_id": "item_call_search_001", "output_item_id": "item_output_search_001", "executor": "codex_client"}, + "streaming": {"added": null, "delta": null, "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "discovery request/result stays client-executed" + }, + "selected_provider_hosted": { + "declaration": {"type": "web_search", "executor": "selected_provider", "provider_scope": "selected_provider_only"}, + "call": {"type": "web_search_call", "item_id": "item_call_hosted_001", "status": "", "action": ""}, + "result": {"type": "web_search_call", "item_id": "item_output_hosted_001", "status": "", "provider_scope": "selected_provider_only"}, + "history": {"call_id": null, "call_item_id": "item_call_hosted_001", "output_item_id": "item_output_hosted_001", "executor": "selected_provider", "cross_provider_proxy": "forbidden"}, + "streaming": {"added": "response.output_item.added", "delta": "", "done": "response.output_item.done", "terminal": "response.completed", "event_order": ["response.output_item.added", "", "response.output_item.done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "observed": false, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden" + }, + "unknown_future_kind": { + "declaration": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "call": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "result": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, + "history": {"call_id": null, "call_item_id": null, "output_item_id": null, "loss_rule": "retain opaque sentinel"}, + "streaming": {"added": "unknown.future_event", "delta": "unknown.future_delta", "done": "unknown.future_done", "terminal": "response.completed", "event_order": ["unknown.future_event", "unknown.future_delta", "unknown.future_done", "response.completed"]}, + "terminal": {"event": "response.completed", "classification": "not_observed"}, + "error": {"event": "response.failed", "classification": "unqualified"}, + "loss_boundary": "retain tag and opaque payload; do not normalize", + "observed": false, + "status": "opaque_sentinel_only" + } + } + } +} diff --git a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json index 9b059970..77a22173 100644 --- a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json +++ b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json @@ -23,150 +23,10 @@ "classification_basis": "upstream_route and catalog_binding; never configured_provider_id alone" }, "provenance": { - "cli_version": "0.146.0", - "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", - "cli_source_commit_status": "published_attested", - "cli_source_tag": "rust-v0.146.0", - "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", - "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", "capture_id": "sanitized-current-side-capture" }, - "runtime_wire_surface": { - "source": "Codex CLI 0.146.0 ToolSpec and ResponseItem contracts", - "declaration_family_order": [ - "plain_function", - "custom_freeform", - "namespace", - "client_executed_tool_discovery", - "selected_provider_hosted", - "unknown_future_kind" - ], - "request_shape": { - "protocol": "responses", - "streaming_fields": [ - "model", - "instructions", - "input", - "tools", - "tool_choice", - "parallel_tool_calls", - "reasoning", - "store", - "stream", - "stream_options", - "include", - "service_tier", - "prompt_cache_key", - "text", - "client_metadata" - ], - "representative": { - "model": "gpt-5.6-sol", - "input": "", - "tools": "", - "tool_choice": "auto", - "parallel_tool_calls": false, - "stream": true, - "store": false - }, - "non_streaming_control": { - "stream": false, - "response_body": "", - "captured": false, - "status": "live_control_required" - } - }, - "response_shape": { - "response_item_types": [ - "message", - "reasoning", - "function_call", - "function_call_output", - "custom_tool_call", - "custom_tool_call_output", - "tool_search_call", - "tool_search_output", - "web_search_call", - "local_shell_call", - "compaction", - "context_compaction", - "unknown" - ], - "stream_event_order": [ - "response.created", - "response.in_progress", - "response.output_item.added", - "response.output_text.delta", - "response.function_call_arguments.delta", - "response.custom_tool_call_input.delta", - "response.output_item.done", - "response.reasoning_summary_part.added", - "response.reasoning_summary_text.delta", - "response.reasoning_summary_text.done", - "response.completed" - ], - "terminal_events": [ - "response.completed", - "response.incomplete", - "response.failed" - ], - "error_shape": { - "event": "response.failed", - "response": {"id": "response_error_001", "status": "failed", "error": ""}, - "classification": "bounded_error_only" - } - }, - "declaration_family_examples": { - "plain_function": { - "declaration": {"type": "function", "name": "function_alpha", "parameters": ""}, - "call": {"type": "function_call", "item_id": "item_call_function_001", "call_id": "call_function_001", "arguments": ""}, - "result": {"type": "function_call_output", "item_id": "item_output_function_001", "call_id": "call_function_001", "output": ""}, - "history": {"call_id": "call_function_001", "call_item_id": "item_call_function_001", "output_item_id": "item_output_function_001"}, - "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "done": "response.output_item.done", "terminal": "response.completed"} - }, - "custom_freeform": { - "declaration": {"type": "custom", "name": "custom_alpha", "format": ""}, - "call": {"type": "custom_tool_call", "item_id": "item_call_custom_001", "call_id": "call_custom_001", "input": ""}, - "result": {"type": "custom_tool_call_output", "item_id": "item_output_custom_001", "call_id": "call_custom_001", "output": ""}, - "history": {"call_id": "call_custom_001", "call_item_id": "item_call_custom_001", "output_item_id": "item_output_custom_001"}, - "streaming": {"added": "response.output_item.added", "delta": "response.custom_tool_call_input.delta", "done": "response.output_item.done", "terminal": "response.completed"} - }, - "namespace": { - "declaration": {"type": "namespace", "name": "namespace_alpha", "tools": [{"type": "function", "name": "function_beta", "parameters": ""}]}, - "call": {"type": "function_call", "item_id": "item_call_namespace_001", "call_id": "call_namespace_001", "namespace": "namespace_alpha", "arguments": ""}, - "result": {"type": "function_call_output", "item_id": "item_output_namespace_001", "call_id": "call_namespace_001", "output": ""}, - "history": {"call_id": "call_namespace_001", "call_item_id": "item_call_namespace_001", "output_item_id": "item_output_namespace_001", "namespace": "namespace_alpha"}, - "streaming": {"added": "response.output_item.added", "delta": "response.function_call_arguments.delta", "done": "response.output_item.done", "terminal": "response.completed"} - }, - "client_executed_tool_discovery": { - "declaration": {"type": "tool_search", "execution": "client", "parameters": ""}, - "call": {"type": "tool_search_call", "item_id": "item_call_search_001", "call_id": "call_search_001", "execution": "client", "arguments": ""}, - "result": {"type": "tool_search_output", "item_id": "item_output_search_001", "call_id": "call_search_001", "execution": "client", "tools": ""}, - "history": {"call_id": "call_search_001", "call_item_id": "item_call_search_001", "output_item_id": "item_output_search_001", "executor": "codex_client"}, - "streaming": {"added": "response.output_item.added", "delta": "response.output_text.delta", "done": "response.output_item.done", "terminal": "response.completed"} - }, - "selected_provider_hosted": { - "declaration": {"type": "web_search", "executor": "selected_provider", "provider_scope": "selected_provider_only"}, - "call": {"type": "web_search_call", "item_id": "item_call_hosted_001", "status": "", "action": ""}, - "result": {"type": "web_search_call", "item_id": "item_output_hosted_001", "status": "", "provider_scope": "selected_provider_only"}, - "history": {"call_id": null, "call_item_id": "item_call_hosted_001", "output_item_id": "item_output_hosted_001", "executor": "selected_provider", "cross_provider_proxy": "forbidden"}, - "streaming": {"added": "response.output_item.added", "delta": "", "done": "response.output_item.done", "terminal": "response.completed"}, - "observed": false, - "status": "selected_provider_control_required", - "provider_scope": "selected_provider_only", - "cross_provider_proxy": "forbidden" - }, - "unknown_future_kind": { - "declaration": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, - "call": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, - "result": {"type": "unknown", "tag": "unknown", "opaque_payload": ""}, - "history": {"call_id": null, "call_item_id": null, "output_item_id": null, "loss_rule": "retain opaque sentinel"}, - "streaming": {"added": "unknown.future_event", "delta": "unknown.future_delta", "done": "unknown.future_done", "terminal": "response.completed"}, - "observed": false, - "status": "opaque_sentinel_only" - } - } - }, "exposure_state_tags": ["Direct", "DirectModelOnly", "Deferred", "Hidden", "hosted-only", "host-unavailable"], "pre_gateway": { "request_id": "request_001", diff --git a/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json b/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json index 60ff68df..fcfedaff 100644 --- a/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json +++ b/docs/evidence/issue-62/current-codexhub-thread-tool-surface.json @@ -4,10 +4,7 @@ "source": { "capture_id": "sanitized-current-side-capture", "originator": "Codex Desktop", - "cli_version": "0.146.0", - "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", - "cli_source_commit_status": "published_attested", - "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", + "cli_version": "0.144.0-alpha.4", "configured_provider_id": "custom", "model": "gpt-5.6-sol", "reasoning_effort": "xhigh", @@ -90,8 +87,7 @@ {"state": "host-unavailable", "plane": "host-binding", "runtime_observed": false, "evidence": "tagged reconciliation sentinel only; not inferred as a ToolExposure enum variant"} ], "planner_gates": { - "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", - "cli_source_tag": "rust-v0.146.0", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", "predicate": "model_info.supports_search_tool && provider.capabilities.namespace_tools", "catalog_source": { "post_switch_catalog_basename": "codexhub-model-catalog.json", @@ -119,62 +115,6 @@ "codex_app_direct_tools": ["navigate_to_codex_page", "read_thread_terminal", "load_workspace_dependencies"], "codex_app_deferred_tools_discoverable_through_tool_search": ["automation_update", "fork_thread", "handoff_thread", "get_handoff_status", "list_projects", "create_thread", "list_threads", "read_thread", "send_message_to_thread", "set_thread_pinned", "set_thread_archived", "set_thread_title"] }, - "declaration_families": [ - { - "family": "plain_function", - "runtime_type": "function", - "wire_declaration_type": "function", - "observed": true, - "observation": "observed", - "executor": "codex_client", - "loss_boundary": "preserve declaration and inverse call/result/history IDs" - }, - { - "family": "custom_freeform", - "runtime_type": "custom", - "wire_declaration_type": "custom", - "observed": true, - "observation": "observed", - "executor": "codex_client", - "loss_boundary": "preserve declaration and inverse call/result/history IDs" - }, - { - "family": "namespace", - "runtime_type": "namespace", - "wire_declaration_type": "namespace", - "observed": true, - "observation": "observed", - "executor": "codex_client", - "loss_boundary": "preserve declaration and inverse call/result/history IDs" - }, - { - "family": "client_executed_tool_discovery", - "runtime_type": "tool_search", - "wire_declaration_type": "tool_search", - "observed": true, - "observation": "observed_client_execution", - "executor": "codex_client", - "loss_boundary": "discovery request/result stays client-executed" - }, - { - "family": "selected_provider_hosted", - "runtime_type": "web_search", - "wire_declaration_type": "web_search", - "observed": false, - "observation": "not_observed_selected_provider_only", - "executor": "selected_provider", - "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly" - }, - { - "family": "unknown_future_kind", - "runtime_type": "unknown", - "wire_declaration_type": "", - "observed": false, - "observation": "opaque_sentinel_only", - "executor": "unknown", - "loss_boundary": "retain tag and opaque payload; do not normalize" - } - ], "runtime_gate_evaluation_for_this_trace": "not_independently_captured" }, "independent_controls": [ diff --git a/docs/evidence/issue-62/read-only-gate-audit.json b/docs/evidence/issue-62/read-only-gate-audit.json index d17337d7..a4c9e8ef 100644 --- a/docs/evidence/issue-62/read-only-gate-audit.json +++ b/docs/evidence/issue-62/read-only-gate-audit.json @@ -1,5 +1,19 @@ { "capture_kind": "sanitized_bounded_read_only_audit", + "provenance": { + "capture_status": "not_observed", + "cli_version": "0.146.0", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", + "cli_source_commit_status": "published_attested", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", + "historical_capture": { + "captured_at": "2026-07-12T14:57:55+08:00", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38" + } + }, "gate_classification": { "choice_controls": "observed", "clean_cold_start_current_binding": "live_control_required", diff --git a/docs/evidence/issue-62/runtime-wire-inventory.json b/docs/evidence/issue-62/runtime-wire-inventory.json index b8e69c99..392a99cb 100644 --- a/docs/evidence/issue-62/runtime-wire-inventory.json +++ b/docs/evidence/issue-62/runtime-wire-inventory.json @@ -11,7 +11,7 @@ "cli_version": "0.146.0", "codex_source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "configured_provider_id": "custom", - "evidence_manifest_sha256": "1e7d07eed427926b69b7bf7d8113f9b3913e02b16be7ffbf17ee64ca58e3d94f", + "evidence_manifest_sha256": "b25da7980235919ddfe19f11d34e2aba2bb8b42ec7828398362e71f01ce79fa9", "inbound_format": "responses", "model": "gpt-5.6-sol", "route_behavior_profile": "official_codex_app_http_passthrough", @@ -22,12 +22,12 @@ "cli_version_floor": "0.146.0", "declaration_families": [ { - "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", "executor": "codex_client", "family": "plain_function", "loss_boundary": "preserve declaration and inverse call/result/history IDs", - "observation": "observed", - "observed": true, + "observation": "not_observed_source_contract_only", + "observed": false, "optional_rule": "preserve_when_supported_else_omit", "representative": { "call": { @@ -41,11 +41,16 @@ "parameters": "", "type": "function" }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, "history": { "call_id": "call_function_001", "call_item_id": "item_call_function_001", "output_item_id": "item_output_function_001" }, + "loss_boundary": "preserve declaration and inverse call/result/history IDs", "result": { "call_id": "call_function_001", "item_id": "item_output_function_001", @@ -54,9 +59,21 @@ }, "streaming": { "added": "response.output_item.added", + "arguments_done": "response.function_call_arguments.done", "delta": "response.function_call_arguments.delta", "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed" + ], "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" } }, "required_rule": "preserve_when_supported_else_required-but-unavailable", @@ -65,12 +82,12 @@ "wire_declaration_type": "function" }, { - "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.custom_freeform", "executor": "codex_client", "family": "custom_freeform", "loss_boundary": "preserve declaration and inverse call/result/history IDs", - "observation": "observed", - "observed": true, + "observation": "not_observed_source_contract_only", + "observed": false, "optional_rule": "preserve_when_supported_else_omit", "representative": { "call": { @@ -84,11 +101,16 @@ "name": "custom_alpha", "type": "custom" }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, "history": { "call_id": "call_custom_001", "call_item_id": "item_call_custom_001", "output_item_id": "item_output_custom_001" }, + "loss_boundary": "preserve declaration and inverse call/result/history IDs", "result": { "call_id": "call_custom_001", "item_id": "item_output_custom_001", @@ -99,7 +121,19 @@ "added": "response.output_item.added", "delta": "response.custom_tool_call_input.delta", "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "response.custom_tool_call_input.delta", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.completed" + ], + "input_done": "response.custom_tool_call_input.done", "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" } }, "required_rule": "preserve_when_supported_else_required-but-unavailable", @@ -108,12 +142,12 @@ "wire_declaration_type": "custom" }, { - "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.namespace", "executor": "codex_client", "family": "namespace", "loss_boundary": "preserve declaration and inverse call/result/history IDs", - "observation": "observed", - "observed": true, + "observation": "not_observed_source_contract_only", + "observed": false, "optional_rule": "preserve_when_supported_else_omit", "representative": { "call": { @@ -134,12 +168,17 @@ ], "type": "namespace" }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, "history": { "call_id": "call_namespace_001", "call_item_id": "item_call_namespace_001", "namespace": "namespace_alpha", "output_item_id": "item_output_namespace_001" }, + "loss_boundary": "preserve declaration and inverse call/result/history IDs", "result": { "call_id": "call_namespace_001", "item_id": "item_output_namespace_001", @@ -148,9 +187,21 @@ }, "streaming": { "added": "response.output_item.added", + "arguments_done": "response.function_call_arguments.done", "delta": "response.function_call_arguments.delta", "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed" + ], "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" } }, "required_rule": "preserve_when_supported_else_required-but-unavailable", @@ -159,12 +210,12 @@ "wire_declaration_type": "namespace" }, { - "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.client_executed_tool_discovery", "executor": "codex_client", "family": "client_executed_tool_discovery", "loss_boundary": "discovery request/result stays client-executed", - "observation": "observed_client_execution", - "observed": true, + "observation": "not_observed_source_contract_only", + "observed": false, "optional_rule": "preserve_client_execution_else_omit", "representative": { "call": { @@ -179,12 +230,17 @@ "parameters": "", "type": "tool_search" }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, "history": { "call_id": "call_search_001", "call_item_id": "item_call_search_001", "executor": "codex_client", "output_item_id": "item_output_search_001" }, + "loss_boundary": "discovery request/result stays client-executed", "result": { "call_id": "call_search_001", "execution": "client", @@ -193,10 +249,18 @@ "type": "tool_search_output" }, "streaming": { - "added": "response.output_item.added", - "delta": "response.output_text.delta", + "added": null, + "delta": null, "done": "response.output_item.done", + "event_order": [ + "response.output_item.done", + "response.completed" + ], "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" } }, "required_rule": "preserve_client_execution_else_required-but-unavailable", @@ -205,11 +269,11 @@ "wire_declaration_type": "tool_search" }, { - "evidence_source": "read-only-gate-audit.json#model_visible_request_plan.declaration_families", + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.selected_provider_hosted", "executor": "selected_provider", "family": "selected_provider_hosted", "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", - "observation": "not_observed_selected_provider_only", + "observation": "not_observed_selected_provider_control_required", "observed": false, "optional_rule": "omit_if_selected_provider_unsupported", "representative": { @@ -225,6 +289,10 @@ "provider_scope": "selected_provider_only", "type": "web_search" }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, "history": { "call_id": null, "call_item_id": "item_call_hosted_001", @@ -232,6 +300,7 @@ "executor": "selected_provider", "output_item_id": "item_output_hosted_001" }, + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", "observed": false, "provider_scope": "selected_provider_only", "result": { @@ -245,7 +314,17 @@ "added": "response.output_item.added", "delta": "", "done": "response.output_item.done", + "event_order": [ + "response.output_item.added", + "", + "response.output_item.done", + "response.completed" + ], "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" } }, "required_rule": "required-but-unavailable_if_selected_provider_unsupported", @@ -254,7 +333,7 @@ "wire_declaration_type": "web_search" }, { - "evidence_source": "codexhub-runtime-wire-fixture.json#runtime_wire_surface.unknown_future", + "evidence_source": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.unknown_future_kind", "executor": "unknown", "family": "unknown_future_kind", "loss_boundary": "retain tag and opaque payload; do not normalize", @@ -272,12 +351,17 @@ "tag": "unknown", "type": "unknown" }, + "error": { + "classification": "unqualified", + "event": "response.failed" + }, "history": { "call_id": null, "call_item_id": null, "loss_rule": "retain opaque sentinel", "output_item_id": null }, + "loss_boundary": "retain tag and opaque payload; do not normalize", "observed": false, "result": { "opaque_payload": "", @@ -289,7 +373,17 @@ "added": "unknown.future_event", "delta": "unknown.future_delta", "done": "unknown.future_done", + "event_order": [ + "unknown.future_event", + "unknown.future_delta", + "unknown.future_done", + "response.completed" + ], "terminal": "response.completed" + }, + "terminal": { + "classification": "not_observed", + "event": "response.completed" } }, "required_rule": "required-but-unavailable", @@ -308,15 +402,19 @@ "evidence_binding": { "audit": { "file": "read-only-gate-audit.json", - "sha256": "eeed7c0ad927d2abf0a10f695530bc38242fafa0ec59ae5898c0bfb307bc83f0" + "sha256": "775efbc763dfae1c5d6a61532d9394e5cd181b2e6803bf2e42f6c886a0a964e3" + }, + "source_contract": { + "file": "codex-0.146-source-contract.json", + "sha256": "6f38b8b07b98c6f28edd7418b63449242ee41d6396694392a70c0d7fb2b70f2c" }, "trace": { "file": "current-codexhub-thread-tool-surface.json", - "sha256": "47bdc0b56a879a734779aeb66aedce6c18f024578219a77548077a25a2976856" + "sha256": "a4367f96fc2aceaa419577ee43f91870fb3fcca4cc95644bc3347cbdd4206ee3" }, "wire_fixture": { "file": "codexhub-runtime-wire-fixture.json", - "sha256": "106cfe30d5933d758fb52f8ff015d578c7c3211107f8c490c3fd1ebd9224d8ce" + "sha256": "3a8654854539799b526bc013c50bc9f16c40c09319cff8da716b75e6deecbc3a" } }, "evidence_sources": { diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index 9642a73f..d953979e 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -43,6 +43,13 @@ CLI_SOURCE_TAG = "rust-v0.146.0" CLI_SOURCE_COMMIT_STATUS = "published_attested" CLI_BINARY_SHA256 = "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb" +CLI_SOURCE_COMMIT = "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" +CANDIDATE_REVISION = "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" +HISTORICAL_CLI_VERSION = "0.144.0-alpha.4" +HISTORICAL_SOURCE_COMMIT = "9e552e9d15ba52bed7077d5357f3e18e330f8f38" +DEFAULT_SOURCE_CONTRACT = Path( + "docs/evidence/issue-62/codex-0.146-source-contract.json" +) STRUCTURAL_FAMILIES = ( "plain_function", "custom_freeform", @@ -604,21 +611,137 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: }, } +STRUCTURAL_EVIDENCE_SOURCES = { + "plain_function": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", + "custom_freeform": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.custom_freeform", + "namespace": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.namespace", + "client_executed_tool_discovery": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.client_executed_tool_discovery", + "selected_provider_hosted": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.selected_provider_hosted", + "unknown_future_kind": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.unknown_future_kind", +} + +STRUCTURAL_STREAM_DONE_EVENTS = { + "plain_function": ("arguments_done", "response.function_call_arguments.done"), + "custom_freeform": ("input_done", "response.custom_tool_call_input.done"), + "namespace": ("arguments_done", "response.function_call_arguments.done"), + "unknown_future_kind": ("done", "unknown.future_done"), +} + + +def _validate_structural_stream_contract( + *, wire: dict[str, Any], examples: dict[str, Any] +) -> None: + """Require family SSE examples to retain done-before-terminal ordering.""" + + response_shape = wire.get("runtime_wire_surface", {}).get("response_shape", {}) + stream_event_order = response_shape.get("stream_event_order") + if not isinstance(stream_event_order, list) or not all( + isinstance(event, str) and event for event in stream_event_order + ): + raise ValueError("wire response_shape.stream_event_order must be a non-empty string array") + if "response.completed" not in stream_event_order: + raise ValueError("wire response_shape.stream_event_order is missing response.completed") + completed_index = stream_event_order.index("response.completed") + output_done_index = ( + stream_event_order.index("response.output_item.done") + if "response.output_item.done" in stream_event_order + else None + ) + for family_name, (_, event) in STRUCTURAL_STREAM_DONE_EVENTS.items(): + if event.startswith("unknown."): + continue + if event not in stream_event_order: + raise ValueError( + f"wire response_shape.stream_event_order is missing {event} for {family_name}" + ) + event_index = stream_event_order.index(event) + if event_index >= completed_index or ( + output_done_index is not None and event_index >= output_done_index + ): + raise ValueError(f"wire stream event {event} must precede terminal/item completion") + + for family_name in STRUCTURAL_FAMILIES: + example = examples[family_name] + streaming = example.get("streaming") + if not isinstance(streaming, dict): + raise ValueError(f"wire runtime streaming example is malformed for {family_name}") + event_order = streaming.get("event_order") + if not isinstance(event_order, list) or not all( + isinstance(event, str) and event for event in event_order + ): + raise ValueError(f"wire runtime streaming event_order is invalid for {family_name}") + if streaming.get("terminal") != "response.completed" or event_order[-1] != "response.completed": + raise ValueError(f"wire runtime streaming terminal ordering is invalid for {family_name}") + if family_name != "unknown_future_kind" and ( + "response.output_item.done" not in event_order + or event_order.index("response.output_item.done") >= len(event_order) - 1 + ): + raise ValueError(f"wire runtime streaming item completion is invalid for {family_name}") + done_spec = STRUCTURAL_STREAM_DONE_EVENTS.get(family_name) + if done_spec is not None: + field, event = done_spec + if streaming.get(field) != event or event not in event_order: + raise ValueError( + f"wire runtime streaming family done event is missing for {family_name}" + ) + if event_order.index(event) >= event_order.index("response.completed"): + raise ValueError(f"wire runtime streaming family done event follows terminal for {family_name}") + if family_name == "client_executed_tool_discovery": + if event_order != ["response.output_item.done", "response.completed"]: + raise ValueError("tool_search_call SSE must contain only item completion and terminal events") + if streaming.get("added") is not None or streaming.get("delta") is not None: + raise ValueError("tool_search_call SSE must not claim a text delta or item-added event") + + +def _validate_structural_evidence_pointers( + *, + source_contract: dict[str, Any], + wire: dict[str, Any], + audit: dict[str, Any], + declaration_families: list[dict[str, Any]], +) -> None: + """Require every declaration-family evidence source to resolve in its fixture.""" + + documents = { + "codex-0.146-source-contract.json": source_contract, + "codexhub-runtime-wire-fixture.json": wire, + "read-only-gate-audit.json": audit, + } + for entry in declaration_families: + family = entry.get("family") + source = entry.get("evidence_source") + expected = STRUCTURAL_EVIDENCE_SOURCES.get(family) + if expected is None or source != expected: + raise ValueError( + f"declaration-family evidence source is invalid for {family!r}: {source!r}" + ) + filename, pointer_text = source.split("#", 1) + document = documents.get(filename) + if document is None: + raise ValueError(f"declaration-family evidence fixture is unknown: {filename}") + try: + _resolve_fixture_pointer(document, tuple(pointer_text.split("."))) + except KeyError as exc: + raise ValueError( + f"declaration-family evidence pointer for {family} is missing: " + f"{filename}#{exc.args[0]}" + ) from exc + def _build_structural_inventory( - *, trace: dict[str, Any], wire: dict[str, Any], audit: dict[str, Any] + *, + source_contract: dict[str, Any], + trace: dict[str, Any], + wire: dict[str, Any], + audit: dict[str, Any], ) -> list[dict[str, Any]]: - """Join exact 0.146 planner family records with sanitized wire examples.""" + """Join the unobserved 0.146 source contract with sanitized examples.""" - audit_families = ( - audit.get("model_visible_request_plan", {}).get("declaration_families", []) + families = source_contract.get("runtime_wire_surface", {}).get( + "declaration_families", [] ) - trace_families = trace.get("planner_gates", {}).get("declaration_families", []) - if audit_families and trace_families and audit_families != trace_families: - raise ValueError("runtime planner declaration families contradict across trace and audit") - families = audit_families or trace_families if not isinstance(families, list): - raise ValueError("runtime planner declaration_families must be an array") + raise ValueError("source contract declaration_families must be an array") by_family: dict[str, dict[str, Any]] = {} for family in families: if not isinstance(family, dict) or not isinstance(family.get("family"), str): @@ -635,8 +758,8 @@ def _build_structural_inventory( f"missing={missing!r} extra={extra!r}" ) - examples = ( - wire.get("runtime_wire_surface", {}).get("declaration_family_examples", {}) + examples = source_contract.get("runtime_wire_surface", {}).get( + "declaration_family_examples", {} ) if not isinstance(examples, dict): raise ValueError("wire runtime declaration_family_examples must be an object") @@ -646,12 +769,31 @@ def _build_structural_inventory( example = examples.get(family_name) if not isinstance(example, dict): raise ValueError(f"wire runtime example missing for {family_name}") - required_example_parts = {"declaration", "call", "result", "history", "streaming"} + required_example_parts = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } if not required_example_parts.issubset(example): missing = sorted(required_example_parts - set(example)) raise ValueError( f"wire runtime example for {family_name} is missing {missing!r}" ) + if not isinstance(example.get("terminal"), dict) or example["terminal"].get( + "classification" + ) not in {"not_observed", "unqualified"}: + raise ValueError(f"runtime terminal classification is invalid for {family_name}") + if not isinstance(example.get("error"), dict) or example["error"].get( + "classification" + ) not in {"not_observed", "unqualified"}: + raise ValueError(f"runtime error classification is invalid for {family_name}") + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"runtime loss boundary is missing for {family_name}") rule = _STRUCTURAL_RULES[family_name] if family.get("executor") != rule["executor"]: raise ValueError( @@ -662,6 +804,24 @@ def _build_structural_inventory( raise ValueError("hosted declaration is not bound to the selected Provider") if example.get("cross_provider_proxy") != "forbidden": raise ValueError("hosted declaration permits a cross-Provider proxy") + if family_name in { + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + }: + call = example["call"] + result = example["result"] + history = example["history"] + if not all(isinstance(part, dict) for part in (call, result, history)): + raise ValueError(f"runtime identity sections are malformed for {family_name}") + if ( + call.get("item_id") != history.get("call_item_id") + or result.get("item_id") != history.get("output_item_id") + or call.get("call_id") != result.get("call_id") + or call.get("call_id") != history.get("call_id") + ): + raise ValueError(f"runtime call/result/history IDs do not reconcile for {family_name}") output.append( { "family": family_name, @@ -674,14 +834,17 @@ def _build_structural_inventory( "optional_rule": rule["optional_rule"], "required_rule": rule["required_rule"], "loss_boundary": family.get("loss_boundary"), - "evidence_source": ( - "read-only-gate-audit.json#model_visible_request_plan.declaration_families" - if family_name != "unknown_future_kind" - else "codexhub-runtime-wire-fixture.json#runtime_wire_surface.unknown_future" - ), + "evidence_source": STRUCTURAL_EVIDENCE_SOURCES[family_name], "representative": example, } ) + _validate_structural_stream_contract(wire=source_contract, examples=examples) + _validate_structural_evidence_pointers( + source_contract=source_contract, + wire=wire, + audit=audit, + declaration_families=output, + ) return output @@ -695,7 +858,16 @@ def _structural_inventory_mismatches(value: Any) -> list[str]: mismatches.append( "declaration_families must contain exactly the six known families" ) - required_example_parts = {"declaration", "call", "result", "history", "streaming"} + required_example_parts = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } for index, family_name in enumerate(STRUCTURAL_FAMILIES): if index >= len(value): break @@ -720,11 +892,7 @@ def _structural_inventory_mismatches(value: Any) -> list[str]: for field in ("selected_protocol_disposition", "optional_rule", "required_rule"): if entry.get(field) != rule[field]: mismatches.append(f"{prefix}.{field} does not match the family rule") - expected_source = ( - "read-only-gate-audit.json#model_visible_request_plan.declaration_families" - if family_name != "unknown_future_kind" - else "codexhub-runtime-wire-fixture.json#runtime_wire_surface.unknown_future" - ) + expected_source = STRUCTURAL_EVIDENCE_SOURCES[family_name] if entry.get("evidence_source") != expected_source: mismatches.append(f"{prefix}.evidence_source does not match the family source") representative = entry.get("representative") @@ -733,6 +901,95 @@ def _structural_inventory_mismatches(value: Any) -> list[str]: continue if not required_example_parts.issubset(representative): mismatches.append(f"{prefix}.representative is missing a wire example section") + terminal = representative.get("terminal") + if not isinstance(terminal, dict): + mismatches.append(f"{prefix}.representative.terminal must be an object") + else: + if terminal.get("event") != "response.completed": + mismatches.append( + f"{prefix}.representative.terminal.event must be response.completed" + ) + if terminal.get("classification") not in {"not_observed", "unqualified"}: + mismatches.append( + f"{prefix}.representative.terminal.classification is invalid" + ) + error = representative.get("error") + if not isinstance(error, dict): + mismatches.append(f"{prefix}.representative.error must be an object") + else: + if error.get("event") != "response.failed": + mismatches.append( + f"{prefix}.representative.error.event must be response.failed" + ) + if error.get("classification") not in {"not_observed", "unqualified"}: + mismatches.append( + f"{prefix}.representative.error.classification is invalid" + ) + if not isinstance(representative.get("loss_boundary"), str) or not representative[ + "loss_boundary" + ]: + mismatches.append(f"{prefix}.representative.loss_boundary is missing or blank") + elif representative["loss_boundary"] != entry.get("loss_boundary"): + mismatches.append( + f"{prefix}.representative.loss_boundary does not match the family boundary" + ) + streaming = representative.get("streaming") + if not isinstance(streaming, dict): + mismatches.append(f"{prefix}.representative.streaming must be an object") + else: + event_order = streaming.get("event_order") + if not isinstance(event_order, list) or not event_order: + mismatches.append(f"{prefix}.representative.streaming.event_order is invalid") + elif streaming.get("terminal") != "response.completed" or event_order[-1] != "response.completed": + mismatches.append( + f"{prefix}.representative.streaming terminal ordering is invalid" + ) + else: + done_spec = STRUCTURAL_STREAM_DONE_EVENTS.get(family_name) + if family_name != "unknown_future_kind" and ( + "response.output_item.done" not in event_order + or event_order.index("response.output_item.done") >= len(event_order) - 1 + ): + mismatches.append( + f"{prefix}.representative.streaming item completion is invalid" + ) + if done_spec is not None: + field, event = done_spec + if streaming.get(field) != event or event not in event_order: + mismatches.append( + f"{prefix}.representative.streaming family done event is missing" + ) + elif event_order.index(event) >= event_order.index("response.completed"): + mismatches.append( + f"{prefix}.representative.streaming family done event follows terminal" + ) + if family_name == "client_executed_tool_discovery": + if event_order != ["response.output_item.done", "response.completed"]: + mismatches.append( + f"{prefix}.representative.streaming tool-search order is invalid" + ) + if streaming.get("added") is not None or streaming.get("delta") is not None: + mismatches.append( + f"{prefix}.representative.streaming tool-search claims text events" + ) + if family_name in { + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + }: + call = representative.get("call") + result = representative.get("result") + history = representative.get("history") + if not all(isinstance(part, dict) for part in (call, result, history)): + mismatches.append(f"{prefix}.representative identity sections are malformed") + elif ( + call.get("item_id") != history.get("call_item_id") + or result.get("item_id") != history.get("output_item_id") + or call.get("call_id") != result.get("call_id") + or call.get("call_id") != history.get("call_id") + ): + mismatches.append(f"{prefix}.representative call/result/history IDs do not reconcile") if family_name == "selected_provider_hosted": if representative.get("provider_scope") != "selected_provider_only": mismatches.append(f"{prefix}.representative is not selected-provider scoped") @@ -1264,36 +1521,50 @@ def _build_qualification( } +def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any]: + if ( + source_contract.get("schema_version") != 1 + or source_contract.get("fixture_kind") != "codex_cli_source_contract" + or source_contract.get("capture_status") != "not_observed" + or source_contract.get("qualification_status") != "unqualified" + or source_contract.get("captured_at") is not None + ): + raise ValueError("Codex 0.146 source contract must remain not_observed and unqualified") + provenance = source_contract.get("provenance", {}) + expected = { + "cli_version": DEFAULT_CLI_FLOOR, + "source_commit": CLI_SOURCE_COMMIT, + "cli_source_tag": CLI_SOURCE_TAG, + "cli_source_commit_status": CLI_SOURCE_COMMIT_STATUS, + "cli_binary_sha256": CLI_BINARY_SHA256, + "candidate_revision": CANDIDATE_REVISION, + } + if any(provenance.get(field) != value for field, value in expected.items()): + raise ValueError("Codex 0.146 source contract provenance is invalid") + return provenance + + def _validate_candidate_binding( *, + source_contract_data: dict[str, Any], trace_data: dict[str, Any], wire_data: dict[str, Any], + audit_data: dict[str, Any], cli_version_floor: str, candidate_cli_version: str | None, candidate_source_commit: str | None, ) -> tuple[dict[str, Any], str]: _validate_supported_floor(cli_version_floor) + contract_provenance = _validate_source_contract(source_contract_data) source = trace_data.get("source", {}) planner_gates = trace_data.get("planner_gates", {}) trace_cli_version = source.get("cli_version") trace_source_commit = planner_gates.get("source_commit") trace_capture_id = source.get("capture_id") - trace_candidate_revision = source.get("candidate_revision") - trace_binary_sha256 = source.get("cli_binary_sha256") - trace_source_status = source.get("cli_source_commit_status") - trace_source_tag = planner_gates.get("cli_source_tag") - if ( - not trace_cli_version - or not trace_source_commit - or not trace_capture_id - or not trace_candidate_revision - or not trace_binary_sha256 - or not trace_source_status - or not trace_source_tag - ): + if not trace_cli_version or not trace_source_commit or not trace_capture_id: raise ValueError( - "trace evidence is missing source.cli_version, source.capture_id, " - "source candidate provenance, or planner_gates.source_commit" + "historical trace evidence is missing source.cli_version, " + "source.capture_id, or planner_gates.source_commit" ) if not isinstance(trace_cli_version, str): raise ValueError("trace source.cli_version must be a string") @@ -1301,47 +1572,27 @@ def _validate_candidate_binding( _version_key(trace_cli_version) except ValueError as exc: raise ValueError("trace source.cli_version is malformed") from exc - if not isinstance(trace_source_commit, str) or not re.fullmatch( - r"[0-9a-f]{40}", trace_source_commit - ): - raise ValueError( - "trace planner_gates.source_commit must be a lowercase 40-character SHA-1" - ) - if not isinstance(trace_candidate_revision, str) or not re.fullmatch( - r"[0-9a-f]{40}", trace_candidate_revision - ): - raise ValueError("trace source.candidate_revision must be a lowercase 40-character SHA-1") - if not isinstance(trace_binary_sha256, str) or not re.fullmatch( - r"[0-9a-f]{64}", trace_binary_sha256 - ): - raise ValueError("trace source.cli_binary_sha256 must be a lowercase 64-character SHA-256") - if trace_binary_sha256 != CLI_BINARY_SHA256: + if trace_cli_version != HISTORICAL_CLI_VERSION or trace_source_commit != HISTORICAL_SOURCE_COMMIT: raise ValueError( - "trace source.cli_binary_sha256 does not match Codex CLI 0.146.0" - ) - if trace_source_status not in {"published_attested", "not_published_by_registry"}: - raise ValueError("trace source.cli_source_commit_status is invalid") - if trace_source_tag != CLI_SOURCE_TAG: - raise ValueError( - f"trace planner_gates.cli_source_tag must bind to {CLI_SOURCE_TAG}" + "trace evidence must retain the historical Codex 0.144.0-alpha.4 provenance" ) if candidate_cli_version is None: - candidate_cli_version = trace_cli_version - elif candidate_cli_version != trace_cli_version: + candidate_cli_version = contract_provenance["cli_version"] + elif candidate_cli_version != contract_provenance["cli_version"]: raise ValueError( - "candidate CLI version does not match trace evidence: " - f"requested={candidate_cli_version!r} observed={trace_cli_version!r}" + "candidate CLI version does not match the source contract: " + f"requested={candidate_cli_version!r} observed={contract_provenance['cli_version']!r}" ) if not isinstance(candidate_cli_version, str): raise ValueError("candidate CLI version must be a string") if candidate_source_commit is None: - candidate_source_commit = trace_source_commit - elif candidate_source_commit != trace_source_commit: + candidate_source_commit = contract_provenance["source_commit"] + elif candidate_source_commit != contract_provenance["source_commit"]: raise ValueError( - "candidate source commit does not match trace evidence: " - f"requested={candidate_source_commit!r} observed={trace_source_commit!r}" + "candidate source commit does not match the source contract: " + f"requested={candidate_source_commit!r} observed={contract_provenance['source_commit']!r}" ) if not isinstance(candidate_source_commit, str) or not re.fullmatch( @@ -1398,12 +1649,14 @@ def _validate_candidate_binding( wire_provenance.get("cli_version") != trace_cli_version or wire_provenance.get("source_commit") != trace_source_commit or wire_provenance.get("capture_id") != trace_capture_id - or wire_provenance.get("candidate_revision") != trace_candidate_revision - or wire_provenance.get("cli_binary_sha256") != trace_binary_sha256 - or wire_provenance.get("cli_source_commit_status") != trace_source_status - or wire_provenance.get("cli_source_tag") != trace_source_tag ): - raise ValueError("wire provenance is not bound to the trace candidate") + raise ValueError("historical wire provenance is not bound to the historical trace") + audit_provenance = audit_data.get("provenance", {}) + if ( + audit_provenance.get("capture_status") != "not_observed" + or any(audit_provenance.get(field) != value for field, value in contract_provenance.items()) + ): + raise ValueError("read-only audit provenance is not bound to the 0.146 source contract") catalog_snapshot = ( trace_data.get("planner_gates", {}) .get("catalog_source", {}) @@ -1439,16 +1692,17 @@ def _validate_candidate_binding( "catalog_snapshot_sha256": wire_route["catalog_snapshot_sha256"], "catalog_model_entry_id": wire_route["catalog_model_entry_id"], "route_behavior_profile": trace_route.get("behavior_profile"), - "candidate_revision": trace_candidate_revision, - "cli_binary_sha256": trace_binary_sha256, - "cli_source_commit_status": trace_source_status, - "cli_source_tag": trace_source_tag, + "candidate_revision": contract_provenance["candidate_revision"], + "cli_binary_sha256": contract_provenance["cli_binary_sha256"], + "cli_source_commit_status": contract_provenance["cli_source_commit_status"], + "cli_source_tag": contract_provenance["cli_source_tag"], } return identity, status def build_inventory( *, + source_contract: Path = DEFAULT_SOURCE_CONTRACT, trace: Path, wire_fixture: Path, audit: Path, @@ -1460,6 +1714,7 @@ def build_inventory( trace_data = _load_json(trace) wire_data = _load_json(wire_fixture) audit_data = _load_json(audit) + source_contract_data = _load_json(source_contract) wire_fixture_sha256 = _sha256_file(wire_fixture) if ( trace_data.get("schema_version") != 4 @@ -1469,6 +1724,7 @@ def build_inventory( or audit_data.get("capture_kind") != "sanitized_bounded_read_only_audit" ): raise ValueError("Issue #62 evidence schema identity is invalid") + _validate_source_contract(source_contract_data) _validate_core_evidence_pointers(wire=wire_data, audit=audit_data) items: list[dict[str, Any]] = [] @@ -1498,13 +1754,16 @@ def build_inventory( items, unknown_tagged_source_count=unknown_tagged_source_count ) structural_families = _build_structural_inventory( + source_contract=source_contract_data, trace=trace_data, wire=wire_data, audit=audit_data, ) candidate_identity, candidate_version_status = _validate_candidate_binding( + source_contract_data=source_contract_data, trace_data=trace_data, wire_data=wire_data, + audit_data=audit_data, cli_version_floor=cli_version_floor, candidate_cli_version=candidate_cli_version, candidate_source_commit=candidate_source_commit, @@ -1516,6 +1775,10 @@ def build_inventory( "sha256": wire_fixture_sha256, }, "audit": {"file": audit.name, "sha256": _sha256_file(audit)}, + "source_contract": { + "file": source_contract.name, + "sha256": _sha256_file(source_contract), + }, } candidate_identity["evidence_manifest_sha256"] = _evidence_manifest_sha256( evidence_binding @@ -1768,7 +2031,7 @@ def reconcile_inventory( if not isinstance(evidence_binding, dict): mismatches.append("evidence_binding must be an object") evidence_binding = {} - for name in ("trace", "wire_fixture", "audit"): + for name in ("source_contract", "trace", "wire_fixture", "audit"): entry = evidence_binding.get(name, {}) if not isinstance(entry, dict) or not entry.get("file") or not re.fullmatch( r"[0-9a-f]{64}", str(entry.get("sha256", "")) @@ -1777,7 +2040,7 @@ def reconcile_inventory( if evidence_binding and all( isinstance(evidence_binding.get(name), dict) - for name in ("trace", "wire_fixture", "audit") + for name in ("source_contract", "trace", "wire_fixture", "audit") ): manifest = _evidence_manifest_sha256(evidence_binding) if candidate_identity.get("evidence_manifest_sha256") != manifest: @@ -1785,7 +2048,7 @@ def reconcile_inventory( if evidence_root is not None and not mismatches: bound_paths: dict[str, Path] = {} - for name in ("trace", "wire_fixture", "audit"): + for name in ("source_contract", "trace", "wire_fixture", "audit"): entry = evidence_binding[name] relative_name = str(entry["file"]) relative_path = Path(relative_name) @@ -1805,7 +2068,11 @@ def reconcile_inventory( trace_data = _load_json(bound_paths["trace"]) wire_data = _load_json(bound_paths["wire_fixture"]) audit_data = _load_json(bound_paths["audit"]) + source_contract_data = _load_json(bound_paths["source_contract"]) if ( + source_contract_data.get("schema_version") != 1 + or source_contract_data.get("fixture_kind") != "codex_cli_source_contract" + or trace_data.get("schema_version") != 4 or wire_data.get("schema_version") != 1 or wire_data.get("fixture_kind") != "sanitized_artifact_backed_replay" @@ -1815,14 +2082,23 @@ def reconcile_inventory( mismatches.append("mutation: bound evidence schema identity is invalid") return {"reconciled": False, "mismatches": mismatches} try: + _validate_source_contract(source_contract_data) + _validate_structural_evidence_pointers( + source_contract=source_contract_data, + wire=wire_data, + audit=audit_data, + declaration_families=inventory.get("declaration_families", []), + ) _validate_core_evidence_pointers(wire=wire_data, audit=audit_data) except ValueError as exc: mismatches.append(f"mutation: bound evidence pointer validation failed: {exc}") return {"reconciled": False, "mismatches": mismatches} try: expected_identity, expected_status_from_evidence = _validate_candidate_binding( + source_contract_data=source_contract_data, trace_data=trace_data, wire_data=wire_data, + audit_data=audit_data, cli_version_floor=str(inventory.get("cli_version_floor", "")), candidate_cli_version=candidate_identity.get("cli_version"), candidate_source_commit=candidate_identity.get("source_commit"), @@ -1859,6 +2135,7 @@ def reconcile_inventory( ) try: generated_inventory = build_inventory( + source_contract=bound_paths["source_contract"], trace=bound_paths["trace"], wire_fixture=bound_paths["wire_fixture"], audit=bound_paths["audit"], @@ -1928,6 +2205,11 @@ def reconcile_inventory( def _build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source-contract", + type=Path, + default=DEFAULT_SOURCE_CONTRACT, + ) parser.add_argument( "--trace", type=Path, @@ -1969,6 +2251,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: def main() -> int: args = _build_arg_parser().parse_args() inventory = build_inventory( + source_contract=args.source_contract, trace=args.trace, wire_fixture=args.wire_fixture, audit=args.audit, diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 3e1edb16..dc16b1fb 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -1,4 +1,5 @@ param( + [string]$SourceContractPath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\codex-0.146-source-contract.json'), [string]$TracePath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\current-codexhub-thread-tool-surface.json'), [string]$WireFixturePath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\codexhub-runtime-wire-fixture.json'), [string]$AuditPath = (Join-Path (Split-Path -Parent $PSScriptRoot) 'docs\evidence\issue-62\read-only-gate-audit.json'), @@ -11,12 +12,13 @@ param( $ErrorActionPreference = 'Stop' -foreach ($path in @($TracePath, $WireFixturePath, $AuditPath, $InventoryPath)) { +foreach ($path in @($SourceContractPath, $TracePath, $WireFixturePath, $AuditPath, $InventoryPath)) { if (-not (Test-Path -LiteralPath $path)) { throw "Evidence file not found: $path" } } +$sourceContract = Get-Content -Raw -LiteralPath $SourceContractPath | ConvertFrom-Json $trace = Get-Content -Raw -LiteralPath $TracePath | ConvertFrom-Json $wire = Get-Content -Raw -LiteralPath $WireFixturePath | ConvertFrom-Json $audit = Get-Content -Raw -LiteralPath $AuditPath | ConvertFrom-Json @@ -39,6 +41,7 @@ if ($null -eq $python) { } else { try { $generatorOutput = & $python.Source $inventoryGenerator ` + --source-contract $SourceContractPath ` --trace $TracePath ` --wire-fixture $WireFixturePath ` --audit $AuditPath ` @@ -675,19 +678,41 @@ if ([string]::IsNullOrWhiteSpace([string]$inventoryCandidate.cli_source_tag)) { Add-Mismatch 'inventory candidate CLI source tag is missing or blank' } if ( + $sourceContract.schema_version -ne 1 -or + $sourceContract.fixture_kind -ne 'codex_cli_source_contract' -or + $sourceContract.capture_status -ne 'not_observed' -or + $sourceContract.qualification_status -ne 'unqualified' -or + $null -ne $sourceContract.captured_at -or + $sourceContract.provenance.cli_version -ne '0.146.0' -or + $sourceContract.provenance.source_commit -ne 'e363b08c9175ac1cbe5893615dd2cb9ddf95043b' -or + $sourceContract.provenance.cli_source_tag -ne 'rust-v0.146.0' -or + $sourceContract.provenance.cli_source_commit_status -ne 'published_attested' -or + $sourceContract.provenance.cli_binary_sha256 -ne 'bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb' -or + $sourceContract.provenance.candidate_revision -ne 'accab8ff6eb4d6ebd93cda84585fb5f6cb89da82' -or [string]::IsNullOrWhiteSpace([string]$trace.source.capture_id) -or [string]::IsNullOrWhiteSpace([string]$wire.provenance.capture_id) ) { - Add-Mismatch 'candidate trace/wire capture identity is missing' + Add-Mismatch 'source contract or historical trace/wire provenance is invalid' } if ( - $inventoryCandidate.cli_version -ne $trace.source.cli_version -or - $inventoryCandidate.source_commit -ne $trace.planner_gates.source_commit -or - $inventoryCandidate.codex_source_commit -ne $trace.planner_gates.source_commit -or - $inventoryCandidate.candidate_revision -ne $trace.source.candidate_revision -or - $inventoryCandidate.cli_binary_sha256 -ne $trace.source.cli_binary_sha256 -or - $inventoryCandidate.cli_source_commit_status -ne $trace.source.cli_source_commit_status -or - $inventoryCandidate.cli_source_tag -ne $trace.planner_gates.cli_source_tag -or + $trace.source.cli_version -ne '0.144.0-alpha.4' -or + $trace.planner_gates.source_commit -ne '9e552e9d15ba52bed7077d5357f3e18e330f8f38' -or + $wire.provenance.cli_version -ne $trace.source.cli_version -or + $wire.provenance.source_commit -ne $trace.planner_gates.source_commit -or + $inventoryCandidate.cli_version -ne $sourceContract.provenance.cli_version -or + $inventoryCandidate.source_commit -ne $sourceContract.provenance.source_commit -or + $inventoryCandidate.codex_source_commit -ne $sourceContract.provenance.source_commit -or + $inventoryCandidate.candidate_revision -ne $sourceContract.provenance.candidate_revision -or + $inventoryCandidate.cli_binary_sha256 -ne $sourceContract.provenance.cli_binary_sha256 -or + $inventoryCandidate.cli_source_commit_status -ne $sourceContract.provenance.cli_source_commit_status -or + $inventoryCandidate.cli_source_tag -ne $sourceContract.provenance.cli_source_tag -or + $audit.provenance.capture_status -ne 'not_observed' -or + $audit.provenance.cli_version -ne $sourceContract.provenance.cli_version -or + $audit.provenance.source_commit -ne $sourceContract.provenance.source_commit -or + $audit.provenance.candidate_revision -ne $sourceContract.provenance.candidate_revision -or + $audit.provenance.cli_binary_sha256 -ne $sourceContract.provenance.cli_binary_sha256 -or + $audit.provenance.cli_source_commit_status -ne $sourceContract.provenance.cli_source_commit_status -or + $audit.provenance.cli_source_tag -ne $sourceContract.provenance.cli_source_tag -or $inventoryCandidate.route_upstream -ne $wire.route.upstream_route -or $inventoryCandidate.inbound_format -ne $wire.route.inbound_format -or $inventoryCandidate.upstream_format -ne $wire.route.upstream_format -or @@ -710,12 +735,6 @@ if ( $wire.route.catalog_snapshot_sha256 -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.sha256 -or $wire.route.catalog_model_entry_id -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.model_entry_id -or $wire.route.catalog_model_supports_search_tool -ne $trace.planner_gates.catalog_source.read_only_snapshot_validation.model_entry_supports_search_tool -or - $wire.provenance.cli_version -ne $trace.source.cli_version -or - $wire.provenance.source_commit -ne $trace.planner_gates.source_commit -or - $wire.provenance.candidate_revision -ne $trace.source.candidate_revision -or - $wire.provenance.cli_binary_sha256 -ne $trace.source.cli_binary_sha256 -or - $wire.provenance.cli_source_commit_status -ne $trace.source.cli_source_commit_status -or - $wire.provenance.cli_source_tag -ne $trace.planner_gates.cli_source_tag -or $wire.provenance.capture_id -ne $trace.source.capture_id -or $wire.pre_gateway.model -ne $trace.source.model -or $wire.post_gateway.model -ne $trace.source.model @@ -723,6 +742,7 @@ if ( Add-Mismatch 'inventory candidate identity does not bind to the exact trace and wire candidate route' } $evidenceBindings = @{ + source_contract = $SourceContractPath trace = $TracePath wire_fixture = $WireFixturePath audit = $AuditPath @@ -739,7 +759,7 @@ foreach ($name in $evidenceBindings.Keys) { Add-Mismatch "inventory evidence binding $name hash does not match the input artifact" } } -$manifestParts = foreach ($name in @('audit','trace','wire_fixture')) { +$manifestParts = foreach ($name in @('audit','source_contract','trace','wire_fixture')) { $binding = $inventory.evidence_binding.$name '{0}:{1}:{2}' -f $name, $binding.file, $binding.sha256 } diff --git a/tests/test_issue_62_runtime_audit.py b/tests/test_issue_62_runtime_audit.py index aee5125a..2b082719 100644 --- a/tests/test_issue_62_runtime_audit.py +++ b/tests/test_issue_62_runtime_audit.py @@ -394,6 +394,10 @@ def test_audit_surfaces_unclassified_items_and_prefix_mismatch(tmp_path: Path) - def test_committed_audit_preserves_the_bounded_fact_and_sanitization_boundary() -> None: audit = json.loads(AUDIT.read_text(encoding="utf-8")) + assert audit["provenance"]["capture_status"] == "not_observed" + assert audit["provenance"]["cli_version"] == "0.146.0" + assert audit["provenance"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" + assert audit["provenance"]["historical_capture"]["cli_version"] == "0.144.0-alpha.4" assert audit["gateway_identity_route"]["request_starts"] == 525 assert audit["gateway_identity_route"]["prefix_equal"] == 525 assert audit["gateway_identity_route"]["prefix_mismatch"] == 0 @@ -414,7 +418,13 @@ def test_committed_audit_preserves_the_bounded_fact_and_sanitization_boundary() assert "https://" not in serialized assert ".codex" not in serialized.lower() assert not re.search(r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}", serialized) - assert not re.search(r'(? None: diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index 774e431a..352ef24b 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -8,6 +8,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "build_issue_62_runtime_inventory.py" INVENTORY = ROOT / "docs" / "evidence" / "issue-62" / "runtime-wire-inventory.json" +SOURCE_CONTRACT = ROOT / "docs" / "evidence" / "issue-62" / "codex-0.146-source-contract.json" TRACE = ROOT / "docs" / "evidence" / "issue-62" / "current-codexhub-thread-tool-surface.json" WIRE_FIXTURE = ROOT / "docs" / "evidence" / "issue-62" / "codexhub-runtime-wire-fixture.json" AUDIT = ROOT / "docs" / "evidence" / "issue-62" / "read-only-gate-audit.json" @@ -86,6 +87,8 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N } assert inventory["evidence_binding"]["trace"]["file"] == TRACE.name assert len(inventory["evidence_binding"]["trace"]["sha256"]) == 64 + assert inventory["evidence_binding"]["source_contract"]["file"] == SOURCE_CONTRACT.name + assert len(inventory["evidence_binding"]["source_contract"]["sha256"]) == 64 def test_inventory_uses_only_final_capability_disposition_vocabulary() -> None: @@ -150,6 +153,43 @@ def test_inventory_records_all_structural_declaration_families() -> None: assert by_family["selected_provider_hosted"]["executor"] == "selected_provider" assert by_family["selected_provider_hosted"]["representative"]["cross_provider_proxy"] == "forbidden" assert by_family["unknown_future_kind"]["selected_protocol_disposition"] == "omit" + assert by_family["plain_function"]["representative"]["streaming"]["arguments_done"] == "response.function_call_arguments.done" + assert by_family["custom_freeform"]["representative"]["streaming"]["input_done"] == "response.custom_tool_call_input.done" + assert by_family["client_executed_tool_discovery"]["representative"]["streaming"]["event_order"] == [ + "response.output_item.done", + "response.completed", + ] + for entry in families: + representative = entry["representative"] + assert representative["terminal"]["classification"] in {"not_observed", "unqualified"} + assert representative["error"]["classification"] in {"not_observed", "unqualified"} + assert representative["loss_boundary"] + + +def test_declaration_family_evidence_sources_resolve_to_bound_fixtures() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + + module._validate_structural_evidence_pointers( + source_contract=json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")), + wire=wire, + audit=audit, + declaration_families=inventory["declaration_families"], + ) + + dangling = json.loads(json.dumps(inventory["declaration_families"])) + dangling[-1]["evidence_source"] = ( + "codex-0.146-source-contract.json#runtime_wire_surface.unknown_future" + ) + with pytest.raises(ValueError, match="evidence source is invalid"): + module._validate_structural_evidence_pointers( + source_contract=json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")), + wire=wire, + audit=audit, + declaration_families=dangling, + ) def test_every_item_carries_an_allowed_disposition() -> None: @@ -360,6 +400,21 @@ def test_build_inventory_rejects_missing_capture_provenance(tmp_path: Path) -> N ) +def test_build_inventory_rejects_audit_candidate_provenance_drift(tmp_path: Path) -> None: + module = load_inventory_module() + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + audit["provenance"]["source_commit"] = "0" * 40 + audit_path = tmp_path / AUDIT.name + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="audit provenance"): + module.build_inventory( + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=audit_path, + ) + + def test_build_inventory_rejects_unbound_response_identity_pointer(tmp_path: Path) -> None: module = load_inventory_module() wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 5eabab58..c317c258 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -9,6 +9,7 @@ ROOT = Path(__file__).resolve().parents[1] TRACE = ROOT / "docs" / "evidence" / "issue-62" / "current-codexhub-thread-tool-surface.json" +SOURCE_CONTRACT = ROOT / "docs" / "evidence" / "issue-62" / "codex-0.146-source-contract.json" WIRE_FIXTURE = ROOT / "docs" / "evidence" / "issue-62" / "codexhub-runtime-wire-fixture.json" AUDIT = ROOT / "docs" / "evidence" / "issue-62" / "read-only-gate-audit.json" REPLAY_SCRIPT = ROOT / "scripts" / "check-codex-thread-tool-surface.ps1" @@ -57,11 +58,9 @@ def test_trace_covers_dynamic_tool_exposure_and_sanitizes_session_identity() -> assert trace["schema_version"] == 4 assert "session_id" not in trace["source"] - assert trace["source"]["cli_version"] == "0.146.0" - assert trace["source"]["candidate_revision"] == "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" - assert trace["source"]["cli_source_commit_status"] == "published_attested" - assert trace["planner_gates"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" - assert trace["planner_gates"]["cli_source_tag"] == "rust-v0.146.0" + assert trace["captured_at"] == "2026-07-12T14:57:55+08:00" + assert trace["source"]["cli_version"] == "0.144.0-alpha.4" + assert trace["planner_gates"]["source_commit"] == "9e552e9d15ba52bed7077d5357f3e18e330f8f38" assert len(trace["registered_codex_app_tools"]) == 15 required = set(trace["required_thread_tools"]) assert required <= set(trace["registered_codex_app_tools"]) @@ -108,7 +107,12 @@ def test_trace_covers_dynamic_tool_exposure_and_sanitizes_session_identity() -> assert snapshot["model_entry_supports_search_tool"] is True assert len(snapshot["sha256"]) == 64 - families = trace["planner_gates"]["declaration_families"] + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + assert source_contract["capture_status"] == "not_observed" + assert source_contract["qualification_status"] == "unqualified" + assert source_contract["provenance"]["cli_version"] == "0.146.0" + assert source_contract["provenance"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" + families = source_contract["runtime_wire_surface"]["declaration_families"] assert [family["family"] for family in families] == [ "plain_function", "custom_freeform", @@ -123,9 +127,8 @@ def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) assert wire["route"]["upstream_route"] == "official" - assert wire["provenance"]["cli_version"] == "0.146.0" - assert wire["provenance"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" - assert wire["provenance"]["candidate_revision"] == "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" + assert wire["provenance"]["cli_version"] == "0.144.0-alpha.4" + assert wire["provenance"]["source_commit"] == "9e552e9d15ba52bed7077d5357f3e18e330f8f38" assert "no full request or response body fingerprint" in wire["evidence_limit"][ "transport_observation" ] @@ -158,7 +161,7 @@ def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: item.get("tag") == "unknown" for item in wire["response"]["non_streaming"]["response_items"] ) - assert wire["runtime_wire_surface"]["declaration_family_order"] == [ + assert json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8"))["runtime_wire_surface"]["declaration_family_order"] == [ "plain_function", "custom_freeform", "namespace", @@ -166,7 +169,7 @@ def test_wire_fixture_keeps_identity_and_unknown_sentinels() -> None: "selected_provider_hosted", "unknown_future_kind", ] - hosted = wire["runtime_wire_surface"]["declaration_family_examples"][ + hosted = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8"))["runtime_wire_surface"]["declaration_family_examples"][ "selected_provider_hosted" ] assert hosted["provider_scope"] == "selected_provider_only" @@ -288,8 +291,10 @@ def test_powershell_accepts_pass_non_direct_state_like_python(tmp_path: Path) -> audit["gate_classification"]["non_direct_states"] = "pass" trace_path = tmp_path / TRACE.name wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name shutil.copyfile(TRACE, trace_path) shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(SOURCE_CONTRACT, source_contract_path) audit_path = tmp_path / "read-only-gate-audit.json" audit_path.write_text( json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8" @@ -313,6 +318,8 @@ def test_powershell_accepts_pass_non_direct_state_like_python(tmp_path: Path) -> "Bypass", "-File", str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), "-TracePath", str(trace_path), "-WireFixturePath", From 2a0ad0e86acbb0a728d886ef2de62da41ab419d6 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 17:54:19 +0800 Subject: [PATCH 03/13] fix(issue-62): bind audit reruns to source contract --- docs/evidence/issue-62/README.md | 16 ++- .../codexhub-runtime-wire-fixture.json | 1 + .../issue-62/runtime-wire-inventory.json | 4 +- scripts/audit_issue_62_runtime_artifacts.py | 128 ++++++++++++++++++ scripts/build_issue_62_runtime_inventory.py | 112 ++++++++++++++- scripts/check-codex-thread-tool-surface.ps1 | 14 +- tests/test_issue_62_runtime_audit.py | 53 +++++++- tests/test_issue_62_runtime_inventory.py | 53 ++++++++ tests/test_issue_62_runtime_trace.py | 45 ++++++ 9 files changed, 412 insertions(+), 14 deletions(-) diff --git a/docs/evidence/issue-62/README.md b/docs/evidence/issue-62/README.md index ed2596cf..143e51ba 100644 --- a/docs/evidence/issue-62/README.md +++ b/docs/evidence/issue-62/README.md @@ -134,6 +134,7 @@ Run the sanitizer with explicit bounded inputs and observation cutoffs: ```powershell python scripts/audit_issue_62_runtime_artifacts.py ` + --source-contract docs/evidence/issue-62/codex-0.146-source-contract.json ` --codex-log-db ` --gateway-db ` --model gpt-5.6-sol ` @@ -161,9 +162,11 @@ The artifact is bound to CLI floor `0.146.0` and to the candidate identity from the unobserved 0.146 source contract (`cli_version=0.146.0`, source commit `e363b08c9175ac1cbe5893615dd2cb9ddf95043b`, candidate revision `accab8ff6eb4d6ebd93cda84585fb5f6cb89da82`, official Responses route). The -historical trace, wire fixture, and audit remain explicitly bound as -0.144.0/historical evidence; they do not become 0.146 runtime captures. The -candidate is version-eligible, but `qualification.ready_for_beta1` remains +historical trace and wire fixture remain explicitly bound as 0.144.0 evidence. +The audit carries the 0.146 source-contract provenance with +`capture_status=not_observed` and nested historical-capture metadata; it is not +a 0.146 runtime capture. The candidate is version-eligible, but +`qualification.ready_for_beta1` remains `false`: planner completeness, clean current-binding cold start, independently fingerprinted full pre/post request and response bodies, non-streaming/terminal/error/hosted/unknown controls, and wire replay evidence @@ -172,8 +175,11 @@ qualification or capability unlock. The generator rejects an explicitly supplied CLI/source value that does not match the trace, binds route/provider/ model fields across trace and wire fixtures (including pre/post models, catalog binding, and route profile), and records a canonical-LF SHA-256 manifest for -all four input artifacts (including the source contract). It never fabricates a -capability disposition for a gate the artifacts do not qualify. +all four input artifacts (including the source contract). The audit sanitizer +must receive that source-contract path so reruns retain the exact +`capture_status=not_observed` 0.146 provenance and nested historical 0.144 +capture metadata. It never fabricates a capability disposition for a gate the +artifacts do not qualify. The qualification also has a separate `wire_identity_replay` gate. A full request/response fingerprint is not treated as replay proof by itself: a diff --git a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json index 77a22173..2d6469d4 100644 --- a/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json +++ b/docs/evidence/issue-62/codexhub-runtime-wire-fixture.json @@ -25,6 +25,7 @@ "provenance": { "cli_version": "0.144.0-alpha.4", "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + "captured_at": "2026-07-12T14:57:55+08:00", "capture_id": "sanitized-current-side-capture" }, "exposure_state_tags": ["Direct", "DirectModelOnly", "Deferred", "Hidden", "hosted-only", "host-unavailable"], diff --git a/docs/evidence/issue-62/runtime-wire-inventory.json b/docs/evidence/issue-62/runtime-wire-inventory.json index 392a99cb..3062d7e6 100644 --- a/docs/evidence/issue-62/runtime-wire-inventory.json +++ b/docs/evidence/issue-62/runtime-wire-inventory.json @@ -11,7 +11,7 @@ "cli_version": "0.146.0", "codex_source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", "configured_provider_id": "custom", - "evidence_manifest_sha256": "b25da7980235919ddfe19f11d34e2aba2bb8b42ec7828398362e71f01ce79fa9", + "evidence_manifest_sha256": "685858e1affdb073df17467f53ab3de538c0c7f2d930b01c10797a94963c6619", "inbound_format": "responses", "model": "gpt-5.6-sol", "route_behavior_profile": "official_codex_app_http_passthrough", @@ -414,7 +414,7 @@ }, "wire_fixture": { "file": "codexhub-runtime-wire-fixture.json", - "sha256": "3a8654854539799b526bc013c50bc9f16c40c09319cff8da716b75e6deecbc3a" + "sha256": "1e3ecfc51f59de7f21bee8a7a0b743bba26d35fb3dc9eaca5f7b4ea3dd57705f" } }, "evidence_sources": { diff --git a/scripts/audit_issue_62_runtime_artifacts.py b/scripts/audit_issue_62_runtime_artifacts.py index f78112cb..bcbb6eb3 100644 --- a/scripts/audit_issue_62_runtime_artifacts.py +++ b/scripts/audit_issue_62_runtime_artifacts.py @@ -15,6 +15,22 @@ TRANSPORT_TARGET = "codex_http_client::transport" +DEFAULT_SOURCE_CONTRACT = Path( + "docs/evidence/issue-62/codex-0.146-source-contract.json" +) +SOURCE_CONTRACT_PROVENANCE = { + "cli_version": "0.146.0", + "source_commit": "e363b08c9175ac1cbe5893615dd2cb9ddf95043b", + "cli_source_tag": "rust-v0.146.0", + "cli_source_commit_status": "published_attested", + "cli_binary_sha256": "bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb", + "candidate_revision": "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82", +} +HISTORICAL_CAPTURE = { + "captured_at": "2026-07-12T14:57:55+08:00", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", +} POST_BODY_PATTERN = re.compile( r": POST to (?Phttps?://.+?): (?P\{.*\})\s*$", re.DOTALL, @@ -155,6 +171,113 @@ def _is_proven_tool_choice(value: Any) -> bool: ) +def _source_contract_provenance(path: Path) -> dict[str, Any]: + """Load and validate the unobserved 0.146 source-contract identity.""" + + try: + source_contract = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"unable to read source contract: {path}") from exc + if not isinstance(source_contract, dict): + raise ValueError("source contract must be a JSON object") + if ( + source_contract.get("schema_version") != 1 + or source_contract.get("fixture_kind") != "codex_cli_source_contract" + or source_contract.get("capture_status") != "not_observed" + or source_contract.get("qualification_status") != "unqualified" + or "captured_at" not in source_contract + or source_contract.get("captured_at") is not None + ): + raise ValueError( + "source contract must remain not_observed, unqualified, and uncaptured" + ) + provenance = source_contract.get("provenance") + if not isinstance(provenance, dict) or any( + provenance.get(field) != value + for field, value in SOURCE_CONTRACT_PROVENANCE.items() + ): + raise ValueError("source contract provenance is invalid") + runtime_surface = source_contract.get("runtime_wire_surface") + if not isinstance(runtime_surface, dict): + raise ValueError("source contract runtime surface is missing") + if runtime_surface.get("declaration_family_order") != list(DECLARATION_FAMILIES): + raise ValueError("source contract declaration family order is invalid") + declaration_families = runtime_surface.get("declaration_families") + if not isinstance(declaration_families, list) or len(declaration_families) != len( + DECLARATION_FAMILIES + ): + raise ValueError("source contract declaration families are invalid") + expected_observations = { + "plain_function": "not_observed_source_contract_only", + "custom_freeform": "not_observed_source_contract_only", + "namespace": "not_observed_source_contract_only", + "client_executed_tool_discovery": "not_observed_source_contract_only", + "selected_provider_hosted": "not_observed_selected_provider_control_required", + "unknown_future_kind": "opaque_sentinel_only", + } + for family, entry in zip(DECLARATION_FAMILIES, declaration_families): + if not isinstance(entry, dict) or entry.get("family") != family: + raise ValueError("source contract declaration family identity is invalid") + if entry.get("observed") is not False: + raise ValueError("source contract cannot claim an observed declaration family") + if entry.get("observation") != expected_observations[family]: + raise ValueError("source contract declaration observation is invalid") + request_shape = runtime_surface.get("request_shape") + non_streaming_control = ( + request_shape.get("non_streaming_control") + if isinstance(request_shape, dict) + else None + ) + if not isinstance(non_streaming_control, dict) or non_streaming_control.get( + "captured" + ) is not False or non_streaming_control.get("status") != "unqualified": + raise ValueError("source contract cannot claim a captured non-streaming response") + examples = runtime_surface.get("declaration_family_examples") + if not isinstance(examples, dict): + raise ValueError("source contract declaration examples are missing") + for family in DECLARATION_FAMILIES: + example = examples.get(family) + if not isinstance(example, dict): + raise ValueError(f"source contract example is missing for {family}") + if ( + not isinstance(example.get("terminal"), dict) + or example["terminal"].get("event") != "response.completed" + or example["terminal"].get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"source contract terminal status is invalid for {family}") + if ( + not isinstance(example.get("error"), dict) + or example["error"].get("event") != "response.failed" + or example["error"].get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"source contract error status is invalid for {family}") + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"source contract loss boundary is invalid for {family}") + response_shape = runtime_surface.get("response_shape") + error_shape = response_shape.get("error_shape") if isinstance(response_shape, dict) else None + if not isinstance(error_shape, dict) or error_shape.get("classification") != "unqualified": + raise ValueError("source contract response error status is invalid") + + def _captured_true(value: Any) -> bool: + if isinstance(value, dict): + return any( + (key in {"captured", "observed"} and child is True) + or _captured_true(child) + for key, child in value.items() + ) + if isinstance(value, list): + return any(_captured_true(child) for child in value) + return False + + if _captured_true(runtime_surface): + raise ValueError("source contract contains a captured response claim") + return { + "capture_status": source_contract["capture_status"], + **SOURCE_CONTRACT_PROVENANCE, + "historical_capture": dict(HISTORICAL_CAPTURE), + } + + def _sanitize_tool(tool: dict[str, Any]) -> dict[str, Any]: tool_type = tool.get("type") if isinstance(tool.get("type"), str) else None name = tool.get("name") if isinstance(tool.get("name"), str) else None @@ -610,7 +733,9 @@ def audit_artifacts( config_written_at: str, catalog_written_at: str, snapshot_ended_at: str, + source_contract: Path = DEFAULT_SOURCE_CONTRACT, ) -> dict[str, Any]: + provenance = _source_contract_provenance(source_contract) codex = _codex_request_evidence( codex_log_db, model=model, @@ -661,6 +786,7 @@ def audit_artifacts( return { "capture_kind": "sanitized_bounded_read_only_audit", + "provenance": provenance, "gate_classification": { "choice_controls": "observed" if choice_observed else "unclassified", "clean_cold_start_current_binding": "met" @@ -715,6 +841,7 @@ def audit_artifacts( def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-contract", type=Path, default=DEFAULT_SOURCE_CONTRACT) parser.add_argument("--codex-log-db", type=Path, required=True) parser.add_argument("--gateway-db", type=Path, required=True) parser.add_argument("--model", required=True) @@ -737,6 +864,7 @@ def main() -> int: config_written_at=args.config_written_at, catalog_written_at=args.catalog_written_at, snapshot_ended_at=args.snapshot_ended_at, + source_contract=args.source_contract, ) print(json.dumps(audit, ensure_ascii=True, indent=2, sort_keys=True)) return 0 diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index d953979e..ed3190ae 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -47,6 +47,7 @@ CANDIDATE_REVISION = "accab8ff6eb4d6ebd93cda84585fb5f6cb89da82" HISTORICAL_CLI_VERSION = "0.144.0-alpha.4" HISTORICAL_SOURCE_COMMIT = "9e552e9d15ba52bed7077d5357f3e18e330f8f38" +HISTORICAL_CAPTURED_AT = "2026-07-12T14:57:55+08:00" DEFAULT_SOURCE_CONTRACT = Path( "docs/evidence/issue-62/codex-0.146-source-contract.json" ) @@ -1527,6 +1528,7 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] or source_contract.get("fixture_kind") != "codex_cli_source_contract" or source_contract.get("capture_status") != "not_observed" or source_contract.get("qualification_status") != "unqualified" + or "captured_at" not in source_contract or source_contract.get("captured_at") is not None ): raise ValueError("Codex 0.146 source contract must remain not_observed and unqualified") @@ -1541,6 +1543,88 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] } if any(provenance.get(field) != value for field, value in expected.items()): raise ValueError("Codex 0.146 source contract provenance is invalid") + runtime_surface = source_contract.get("runtime_wire_surface") + if not isinstance(runtime_surface, dict): + raise ValueError("Codex 0.146 source contract runtime surface is missing") + if runtime_surface.get("declaration_family_order") != list(STRUCTURAL_FAMILIES): + raise ValueError("Codex 0.146 source contract declaration family order is invalid") + declaration_families = runtime_surface.get("declaration_families") + if not isinstance(declaration_families, list) or len(declaration_families) != len( + STRUCTURAL_FAMILIES + ): + raise ValueError("Codex 0.146 source contract declaration families are invalid") + expected_observations = { + "plain_function": "not_observed_source_contract_only", + "custom_freeform": "not_observed_source_contract_only", + "namespace": "not_observed_source_contract_only", + "client_executed_tool_discovery": "not_observed_source_contract_only", + "selected_provider_hosted": "not_observed_selected_provider_control_required", + "unknown_future_kind": "opaque_sentinel_only", + } + for family, expected_family in zip(STRUCTURAL_FAMILIES, declaration_families): + if not isinstance(expected_family, dict) or expected_family.get("family") != family: + raise ValueError("Codex 0.146 source contract declaration family identity is invalid") + if expected_family.get("observed") is not False: + raise ValueError( + "Codex 0.146 source contract cannot claim an observed declaration family" + ) + if expected_family.get("observation") != expected_observations[family]: + raise ValueError( + "Codex 0.146 source contract declaration observation is invalid" + ) + request_shape = runtime_surface.get("request_shape") + non_streaming_control = ( + request_shape.get("non_streaming_control") + if isinstance(request_shape, dict) + else None + ) + if not isinstance(non_streaming_control, dict) or non_streaming_control.get( + "captured" + ) is not False or non_streaming_control.get("status") != "unqualified": + raise ValueError( + "Codex 0.146 source contract cannot claim a captured non-streaming response" + ) + examples = runtime_surface.get("declaration_family_examples") + if not isinstance(examples, dict): + raise ValueError("Codex 0.146 source contract declaration examples are missing") + for family in STRUCTURAL_FAMILIES: + example = examples.get(family) + if not isinstance(example, dict): + raise ValueError(f"Codex 0.146 source contract example is missing for {family}") + terminal = example.get("terminal") + error = example.get("error") + if ( + not isinstance(terminal, dict) + or terminal.get("event") != "response.completed" + or terminal.get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"Codex 0.146 source contract terminal status is invalid for {family}") + if ( + not isinstance(error, dict) + or error.get("event") != "response.failed" + or error.get("classification") not in {"not_observed", "unqualified"} + ): + raise ValueError(f"Codex 0.146 source contract error status is invalid for {family}") + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"Codex 0.146 source contract loss boundary is invalid for {family}") + response_shape = runtime_surface.get("response_shape") + error_shape = response_shape.get("error_shape") if isinstance(response_shape, dict) else None + if not isinstance(error_shape, dict) or error_shape.get("classification") != "unqualified": + raise ValueError("Codex 0.146 source contract response error status is invalid") + + def _captured_true(value: Any) -> bool: + if isinstance(value, dict): + return any( + (key in {"captured", "observed"} and child is True) + or _captured_true(child) + for key, child in value.items() + ) + if isinstance(value, list): + return any(_captured_true(child) for child in value) + return False + + if _captured_true(runtime_surface): + raise ValueError("Codex 0.146 source contract contains a captured response claim") return provenance @@ -1561,10 +1645,16 @@ def _validate_candidate_binding( trace_cli_version = source.get("cli_version") trace_source_commit = planner_gates.get("source_commit") trace_capture_id = source.get("capture_id") - if not trace_cli_version or not trace_source_commit or not trace_capture_id: + trace_captured_at = trace_data.get("captured_at") + if ( + not trace_cli_version + or not trace_source_commit + or not trace_capture_id + or not trace_captured_at + ): raise ValueError( "historical trace evidence is missing source.cli_version, " - "source.capture_id, or planner_gates.source_commit" + "source.capture_id, planner_gates.source_commit, or captured_at" ) if not isinstance(trace_cli_version, str): raise ValueError("trace source.cli_version must be a string") @@ -1572,7 +1662,11 @@ def _validate_candidate_binding( _version_key(trace_cli_version) except ValueError as exc: raise ValueError("trace source.cli_version is malformed") from exc - if trace_cli_version != HISTORICAL_CLI_VERSION or trace_source_commit != HISTORICAL_SOURCE_COMMIT: + if ( + trace_cli_version != HISTORICAL_CLI_VERSION + or trace_source_commit != HISTORICAL_SOURCE_COMMIT + or trace_captured_at != HISTORICAL_CAPTURED_AT + ): raise ValueError( "trace evidence must retain the historical Codex 0.144.0-alpha.4 provenance" ) @@ -1649,6 +1743,7 @@ def _validate_candidate_binding( wire_provenance.get("cli_version") != trace_cli_version or wire_provenance.get("source_commit") != trace_source_commit or wire_provenance.get("capture_id") != trace_capture_id + or wire_provenance.get("captured_at") != trace_captured_at ): raise ValueError("historical wire provenance is not bound to the historical trace") audit_provenance = audit_data.get("provenance", {}) @@ -1657,6 +1752,17 @@ def _validate_candidate_binding( or any(audit_provenance.get(field) != value for field, value in contract_provenance.items()) ): raise ValueError("read-only audit provenance is not bound to the 0.146 source contract") + historical_capture = audit_provenance.get("historical_capture") + if ( + not isinstance(historical_capture, dict) + or historical_capture.get("captured_at") != trace_captured_at + or historical_capture.get("cli_version") != trace_cli_version + or historical_capture.get("source_commit") != trace_source_commit + or historical_capture.get("captured_at") != wire_provenance.get("captured_at") + or historical_capture.get("cli_version") != wire_provenance.get("cli_version") + or historical_capture.get("source_commit") != wire_provenance.get("source_commit") + ): + raise ValueError("read-only audit historical capture provenance is not bound") catalog_snapshot = ( trace_data.get("planner_gates", {}) .get("catalog_source", {}) diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index dc16b1fb..3ad09e29 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -689,16 +689,25 @@ if ( $sourceContract.provenance.cli_source_commit_status -ne 'published_attested' -or $sourceContract.provenance.cli_binary_sha256 -ne 'bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb' -or $sourceContract.provenance.candidate_revision -ne 'accab8ff6eb4d6ebd93cda84585fb5f6cb89da82' -or + $sourceContract.runtime_wire_surface.declaration_families.Count -ne 6 -or + ($sourceContract.runtime_wire_surface.declaration_families | Where-Object { $_.observed -ne $false }).Count -ne 0 -or + ($sourceContract.runtime_wire_surface.declaration_families | Where-Object { $_.observation -notin @('not_observed_source_contract_only', 'not_observed_selected_provider_control_required', 'opaque_sentinel_only') }).Count -ne 0 -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.captured -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.status -ne 'unqualified' -or [string]::IsNullOrWhiteSpace([string]$trace.source.capture_id) -or - [string]::IsNullOrWhiteSpace([string]$wire.provenance.capture_id) + [string]::IsNullOrWhiteSpace([string]$wire.provenance.capture_id) -or + $trace.captured_at -ne '2026-07-12T14:57:55+08:00' -or + $wire.provenance.captured_at -ne '2026-07-12T14:57:55+08:00' ) { Add-Mismatch 'source contract or historical trace/wire provenance is invalid' } if ( $trace.source.cli_version -ne '0.144.0-alpha.4' -or $trace.planner_gates.source_commit -ne '9e552e9d15ba52bed7077d5357f3e18e330f8f38' -or + $trace.captured_at -ne '2026-07-12T14:57:55+08:00' -or $wire.provenance.cli_version -ne $trace.source.cli_version -or $wire.provenance.source_commit -ne $trace.planner_gates.source_commit -or + $wire.provenance.captured_at -ne $trace.captured_at -or $inventoryCandidate.cli_version -ne $sourceContract.provenance.cli_version -or $inventoryCandidate.source_commit -ne $sourceContract.provenance.source_commit -or $inventoryCandidate.codex_source_commit -ne $sourceContract.provenance.source_commit -or @@ -713,6 +722,9 @@ if ( $audit.provenance.cli_binary_sha256 -ne $sourceContract.provenance.cli_binary_sha256 -or $audit.provenance.cli_source_commit_status -ne $sourceContract.provenance.cli_source_commit_status -or $audit.provenance.cli_source_tag -ne $sourceContract.provenance.cli_source_tag -or + $audit.provenance.historical_capture.captured_at -ne $trace.captured_at -or + $audit.provenance.historical_capture.cli_version -ne $trace.source.cli_version -or + $audit.provenance.historical_capture.source_commit -ne $trace.planner_gates.source_commit -or $inventoryCandidate.route_upstream -ne $wire.route.upstream_route -or $inventoryCandidate.inbound_format -ne $wire.route.inbound_format -or $inventoryCandidate.upstream_format -ne $wire.route.upstream_format -or diff --git a/tests/test_issue_62_runtime_audit.py b/tests/test_issue_62_runtime_audit.py index 2b082719..af3cf6fc 100644 --- a/tests/test_issue_62_runtime_audit.py +++ b/tests/test_issue_62_runtime_audit.py @@ -10,6 +10,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "audit_issue_62_runtime_artifacts.py" AUDIT = ROOT / "docs" / "evidence" / "issue-62" / "read-only-gate-audit.json" +SOURCE_CONTRACT = ROOT / "docs" / "evidence" / "issue-62" / "codex-0.146-source-contract.json" def load_audit_module(): @@ -27,6 +28,7 @@ def run_audit( gateway_db: Path, *, config_written_at: str = "1970-01-01T00:06:00Z", + source_contract: Path = SOURCE_CONTRACT, ): return module.audit_artifacts( codex_log_db=codex_db, @@ -37,6 +39,7 @@ def run_audit( config_written_at=config_written_at, catalog_written_at="1970-01-01T00:02:00Z", snapshot_ended_at="1970-01-01T00:10:00Z", + source_contract=source_contract, ) @@ -393,11 +396,17 @@ def test_audit_surfaces_unclassified_items_and_prefix_mismatch(tmp_path: Path) - def test_committed_audit_preserves_the_bounded_fact_and_sanitization_boundary() -> None: audit = json.loads(AUDIT.read_text(encoding="utf-8")) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) assert audit["provenance"]["capture_status"] == "not_observed" - assert audit["provenance"]["cli_version"] == "0.146.0" - assert audit["provenance"]["source_commit"] == "e363b08c9175ac1cbe5893615dd2cb9ddf95043b" - assert audit["provenance"]["historical_capture"]["cli_version"] == "0.144.0-alpha.4" + assert audit["provenance"]["capture_status"] == source_contract["capture_status"] + for field, value in source_contract["provenance"].items(): + assert audit["provenance"][field] == value + assert audit["provenance"]["historical_capture"] == { + "captured_at": "2026-07-12T14:57:55+08:00", + "cli_version": "0.144.0-alpha.4", + "source_commit": "9e552e9d15ba52bed7077d5357f3e18e330f8f38", + } assert audit["gateway_identity_route"]["request_starts"] == 525 assert audit["gateway_identity_route"]["prefix_equal"] == 525 assert audit["gateway_identity_route"]["prefix_mismatch"] == 0 @@ -427,6 +436,44 @@ def test_committed_audit_preserves_the_bounded_fact_and_sanitization_boundary() ] +def test_audit_rejects_source_contract_provenance_drift(tmp_path: Path) -> None: + module = load_audit_module() + codex_db = tmp_path / "codex.sqlite" + gateway_db = tmp_path / "gateway.sqlite" + create_codex_log_db(codex_db) + create_gateway_db(gateway_db) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["provenance"]["source_commit"] = "0" * 40 + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="source contract provenance"): + run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) + + +def test_audit_rejects_captured_source_contract_claim(tmp_path: Path) -> None: + module = load_audit_module() + codex_db = tmp_path / "codex.sqlite" + gateway_db = tmp_path / "gateway.sqlite" + create_codex_log_db(codex_db) + create_gateway_db(gateway_db) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["request_shape"]["non_streaming_control"][ + "captured" + ] = True + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="captured non-streaming"): + run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) + + def test_audit_detects_generic_response_body_fingerprint_fields(tmp_path: Path) -> None: module = load_audit_module() codex_db = tmp_path / "codex.sqlite" diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index 352ef24b..875e8d42 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -415,6 +415,59 @@ def test_build_inventory_rejects_audit_candidate_provenance_drift(tmp_path: Path ) +def test_build_inventory_rejects_audit_historical_capture_drift(tmp_path: Path) -> None: + module = load_inventory_module() + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + audit["provenance"]["historical_capture"]["captured_at"] = "2026-07-13T14:57:55+08:00" + audit_path = tmp_path / AUDIT.name + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="historical capture"): + module.build_inventory( + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=audit_path, + ) + + +def test_build_inventory_rejects_observed_source_contract_family(tmp_path: Path) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["declaration_families"][0]["observed"] = True + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="cannot claim an observed"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +def test_build_inventory_rejects_captured_source_contract_control(tmp_path: Path) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["request_shape"]["non_streaming_control"][ + "captured" + ] = True + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="captured non-streaming"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + def test_build_inventory_rejects_unbound_response_identity_pointer(tmp_path: Path) -> None: module = load_inventory_module() wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index c317c258..3b912213 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -338,6 +338,51 @@ def test_powershell_accepts_pass_non_direct_state_like_python(tmp_path: Path) -> assert result.returncode == 0, result.stderr +def test_powershell_rejects_historical_capture_provenance_drift(tmp_path: Path) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(SOURCE_CONTRACT, source_contract_path) + audit = json.loads(AUDIT.read_text(encoding="utf-8")) + audit["provenance"]["historical_capture"]["captured_at"] = "2026-07-13T14:57:55+08:00" + audit_path.write_text( + json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) def test_negative_inventory_replays_fail_visibly(case: str) -> None: result = run_inventory_replay(case) From 2f983163a2e06cb6c6dac2205982cfe66f4d8d3c Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 18:27:47 +0800 Subject: [PATCH 04/13] fix(issue-62): enforce declaration family schemas --- scripts/build_issue_62_runtime_inventory.py | 224 ++++++++++++++++++++ scripts/check-codex-thread-tool-surface.ps1 | 46 ++++ tests/test_issue_62_runtime_inventory.py | 46 ++++ tests/test_issue_62_runtime_trace.py | 47 ++++ 4 files changed, 363 insertions(+) diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index ed3190ae..7ff450d8 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -612,6 +612,206 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: }, } +# The source contract is intentionally closed over the six declaration +# families. Keep the wire/runtime names and identity-bearing sections +# canonical so a regenerated inventory cannot turn an arbitrary fixture value +# into a qualified family claim. +STRUCTURAL_FAMILY_SCHEMAS = { + "plain_function": { + "runtime_type": "function", + "wire_declaration_type": "function", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "declaration": {"type": "function", "required": ("name", "parameters")}, + "call": { + "type": "function_call", + "required": ("item_id", "call_id", "arguments"), + }, + "result": { + "type": "function_call_output", + "required": ("item_id", "call_id", "output"), + }, + "history": {"required": ("call_id", "call_item_id", "output_item_id")}, + }, + "custom_freeform": { + "runtime_type": "custom", + "wire_declaration_type": "custom", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "declaration": {"type": "custom", "required": ("name", "format")}, + "call": { + "type": "custom_tool_call", + "required": ("item_id", "call_id", "input"), + }, + "result": { + "type": "custom_tool_call_output", + "required": ("item_id", "call_id", "output"), + }, + "history": {"required": ("call_id", "call_item_id", "output_item_id")}, + }, + "namespace": { + "runtime_type": "namespace", + "wire_declaration_type": "namespace", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "preserve declaration and inverse call/result/history IDs", + "declaration": {"type": "namespace", "required": ("name", "tools")}, + "call": { + "type": "function_call", + "required": ("item_id", "call_id", "namespace", "arguments"), + }, + "result": { + "type": "function_call_output", + "required": ("item_id", "call_id", "output"), + }, + "history": { + "required": ("call_id", "call_item_id", "output_item_id", "namespace") + }, + }, + "client_executed_tool_discovery": { + "runtime_type": "tool_search", + "wire_declaration_type": "tool_search", + "executor": "codex_client", + "observation": "not_observed_source_contract_only", + "loss_boundary": "discovery request/result stays client-executed", + "declaration": { + "type": "tool_search", + "required": ("execution", "parameters"), + "equals": {"execution": "client"}, + }, + "call": { + "type": "tool_search_call", + "required": ("item_id", "call_id", "execution", "arguments"), + "equals": {"execution": "client"}, + }, + "result": { + "type": "tool_search_output", + "required": ("item_id", "call_id", "execution", "tools"), + "equals": {"execution": "client"}, + }, + "history": { + "required": ("call_id", "call_item_id", "output_item_id", "executor"), + "equals": {"executor": "codex_client"}, + }, + }, + "selected_provider_hosted": { + "runtime_type": "web_search", + "wire_declaration_type": "web_search", + "executor": "selected_provider", + "observation": "not_observed_selected_provider_control_required", + "loss_boundary": "optional unsupported hosted capability is omitted; required capability fails visibly", + "declaration": { + "type": "web_search", + "required": ("executor", "provider_scope"), + "equals": { + "executor": "selected_provider", + "provider_scope": "selected_provider_only", + }, + }, + "call": { + "type": "web_search_call", + "required": ("item_id", "status", "action"), + }, + "result": { + "type": "web_search_call", + "required": ("item_id", "status", "provider_scope"), + "equals": {"provider_scope": "selected_provider_only"}, + }, + "history": { + "required": ( + "call_id", + "call_item_id", + "output_item_id", + "executor", + "cross_provider_proxy", + ), + "equals": { + "executor": "selected_provider", + "cross_provider_proxy": "forbidden", + }, + "nullable": ("call_id",), + }, + }, + "unknown_future_kind": { + "runtime_type": "unknown", + "wire_declaration_type": "", + "executor": "unknown", + "observation": "opaque_sentinel_only", + "loss_boundary": "retain tag and opaque payload; do not normalize", + "declaration": { + "type": "unknown", + "required": ("tag", "opaque_payload"), + "equals": {"tag": "unknown"}, + }, + "call": { + "type": "unknown", + "required": ("tag", "opaque_payload"), + "equals": {"tag": "unknown"}, + }, + "result": { + "type": "unknown", + "required": ("tag", "opaque_payload"), + "equals": {"tag": "unknown"}, + }, + "history": { + "required": ("call_id", "call_item_id", "output_item_id", "loss_rule"), + "nullable": ("call_id", "call_item_id", "output_item_id"), + "equals": { + "call_id": None, + "call_item_id": None, + "output_item_id": None, + "loss_rule": "retain opaque sentinel", + }, + }, + }, +} + + +def _validate_structural_family_schema( + family_name: str, + family: dict[str, Any], + example: dict[str, Any], + *, + context: str, +) -> None: + schema = STRUCTURAL_FAMILY_SCHEMAS.get(family_name) + if schema is None: + raise ValueError(f"{context} has an unknown family schema") + for field in ("runtime_type", "wire_declaration_type", "executor", "observation", "loss_boundary"): + if family.get(field) != schema[field]: + raise ValueError(f"{context}.{field} does not match the canonical family schema") + for section_name in ("declaration", "call", "result", "history"): + section = example.get(section_name) + section_schema = schema[section_name] + if not isinstance(section, dict): + raise ValueError(f"{context}.{section_name} must be an object") + expected_type = section_schema.get("type") + if expected_type is not None and section.get("type") != expected_type: + raise ValueError(f"{context}.{section_name}.type does not match the canonical family schema") + nullable = set(section_schema.get("nullable", ())) + for key in section_schema.get("required", ()): + if key not in section: + raise ValueError(f"{context}.{section_name}.{key} is required") + if key not in nullable and section[key] is None: + raise ValueError(f"{context}.{section_name}.{key} cannot be null") + for key, expected in section_schema.get("equals", {}).items(): + if section.get(key) != expected: + raise ValueError(f"{context}.{section_name}.{key} does not match the canonical family schema") + if family_name == "namespace": + tools = example["declaration"].get("tools") + if not isinstance(tools, list) or not tools: + raise ValueError(f"{context}.declaration.tools must contain a function") + for nested in tools: + if ( + not isinstance(nested, dict) + or nested.get("type") != "function" + or not nested.get("name") + or "parameters" not in nested + ): + raise ValueError(f"{context}.declaration.tools contains an invalid function") + STRUCTURAL_EVIDENCE_SOURCES = { "plain_function": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", "custom_freeform": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.custom_freeform", @@ -785,6 +985,12 @@ def _build_structural_inventory( raise ValueError( f"wire runtime example for {family_name} is missing {missing!r}" ) + _validate_structural_family_schema( + family_name, + family, + example, + context=f"runtime planner family {family_name}", + ) if not isinstance(example.get("terminal"), dict) or example["terminal"].get( "classification" ) not in {"not_observed", "unqualified"}: @@ -902,6 +1108,16 @@ def _structural_inventory_mismatches(value: Any) -> list[str]: continue if not required_example_parts.issubset(representative): mismatches.append(f"{prefix}.representative is missing a wire example section") + else: + try: + _validate_structural_family_schema( + family_name, + entry, + representative, + context=prefix, + ) + except ValueError as exc: + mismatches.append(str(exc)) terminal = representative.get("terminal") if not isinstance(terminal, dict): mismatches.append(f"{prefix}.representative.terminal must be an object") @@ -1553,6 +1769,7 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] STRUCTURAL_FAMILIES ): raise ValueError("Codex 0.146 source contract declaration families are invalid") + declaration_families_by_name: dict[str, dict[str, Any]] = {} expected_observations = { "plain_function": "not_observed_source_contract_only", "custom_freeform": "not_observed_source_contract_only", @@ -1564,6 +1781,7 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] for family, expected_family in zip(STRUCTURAL_FAMILIES, declaration_families): if not isinstance(expected_family, dict) or expected_family.get("family") != family: raise ValueError("Codex 0.146 source contract declaration family identity is invalid") + declaration_families_by_name[family] = expected_family if expected_family.get("observed") is not False: raise ValueError( "Codex 0.146 source contract cannot claim an observed declaration family" @@ -1591,6 +1809,12 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] example = examples.get(family) if not isinstance(example, dict): raise ValueError(f"Codex 0.146 source contract example is missing for {family}") + _validate_structural_family_schema( + family, + declaration_families_by_name[family], + example, + context=f"Codex 0.146 source contract family {family}", + ) terminal = example.get("terminal") error = example.get("error") if ( diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 3ad09e29..0e040ed3 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -30,6 +30,51 @@ function Add-Mismatch { $script:mismatches.Add($Message) } +$expectedFamilySchemas = @( + @{ Name = 'plain_function'; RuntimeType = 'function'; WireType = 'function'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'preserve declaration and inverse call/result/history IDs'; DeclarationType = 'function'; CallType = 'function_call'; ResultType = 'function_call_output'; DeclarationRequired = @('name', 'parameters'); CallRequired = @('item_id', 'call_id', 'arguments'); ResultRequired = @('item_id', 'call_id', 'output'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id') }, + @{ Name = 'custom_freeform'; RuntimeType = 'custom'; WireType = 'custom'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'preserve declaration and inverse call/result/history IDs'; DeclarationType = 'custom'; CallType = 'custom_tool_call'; ResultType = 'custom_tool_call_output'; DeclarationRequired = @('name', 'format'); CallRequired = @('item_id', 'call_id', 'input'); ResultRequired = @('item_id', 'call_id', 'output'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id') }, + @{ Name = 'namespace'; RuntimeType = 'namespace'; WireType = 'namespace'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'preserve declaration and inverse call/result/history IDs'; DeclarationType = 'namespace'; CallType = 'function_call'; ResultType = 'function_call_output'; DeclarationRequired = @('name', 'tools'); CallRequired = @('item_id', 'call_id', 'namespace', 'arguments'); ResultRequired = @('item_id', 'call_id', 'output'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'namespace') }, + @{ Name = 'client_executed_tool_discovery'; RuntimeType = 'tool_search'; WireType = 'tool_search'; Executor = 'codex_client'; Observation = 'not_observed_source_contract_only'; LossBoundary = 'discovery request/result stays client-executed'; DeclarationType = 'tool_search'; CallType = 'tool_search_call'; ResultType = 'tool_search_output'; DeclarationRequired = @('execution', 'parameters'); CallRequired = @('item_id', 'call_id', 'execution', 'arguments'); ResultRequired = @('item_id', 'call_id', 'execution', 'tools'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'executor') }, + @{ Name = 'selected_provider_hosted'; RuntimeType = 'web_search'; WireType = 'web_search'; Executor = 'selected_provider'; Observation = 'not_observed_selected_provider_control_required'; LossBoundary = 'optional unsupported hosted capability is omitted; required capability fails visibly'; DeclarationType = 'web_search'; CallType = 'web_search_call'; ResultType = 'web_search_call'; DeclarationRequired = @('executor', 'provider_scope'); CallRequired = @('item_id', 'status', 'action'); ResultRequired = @('item_id', 'status', 'provider_scope'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'executor', 'cross_provider_proxy') }, + @{ Name = 'unknown_future_kind'; RuntimeType = 'unknown'; WireType = ''; Executor = 'unknown'; Observation = 'opaque_sentinel_only'; LossBoundary = 'retain tag and opaque payload; do not normalize'; DeclarationType = 'unknown'; CallType = 'unknown'; ResultType = 'unknown'; DeclarationRequired = @('tag', 'opaque_payload'); CallRequired = @('tag', 'opaque_payload'); ResultRequired = @('tag', 'opaque_payload'); HistoryRequired = @('call_id', 'call_item_id', 'output_item_id', 'loss_rule') } +) +$sourceContractSchemaValid = $true +$sourceContractFamilies = @($sourceContract.runtime_wire_surface.declaration_families) +$sourceContractExamples = $sourceContract.runtime_wire_surface.declaration_family_examples +foreach ($expected in $expectedFamilySchemas) { + $family = @($sourceContractFamilies | Where-Object { $_.family -eq $expected.Name }) | Select-Object -First 1 + $exampleProperty = if ($null -ne $sourceContractExamples) { $sourceContractExamples.PSObject.Properties[$expected.Name] } else { $null } + $example = if ($null -ne $exampleProperty) { $exampleProperty.Value } else { $null } + if ($null -eq $family -or $null -eq $example -or + $family.runtime_type -ne $expected.RuntimeType -or + $family.wire_declaration_type -ne $expected.WireType -or + $family.executor -ne $expected.Executor -or + $family.observation -ne $expected.Observation -or + $family.loss_boundary -ne $expected.LossBoundary) { + $sourceContractSchemaValid = $false + continue + } + $sections = @( + @{ Name = 'declaration'; Type = $expected.DeclarationType; Required = $expected.DeclarationRequired }, + @{ Name = 'call'; Type = $expected.CallType; Required = $expected.CallRequired }, + @{ Name = 'result'; Type = $expected.ResultType; Required = $expected.ResultRequired }, + @{ Name = 'history'; Type = $null; Required = $expected.HistoryRequired } + ) + foreach ($sectionSpec in $sections) { + $sectionProperty = $example.PSObject.Properties[$sectionSpec.Name] + $section = if ($null -ne $sectionProperty) { $sectionProperty.Value } else { $null } + if ($null -eq $section -or ($null -ne $sectionSpec.Type -and $section.type -ne $sectionSpec.Type)) { + $sourceContractSchemaValid = $false + continue + } + foreach ($required in $sectionSpec.Required) { + if ($null -eq $section.PSObject.Properties[$required]) { + $sourceContractSchemaValid = $false + } + } + } +} + # Rebuild the inventory from the bound evidence and compare it with the # committed artifact. The PowerShell checks below remain an independent # reconciliation, while this call catches stale generated fields/notes that @@ -689,6 +734,7 @@ if ( $sourceContract.provenance.cli_source_commit_status -ne 'published_attested' -or $sourceContract.provenance.cli_binary_sha256 -ne 'bc343ba420dc2e2e9f59e6fc5e5bf0aae1cd8c771fc319665241fc9c0271fddb' -or $sourceContract.provenance.candidate_revision -ne 'accab8ff6eb4d6ebd93cda84585fb5f6cb89da82' -or + -not $sourceContractSchemaValid -or $sourceContract.runtime_wire_surface.declaration_families.Count -ne 6 -or ($sourceContract.runtime_wire_surface.declaration_families | Where-Object { $_.observed -ne $false }).Count -ne 0 -or ($sourceContract.runtime_wire_surface.declaration_families | Where-Object { $_.observation -notin @('not_observed_source_contract_only', 'not_observed_selected_provider_control_required', 'opaque_sentinel_only') }).Count -ne 0 -or diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index 875e8d42..a377d774 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -468,6 +468,52 @@ def test_build_inventory_rejects_captured_source_contract_control(tmp_path: Path ) +@pytest.mark.parametrize( + "mutation", + [ + "runtime_type", + "wire_declaration_type", + "declaration_type", + "call_type", + "result_type", + ], +) +def test_build_inventory_rejects_source_contract_schema_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation in {"runtime_type", "wire_declaration_type"}: + source_contract["runtime_wire_surface"]["declaration_families"][0][mutation] = "bogus" + else: + source_contract["runtime_wire_surface"]["declaration_family_examples"]["plain_function"][ + {"declaration_type": "declaration", "call_type": "call", "result_type": "result"}[mutation] + ]["type"] = "web_search_call" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="canonical family schema"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["declaration_families"][0]["representative"]["call"]["type"] = "web_search_call" + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("canonical family schema" in mismatch for mismatch in report["mismatches"]) + + def test_build_inventory_rejects_unbound_response_identity_pointer(tmp_path: Path) -> None: module = load_inventory_module() wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 3b912213..327aa889 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -383,6 +383,53 @@ def test_powershell_rejects_historical_capture_provenance_drift(tmp_path: Path) assert "generated inventory drift check failed" in result.stderr.lower() +def test_powershell_rejects_source_contract_family_schema_drift(tmp_path: Path) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["declaration_family_examples"]["plain_function"][ + "call" + ]["type"] = "web_search_call" + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) def test_negative_inventory_replays_fail_visibly(case: str) -> None: result = run_inventory_replay(case) From ec631713881025a3de85d99bd1e58d43fbf0c0fd Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 18:29:59 +0800 Subject: [PATCH 05/13] fix(issue-62): reconcile family owner identities --- scripts/build_issue_62_runtime_inventory.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index 7ff450d8..f42bcb78 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -811,6 +811,25 @@ def _validate_structural_family_schema( or "parameters" not in nested ): raise ValueError(f"{context}.declaration.tools contains an invalid function") + namespace = example["declaration"].get("name") + if ( + example["call"].get("namespace") != namespace + or example["history"].get("namespace") != namespace + ): + raise ValueError(f"{context} namespace ownership does not reconcile") + if family_name in { + "plain_function", + "custom_freeform", + "namespace", + "client_executed_tool_discovery", + "selected_provider_hosted", + }: + if ( + example["call"].get("item_id") != example["history"].get("call_item_id") + or example["result"].get("item_id") + != example["history"].get("output_item_id") + ): + raise ValueError(f"{context} call/result item IDs do not reconcile") STRUCTURAL_EVIDENCE_SOURCES = { "plain_function": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", From 6da52af474099178efdfac945529d17cc0b7e8d5 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 18:50:10 +0800 Subject: [PATCH 06/13] fix(issue-62): enforce identity and SSE labels --- scripts/build_issue_62_runtime_inventory.py | 141 ++++++++++++++++++++ scripts/check-codex-thread-tool-surface.ps1 | 39 +++++- tests/test_issue_62_runtime_inventory.py | 57 ++++++++ 3 files changed, 236 insertions(+), 1 deletion(-) diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index f42bcb78..9a125f2b 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -767,6 +767,22 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: }, }, } +STRUCTURAL_NONEMPTY_STRING_FIELDS = frozenset( + { + "name", + "namespace", + "executor", + "execution", + "provider_scope", + "cross_provider_proxy", + "tag", + "loss_rule", + "item_id", + "call_id", + "call_item_id", + "output_item_id", + } +) def _validate_structural_family_schema( @@ -794,6 +810,12 @@ def _validate_structural_family_schema( for key in section_schema.get("required", ()): if key not in section: raise ValueError(f"{context}.{section_name}.{key} is required") + if key in STRUCTURAL_NONEMPTY_STRING_FIELDS and section[key] is not None and ( + not isinstance(section[key], str) or not section[key] + ): + raise ValueError( + f"{context}.{section_name}.{key} must be a non-empty string" + ) if key not in nullable and section[key] is None: raise ValueError(f"{context}.{section_name}.{key} cannot be null") for key, expected in section_schema.get("equals", {}).items(): @@ -807,6 +829,7 @@ def _validate_structural_family_schema( if ( not isinstance(nested, dict) or nested.get("type") != "function" + or not isinstance(nested.get("name"), str) or not nested.get("name") or "parameters" not in nested ): @@ -830,6 +853,11 @@ def _validate_structural_family_schema( != example["history"].get("output_item_id") ): raise ValueError(f"{context} call/result item IDs do not reconcile") + _validate_structural_stream_example( + family_name, + example.get("streaming"), + context=context, + ) STRUCTURAL_EVIDENCE_SOURCES = { "plain_function": "codex-0.146-source-contract.json#runtime_wire_surface.declaration_family_examples.plain_function", @@ -847,6 +875,114 @@ def _validate_structural_family_schema( "unknown_future_kind": ("done", "unknown.future_done"), } +STRUCTURAL_STREAM_SCHEMAS = { + "plain_function": { + "added": "response.output_item.added", + "delta": "response.function_call_arguments.delta", + "done_field": "arguments_done", + "done": "response.function_call_arguments.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed", + ], + }, + "custom_freeform": { + "added": "response.output_item.added", + "delta": "response.custom_tool_call_input.delta", + "done_field": "input_done", + "done": "response.custom_tool_call_input.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "response.custom_tool_call_input.delta", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.completed", + ], + }, + "namespace": { + "added": "response.output_item.added", + "delta": "response.function_call_arguments.delta", + "done_field": "arguments_done", + "done": "response.function_call_arguments.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done", + "response.completed", + ], + }, + "client_executed_tool_discovery": { + "added": None, + "delta": None, + "done_field": "done", + "done": "response.output_item.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": ["response.output_item.done", "response.completed"], + }, + "selected_provider_hosted": { + "added": "response.output_item.added", + "delta": "", + "done_field": "done", + "done": "response.output_item.done", + "item_done": "response.output_item.done", + "terminal": "response.completed", + "event_order": [ + "response.output_item.added", + "", + "response.output_item.done", + "response.completed", + ], + }, + "unknown_future_kind": { + "added": "unknown.future_event", + "delta": "unknown.future_delta", + "done_field": "done", + "done": "unknown.future_done", + "item_done": "unknown.future_done", + "terminal": "response.completed", + "event_order": [ + "unknown.future_event", + "unknown.future_delta", + "unknown.future_done", + "response.completed", + ], + }, +} + + +def _validate_structural_stream_example( + family_name: str, + streaming: Any, + *, + context: str, +) -> None: + schema = STRUCTURAL_STREAM_SCHEMAS.get(family_name) + if schema is None: + raise ValueError(f"{context} has an unknown stream schema") + if not isinstance(streaming, dict): + raise ValueError(f"{context}.streaming must be an object") + for field in ("added", "delta", "terminal"): + if streaming.get(field) != schema[field]: + raise ValueError(f"{context}.streaming.{field} does not match the canonical SSE schema") + done_field = schema["done_field"] + if streaming.get(done_field) != schema["done"]: + raise ValueError(f"{context}.streaming.{done_field} does not match the canonical SSE schema") + if streaming.get("done") != schema["item_done"]: + raise ValueError(f"{context}.streaming.done does not match the canonical SSE schema") + if streaming.get("event_order") != schema["event_order"]: + raise ValueError(f"{context}.streaming.event_order does not match the canonical SSE schema") + def _validate_structural_stream_contract( *, wire: dict[str, Any], examples: dict[str, Any] @@ -883,6 +1019,11 @@ def _validate_structural_stream_contract( for family_name in STRUCTURAL_FAMILIES: example = examples[family_name] streaming = example.get("streaming") + _validate_structural_stream_example( + family_name, + streaming, + context=f"wire runtime family {family_name}", + ) if not isinstance(streaming, dict): raise ValueError(f"wire runtime streaming example is malformed for {family_name}") event_order = streaming.get("event_order") diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 0e040ed3..997e1a3c 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -41,6 +41,19 @@ $expectedFamilySchemas = @( $sourceContractSchemaValid = $true $sourceContractFamilies = @($sourceContract.runtime_wire_surface.declaration_families) $sourceContractExamples = $sourceContract.runtime_wire_surface.declaration_family_examples +$nullableByFamily = @{ + selected_provider_hosted = @{ history = @('call_id') } + unknown_future_kind = @{ history = @('call_id', 'call_item_id', 'output_item_id') } +} +$nonEmptyStringFields = @('name', 'namespace', 'executor', 'execution', 'provider_scope', 'cross_provider_proxy', 'tag', 'loss_rule', 'item_id', 'call_id', 'call_item_id', 'output_item_id') +$expectedSseSchemas = @{ + plain_function = @{ Added = 'response.output_item.added'; Delta = 'response.function_call_arguments.delta'; DoneField = 'arguments_done'; Done = 'response.function_call_arguments.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', 'response.function_call_arguments.delta', 'response.function_call_arguments.done', 'response.output_item.done', 'response.completed') } + custom_freeform = @{ Added = 'response.output_item.added'; Delta = 'response.custom_tool_call_input.delta'; DoneField = 'input_done'; Done = 'response.custom_tool_call_input.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', 'response.custom_tool_call_input.delta', 'response.custom_tool_call_input.done', 'response.output_item.done', 'response.completed') } + namespace = @{ Added = 'response.output_item.added'; Delta = 'response.function_call_arguments.delta'; DoneField = 'arguments_done'; Done = 'response.function_call_arguments.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', 'response.function_call_arguments.delta', 'response.function_call_arguments.done', 'response.output_item.done', 'response.completed') } + client_executed_tool_discovery = @{ Added = $null; Delta = $null; DoneField = 'done'; Done = 'response.output_item.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.done', 'response.completed') } + selected_provider_hosted = @{ Added = 'response.output_item.added'; Delta = ''; DoneField = 'done'; Done = 'response.output_item.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', '', 'response.output_item.done', 'response.completed') } + unknown_future_kind = @{ Added = 'unknown.future_event'; Delta = 'unknown.future_delta'; DoneField = 'done'; Done = 'unknown.future_done'; ItemDone = 'unknown.future_done'; Terminal = 'response.completed'; Order = @('unknown.future_event', 'unknown.future_delta', 'unknown.future_done', 'response.completed') } +} foreach ($expected in $expectedFamilySchemas) { $family = @($sourceContractFamilies | Where-Object { $_.family -eq $expected.Name }) | Select-Object -First 1 $exampleProperty = if ($null -ne $sourceContractExamples) { $sourceContractExamples.PSObject.Properties[$expected.Name] } else { $null } @@ -68,11 +81,35 @@ foreach ($expected in $expectedFamilySchemas) { continue } foreach ($required in $sectionSpec.Required) { - if ($null -eq $section.PSObject.Properties[$required]) { + $requiredProperty = $section.PSObject.Properties[$required] + $nullableFields = @() + if ($nullableByFamily.ContainsKey($expected.Name) -and $nullableByFamily[$expected.Name].ContainsKey($sectionSpec.Name)) { + $nullableFields = @($nullableByFamily[$expected.Name][$sectionSpec.Name]) + } + if ($null -eq $requiredProperty -or + ($required -in $nonEmptyStringFields -and + $required -notin $nullableFields -and + ($null -eq $requiredProperty.Value -or + $requiredProperty.Value -isnot [string] -or + [string]::IsNullOrEmpty($requiredProperty.Value)))) { $sourceContractSchemaValid = $false } } } + $sseSchema = $expectedSseSchemas[$expected.Name] + $streamProperty = $example.PSObject.Properties['streaming'] + $stream = if ($null -ne $streamProperty) { $streamProperty.Value } else { $null } + $doneProperty = if ($null -ne $stream) { $stream.PSObject.Properties[$sseSchema.DoneField] } else { $null } + if ($null -eq $stream -or + $stream.added -ne $sseSchema.Added -or + $stream.delta -ne $sseSchema.Delta -or + $stream.terminal -ne $sseSchema.Terminal -or + $stream.done -ne $sseSchema.ItemDone -or + $null -eq $doneProperty -or + $doneProperty.Value -ne $sseSchema.Done -or + (($stream.event_order -join '|') -ne ($sseSchema.Order -join '|'))) { + $sourceContractSchemaValid = $false + } } # Rebuild the inventory from the bound evidence and compare it with the diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index a377d774..a77c7a4e 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -503,6 +503,57 @@ def test_build_inventory_rejects_source_contract_schema_mutations( ) +@pytest.mark.parametrize( + "mutation", + [ + "plain_call_item_id_empty", + "plain_history_call_id_empty", + "namespace_owner_empty", + "plain_added_event", + "custom_delta_event", + "namespace_done_event", + "tool_search_added_event", + "unknown_event_label", + ], +) +def test_build_inventory_rejects_source_contract_identity_and_sse_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + examples = source_contract["runtime_wire_surface"]["declaration_family_examples"] + if mutation == "plain_call_item_id_empty": + examples["plain_function"]["call"]["item_id"] = "" + elif mutation == "plain_history_call_id_empty": + examples["plain_function"]["history"]["call_id"] = "" + elif mutation == "namespace_owner_empty": + examples["namespace"]["call"]["namespace"] = "" + elif mutation == "plain_added_event": + examples["plain_function"]["streaming"]["added"] = "bogus.event" + elif mutation == "custom_delta_event": + examples["custom_freeform"]["streaming"]["delta"] = "bogus.event" + elif mutation == "namespace_done_event": + examples["namespace"]["streaming"]["arguments_done"] = "bogus.event" + elif mutation == "tool_search_added_event": + examples["client_executed_tool_discovery"]["streaming"]["added"] = ( + "response.output_item.added" + ) + elif mutation == "unknown_event_label": + examples["unknown_future_kind"]["streaming"]["event_order"][0] = "bogus.event" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="(?:non-empty string|canonical SSE)"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root() -> None: module = load_inventory_module() inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) @@ -513,6 +564,12 @@ def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root assert report["reconciled"] is False assert any("canonical family schema" in mismatch for mismatch in report["mismatches"]) + id_mutation = json.loads(INVENTORY.read_text(encoding="utf-8")) + id_mutation["declaration_families"][0]["representative"]["call"]["item_id"] = "" + id_report = module.reconcile_inventory(id_mutation) + assert id_report["reconciled"] is False + assert any("non-empty string" in mismatch for mismatch in id_report["mismatches"]) + def test_build_inventory_rejects_unbound_response_identity_pointer(tmp_path: Path) -> None: module = load_inventory_module() From fb6612cda04e25703ee37c4de5d7fa49ef8efb10 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 19:17:45 +0800 Subject: [PATCH 07/13] fix(issue-62): harden beta2 inventory replay contract --- docs/evidence/issue-62/README.md | 4 +- .../issue-62/runtime-wire-inventory.json | 2 +- scripts/build_issue_62_runtime_inventory.py | 118 ++++++++++++++++-- scripts/check-codex-thread-tool-surface.ps1 | 7 +- tests/test_issue_62_runtime_inventory.py | 115 ++++++++++++++++- 5 files changed, 229 insertions(+), 17 deletions(-) diff --git a/docs/evidence/issue-62/README.md b/docs/evidence/issue-62/README.md index 143e51ba..1785d386 100644 --- a/docs/evidence/issue-62/README.md +++ b/docs/evidence/issue-62/README.md @@ -166,7 +166,7 @@ historical trace and wire fixture remain explicitly bound as 0.144.0 evidence. The audit carries the 0.146 source-contract provenance with `capture_status=not_observed` and nested historical-capture metadata; it is not a 0.146 runtime capture. The candidate is version-eligible, but -`qualification.ready_for_beta1` remains +`qualification.ready_for_beta2` remains `false`: planner completeness, clean current-binding cold start, independently fingerprinted full pre/post request and response bodies, non-streaming/terminal/error/hosted/unknown controls, and wire replay evidence @@ -241,7 +241,7 @@ reconciliation invokes the same drift check, then independently checks the input fingerprints, rejects duplicate scopes, and requires each core scope to point at its declared evidence path. A zero `unclassified_core_items` count therefore describes vocabulary validity only; -`qualification.ready_for_beta1` is the separate completion gate. That gate also +`qualification.ready_for_beta2` is the separate completion gate. That gate also consumes planner completeness, current-binding cold-start, full-wire fingerprinting, non-streaming, and identity-replay statuses; item dispositions alone cannot make an incomplete evidence set ready. diff --git a/docs/evidence/issue-62/runtime-wire-inventory.json b/docs/evidence/issue-62/runtime-wire-inventory.json index 3062d7e6..af3e99d3 100644 --- a/docs/evidence/issue-62/runtime-wire-inventory.json +++ b/docs/evidence/issue-62/runtime-wire-inventory.json @@ -633,7 +633,7 @@ "terminal_events": "not_captured", "wire_identity_replay": "not_captured" }, - "ready_for_beta1": false + "ready_for_beta2": false }, "schema_version": 1 } diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index 9a125f2b..19cac9b1 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -294,6 +294,35 @@ "cli_source_tag", ) +# This artifact is deliberately bound to the current Issue #62 evidence +# package. A standalone reconciliation has no evidence root to reopen, so +# it must still reject a hand-edited candidate or pointer that merely has the +# right type/shape but no longer names the retained package. +EXPECTED_CANDIDATE_VALUES = { + "cli_version": DEFAULT_CLI_FLOOR, + "source_commit": CLI_SOURCE_COMMIT, + "codex_source_commit": CLI_SOURCE_COMMIT, + "route_upstream": "official", + "inbound_format": "responses", + "upstream_format": "responses", + "configured_provider_id": "custom", + "model": "gpt-5.6-sol", + "catalog_binding": "official Codex catalog entry for openai/gpt-5.6-sol", + "catalog_model_entry_id": "gpt-5.6-sol", + "route_behavior_profile": "official_codex_app_http_passthrough", + "candidate_revision": CANDIDATE_REVISION, + "cli_binary_sha256": CLI_BINARY_SHA256, + "cli_source_commit_status": CLI_SOURCE_COMMIT_STATUS, + "cli_source_tag": CLI_SOURCE_TAG, +} + +EXPECTED_EVIDENCE_BINDING_FILES = { + "source_contract": "codex-0.146-source-contract.json", + "trace": "current-codexhub-thread-tool-surface.json", + "wire_fixture": "codexhub-runtime-wire-fixture.json", + "audit": "read-only-gate-audit.json", +} + def _sha256_file(path: Path) -> str: # Evidence is JSON text; hash its canonical LF representation so a @@ -355,7 +384,9 @@ def _validate_supported_floor(value: str) -> str: return value -def _candidate_identity_mismatches(candidate_identity: Any) -> list[str]: +def _candidate_identity_mismatches( + candidate_identity: Any, *, enforce_retained: bool = False +) -> list[str]: """Validate candidate metadata that is self-contained in the inventory. Reconciliation is also used for in-memory replay controls where no evidence @@ -424,6 +455,13 @@ def _candidate_identity_mismatches(candidate_identity: Any) -> list[str]: if not isinstance(source_tag, str) or not source_tag.strip(): mismatches.append("candidate_identity.cli_source_tag is missing or blank") + if enforce_retained: + for field, expected in EXPECTED_CANDIDATE_VALUES.items(): + if candidate_identity.get(field) != expected: + mismatches.append( + f"candidate_identity.{field} does not match the retained Issue #62 candidate" + ) + for field in ( "catalog_snapshot_sha256", "evidence_manifest_sha256", @@ -1215,7 +1253,9 @@ def _build_structural_inventory( return output -def _structural_inventory_mismatches(value: Any) -> list[str]: +def _structural_inventory_mismatches( + value: Any, *, require_unobserved: bool = False +) -> list[str]: """Validate the stable shape of the emitted declaration-family inventory.""" if not isinstance(value, list): @@ -1256,6 +1296,10 @@ def _structural_inventory_mismatches(value: Any) -> list[str]: mismatches.append(f"{prefix}.{field} is missing or blank") if not isinstance(entry.get("observed"), bool): mismatches.append(f"{prefix}.observed must be boolean") + elif require_unobserved and entry.get("observed") is not False: + mismatches.append( + f"{prefix}.observed must remain false for the unobserved 0.146 source contract" + ) for field in ("selected_protocol_disposition", "optional_rule", "required_rule"): if entry.get(field) != rule[field]: mismatches.append(f"{prefix}.{field} does not match the family rule") @@ -1892,7 +1936,7 @@ def _build_qualification( "blocking_scopes": blocking_scopes, "evidence_gates": evidence_gates, "blocking_gates": blocking_gates, - "ready_for_beta1": candidate_version_eligible + "ready_for_beta2": candidate_version_eligible and not blocking_scopes and not blocking_gates, } @@ -1941,6 +1985,18 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] for family, expected_family in zip(STRUCTURAL_FAMILIES, declaration_families): if not isinstance(expected_family, dict) or expected_family.get("family") != family: raise ValueError("Codex 0.146 source contract declaration family identity is invalid") + if set(expected_family) != { + "family", + "runtime_type", + "wire_declaration_type", + "observed", + "observation", + "executor", + "loss_boundary", + }: + raise ValueError( + "Codex 0.146 source contract declaration family has unknown fields" + ) declaration_families_by_name[family] = expected_family if expected_family.get("observed") is not False: raise ValueError( @@ -1969,6 +2025,29 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] example = examples.get(family) if not isinstance(example, dict): raise ValueError(f"Codex 0.146 source contract example is missing for {family}") + expected_status_fields = { + "selected_provider_hosted": { + "observed": False, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden", + }, + "unknown_future_kind": { + "observed": False, + "status": "opaque_sentinel_only", + }, + }.get(family, {}) + for field, expected_value in expected_status_fields.items(): + if example.get(field) != expected_value: + raise ValueError( + f"Codex 0.146 source contract {family} {field} status is invalid" + ) + if not expected_status_fields and any( + field in example for field in ("observed", "status") + ): + raise ValueError( + f"Codex 0.146 source contract {family} has an unknown status field" + ) _validate_structural_family_schema( family, declaration_families_by_name[family], @@ -2358,7 +2437,8 @@ def reconcile_inventory( mismatches.extend( f"mutation: {message}" for message in _structural_inventory_mismatches( - inventory.get("declaration_families") + inventory.get("declaration_families"), + require_unobserved=evidence_root is None, ) ) @@ -2419,7 +2499,11 @@ def reconcile_inventory( mismatches.append(f"deletion: missing scopes {missing}") candidate_identity = inventory.get("candidate_identity", {}) - mismatches.extend(_candidate_identity_mismatches(candidate_identity)) + mismatches.extend( + _candidate_identity_mismatches( + candidate_identity, enforce_retained=evidence_root is None + ) + ) if not isinstance(candidate_identity, dict): candidate_identity = {} for field in REQUIRED_CANDIDATE_FIELDS: @@ -2484,6 +2568,10 @@ def reconcile_inventory( } if set(evidence_gates) != expected_gate_keys: mismatches.append("qualification.evidence_gates has an unexpected key set") + if "ready_for_beta1" in qualification: + mismatches.append( + "qualification.ready_for_beta1 is stale; use ready_for_beta2" + ) accepted_gate_statuses = { "complete_model_visible_plan": {"complete"}, "clean_cold_start_current_binding": {"complete", "pass"}, @@ -2512,21 +2600,35 @@ def reconcile_inventory( and not actual_blocking_scopes and not actual_blocking_gates ) - if qualification.get("ready_for_beta1") is not expected_ready: + if qualification.get("ready_for_beta2") is not expected_ready: + mismatches.append( + "qualification.ready_for_beta2 is inconsistent with candidate eligibility and blockers" + ) + if evidence_root is None and qualification.get("ready_for_beta2") is True: mismatches.append( - "qualification.ready_for_beta1 is inconsistent with candidate eligibility and blockers" + "qualification.ready_for_beta2 cannot be asserted without bound evidence" + ) + if evidence_root is None and not actual_blocking_gates: + mismatches.append( + "qualification.blocking_gates cannot be empty without bound evidence" ) evidence_binding = inventory.get("evidence_binding", {}) if not isinstance(evidence_binding, dict): mismatches.append("evidence_binding must be an object") evidence_binding = {} + elif set(evidence_binding) != set(EXPECTED_EVIDENCE_BINDING_FILES): + mismatches.append("evidence_binding has an unexpected key set") for name in ("source_contract", "trace", "wire_fixture", "audit"): entry = evidence_binding.get(name, {}) if not isinstance(entry, dict) or not entry.get("file") or not re.fullmatch( r"[0-9a-f]{64}", str(entry.get("sha256", "")) ): mismatches.append(f"loss: evidence_binding.{name} is missing or malformed") + elif entry.get("file") != EXPECTED_EVIDENCE_BINDING_FILES[name]: + mismatches.append( + f"mutation: evidence_binding.{name}.file does not name the retained fixture" + ) if evidence_binding and all( isinstance(evidence_binding.get(name), dict) @@ -2617,7 +2719,7 @@ def reconcile_inventory( "evidence_gates", "blocking_gates", "blocking_scopes", - "ready_for_beta1", + "ready_for_beta2", ): if qualification.get(field) != expected_qualification[field]: mismatches.append( diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 997e1a3c..b16888cd 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -958,6 +958,9 @@ $qualification = $inventory.qualification if (-not $qualification -or $qualification.candidate_version_status -notin @('eligible','legacy_below_floor')) { Add-Mismatch 'inventory qualification candidate version status is invalid' } +if ($qualification.PSObject.Properties.Name -contains 'ready_for_beta1') { + Add-Mismatch 'inventory qualification ready_for_beta1 is stale; use ready_for_beta2' +} $blockingScopes = @( $coreEvidence.Keys + $requiredLiveControlScopes + $requiredChoiceScope | Sort-Object -Unique @@ -1031,8 +1034,8 @@ if ((($qualification.blocking_gates | Sort-Object) -join '|') -ne (($observedBlo Add-Mismatch 'inventory qualification blocking_gates does not match trace/audit' } $expectedReady = $expectedCandidateEligible -and $observedBlockingScopes.Count -eq 0 -and $observedBlockingGates.Count -eq 0 -if ([bool]$qualification.ready_for_beta1 -ne $expectedReady) { - Add-Mismatch 'inventory qualification ready_for_beta1 is inconsistent with evidence blockers' +if ([bool]$qualification.ready_for_beta2 -ne $expectedReady) { + Add-Mismatch 'inventory qualification ready_for_beta2 is inconsistent with evidence blockers' } $advancedScopes = @('code_mode','tool_search','collaboration_v2','chat_conversion') foreach ($scope in $advancedScopes) { diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index a77c7a4e..ea4cf1c4 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -55,7 +55,7 @@ def test_inventory_is_bound_to_supported_cli_floor_and_candidate_identity() -> N assert len(candidate["evidence_manifest_sha256"]) == 64 assert inventory["qualification"]["candidate_version_status"] == "eligible" assert inventory["qualification"]["candidate_version_eligible"] is True - assert inventory["qualification"]["ready_for_beta1"] is False + assert inventory["qualification"]["ready_for_beta2"] is False assert inventory["identity_control"]["unknown_tagged_source_count"] == 2 assert inventory["qualification"]["blocking_gates"] == [ "clean_cold_start_current_binding", @@ -259,7 +259,7 @@ def test_inventory_reports_zero_unclassified_core_items() -> None: assert inventory["identity_control"]["unclassified_core_items"] == 0 assert inventory["identity_control"]["unclassified_scopes"] == [] - assert inventory["qualification"]["ready_for_beta1"] is False + assert inventory["qualification"]["ready_for_beta2"] is False assert "core_function_replay" in inventory["qualification"]["blocking_scopes"] @@ -448,6 +448,36 @@ def test_build_inventory_rejects_observed_source_contract_family(tmp_path: Path) ) +@pytest.mark.parametrize( + "family, field, value", + [ + ("selected_provider_hosted", "status", "captured"), + ("unknown_future_kind", "status", "captured"), + ("plain_function", "status", "future_status"), + ], +) +def test_build_inventory_rejects_source_contract_status_or_unknown_fields( + tmp_path: Path, family: str, field: str, value: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + source_contract["runtime_wire_surface"]["declaration_family_examples"][family][ + field + ] = value + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="(?:status is invalid|unknown status field)"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + def test_build_inventory_rejects_captured_source_contract_control(tmp_path: Path) -> None: module = load_inventory_module() source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) @@ -571,6 +601,83 @@ def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root assert any("non-empty string" in mismatch for mismatch in id_report["mismatches"]) +def test_inventory_reconcile_rejects_observed_family_without_evidence_root() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["declaration_families"][0]["observed"] = True + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("observed must remain false" in mismatch for mismatch in report["mismatches"]) + + +def test_structural_reconcile_allows_observed_family_when_bound_evidence_is_authoritative() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["declaration_families"][0]["observed"] = True + + assert not any( + "observed must remain false" + in mismatch + for mismatch in module._structural_inventory_mismatches( + inventory["declaration_families"], require_unobserved=False + ) + ) + + +@pytest.mark.parametrize( + "field, value", + [ + ("source_commit", "0" * 40), + ("route_upstream", "custom"), + ("model", "gpt-5.5"), + ], +) +def test_inventory_reconcile_rejects_candidate_binding_mutation_without_evidence_root( + field: str, value: str +) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["candidate_identity"][field] = value + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained Issue #62 candidate" in mismatch for mismatch in report["mismatches"]) + + +def test_inventory_reconcile_rejects_evidence_pointer_and_self_reported_readiness_mutation() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["evidence_binding"]["trace"]["file"] = "evil.json" + inventory["qualification"]["evidence_gates"] = { + gate: "complete" for gate in inventory["qualification"]["evidence_gates"] + } + inventory["qualification"]["blocking_gates"] = [] + inventory["qualification"]["ready_for_beta2"] = True + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained fixture" in mismatch for mismatch in report["mismatches"]) + assert any("cannot be asserted without bound evidence" in mismatch for mismatch in report["mismatches"]) + assert any("blocking_gates cannot be empty" in mismatch for mismatch in report["mismatches"]) + + +def test_inventory_reconcile_rejects_legacy_beta1_readiness_key() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["qualification"]["ready_for_beta1"] = inventory["qualification"].pop( + "ready_for_beta2" + ) + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("ready_for_beta1 is stale" in mismatch for mismatch in report["mismatches"]) + + def test_build_inventory_rejects_unbound_response_identity_pointer(tmp_path: Path) -> None: module = load_inventory_module() wire = json.loads(WIRE_FIXTURE.read_text(encoding="utf-8")) @@ -619,7 +726,7 @@ def test_captured_sse_status_does_not_satisfy_independent_sse_gate() -> None: ) assert qualification["evidence_gates"]["sse_identity"] == "captured" assert "sse_identity" in qualification["blocking_gates"] - assert qualification["ready_for_beta1"] is False + assert qualification["ready_for_beta2"] is False def test_committed_inventory_matches_generator_output() -> None: @@ -873,7 +980,7 @@ def test_qualification_accepts_audit_met_status_when_all_gates_are_complete() -> }, ) assert qualification["blocking_gates"] == [] - assert qualification["ready_for_beta1"] is True + assert qualification["ready_for_beta2"] is True @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) From ec97a7203a1f219abc3038dba66c0a9b9b3386ce Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 19:25:29 +0800 Subject: [PATCH 08/13] test(issue-62): reject unknown source contract fields --- scripts/build_issue_62_runtime_inventory.py | 17 ++++++++++++++++- tests/test_issue_62_runtime_inventory.py | 5 ++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index 19cac9b1..20034988 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -2036,7 +2036,22 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] "observed": False, "status": "opaque_sentinel_only", }, - }.get(family, {}) + }.get(family, {}) + expected_example_fields = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + *expected_status_fields, + } + if set(example) != expected_example_fields: + raise ValueError( + f"Codex 0.146 source contract {family} has unknown example fields" + ) for field, expected_value in expected_status_fields.items(): if example.get(field) != expected_value: raise ValueError( diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index ea4cf1c4..4a042c47 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -454,6 +454,7 @@ def test_build_inventory_rejects_observed_source_contract_family(tmp_path: Path) ("selected_provider_hosted", "status", "captured"), ("unknown_future_kind", "status", "captured"), ("plain_function", "status", "future_status"), + ("plain_function", "unknown_field", "future_status"), ], ) def test_build_inventory_rejects_source_contract_status_or_unknown_fields( @@ -469,7 +470,9 @@ def test_build_inventory_rejects_source_contract_status_or_unknown_fields( json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) - with pytest.raises(ValueError, match="(?:status is invalid|unknown status field)"): + with pytest.raises( + ValueError, match="(?:status is invalid|unknown status field|unknown example fields)" + ): module.build_inventory( source_contract=source_contract_path, trace=TRACE, From 81a3f576b09f6b13809618e493c32f3be45fc6d4 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 20:01:37 +0800 Subject: [PATCH 09/13] fix(issue-62): harden standalone replay provenance --- scripts/audit_issue_62_runtime_artifacts.py | 101 +++++++- scripts/build_issue_62_runtime_inventory.py | 268 ++++++++++++++++++++ scripts/check-codex-thread-tool-surface.ps1 | 98 +++++++ tests/test_issue_62_runtime_audit.py | 35 +++ tests/test_issue_62_runtime_inventory.py | 76 ++++++ tests/test_issue_62_runtime_trace.py | 62 +++++ 6 files changed, 638 insertions(+), 2 deletions(-) diff --git a/scripts/audit_issue_62_runtime_artifacts.py b/scripts/audit_issue_62_runtime_artifacts.py index bcbb6eb3..66642dbc 100644 --- a/scripts/audit_issue_62_runtime_artifacts.py +++ b/scripts/audit_issue_62_runtime_artifacts.py @@ -180,6 +180,16 @@ def _source_contract_provenance(path: Path) -> dict[str, Any]: raise ValueError(f"unable to read source contract: {path}") from exc if not isinstance(source_contract, dict): raise ValueError("source contract must be a JSON object") + if set(source_contract) != { + "schema_version", + "fixture_kind", + "capture_status", + "qualification_status", + "captured_at", + "provenance", + "runtime_wire_surface", + }: + raise ValueError("source contract has unknown top-level fields") if ( source_contract.get("schema_version") != 1 or source_contract.get("fixture_kind") != "codex_cli_source_contract" @@ -192,14 +202,23 @@ def _source_contract_provenance(path: Path) -> dict[str, Any]: "source contract must remain not_observed, unqualified, and uncaptured" ) provenance = source_contract.get("provenance") - if not isinstance(provenance, dict) or any( + if not isinstance(provenance, dict) or set(provenance) != set(SOURCE_CONTRACT_PROVENANCE) or any( provenance.get(field) != value for field, value in SOURCE_CONTRACT_PROVENANCE.items() ): - raise ValueError("source contract provenance is invalid") + raise ValueError("source contract provenance is invalid or has unknown fields") runtime_surface = source_contract.get("runtime_wire_surface") if not isinstance(runtime_surface, dict): raise ValueError("source contract runtime surface is missing") + if set(runtime_surface) != { + "source", + "declaration_family_order", + "declaration_families", + "request_shape", + "response_shape", + "declaration_family_examples", + }: + raise ValueError("source contract runtime_wire_surface has unknown fields") if runtime_surface.get("declaration_family_order") != list(DECLARATION_FAMILIES): raise ValueError("source contract declaration family order is invalid") declaration_families = runtime_surface.get("declaration_families") @@ -218,16 +237,52 @@ def _source_contract_provenance(path: Path) -> dict[str, Any]: for family, entry in zip(DECLARATION_FAMILIES, declaration_families): if not isinstance(entry, dict) or entry.get("family") != family: raise ValueError("source contract declaration family identity is invalid") + if set(entry) != { + "family", + "runtime_type", + "wire_declaration_type", + "observed", + "observation", + "executor", + "loss_boundary", + }: + raise ValueError("source contract declaration family has unknown fields") if entry.get("observed") is not False: raise ValueError("source contract cannot claim an observed declaration family") if entry.get("observation") != expected_observations[family]: raise ValueError("source contract declaration observation is invalid") request_shape = runtime_surface.get("request_shape") + if not isinstance(request_shape, dict) or set(request_shape) != { + "protocol", + "streaming_fields", + "representative", + "non_streaming_control", + }: + raise ValueError("source contract request_shape has unknown fields") + if not isinstance(request_shape.get("representative"), dict) or set( + request_shape["representative"] + ) != { + "model", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "stream", + "store", + }: + raise ValueError("source contract request_shape.representative has unknown fields") non_streaming_control = ( request_shape.get("non_streaming_control") if isinstance(request_shape, dict) else None ) + if not isinstance(non_streaming_control, dict) or set(non_streaming_control) != { + "stream", + "response_body", + "captured", + "status", + }: + raise ValueError("source contract request_shape has unknown fields") if not isinstance(non_streaming_control, dict) or non_streaming_control.get( "captured" ) is not False or non_streaming_control.get("status") != "unqualified": @@ -235,10 +290,33 @@ def _source_contract_provenance(path: Path) -> dict[str, Any]: examples = runtime_surface.get("declaration_family_examples") if not isinstance(examples, dict): raise ValueError("source contract declaration examples are missing") + if set(examples) != set(DECLARATION_FAMILIES): + raise ValueError("source contract declaration examples have unknown fields") for family in DECLARATION_FAMILIES: example = examples.get(family) if not isinstance(example, dict): raise ValueError(f"source contract example is missing for {family}") + expected_example_fields = { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } + if family == "selected_provider_hosted": + expected_example_fields |= { + "observed", + "status", + "provider_scope", + "cross_provider_proxy", + } + elif family == "unknown_future_kind": + expected_example_fields |= {"observed", "status"} + if set(example) != expected_example_fields: + raise ValueError(f"source contract {family} has unknown example fields") if ( not isinstance(example.get("terminal"), dict) or example["terminal"].get("event") != "response.completed" @@ -254,7 +332,26 @@ def _source_contract_provenance(path: Path) -> dict[str, Any]: if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: raise ValueError(f"source contract loss boundary is invalid for {family}") response_shape = runtime_surface.get("response_shape") + if not isinstance(response_shape, dict) or set(response_shape) != { + "response_item_types", + "stream_event_order", + "terminal_events", + "error_shape", + }: + raise ValueError("source contract response_shape has unknown fields") error_shape = response_shape.get("error_shape") if isinstance(response_shape, dict) else None + if not isinstance(error_shape, dict) or set(error_shape) != { + "event", + "response", + "classification", + }: + raise ValueError("source contract response_shape.error_shape has unknown fields") + if not isinstance(error_shape.get("response"), dict) or set( + error_shape["response"] + ) != {"id", "status", "error"}: + raise ValueError( + "source contract response_shape.error_shape.response has unknown fields" + ) if not isinstance(error_shape, dict) or error_shape.get("classification") != "unqualified": raise ValueError("source contract response error status is invalid") diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index 20034988..758868bd 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -323,6 +323,74 @@ "audit": "read-only-gate-audit.json", } +# ``reconcile_inventory`` is also used as a standalone replay check. In that +# mode there is deliberately no evidence root to reopen, so hashes, manifests, +# and unobserved qualification fields must be compared with the retained +# package identity rather than recalculated from the candidate's own fields. +# The evidence-root path below remains authoritative for a later observed +# capture and does not use these retained-only assertions. +RETAINED_EVIDENCE_BINDING = { + "source_contract": { + "file": "codex-0.146-source-contract.json", + "sha256": "6f38b8b07b98c6f28edd7418b63449242ee41d6396694392a70c0d7fb2b70f2c", + }, + "trace": { + "file": "current-codexhub-thread-tool-surface.json", + "sha256": "a4367f96fc2aceaa419577ee43f91870fb3fcca4cc95644bc3347cbdd4206ee3", + }, + "wire_fixture": { + "file": "codexhub-runtime-wire-fixture.json", + "sha256": "1e3ecfc51f59de7f21bee8a7a0b743bba26d35fb3dc9eaca5f7b4ea3dd57705f", + }, + "audit": { + "file": "read-only-gate-audit.json", + "sha256": "775efbc763dfae1c5d6a61532d9394e5cd181b2e6803bf2e42f6c886a0a964e3", + }, +} +RETAINED_EVIDENCE_MANIFEST_SHA256 = ( + "685858e1affdb073df17467f53ab3de538c0c7f2d930b01c10797a94963c6619" +) +RETAINED_UNOBSERVED_EVIDENCE_GATES = { + "clean_cold_start_current_binding": "not_run", + "complete_model_visible_plan": "partial", + "error_events": "not_captured", + "full_pre_post_request_response": "live_control_required", + "full_request_fingerprint": "not_captured", + "full_response_fingerprint": "not_captured", + "identity_replay": "partial", + "non_streaming": "live_control_required", + "non_streaming_fixture": "not_captured", + "sse_identity": "not_captured", + "terminal_events": "not_captured", + "wire_identity_replay": "not_captured", +} +RETAINED_UNOBSERVED_BLOCKING_GATES = [ + "clean_cold_start_current_binding", + "complete_model_visible_plan", + "error_events", + "full_pre_post_request_response", + "full_request_fingerprint", + "full_response_fingerprint", + "identity_replay", + "non_streaming", + "non_streaming_fixture", + "sse_identity", + "terminal_events", + "wire_identity_replay", +] +RETAINED_UNOBSERVED_BLOCKING_SCOPES = [ + "choice_controls", + "core_function_replay", + "core_sse_errors", + "core_sse_terminal_events", + "core_text_non_streaming", + "default_runtime_fields", + "errors", + "hosted_only_declarations", + "terminal_events", + "unknown_tagged_sentinels", +] + def _sha256_file(path: Path) -> str: # Evidence is JSON text; hash its canonical LF representation so a @@ -475,6 +543,97 @@ def _candidate_identity_mismatches( return mismatches +def _standalone_retained_mismatches( + *, + inventory: dict[str, Any], + candidate_identity: Any, + qualification: Any, +) -> list[str]: + """Reject self-consistent edits when retained evidence cannot be reopened. + + A no-root replay has no trustworthy input from which to recompute a hash, + gate, or readiness value. Compare those fields with the immutable retained + package identity instead. Observed-capture reconciliation supplies an + evidence root and intentionally takes the dynamic path below. + """ + + mismatches: list[str] = [] + evidence_binding = inventory.get("evidence_binding") + if isinstance(evidence_binding, dict): + for name, expected in RETAINED_EVIDENCE_BINDING.items(): + actual = evidence_binding.get(name) + if actual != expected: + mismatches.append( + f"mutation: evidence_binding.{name} does not match the retained evidence artifact" + ) + if isinstance(candidate_identity, dict) and candidate_identity.get( + "evidence_manifest_sha256" + ) != RETAINED_EVIDENCE_MANIFEST_SHA256: + mismatches.append( + "mutation: candidate_identity.evidence_manifest_sha256 does not match the retained evidence manifest" + ) + + if isinstance(qualification, dict): + if qualification.get("evidence_gates") != RETAINED_UNOBSERVED_EVIDENCE_GATES: + mismatches.append( + "mutation: qualification.evidence_gates does not match the retained unobserved evidence" + ) + if qualification.get("blocking_gates") != RETAINED_UNOBSERVED_BLOCKING_GATES: + mismatches.append( + "mutation: qualification.blocking_gates does not match the retained unobserved evidence" + ) + if qualification.get("blocking_scopes") != RETAINED_UNOBSERVED_BLOCKING_SCOPES: + mismatches.append( + "mutation: qualification.blocking_scopes does not match the retained unobserved evidence" + ) + if qualification.get("ready_for_beta2") is not False: + mismatches.append( + "mutation: qualification.ready_for_beta2 does not match the retained unobserved evidence" + ) + + identity_control = inventory.get("identity_control") + if isinstance(identity_control, dict): + if identity_control.get("unknown_tagged_source_count") != 2: + mismatches.append( + "mutation: identity_control.unknown_tagged_source_count does not match the retained wire evidence" + ) + if identity_control.get("unclassified_core_items") != 0: + mismatches.append( + "mutation: identity_control.unclassified_core_items does not match the retained evidence" + ) + if identity_control.get("unclassified_scopes") != []: + mismatches.append( + "mutation: identity_control.unclassified_scopes does not match the retained evidence" + ) + + families = inventory.get("declaration_families") + if isinstance(families, list): + expected_observations = { + family: STRUCTURAL_FAMILY_SCHEMAS[family]["observation"] + for family in STRUCTURAL_FAMILIES + } + for index, family in enumerate(STRUCTURAL_FAMILIES): + if index >= len(families) or not isinstance(families[index], dict): + continue + entry = families[index] + representative = entry.get("representative") + if entry.get("observed") is not False or entry.get("observation") != expected_observations[family]: + mismatches.append( + f"mutation: declaration_families[{index}] does not match the retained unobserved source contract" + ) + if not isinstance(representative, dict): + continue + expected_statuses = {"terminal": "not_observed", "error": "unqualified"} + for section, expected in expected_statuses.items(): + actual_section = representative.get(section) + if not isinstance(actual_section, dict) or actual_section.get("classification") != expected: + mismatches.append( + f"mutation: declaration_families[{index}].representative.{section} classification does not match the retained unobserved source contract" + ) + + return mismatches + + def _evidence_source_allowed(scope: str, evidence_source: Any) -> bool: expected = SCOPE_EVIDENCE_SOURCES.get(scope) if expected is None: @@ -1943,6 +2102,17 @@ def _build_qualification( def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any]: + expected_top_level_fields = { + "schema_version", + "fixture_kind", + "capture_status", + "qualification_status", + "captured_at", + "provenance", + "runtime_wire_surface", + } + if set(source_contract) != expected_top_level_fields: + raise ValueError("Codex 0.146 source contract has unknown top-level fields") if ( source_contract.get("schema_version") != 1 or source_contract.get("fixture_kind") != "codex_cli_source_contract" @@ -1953,6 +2123,16 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] ): raise ValueError("Codex 0.146 source contract must remain not_observed and unqualified") provenance = source_contract.get("provenance", {}) + expected_provenance_fields = { + "cli_version", + "source_commit", + "cli_source_tag", + "cli_source_commit_status", + "cli_binary_sha256", + "candidate_revision", + } + if not isinstance(provenance, dict) or set(provenance) != expected_provenance_fields: + raise ValueError("Codex 0.146 source contract provenance has unknown fields") expected = { "cli_version": DEFAULT_CLI_FLOOR, "source_commit": CLI_SOURCE_COMMIT, @@ -1966,6 +2146,18 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] runtime_surface = source_contract.get("runtime_wire_surface") if not isinstance(runtime_surface, dict): raise ValueError("Codex 0.146 source contract runtime surface is missing") + expected_runtime_surface_fields = { + "source", + "declaration_family_order", + "declaration_families", + "request_shape", + "response_shape", + "declaration_family_examples", + } + if set(runtime_surface) != expected_runtime_surface_fields: + raise ValueError( + "Codex 0.146 source contract runtime_wire_surface has unknown fields" + ) if runtime_surface.get("declaration_family_order") != list(STRUCTURAL_FAMILIES): raise ValueError("Codex 0.146 source contract declaration family order is invalid") declaration_families = runtime_surface.get("declaration_families") @@ -2007,11 +2199,50 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] "Codex 0.146 source contract declaration observation is invalid" ) request_shape = runtime_surface.get("request_shape") + expected_request_shape_fields = { + "protocol", + "streaming_fields", + "representative", + "non_streaming_control", + } + if not isinstance(request_shape, dict) or set(request_shape) != expected_request_shape_fields: + raise ValueError( + "Codex 0.146 source contract request_shape has unknown fields" + ) + expected_request_representative_fields = { + "model", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "stream", + "store", + } + if ( + not isinstance(request_shape.get("representative"), dict) + or set(request_shape["representative"]) != expected_request_representative_fields + ): + raise ValueError( + "Codex 0.146 source contract request_shape.representative has unknown fields" + ) + expected_non_streaming_control_fields = { + "stream", + "response_body", + "captured", + "status", + } non_streaming_control = ( request_shape.get("non_streaming_control") if isinstance(request_shape, dict) else None ) + if ( + not isinstance(non_streaming_control, dict) + or set(non_streaming_control) != expected_non_streaming_control_fields + ): + raise ValueError( + "Codex 0.146 source contract request_shape.non_streaming_control has unknown fields" + ) if not isinstance(non_streaming_control, dict) or non_streaming_control.get( "captured" ) is not False or non_streaming_control.get("status") != "unqualified": @@ -2021,6 +2252,10 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] examples = runtime_surface.get("declaration_family_examples") if not isinstance(examples, dict): raise ValueError("Codex 0.146 source contract declaration examples are missing") + if set(examples) != set(STRUCTURAL_FAMILIES): + raise ValueError( + "Codex 0.146 source contract declaration_family_examples has unknown fields" + ) for family in STRUCTURAL_FAMILIES: example = examples.get(family) if not isinstance(example, dict): @@ -2086,7 +2321,31 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: raise ValueError(f"Codex 0.146 source contract loss boundary is invalid for {family}") response_shape = runtime_surface.get("response_shape") + expected_response_shape_fields = { + "response_item_types", + "stream_event_order", + "terminal_events", + "error_shape", + } + if not isinstance(response_shape, dict) or set(response_shape) != expected_response_shape_fields: + raise ValueError( + "Codex 0.146 source contract response_shape has unknown fields" + ) error_shape = response_shape.get("error_shape") if isinstance(response_shape, dict) else None + expected_error_shape_fields = {"event", "response", "classification"} + if not isinstance(error_shape, dict) or set(error_shape) != expected_error_shape_fields: + raise ValueError( + "Codex 0.146 source contract response_shape.error_shape has unknown fields" + ) + error_response = error_shape.get("response") + if not isinstance(error_response, dict) or set(error_response) != { + "id", + "status", + "error", + }: + raise ValueError( + "Codex 0.146 source contract response_shape.error_shape.response has unknown fields" + ) if not isinstance(error_shape, dict) or error_shape.get("classification") != "unqualified": raise ValueError("Codex 0.146 source contract response error status is invalid") @@ -2653,6 +2912,15 @@ def reconcile_inventory( if candidate_identity.get("evidence_manifest_sha256") != manifest: mismatches.append("loss: candidate_identity.evidence_manifest_sha256 is stale") + if evidence_root is None: + mismatches.extend( + _standalone_retained_mismatches( + inventory=inventory, + candidate_identity=candidate_identity, + qualification=qualification, + ) + ) + if evidence_root is not None and not mismatches: bound_paths: dict[str, Path] = {} for name in ("source_contract", "trace", "wire_fixture", "audit"): diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index b16888cd..cd722982 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -41,6 +41,86 @@ $expectedFamilySchemas = @( $sourceContractSchemaValid = $true $sourceContractFamilies = @($sourceContract.runtime_wire_surface.declaration_families) $sourceContractExamples = $sourceContract.runtime_wire_surface.declaration_family_examples + +function Test-ExactPropertySet { + param( + [object]$Value, + [string[]]$Expected + ) + + if ($null -eq $Value) { + return $false + } + $actual = @($Value.PSObject.Properties.Name) + foreach ($name in $Expected) { + if ($actual -notcontains $name) { + return $false + } + } + foreach ($name in $actual) { + if ($Expected -notcontains $name) { + return $false + } + } + return $true +} + +if (-not (Test-ExactPropertySet -Value $sourceContract -Expected @( + 'schema_version', 'fixture_kind', 'capture_status', + 'qualification_status', 'captured_at', 'provenance', + 'runtime_wire_surface' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.provenance -Expected @( + 'cli_version', 'source_commit', 'cli_source_tag', + 'cli_source_commit_status', 'cli_binary_sha256', 'candidate_revision' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface -Expected @( + 'source', 'declaration_family_order', 'declaration_families', + 'request_shape', 'response_shape', 'declaration_family_examples' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.request_shape -Expected @( + 'protocol', 'streaming_fields', 'representative', 'non_streaming_control' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.request_shape.representative -Expected @( + 'model', 'input', 'tools', 'tool_choice', 'parallel_tool_calls', 'stream', 'store' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.request_shape.non_streaming_control -Expected @( + 'stream', 'response_body', 'captured', 'status' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape -Expected @( + 'response_item_types', 'stream_event_order', 'terminal_events', 'error_shape' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape.error_shape -Expected @( + 'event', 'response', 'classification' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape.error_shape.response -Expected @( + 'id', 'status', 'error' + ))) { + $sourceContractSchemaValid = $false +} +if (-not (Test-ExactPropertySet -Value $sourceContractExamples -Expected @( + 'plain_function', 'custom_freeform', 'namespace', + 'client_executed_tool_discovery', 'selected_provider_hosted', + 'unknown_future_kind' + ))) { + $sourceContractSchemaValid = $false +} $nullableByFamily = @{ selected_provider_hosted = @{ history = @('call_id') } unknown_future_kind = @{ history = @('call_id', 'call_item_id', 'output_item_id') } @@ -67,6 +147,24 @@ foreach ($expected in $expectedFamilySchemas) { $sourceContractSchemaValid = $false continue } + if (-not (Test-ExactPropertySet -Value $family -Expected @( + 'family', 'runtime_type', 'wire_declaration_type', 'observed', + 'observation', 'executor', 'loss_boundary' + ))) { + $sourceContractSchemaValid = $false + } + $expectedExampleFields = @( + 'declaration', 'call', 'result', 'history', 'streaming', + 'terminal', 'error', 'loss_boundary' + ) + if ($expected.Name -eq 'selected_provider_hosted') { + $expectedExampleFields += @('observed', 'status', 'provider_scope', 'cross_provider_proxy') + } elseif ($expected.Name -eq 'unknown_future_kind') { + $expectedExampleFields += @('observed', 'status') + } + if (-not (Test-ExactPropertySet -Value $example -Expected $expectedExampleFields)) { + $sourceContractSchemaValid = $false + } $sections = @( @{ Name = 'declaration'; Type = $expected.DeclarationType; Required = $expected.DeclarationRequired }, @{ Name = 'call'; Type = $expected.CallType; Required = $expected.CallRequired }, diff --git a/tests/test_issue_62_runtime_audit.py b/tests/test_issue_62_runtime_audit.py index af3cf6fc..dbee054d 100644 --- a/tests/test_issue_62_runtime_audit.py +++ b/tests/test_issue_62_runtime_audit.py @@ -474,6 +474,41 @@ def test_audit_rejects_captured_source_contract_claim(tmp_path: Path) -> None: run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) +@pytest.mark.parametrize( + "mutation", + ["top_level", "runtime_wire_surface", "request_shape", "response_shape"], +) +def test_audit_rejects_unknown_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + module = load_audit_module() + codex_db = tmp_path / "codex.sqlite" + gateway_db = tmp_path / "gateway.sqlite" + create_codex_log_db(codex_db) + create_gateway_db(gateway_db) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation == "top_level": + source_contract["future_field"] = "must not be accepted" + elif mutation == "runtime_wire_surface": + source_contract["runtime_wire_surface"]["future_field"] = "must not be accepted" + elif mutation == "request_shape": + source_contract["runtime_wire_surface"]["request_shape"][ + "future_field" + ] = "must not be accepted" + else: + source_contract["runtime_wire_surface"]["response_shape"][ + "future_field" + ] = "must not be accepted" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unknown"): + run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) + + def test_audit_detects_generic_response_body_fingerprint_fields(tmp_path: Path) -> None: module = load_audit_module() codex_db = tmp_path / "codex.sqlite" diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index 4a042c47..82dab12b 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -501,6 +501,46 @@ def test_build_inventory_rejects_captured_source_contract_control(tmp_path: Path ) +@pytest.mark.parametrize( + "mutation", + [ + "top_level", + "runtime_wire_surface", + "request_shape", + "response_shape", + ], +) +def test_build_inventory_rejects_unknown_regenerated_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation == "top_level": + source_contract["future_field"] = "must not be accepted" + elif mutation == "runtime_wire_surface": + source_contract["runtime_wire_surface"]["future_field"] = "must not be accepted" + elif mutation == "request_shape": + source_contract["runtime_wire_surface"]["request_shape"][ + "future_field" + ] = "must not be accepted" + else: + source_contract["runtime_wire_surface"]["response_shape"][ + "future_field" + ] = "must not be accepted" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="unknown"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + @pytest.mark.parametrize( "mutation", [ @@ -668,6 +708,42 @@ def test_inventory_reconcile_rejects_evidence_pointer_and_self_reported_readines assert any("blocking_gates cannot be empty" in mismatch for mismatch in report["mismatches"]) +def test_inventory_reconcile_standalone_rejects_self_consistent_hash_manifest_edit() -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + inventory["evidence_binding"]["trace"]["sha256"] = "0" * 64 + inventory["candidate_identity"]["evidence_manifest_sha256"] = module._evidence_manifest_sha256( + inventory["evidence_binding"] + ) + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained evidence artifact" in mismatch for mismatch in report["mismatches"]) + assert any("retained evidence manifest" in mismatch for mismatch in report["mismatches"]) + + +@pytest.mark.parametrize("mutation", ["gate", "terminal", "readiness"]) +def test_inventory_reconcile_standalone_rejects_unobserved_status_edits( + mutation: str, +) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + if mutation == "gate": + inventory["qualification"]["evidence_gates"]["terminal_events"] = "met" + elif mutation == "terminal": + inventory["declaration_families"][0]["representative"]["terminal"][ + "classification" + ] = "unqualified" + else: + inventory["qualification"]["ready_for_beta2"] = True + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("retained unobserved" in mismatch for mismatch in report["mismatches"]) + + def test_inventory_reconcile_rejects_legacy_beta1_readiness_key() -> None: module = load_inventory_module() inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 327aa889..6c1f0a9e 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -430,6 +430,68 @@ def test_powershell_rejects_source_contract_family_schema_drift(tmp_path: Path) assert "generated inventory drift check failed" in result.stderr.lower() +@pytest.mark.parametrize( + "mutation", + ["top_level", "runtime_wire_surface", "request_shape", "response_shape"], +) +def test_powershell_rejects_unknown_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + if mutation == "top_level": + source_contract["future_field"] = "must not be accepted" + elif mutation == "runtime_wire_surface": + source_contract["runtime_wire_surface"]["future_field"] = "must not be accepted" + elif mutation == "request_shape": + source_contract["runtime_wire_surface"]["request_shape"][ + "future_field" + ] = "must not be accepted" + else: + source_contract["runtime_wire_surface"]["response_shape"][ + "future_field" + ] = "must not be accepted" + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) def test_negative_inventory_replays_fail_visibly(case: str) -> None: result = run_inventory_replay(case) From 6ecec73fb5dd0acc069ce3fba5626f1727b42bdc Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 20:01:37 +0800 Subject: [PATCH 10/13] fix(issue-62): harden standalone replay provenance --- scripts/build_issue_62_runtime_inventory.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index 758868bd..b5abec72 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -327,8 +327,10 @@ # mode there is deliberately no evidence root to reopen, so hashes, manifests, # and unobserved qualification fields must be compared with the retained # package identity rather than recalculated from the candidate's own fields. -# The evidence-root path below remains authoritative for a later observed -# capture and does not use these retained-only assertions. +# The evidence-root path below validates the currently retained, unobserved +# source-contract package and does not use these retained-only assertions. A +# later observed source contract requires a deliberate versioned schema update; +# this branch does not claim to admit that transition. RETAINED_EVIDENCE_BINDING = { "source_contract": { "file": "codex-0.146-source-contract.json", @@ -553,8 +555,11 @@ def _standalone_retained_mismatches( A no-root replay has no trustworthy input from which to recompute a hash, gate, or readiness value. Compare those fields with the immutable retained - package identity instead. Observed-capture reconciliation supplies an - evidence root and intentionally takes the dynamic path below. + package identity instead. Bound-evidence reconciliation supplies an + evidence root and intentionally takes the dynamic path below. The + source-contract validator remains versioned and unobserved-only; an + observed source contract needs a separate schema transition rather than + silently bypassing these retained checks. """ mismatches: list[str] = [] From 2d1813bb6ded81d19268b427cfe6bf9428d84e5c Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 20:31:46 +0800 Subject: [PATCH 11/13] fix(issue-62): close nested source-contract schemas --- scripts/build_issue_62_runtime_inventory.py | 304 ++++++++++++++++---- scripts/check-codex-thread-tool-surface.ps1 | 115 ++++++++ 2 files changed, 361 insertions(+), 58 deletions(-) diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index b5abec72..f99d6649 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -969,6 +969,78 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: }, }, } +STRUCTURAL_EXAMPLE_BASE_FIELDS = frozenset( + { + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "loss_boundary", + } +) +STRUCTURAL_EXAMPLE_STATUS_FIELDS = { + "selected_provider_hosted": { + "observed": False, + "status": "selected_provider_control_required", + "provider_scope": "selected_provider_only", + "cross_provider_proxy": "forbidden", + }, + "unknown_future_kind": { + "observed": False, + "status": "opaque_sentinel_only", + }, +} +STRUCTURAL_REDACTED_FIELDS = frozenset( + { + "parameters", + "arguments", + "input", + "output", + "format", + "action", + "status", + "tools", + "opaque_payload", + } +) +STRUCTURAL_RESPONSE_ITEM_TYPES = [ + "message", + "reasoning", + "function_call", + "function_call_output", + "custom_tool_call", + "custom_tool_call_output", + "tool_search_call", + "tool_search_output", + "web_search_call", + "local_shell_call", + "compaction", + "context_compaction", + "unknown", +] +STRUCTURAL_RESPONSE_STREAM_EVENT_ORDER = [ + "response.created", + "response.in_progress", + "response.output_item.added", + "response.output_text.delta", + "response.function_call_arguments.delta", + "response.custom_tool_call_input.delta", + "response.function_call_arguments.done", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.completed", +] +STRUCTURAL_RESPONSE_TERMINAL_EVENTS = [ + "response.completed", + "response.incomplete", + "response.failed", +] STRUCTURAL_NONEMPTY_STRING_FIELDS = frozenset( { "name", @@ -987,6 +1059,147 @@ def _streaming_events(wire: dict[str, Any]) -> list[dict[str, Any]]: ) +def _require_exact_fields( + value: dict[str, Any], expected: set[str] | frozenset[str], *, context: str +) -> None: + actual = set(value) + if actual != set(expected): + missing = sorted(set(expected) - actual) + unknown = sorted(actual - set(expected)) + raise ValueError( + f"{context} has unknown or missing fields " + f"(missing={missing!r}, unknown={unknown!r})" + ) + + +def _validate_structural_section( + family_name: str, section_name: str, section: dict[str, Any], *, context: str +) -> None: + section_schema = STRUCTURAL_FAMILY_SCHEMAS[family_name][section_name] + expected_fields = set(section_schema.get("required", ())) + if section_schema.get("type") is not None: + expected_fields.add("type") + _require_exact_fields(section, expected_fields, context=context) + nullable = set(section_schema.get("nullable", ())) + equals = section_schema.get("equals", {}) + for key in expected_fields: + value = section[key] + if key in equals: + if value != equals[key]: + raise ValueError( + f"{context}.{key} does not match the canonical family schema" + ) + continue + if key == "type": + if value != section_schema["type"]: + raise ValueError( + f"{context}.type does not match the canonical family schema" + ) + continue + if value is None: + if key not in nullable: + raise ValueError(f"{context}.{key} cannot be null") + continue + if family_name == "namespace" and section_name == "declaration" and key == "tools": + continue + if key in STRUCTURAL_REDACTED_FIELDS: + if value != "": + raise ValueError(f"{context}.{key} must remain the redacted sentinel") + continue + if key in STRUCTURAL_NONEMPTY_STRING_FIELDS and ( + not isinstance(value, str) or not value + ): + raise ValueError(f"{context}.{key} must be a non-empty string") + if not isinstance(value, str): + raise ValueError(f"{context}.{key} must be a string") + + if family_name == "namespace" and section_name == "declaration": + tools = section["tools"] + if not isinstance(tools, list) or not tools: + raise ValueError(f"{context}.tools must contain a function") + for index, nested in enumerate(tools): + nested_context = f"{context}.tools[{index}]" + if not isinstance(nested, dict): + raise ValueError(f"{nested_context} must be an object") + _require_exact_fields( + nested, + {"type", "name", "parameters"}, + context=nested_context, + ) + if nested.get("type") != "function": + raise ValueError(f"{nested_context}.type must be function") + if not isinstance(nested.get("name"), str) or not nested["name"]: + raise ValueError(f"{nested_context}.name must be a non-empty string") + if nested.get("parameters") != "": + raise ValueError( + f"{nested_context}.parameters must remain the redacted sentinel" + ) + + +def _validate_structural_terminal_error( + example: dict[str, Any], *, context: str +) -> None: + terminal = example.get("terminal") + if not isinstance(terminal, dict): + raise ValueError(f"{context}.terminal must be an object") + _require_exact_fields(terminal, {"event", "classification"}, context=f"{context}.terminal") + if terminal.get("event") != "response.completed": + raise ValueError(f"{context}.terminal.event is invalid") + if terminal.get("classification") not in {"not_observed", "unqualified"}: + raise ValueError(f"{context}.terminal.classification is invalid") + + error = example.get("error") + if not isinstance(error, dict): + raise ValueError(f"{context}.error must be an object") + _require_exact_fields(error, {"event", "classification"}, context=f"{context}.error") + if error.get("event") != "response.failed": + raise ValueError(f"{context}.error.event is invalid") + if error.get("classification") not in {"not_observed", "unqualified"}: + raise ValueError(f"{context}.error.classification is invalid") + + +def _validate_response_shape_schema(response_shape: Any, *, context: str) -> None: + if not isinstance(response_shape, dict): + raise ValueError(f"{context} must be an object") + _require_exact_fields( + response_shape, + {"response_item_types", "stream_event_order", "terminal_events", "error_shape"}, + context=context, + ) + if response_shape.get("response_item_types") != STRUCTURAL_RESPONSE_ITEM_TYPES: + raise ValueError(f"{context}.response_item_types does not match the canonical list") + if response_shape.get("stream_event_order") != STRUCTURAL_RESPONSE_STREAM_EVENT_ORDER: + raise ValueError(f"{context}.stream_event_order does not match the canonical list") + if response_shape.get("terminal_events") != STRUCTURAL_RESPONSE_TERMINAL_EVENTS: + raise ValueError(f"{context}.terminal_events does not match the canonical list") + error_shape = response_shape.get("error_shape") + if not isinstance(error_shape, dict): + raise ValueError(f"{context}.error_shape must be an object") + _require_exact_fields( + error_shape, + {"event", "response", "classification"}, + context=f"{context}.error_shape", + ) + if error_shape.get("event") != "response.failed": + raise ValueError(f"{context}.error_shape.event is invalid") + if error_shape.get("classification") != "unqualified": + raise ValueError(f"{context}.error_shape.classification is invalid") + error_response = error_shape.get("response") + if not isinstance(error_response, dict): + raise ValueError(f"{context}.error_shape.response must be an object") + _require_exact_fields( + error_response, + {"id", "status", "error"}, + context=f"{context}.error_shape.response", + ) + if error_response.get("id") != "response_error_001": + raise ValueError(f"{context}.error_shape.response.id is invalid") + if error_response.get("status") != "failed": + raise ValueError(f"{context}.error_shape.response.status is invalid") + if error_response.get("error") != "": + raise ValueError(f"{context}.error_shape.response.error must remain redacted") + + def _validate_structural_family_schema( family_name: str, family: dict[str, Any], @@ -1000,42 +1213,31 @@ def _validate_structural_family_schema( for field in ("runtime_type", "wire_declaration_type", "executor", "observation", "loss_boundary"): if family.get(field) != schema[field]: raise ValueError(f"{context}.{field} does not match the canonical family schema") + if not isinstance(example, dict): + raise ValueError(f"{context} must be an object") + expected_example_fields = STRUCTURAL_EXAMPLE_BASE_FIELDS | set( + STRUCTURAL_EXAMPLE_STATUS_FIELDS.get(family_name, {}) + ) + _require_exact_fields(example, expected_example_fields, context=context) + for field, expected in STRUCTURAL_EXAMPLE_STATUS_FIELDS.get(family_name, {}).items(): + if example.get(field) != expected: + raise ValueError(f"{context}.{field} does not match the canonical status") for section_name in ("declaration", "call", "result", "history"): section = example.get(section_name) - section_schema = schema[section_name] if not isinstance(section, dict): raise ValueError(f"{context}.{section_name} must be an object") - expected_type = section_schema.get("type") - if expected_type is not None and section.get("type") != expected_type: - raise ValueError(f"{context}.{section_name}.type does not match the canonical family schema") - nullable = set(section_schema.get("nullable", ())) - for key in section_schema.get("required", ()): - if key not in section: - raise ValueError(f"{context}.{section_name}.{key} is required") - if key in STRUCTURAL_NONEMPTY_STRING_FIELDS and section[key] is not None and ( - not isinstance(section[key], str) or not section[key] - ): - raise ValueError( - f"{context}.{section_name}.{key} must be a non-empty string" - ) - if key not in nullable and section[key] is None: - raise ValueError(f"{context}.{section_name}.{key} cannot be null") - for key, expected in section_schema.get("equals", {}).items(): - if section.get(key) != expected: - raise ValueError(f"{context}.{section_name}.{key} does not match the canonical family schema") + _validate_structural_section( + family_name, + section_name, + section, + context=f"{context}.{section_name}", + ) + if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: + raise ValueError(f"{context}.loss_boundary is missing or blank") + if example["loss_boundary"] != schema["loss_boundary"]: + raise ValueError(f"{context}.loss_boundary does not match the canonical family schema") + _validate_structural_terminal_error(example, context=context) if family_name == "namespace": - tools = example["declaration"].get("tools") - if not isinstance(tools, list) or not tools: - raise ValueError(f"{context}.declaration.tools must contain a function") - for nested in tools: - if ( - not isinstance(nested, dict) - or nested.get("type") != "function" - or not isinstance(nested.get("name"), str) - or not nested.get("name") - or "parameters" not in nested - ): - raise ValueError(f"{context}.declaration.tools contains an invalid function") namespace = example["declaration"].get("name") if ( example["call"].get("namespace") != namespace @@ -1174,6 +1376,15 @@ def _validate_structural_stream_example( raise ValueError(f"{context} has an unknown stream schema") if not isinstance(streaming, dict): raise ValueError(f"{context}.streaming must be an object") + expected_fields = { + "added", + "delta", + "done", + "terminal", + "event_order", + schema["done_field"], + } + _require_exact_fields(streaming, expected_fields, context=f"{context}.streaming") for field in ("added", "delta", "terminal"): if streaming.get(field) != schema[field]: raise ValueError(f"{context}.streaming.{field} does not match the canonical SSE schema") @@ -2326,33 +2537,10 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] if not isinstance(example.get("loss_boundary"), str) or not example["loss_boundary"]: raise ValueError(f"Codex 0.146 source contract loss boundary is invalid for {family}") response_shape = runtime_surface.get("response_shape") - expected_response_shape_fields = { - "response_item_types", - "stream_event_order", - "terminal_events", - "error_shape", - } - if not isinstance(response_shape, dict) or set(response_shape) != expected_response_shape_fields: - raise ValueError( - "Codex 0.146 source contract response_shape has unknown fields" - ) - error_shape = response_shape.get("error_shape") if isinstance(response_shape, dict) else None - expected_error_shape_fields = {"event", "response", "classification"} - if not isinstance(error_shape, dict) or set(error_shape) != expected_error_shape_fields: - raise ValueError( - "Codex 0.146 source contract response_shape.error_shape has unknown fields" - ) - error_response = error_shape.get("response") - if not isinstance(error_response, dict) or set(error_response) != { - "id", - "status", - "error", - }: - raise ValueError( - "Codex 0.146 source contract response_shape.error_shape.response has unknown fields" - ) - if not isinstance(error_shape, dict) or error_shape.get("classification") != "unqualified": - raise ValueError("Codex 0.146 source contract response error status is invalid") + _validate_response_shape_schema( + response_shape, + context="Codex 0.146 source contract response_shape", + ) def _captured_true(value: Any) -> bool: if isinstance(value, dict): diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index cd722982..56793b14 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -99,6 +99,21 @@ if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.requ ))) { $sourceContractSchemaValid = $false } +$expectedResponseItemTypes = @( + 'message', 'reasoning', 'function_call', 'function_call_output', + 'custom_tool_call', 'custom_tool_call_output', 'tool_search_call', + 'tool_search_output', 'web_search_call', 'local_shell_call', + 'compaction', 'context_compaction', 'unknown' +) +$expectedResponseStreamEventOrder = @( + 'response.created', 'response.in_progress', 'response.output_item.added', + 'response.output_text.delta', 'response.function_call_arguments.delta', + 'response.custom_tool_call_input.delta', 'response.function_call_arguments.done', + 'response.custom_tool_call_input.done', 'response.output_item.done', + 'response.reasoning_summary_part.added', 'response.reasoning_summary_text.delta', + 'response.reasoning_summary_text.done', 'response.completed' +) +$expectedResponseTerminalEvents = @('response.completed', 'response.incomplete', 'response.failed') if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.response_shape -Expected @( 'response_item_types', 'stream_event_order', 'terminal_events', 'error_shape' ))) { @@ -114,6 +129,24 @@ if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.resp ))) { $sourceContractSchemaValid = $false } +if (($sourceContract.runtime_wire_surface.response_shape.response_item_types -join '|') -ne ($expectedResponseItemTypes -join '|')) { + $sourceContractSchemaValid = $false +} +if (($sourceContract.runtime_wire_surface.response_shape.stream_event_order -join '|') -ne ($expectedResponseStreamEventOrder -join '|')) { + $sourceContractSchemaValid = $false +} +if (($sourceContract.runtime_wire_surface.response_shape.terminal_events -join '|') -ne ($expectedResponseTerminalEvents -join '|')) { + $sourceContractSchemaValid = $false +} +if ( + $sourceContract.runtime_wire_surface.response_shape.error_shape.event -ne 'response.failed' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.classification -ne 'unqualified' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.response.id -ne 'response_error_001' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.response.status -ne 'failed' -or + $sourceContract.runtime_wire_surface.response_shape.error_shape.response.error -ne '' +) { + $sourceContractSchemaValid = $false +} if (-not (Test-ExactPropertySet -Value $sourceContractExamples -Expected @( 'plain_function', 'custom_freeform', 'namespace', 'client_executed_tool_discovery', 'selected_provider_hosted', @@ -134,6 +167,8 @@ $expectedSseSchemas = @{ selected_provider_hosted = @{ Added = 'response.output_item.added'; Delta = ''; DoneField = 'done'; Done = 'response.output_item.done'; ItemDone = 'response.output_item.done'; Terminal = 'response.completed'; Order = @('response.output_item.added', '', 'response.output_item.done', 'response.completed') } unknown_future_kind = @{ Added = 'unknown.future_event'; Delta = 'unknown.future_delta'; DoneField = 'done'; Done = 'unknown.future_done'; ItemDone = 'unknown.future_done'; Terminal = 'response.completed'; Order = @('unknown.future_event', 'unknown.future_delta', 'unknown.future_done', 'response.completed') } } +$redactedStructuralFields = @('parameters', 'arguments', 'input', 'output', 'format', 'action', 'status', 'tools', 'opaque_payload') +$nonEmptyStructuralFields = @('name', 'namespace', 'executor', 'execution', 'provider_scope', 'cross_provider_proxy', 'tag', 'loss_rule', 'item_id', 'call_id', 'call_item_id', 'output_item_id') foreach ($expected in $expectedFamilySchemas) { $family = @($sourceContractFamilies | Where-Object { $_.family -eq $expected.Name }) | Select-Object -First 1 $exampleProperty = if ($null -ne $sourceContractExamples) { $sourceContractExamples.PSObject.Properties[$expected.Name] } else { $null } @@ -165,6 +200,16 @@ foreach ($expected in $expectedFamilySchemas) { if (-not (Test-ExactPropertySet -Value $example -Expected $expectedExampleFields)) { $sourceContractSchemaValid = $false } + if ($expected.Name -eq 'selected_provider_hosted' -and + ($example.observed -ne $false -or + $example.status -ne 'selected_provider_control_required' -or + $example.provider_scope -ne 'selected_provider_only' -or + $example.cross_provider_proxy -ne 'forbidden')) { + $sourceContractSchemaValid = $false + } elseif ($expected.Name -eq 'unknown_future_kind' -and + ($example.observed -ne $false -or $example.status -ne 'opaque_sentinel_only')) { + $sourceContractSchemaValid = $false + } $sections = @( @{ Name = 'declaration'; Type = $expected.DeclarationType; Required = $expected.DeclarationRequired }, @{ Name = 'call'; Type = $expected.CallType; Required = $expected.CallRequired }, @@ -178,6 +223,13 @@ foreach ($expected in $expectedFamilySchemas) { $sourceContractSchemaValid = $false continue } + $expectedSectionFields = @($sectionSpec.Required) + if ($null -ne $sectionSpec.Type) { + $expectedSectionFields += 'type' + } + if (-not (Test-ExactPropertySet -Value $section -Expected $expectedSectionFields)) { + $sourceContractSchemaValid = $false + } foreach ($required in $sectionSpec.Required) { $requiredProperty = $section.PSObject.Properties[$required] $nullableFields = @() @@ -193,11 +245,74 @@ foreach ($expected in $expectedFamilySchemas) { $sourceContractSchemaValid = $false } } + foreach ($property in $section.PSObject.Properties) { + if ($property.Name -eq 'type' -or $property.Name -in $nonEmptyStructuralFields) { + continue + } + if ($property.Name -eq 'tools' -and $expected.Name -eq 'namespace' -and $sectionSpec.Name -eq 'declaration') { + if ($property.Value -isnot [array] -or @($property.Value).Count -eq 0) { + $sourceContractSchemaValid = $false + continue + } + foreach ($nested in @($property.Value)) { + if (-not (Test-ExactPropertySet -Value $nested -Expected @('type', 'name', 'parameters')) -or + $nested.type -ne 'function' -or + $nested.name -isnot [string] -or [string]::IsNullOrEmpty($nested.name) -or + $nested.parameters -ne '') { + $sourceContractSchemaValid = $false + } + } + continue + } + if ($property.Name -in $redactedStructuralFields -and $property.Value -ne '') { + $sourceContractSchemaValid = $false + } elseif ($property.Value -isnot [string]) { + $sourceContractSchemaValid = $false + } + } + } + if ($expected.Name -eq 'client_executed_tool_discovery' -and + ($example.declaration.execution -ne 'client' -or + $example.call.execution -ne 'client' -or + $example.result.execution -ne 'client' -or + $example.history.executor -ne 'codex_client')) { + $sourceContractSchemaValid = $false + } + if ($expected.Name -eq 'selected_provider_hosted' -and + ($example.declaration.executor -ne 'selected_provider' -or + $example.declaration.provider_scope -ne 'selected_provider_only' -or + $example.result.provider_scope -ne 'selected_provider_only' -or + $example.history.executor -ne 'selected_provider' -or + $example.history.cross_provider_proxy -ne 'forbidden')) { + $sourceContractSchemaValid = $false + } + if ($expected.Name -eq 'unknown_future_kind' -and + ($example.declaration.tag -ne 'unknown' -or + $example.call.tag -ne 'unknown' -or + $example.result.tag -ne 'unknown' -or + $example.history.call_id -ne $null -or + $example.history.call_item_id -ne $null -or + $example.history.output_item_id -ne $null -or + $example.history.loss_rule -ne 'retain opaque sentinel')) { + $sourceContractSchemaValid = $false + } + if (-not (Test-ExactPropertySet -Value $example.terminal -Expected @('event', 'classification')) -or + $example.terminal.event -ne 'response.completed' -or + $example.terminal.classification -notin @('not_observed', 'unqualified') -or + -not (Test-ExactPropertySet -Value $example.error -Expected @('event', 'classification')) -or + $example.error.event -ne 'response.failed' -or + $example.error.classification -notin @('not_observed', 'unqualified') -or + $example.loss_boundary -isnot [string] -or [string]::IsNullOrEmpty($example.loss_boundary)) { + $sourceContractSchemaValid = $false } $sseSchema = $expectedSseSchemas[$expected.Name] $streamProperty = $example.PSObject.Properties['streaming'] $stream = if ($null -ne $streamProperty) { $streamProperty.Value } else { $null } $doneProperty = if ($null -ne $stream) { $stream.PSObject.Properties[$sseSchema.DoneField] } else { $null } + $expectedStreamFields = @('added', 'delta', 'done', 'terminal', 'event_order', $sseSchema.DoneField) | Select-Object -Unique + if (-not (Test-ExactPropertySet -Value $stream -Expected $expectedStreamFields)) { + $sourceContractSchemaValid = $false + } if ($null -eq $stream -or $stream.added -ne $sseSchema.Added -or $stream.delta -ne $sseSchema.Delta -or From b88cf5c13bdb60894bdbe67cddd59716c91a0344 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 20:36:02 +0800 Subject: [PATCH 12/13] test(issue-62): reject nested source-contract edits --- tests/test_issue_62_runtime_inventory.py | 100 +++++++++++++++++++++++ tests/test_issue_62_runtime_trace.py | 81 ++++++++++++++++++ 2 files changed, 181 insertions(+) diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index 82dab12b..85bd4613 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -627,6 +627,89 @@ def test_build_inventory_rejects_source_contract_identity_and_sse_mutations( ) +@pytest.mark.parametrize( + "mutation", + [ + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "namespace_tool", + ], +) +def test_build_inventory_rejects_unknown_nested_family_fields( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + examples = source_contract["runtime_wire_surface"]["declaration_family_examples"] + if mutation == "declaration": + examples["plain_function"]["declaration"]["future_field"] = "must fail" + elif mutation == "call": + examples["plain_function"]["call"]["future_field"] = "must fail" + elif mutation == "result": + examples["custom_freeform"]["result"]["future_field"] = "must fail" + elif mutation == "history": + examples["client_executed_tool_discovery"]["history"]["future_field"] = "must fail" + elif mutation == "streaming": + examples["plain_function"]["streaming"]["future_field"] = "must fail" + elif mutation == "terminal": + examples["plain_function"]["terminal"]["future_field"] = "must fail" + elif mutation == "error": + examples["plain_function"]["error"]["future_field"] = "must fail" + else: + examples["namespace"]["declaration"]["tools"][0]["future_field"] = "must fail" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="unknown or missing fields"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + +@pytest.mark.parametrize( + "mutation", + ["redacted_value", "hosted_status", "response_item_type"], +) +def test_build_inventory_rejects_nested_family_value_extensions( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + surface = source_contract["runtime_wire_surface"] + examples = surface["declaration_family_examples"] + if mutation == "redacted_value": + examples["plain_function"]["call"]["arguments"] = "future-arguments" + elif mutation == "hosted_status": + examples["selected_provider_hosted"]["status"] = "captured" + else: + surface["response_shape"]["response_item_types"].append("future_item") + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises( + ValueError, + match="(?:redacted sentinel|canonical status|canonical list|status is invalid)", + ): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root() -> None: module = load_inventory_module() inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) @@ -644,6 +727,23 @@ def test_inventory_reconcile_rejects_mutated_family_schema_without_evidence_root assert any("non-empty string" in mismatch for mismatch in id_report["mismatches"]) +@pytest.mark.parametrize("mutation", ["representative_future_field", "representative_status"]) +def test_inventory_reconcile_rejects_nested_representative_edits(mutation: str) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + representative = inventory["declaration_families"][0]["representative"] + if mutation == "representative_future_field": + representative["future_field"] = "must fail" + else: + representative["status"] = "captured" + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert report["mismatches"] + assert any("unknown or missing fields" in mismatch for mismatch in report["mismatches"]) + + def test_inventory_reconcile_rejects_observed_family_without_evidence_root() -> None: module = load_inventory_module() inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 6c1f0a9e..826eb8c8 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -492,6 +492,87 @@ def test_powershell_rejects_unknown_source_contract_fields( assert "generated inventory drift check failed" in result.stderr.lower() +@pytest.mark.parametrize( + "mutation", + [ + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "namespace_tool", + "response_item_type", + ], +) +def test_powershell_rejects_unknown_nested_source_contract_fields( + tmp_path: Path, mutation: str +) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + examples = source_contract["runtime_wire_surface"]["declaration_family_examples"] + if mutation == "declaration": + examples["plain_function"]["declaration"]["future_field"] = "must fail" + elif mutation == "call": + examples["plain_function"]["call"]["future_field"] = "must fail" + elif mutation == "result": + examples["custom_freeform"]["result"]["future_field"] = "must fail" + elif mutation == "history": + examples["client_executed_tool_discovery"]["history"]["future_field"] = "must fail" + elif mutation == "streaming": + examples["plain_function"]["streaming"]["future_field"] = "must fail" + elif mutation == "terminal": + examples["plain_function"]["terminal"]["future_field"] = "must fail" + elif mutation == "error": + examples["plain_function"]["error"]["future_field"] = "must fail" + elif mutation == "namespace_tool": + examples["namespace"]["declaration"]["tools"][0]["future_field"] = "must fail" + else: + source_contract["runtime_wire_surface"]["response_shape"]["response_item_types"].append( + "future_item" + ) + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) def test_negative_inventory_replays_fail_visibly(case: str) -> None: result = run_inventory_replay(case) From 69f6ccebb13e7531e4cc6e35c06f19dd440c4e64 Mon Sep 17 00:00:00 2001 From: Paul Qu Date: Tue, 4 Aug 2026 21:12:15 +0800 Subject: [PATCH 13/13] fix(issue-62): close source contract value parity --- scripts/audit_issue_62_runtime_artifacts.py | 28 +++ scripts/build_issue_62_runtime_inventory.py | 258 ++++++++++++++++++-- scripts/check-codex-thread-tool-surface.ps1 | 30 +++ tests/test_issue_62_runtime_audit.py | 108 ++++++++ tests/test_issue_62_runtime_inventory.py | 100 ++++++++ tests/test_issue_62_runtime_trace.py | 83 +++++++ 6 files changed, 588 insertions(+), 19 deletions(-) diff --git a/scripts/audit_issue_62_runtime_artifacts.py b/scripts/audit_issue_62_runtime_artifacts.py index 66642dbc..afea3521 100644 --- a/scripts/audit_issue_62_runtime_artifacts.py +++ b/scripts/audit_issue_62_runtime_artifacts.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import importlib.util import json import re import sqlite3 @@ -171,6 +172,29 @@ def _is_proven_tool_choice(value: Any) -> bool: ) +def _validate_source_contract_with_inventory(source_contract: dict[str, Any]) -> dict[str, Any]: + """Apply the canonical source-contract validator used by inventory replay. + + The sanitizer and inventory are independent entry points, but they must + reject the same unobserved source-contract mutations. Load the sibling + generator by path so this module remains directly executable and importable + from tests without relying on ``scripts`` being a Python package. + """ + + validator_path = Path(__file__).with_name("build_issue_62_runtime_inventory.py") + spec = importlib.util.spec_from_file_location( + "_issue_62_runtime_inventory_contract_validator", validator_path + ) + if spec is None or spec.loader is None: + raise ValueError("unable to load canonical source-contract validator") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + validator = getattr(module, "_validate_source_contract", None) + if not callable(validator): + raise ValueError("canonical source-contract validator is unavailable") + return validator(source_contract) + + def _source_contract_provenance(path: Path) -> dict[str, Any]: """Load and validate the unobserved 0.146 source-contract identity.""" @@ -180,6 +204,10 @@ def _source_contract_provenance(path: Path) -> dict[str, Any]: raise ValueError(f"unable to read source contract: {path}") from exc if not isinstance(source_contract, dict): raise ValueError("source contract must be a JSON object") + try: + _validate_source_contract_with_inventory(source_contract) + except ValueError as exc: + raise ValueError(str(exc)) from exc if set(source_contract) != { "schema_version", "fixture_kind", diff --git a/scripts/build_issue_62_runtime_inventory.py b/scripts/build_issue_62_runtime_inventory.py index f99d6649..6995eb2c 100644 --- a/scripts/build_issue_62_runtime_inventory.py +++ b/scripts/build_issue_62_runtime_inventory.py @@ -60,6 +60,40 @@ "unknown_future_kind", ) +REQUEST_SHAPE_PROTOCOL = "responses" +REQUEST_SHAPE_STREAMING_FIELDS = [ + "model", + "instructions", + "input", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "store", + "stream", + "stream_options", + "include", + "service_tier", + "prompt_cache_key", + "text", + "client_metadata", +] +REQUEST_SHAPE_REPRESENTATIVE = { + "model": "gpt-5.6-sol", + "input": "", + "tools": "", + "tool_choice": "auto", + "parallel_tool_calls": False, + "stream": True, + "store": False, +} +REQUEST_SHAPE_NON_STREAMING_CONTROL = { + "stream": False, + "response_body": "", + "captured": False, + "status": "unqualified", +} + ALLOWED_DISPOSITIONS = ( "preserved", "reversibly_adapted", @@ -322,6 +356,104 @@ "wire_fixture": "codexhub-runtime-wire-fixture.json", "audit": "read-only-gate-audit.json", } +INVENTORY_TOP_LEVEL_FIELDS = frozenset( + { + "schema_version", + "artifact_kind", + "cli_version_floor", + "candidate_identity", + "declaration_families", + "qualification", + "disposition_vocabulary", + "items", + "identity_control", + "feeds", + "evidence_sources", + "evidence_binding", + } +) +INVENTORY_CANDIDATE_FIELDS = frozenset( + { + "candidate_revision", + "catalog_binding", + "catalog_model_entry_id", + "catalog_snapshot_sha256", + "cli_binary_sha256", + "cli_source_commit_status", + "cli_source_tag", + "cli_version", + "codex_source_commit", + "configured_provider_id", + "evidence_manifest_sha256", + "inbound_format", + "model", + "route_behavior_profile", + "route_upstream", + "source_commit", + "upstream_format", + } +) +INVENTORY_FAMILY_FIELDS = frozenset( + { + "family", + "runtime_type", + "wire_declaration_type", + "observed", + "observation", + "executor", + "loss_boundary", + "selected_protocol_disposition", + "optional_rule", + "required_rule", + "evidence_source", + "representative", + } +) +INVENTORY_ITEM_FIELDS = frozenset({"scope", "disposition", "evidence_source"}) +INVENTORY_QUALIFICATION_FIELDS = frozenset( + { + "candidate_version_status", + "candidate_version_eligible", + "blocking_scopes", + "evidence_gates", + "blocking_gates", + "ready_for_beta2", + } +) +INVENTORY_EVIDENCE_GATES = frozenset( + { + "complete_model_visible_plan", + "clean_cold_start_current_binding", + "full_pre_post_request_response", + "full_request_fingerprint", + "full_response_fingerprint", + "sse_identity", + "terminal_events", + "error_events", + "non_streaming", + "non_streaming_fixture", + "identity_replay", + "wire_identity_replay", + } +) +INVENTORY_IDENTITY_CONTROL_FIELDS = frozenset( + { + "fail_closed", + "replay_cases", + "unclassified_core_items", + "unclassified_scopes", + "unknown_tagged_source_count", + } +) +INVENTORY_EVIDENCE_SOURCES_FIELDS = frozenset({"trace", "wire_fixture", "audit"}) +INVENTORY_FEEDS_FIELDS = frozenset({"capability_gate", "chat_conversion_matrix"}) +INVENTORY_EVIDENCE_BINDING_ENTRY_FIELDS = frozenset({"file", "sha256"}) +EXPECTED_EVIDENCE_SOURCES = { + "trace": "current-codexhub-thread-tool-surface.json", + "wire_fixture": "codexhub-runtime-wire-fixture.json", + "audit": "read-only-gate-audit.json", +} +EXPECTED_FEEDS = {"capability_gate": "#249", "chat_conversion_matrix": "#66"} # ``reconcile_inventory`` is also used as a standalone replay check. In that # mode there is deliberately no evidence root to reopen, so hashes, manifests, @@ -1072,6 +1204,25 @@ def _require_exact_fields( ) +def _exact_inventory_fields_mismatch( + value: Any, + expected: set[str] | frozenset[str], + *, + context: str, + optional: set[str] | frozenset[str] = frozenset(), +) -> str | None: + """Return a fail-closed mismatch for an inventory object with extra keys.""" + + if not isinstance(value, dict): + return f"{context} must be an object" + actual = set(value) + if set(expected) <= actual and not (actual - set(expected) - set(optional)): + return None + missing = sorted(set(expected) - actual) + unknown = sorted(actual - set(expected) - set(optional)) + return f"{context} has unknown or missing fields (missing={missing!r}, unknown={unknown!r})" + + def _validate_structural_section( family_name: str, section_name: str, section: dict[str, Any], *, context: str ) -> None: @@ -1658,6 +1809,11 @@ def _structural_inventory_mismatches( if not isinstance(entry, dict): mismatches.append(f"{prefix} must be an object") continue + extra_fields = _exact_inventory_fields_mismatch( + entry, INVENTORY_FAMILY_FIELDS, context=prefix + ) + if extra_fields: + mismatches.append(extra_fields) if entry.get("family") != family_name: mismatches.append( f"{prefix}.family must be {family_name!r} in the canonical order" @@ -2425,6 +2581,12 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] raise ValueError( "Codex 0.146 source contract request_shape has unknown fields" ) + if request_shape.get("protocol") != REQUEST_SHAPE_PROTOCOL: + raise ValueError("Codex 0.146 source contract request_shape.protocol is invalid") + if request_shape.get("streaming_fields") != REQUEST_SHAPE_STREAMING_FIELDS: + raise ValueError( + "Codex 0.146 source contract request_shape.streaming_fields is invalid" + ) expected_request_representative_fields = { "model", "input", @@ -2441,6 +2603,10 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] raise ValueError( "Codex 0.146 source contract request_shape.representative has unknown fields" ) + if request_shape["representative"] != REQUEST_SHAPE_REPRESENTATIVE: + raise ValueError( + "Codex 0.146 source contract request_shape.representative values are invalid" + ) expected_non_streaming_control_fields = { "stream", "response_body", @@ -2459,11 +2625,10 @@ def _validate_source_contract(source_contract: dict[str, Any]) -> dict[str, Any] raise ValueError( "Codex 0.146 source contract request_shape.non_streaming_control has unknown fields" ) - if not isinstance(non_streaming_control, dict) or non_streaming_control.get( - "captured" - ) is not False or non_streaming_control.get("status") != "unqualified": + if non_streaming_control != REQUEST_SHAPE_NON_STREAMING_CONTROL: raise ValueError( - "Codex 0.146 source contract cannot claim a captured non-streaming response" + "Codex 0.146 source contract cannot claim a captured non-streaming response; " + "request_shape.non_streaming_control values are invalid" ) examples = runtime_surface.get("declaration_family_examples") if not isinstance(examples, dict): @@ -2895,6 +3060,14 @@ def reconcile_inventory( """Reconcile an inventory (possibly mutated) against the identity contract.""" mismatches: list[str] = [] + if not isinstance(inventory, dict): + return {"reconciled": False, "mismatches": ["inventory must be an object"]} + top_level_fields = _exact_inventory_fields_mismatch( + inventory, INVENTORY_TOP_LEVEL_FIELDS, context="inventory" + ) + if top_level_fields: + mismatches.append(top_level_fields) + if inventory.get("schema_version") != SCHEMA_VERSION: mismatches.append("inventory schema_version is invalid") if inventory.get("artifact_kind") != ARTIFACT_KIND: @@ -2918,6 +3091,16 @@ def reconcile_inventory( if not isinstance(item, dict): mismatches.append("item must be an object") continue + item_fields = _exact_inventory_fields_mismatch( + item, + INVENTORY_ITEM_FIELDS, + context="inventory item", + optional={"notes"}, + ) + if item_fields: + mismatches.append(item_fields) + if "notes" in item and not isinstance(item.get("notes"), str): + mismatches.append("inventory item notes must be a string") scope = item.get("scope") if not scope: mismatches.append("item missing scope") @@ -2966,6 +3149,11 @@ def reconcile_inventory( mismatches.append(f"deletion: missing scopes {missing}") candidate_identity = inventory.get("candidate_identity", {}) + candidate_fields = _exact_inventory_fields_mismatch( + candidate_identity, INVENTORY_CANDIDATE_FIELDS, context="candidate_identity" + ) + if candidate_fields: + mismatches.append(candidate_fields) mismatches.extend( _candidate_identity_mismatches( candidate_identity, enforce_retained=evidence_root is None @@ -2978,6 +3166,11 @@ def reconcile_inventory( mismatches.append(f"loss: candidate_identity.{field} is missing") qualification = inventory.get("qualification", {}) + qualification_fields = _exact_inventory_fields_mismatch( + qualification, INVENTORY_QUALIFICATION_FIELDS, context="qualification" + ) + if qualification_fields: + mismatches.append(qualification_fields) if not isinstance(qualification, dict): mismatches.append("qualification must be an object") qualification = {} @@ -3019,21 +3212,9 @@ def reconcile_inventory( ) evidence_gates = qualification.get("evidence_gates", {}) - expected_gate_keys = { - "complete_model_visible_plan", - "clean_cold_start_current_binding", - "full_pre_post_request_response", - "full_request_fingerprint", - "full_response_fingerprint", - "sse_identity", - "terminal_events", - "error_events", - "non_streaming", - "non_streaming_fixture", - "identity_replay", - "wire_identity_replay", - } - if set(evidence_gates) != expected_gate_keys: + if not isinstance(evidence_gates, dict) or set(evidence_gates) != set( + INVENTORY_EVIDENCE_GATES + ): mismatches.append("qualification.evidence_gates has an unexpected key set") if "ready_for_beta1" in qualification: mismatches.append( @@ -3088,6 +3269,13 @@ def reconcile_inventory( mismatches.append("evidence_binding has an unexpected key set") for name in ("source_contract", "trace", "wire_fixture", "audit"): entry = evidence_binding.get(name, {}) + entry_fields = _exact_inventory_fields_mismatch( + entry, + INVENTORY_EVIDENCE_BINDING_ENTRY_FIELDS, + context=f"evidence_binding.{name}", + ) + if entry_fields: + mismatches.append(entry_fields) if not isinstance(entry, dict) or not entry.get("file") or not re.fullmatch( r"[0-9a-f]{64}", str(entry.get("sha256", "")) ): @@ -3097,6 +3285,31 @@ def reconcile_inventory( f"mutation: evidence_binding.{name}.file does not name the retained fixture" ) + evidence_sources = inventory.get("evidence_sources") + evidence_sources_fields = _exact_inventory_fields_mismatch( + evidence_sources, INVENTORY_EVIDENCE_SOURCES_FIELDS, context="evidence_sources" + ) + if evidence_sources_fields: + mismatches.append(evidence_sources_fields) + if isinstance(evidence_sources, dict): + for name, expected in EXPECTED_EVIDENCE_SOURCES.items(): + if evidence_sources.get(name) != expected: + mismatches.append( + f"mutation: evidence_sources.{name} does not match the retained fixture" + ) + feeds = inventory.get("feeds") + feeds_fields = _exact_inventory_fields_mismatch( + feeds, INVENTORY_FEEDS_FIELDS, context="feeds" + ) + if feeds_fields: + mismatches.append(feeds_fields) + if isinstance(feeds, dict): + for name, expected in EXPECTED_FEEDS.items(): + if feeds.get(name) != expected: + mismatches.append( + f"mutation: feeds.{name} does not match the retained issue feed" + ) + if evidence_binding and all( isinstance(evidence_binding.get(name), dict) for name in ("source_contract", "trace", "wire_fixture", "audit") @@ -3222,6 +3435,13 @@ def reconcile_inventory( ) identity_control = inventory.get("identity_control", {}) + identity_control_fields = _exact_inventory_fields_mismatch( + identity_control, + INVENTORY_IDENTITY_CONTROL_FIELDS, + context="identity_control", + ) + if identity_control_fields: + mismatches.append(identity_control_fields) if not isinstance(identity_control, dict): mismatches.append("identity_control must be an object") identity_control = {} diff --git a/scripts/check-codex-thread-tool-surface.ps1 b/scripts/check-codex-thread-tool-surface.ps1 index 56793b14..34b04e74 100644 --- a/scripts/check-codex-thread-tool-surface.ps1 +++ b/scripts/check-codex-thread-tool-surface.ps1 @@ -99,6 +99,36 @@ if (-not (Test-ExactPropertySet -Value $sourceContract.runtime_wire_surface.requ ))) { $sourceContractSchemaValid = $false } +$expectedRequestStreamingFields = @( + 'model', 'instructions', 'input', 'tools', 'tool_choice', + 'parallel_tool_calls', 'reasoning', 'store', 'stream', 'stream_options', + 'include', 'service_tier', 'prompt_cache_key', 'text', 'client_metadata' +) +if ( + $sourceContract.runtime_wire_surface.request_shape.protocol -ne 'responses' -or + ($sourceContract.runtime_wire_surface.request_shape.streaming_fields -join '|') -ne ($expectedRequestStreamingFields -join '|') +) { + $sourceContractSchemaValid = $false +} +if ( + $sourceContract.runtime_wire_surface.request_shape.representative.model -ne 'gpt-5.6-sol' -or + $sourceContract.runtime_wire_surface.request_shape.representative.input -ne '' -or + $sourceContract.runtime_wire_surface.request_shape.representative.tools -ne '' -or + $sourceContract.runtime_wire_surface.request_shape.representative.tool_choice -ne 'auto' -or + $sourceContract.runtime_wire_surface.request_shape.representative.parallel_tool_calls -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.representative.stream -ne $true -or + $sourceContract.runtime_wire_surface.request_shape.representative.store -ne $false +) { + $sourceContractSchemaValid = $false +} +if ( + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.stream -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.response_body -ne '' -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.captured -ne $false -or + $sourceContract.runtime_wire_surface.request_shape.non_streaming_control.status -ne 'unqualified' +) { + $sourceContractSchemaValid = $false +} $expectedResponseItemTypes = @( 'message', 'reasoning', 'function_call', 'function_call_output', 'custom_tool_call', 'custom_tool_call_output', 'tool_search_call', diff --git a/tests/test_issue_62_runtime_audit.py b/tests/test_issue_62_runtime_audit.py index dbee054d..84044777 100644 --- a/tests/test_issue_62_runtime_audit.py +++ b/tests/test_issue_62_runtime_audit.py @@ -474,6 +474,114 @@ def test_audit_rejects_captured_source_contract_claim(tmp_path: Path) -> None: run_audit(module, codex_db, gateway_db, source_contract=source_contract_path) +@pytest.mark.parametrize( + "mutation", + [ + "protocol", + "streaming_fields", + "representative_model", + "representative_input", + "representative_tools", + "representative_tool_choice", + "representative_parallel_tool_calls", + "representative_stream", + "representative_store", + "non_streaming_stream", + "non_streaming_response_body", + ], +) +def test_audit_rejects_source_contract_request_shape_value_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_audit_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + request_shape = source_contract["runtime_wire_surface"]["request_shape"] + if mutation == "protocol": + request_shape["protocol"] = "chat_completions" + elif mutation == "streaming_fields": + request_shape["streaming_fields"] = ["model"] + elif mutation.startswith("representative_"): + field = mutation.removeprefix("representative_") + values = { + "model": "gpt-5.5", + "input": "not-redacted", + "tools": [], + "tool_choice": "required", + "parallel_tool_calls": True, + "stream": False, + "store": True, + } + request_shape["representative"][field] = values[field] + elif mutation == "non_streaming_stream": + request_shape["non_streaming_control"]["stream"] = True + else: + request_shape["non_streaming_control"]["response_body"] = "captured-body" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="request_shape"): + module._source_contract_provenance(source_contract_path) + + +@pytest.mark.parametrize( + "mutation", + [ + "family_runtime_type", + "family_wire_type", + "family_executor", + "declaration", + "call", + "result", + "history", + "streaming", + "terminal", + "error", + "namespace_tool", + "response_item_type", + ], +) +def test_audit_rejects_nested_source_contract_schema_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_audit_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + surface = source_contract["runtime_wire_surface"] + examples = surface["declaration_family_examples"] + if mutation == "family_runtime_type": + surface["declaration_families"][0]["runtime_type"] = "bogus" + elif mutation == "family_wire_type": + surface["declaration_families"][0]["wire_declaration_type"] = "bogus" + elif mutation == "family_executor": + surface["declaration_families"][0]["executor"] = "bogus" + elif mutation == "declaration": + examples["plain_function"]["declaration"]["type"] = "bogus" + elif mutation == "call": + examples["plain_function"]["call"]["type"] = "bogus" + elif mutation == "result": + examples["plain_function"]["result"]["type"] = "bogus" + elif mutation == "history": + examples["plain_function"]["history"]["future_field"] = "must fail" + elif mutation == "streaming": + examples["plain_function"]["streaming"]["future_field"] = "must fail" + elif mutation == "terminal": + examples["plain_function"]["terminal"]["event"] = "bogus" + elif mutation == "error": + examples["plain_function"]["error"]["event"] = "bogus" + elif mutation == "namespace_tool": + examples["namespace"]["declaration"]["tools"][0]["future_field"] = "must fail" + else: + surface["response_shape"]["response_item_types"].append("future_item") + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError): + module._source_contract_provenance(source_contract_path) + + @pytest.mark.parametrize( "mutation", ["top_level", "runtime_wire_surface", "request_shape", "response_shape"], diff --git a/tests/test_issue_62_runtime_inventory.py b/tests/test_issue_62_runtime_inventory.py index 85bd4613..b85bfee7 100644 --- a/tests/test_issue_62_runtime_inventory.py +++ b/tests/test_issue_62_runtime_inventory.py @@ -501,6 +501,62 @@ def test_build_inventory_rejects_captured_source_contract_control(tmp_path: Path ) +@pytest.mark.parametrize( + "mutation", + [ + "protocol", + "streaming_fields", + "representative_model", + "representative_input", + "representative_tools", + "representative_tool_choice", + "representative_parallel_tool_calls", + "representative_stream", + "representative_store", + "non_streaming_stream", + "non_streaming_response_body", + ], +) +def test_build_inventory_rejects_source_contract_request_shape_value_mutations( + tmp_path: Path, mutation: str +) -> None: + module = load_inventory_module() + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + request_shape = source_contract["runtime_wire_surface"]["request_shape"] + if mutation == "protocol": + request_shape["protocol"] = "chat_completions" + elif mutation == "streaming_fields": + request_shape["streaming_fields"] = ["model"] + elif mutation.startswith("representative_"): + field = mutation.removeprefix("representative_") + values = { + "model": "gpt-5.5", + "input": "not-redacted", + "tools": [], + "tool_choice": "required", + "parallel_tool_calls": True, + "stream": False, + "store": True, + } + request_shape["representative"][field] = values[field] + elif mutation == "non_streaming_stream": + request_shape["non_streaming_control"]["stream"] = True + else: + request_shape["non_streaming_control"]["response_body"] = "captured-body" + source_contract_path = tmp_path / SOURCE_CONTRACT.name + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + with pytest.raises(ValueError, match="request_shape"): + module.build_inventory( + source_contract=source_contract_path, + trace=TRACE, + wire_fixture=WIRE_FIXTURE, + audit=AUDIT, + ) + + @pytest.mark.parametrize( "mutation", [ @@ -744,6 +800,50 @@ def test_inventory_reconcile_rejects_nested_representative_edits(mutation: str) assert any("unknown or missing fields" in mismatch for mismatch in report["mismatches"]) +@pytest.mark.parametrize( + "mutation", + [ + "top_level", + "candidate_identity", + "family", + "item", + "qualification", + "identity_control", + "evidence_sources", + "feeds", + "evidence_binding_entry", + ], +) +def test_inventory_reconcile_standalone_rejects_unknown_container_fields( + mutation: str, +) -> None: + module = load_inventory_module() + inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) + if mutation == "top_level": + inventory["future_field"] = "must fail" + elif mutation == "candidate_identity": + inventory["candidate_identity"]["future_field"] = "must fail" + elif mutation == "family": + inventory["declaration_families"][0]["future_field"] = "must fail" + elif mutation == "item": + inventory["items"][0]["future_field"] = "must fail" + elif mutation == "qualification": + inventory["qualification"]["future_field"] = "must fail" + elif mutation == "identity_control": + inventory["identity_control"]["future_field"] = "must fail" + elif mutation == "evidence_sources": + inventory["evidence_sources"]["future_field"] = "must fail" + elif mutation == "feeds": + inventory["feeds"]["future_field"] = "must fail" + else: + inventory["evidence_binding"]["trace"]["future_field"] = "must fail" + + report = module.reconcile_inventory(inventory) + + assert report["reconciled"] is False + assert any("unknown or missing fields" in mismatch for mismatch in report["mismatches"]) + + def test_inventory_reconcile_rejects_observed_family_without_evidence_root() -> None: module = load_inventory_module() inventory = json.loads(INVENTORY.read_text(encoding="utf-8")) diff --git a/tests/test_issue_62_runtime_trace.py b/tests/test_issue_62_runtime_trace.py index 826eb8c8..b6cd837b 100644 --- a/tests/test_issue_62_runtime_trace.py +++ b/tests/test_issue_62_runtime_trace.py @@ -573,6 +573,89 @@ def test_powershell_rejects_unknown_nested_source_contract_fields( assert "generated inventory drift check failed" in result.stderr.lower() +@pytest.mark.parametrize( + "mutation", + [ + "protocol", + "streaming_fields", + "representative_model", + "representative_input", + "representative_tools", + "representative_tool_choice", + "representative_parallel_tool_calls", + "representative_stream", + "representative_store", + "non_streaming_stream", + "non_streaming_response_body", + ], +) +def test_powershell_rejects_source_contract_request_shape_value_mutations( + tmp_path: Path, mutation: str +) -> None: + trace_path = tmp_path / TRACE.name + wire_path = tmp_path / WIRE_FIXTURE.name + source_contract_path = tmp_path / SOURCE_CONTRACT.name + audit_path = tmp_path / AUDIT.name + inventory_path = tmp_path / "runtime-wire-inventory.json" + shutil.copyfile(TRACE, trace_path) + shutil.copyfile(WIRE_FIXTURE, wire_path) + shutil.copyfile(AUDIT, audit_path) + shutil.copyfile(ROOT / "docs/evidence/issue-62/runtime-wire-inventory.json", inventory_path) + source_contract = json.loads(SOURCE_CONTRACT.read_text(encoding="utf-8")) + request_shape = source_contract["runtime_wire_surface"]["request_shape"] + if mutation == "protocol": + request_shape["protocol"] = "chat_completions" + elif mutation == "streaming_fields": + request_shape["streaming_fields"] = ["model"] + elif mutation.startswith("representative_"): + field = mutation.removeprefix("representative_") + values = { + "model": "gpt-5.5", + "input": "not-redacted", + "tools": [], + "tool_choice": "required", + "parallel_tool_calls": True, + "stream": False, + "store": True, + } + request_shape["representative"][field] = values[field] + elif mutation == "non_streaming_stream": + request_shape["non_streaming_control"]["stream"] = True + else: + request_shape["non_streaming_control"]["response_body"] = "captured-body" + source_contract_path.write_text( + json.dumps(source_contract, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + result = subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(REPLAY_SCRIPT), + "-SourceContractPath", + str(source_contract_path), + "-TracePath", + str(trace_path), + "-WireFixturePath", + str(wire_path), + "-AuditPath", + str(audit_path), + "-InventoryPath", + str(inventory_path), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "generated inventory drift check failed" in result.stderr.lower() + + @pytest.mark.parametrize("case", ["mutation", "deletion", "loss"]) def test_negative_inventory_replays_fail_visibly(case: str) -> None: result = run_inventory_replay(case)