audit: revalidate cross-framework comparisons

.github/workflows/ci.yml

@@ -178,9 +178,19 @@ jobs:
             release-performance-evidence.json
           if-no-files-found: error
 
-  contracts:
-    name: Cross-Engine Contracts
+  public-parity:
+    name: Public Parity (${{ matrix.framework }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - framework: vectorbt_oss
+            environment: .venv-vectorbt-oss
+          - framework: backtrader
+            environment: .venv-backtrader
+          - framework: zipline
+            environment: .venv-zipline
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
@@ -192,22 +202,50 @@ jobs:
       - name: Set up Python
         run: uv python install ${{ env.PYTHON_VERSION }}
 
-      - name: Install dependencies (with comparison frameworks)
-        run: uv sync --dev --extra comparison
+      - name: Build the locked framework environment
+        run: >-
+          python validation/build_framework_env.py
+          --framework ${{ matrix.framework }}
+          --evidence-output environment-evidence.json
 
-      - name: Run cross-engine contract tests
-        env:
-          ML4T_COMPARISON_INPROC: "1"
+      - name: Measure native VectorBT OSS behavior
+        if: matrix.framework == 'vectorbt_oss'
         run: >-
-          uv run pytest tests/contracts/test_cross_engine_contracts.py -v --tb=short --no-cov
-          --junitxml=parity-contracts.xml
+          "${{ matrix.environment }}/bin/python"
+          validation/native/vectorbt_behavior.py
+          --framework vectorbt_oss
+          --output native-evidence.json
 
-      - name: Retain cross-engine parity evidence
+      - name: Measure native Backtrader behavior
+        if: matrix.framework == 'backtrader'
+        run: >-
+          "${{ matrix.environment }}/bin/python"
+          validation/native/backtrader_behavior.py
+          --output native-evidence.json
+
+      - name: Measure native Zipline behavior
+        if: matrix.framework == 'zipline'
+        run: >-
+          "${{ matrix.environment }}/bin/python"
+          validation/native/zipline_behavior.py
+          --output native-evidence.json
+
+      - name: Run the complete required scenario matrix
+        run: >-
+          "${{ matrix.environment }}/bin/python"
+          validation/run_all_correctness.py
+          --framework ${{ matrix.framework }}
+          --output correctness-candidate.json
+
+      - name: Retain public parity evidence
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: parity-contracts
-          path: parity-contracts.xml
+          name: parity-${{ matrix.framework }}
+          path: |
+            environment-evidence.json
+            correctness-candidate.json
+            native-evidence.json
           if-no-files-found: error
 
   documentation:
@@ -249,7 +287,7 @@ jobs:
     name: Build Package
     runs-on: ubuntu-latest
     needs:
-      [lint, typecheck, compatibility, security, coverage, runtime, contracts, documentation]
+      [lint, typecheck, compatibility, security, coverage, runtime, public-parity, documentation]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -289,12 +327,12 @@ jobs:
             --commit "$GITHUB_SHA" \
             --repository "$GITHUB_REPOSITORY" \
             --gate compatibility=${{ needs.compatibility.result }} \
-            --gate correctness=${{ needs.coverage.result }} \
+            --gate correctness=${{ needs.public-parity.result }} \
             --gate coverage=${{ needs.coverage.result }} \
             --gate documentation=${{ needs.documentation.result }} \
             --gate lint=${{ needs.lint.result }} \
             --gate packaging=success \
-            --gate parity=${{ needs.contracts.result }} \
+            --gate parity=${{ needs.public-parity.result }} \
             --gate performance=${{ needs.runtime.result }} \
             --gate security=${{ needs.security.result }} \
             --gate typecheck=${{ needs.typecheck.result }} \

.github/workflows/private-comparisons.yml

@@ -0,0 +1,115 @@
+name: Private Comparison Evidence
+
+on:
+  workflow_call:
+    secrets:
+      VECTORBT_PRO_DEPLOY_KEY:
+        required: true
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  vectorbt-pro:
+    name: VectorBT Pro Parity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install ${{ env.PYTHON_VERSION }}
+
+      - name: Configure licensed source access
+        env:
+          VECTORBT_PRO_DEPLOY_KEY: ${{ secrets.VECTORBT_PRO_DEPLOY_KEY }}
+        run: |
+          install -m 700 -d "$HOME/.ssh"
+          printf '%s\n' "$VECTORBT_PRO_DEPLOY_KEY" > "$HOME/.ssh/id_ed25519"
+          chmod 600 "$HOME/.ssh/id_ed25519"
+          ssh-keyscan github.com >> "$HOME/.ssh/known_hosts"
+
+      - name: Build the locked VectorBT Pro environment
+        run: >-
+          python validation/build_framework_env.py
+          --framework vectorbt_pro
+          --evidence-output vectorbt-pro-environment.json
+
+      - name: Measure native VectorBT Pro behavior
+        run: >-
+          .venv-vectorbt-pro/bin/python
+          validation/native/vectorbt_behavior.py
+          --framework vectorbt_pro
+          --output vectorbt-pro-native.json
+
+      - name: Run the complete VectorBT Pro scenario matrix
+        run: >-
+          .venv-vectorbt-pro/bin/python
+          validation/run_all_correctness.py
+          --framework vectorbt_pro
+          --output vectorbt-pro-correctness.json
+
+      - name: Retain VectorBT Pro evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: vectorbt-pro-comparison-${{ github.sha }}
+          path: |
+            vectorbt-pro-environment.json
+            vectorbt-pro-native.json
+            vectorbt-pro-correctness.json
+          if-no-files-found: error
+          retention-days: 90
+
+  lean:
+    name: LEAN Native Behavior
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install ${{ env.PYTHON_VERSION }}
+
+      - name: Verify locked LEAN CLI and engine image
+        run: >-
+          python validation/build_framework_env.py
+          --framework lean
+          --evidence-output lean-environment.json
+
+      - name: Measure native LEAN behavior
+        run: >-
+          .venv-lean/bin/python
+          validation/native/lean_behavior.py
+          --lean-command .venv-lean/bin/lean
+          --output lean-native.json
+
+      - name: Run the LEAN Chapter 16 comparisons
+        run: >-
+          .venv-lean/bin/python
+          validation/run_lean_case_studies.py
+          --lean-command .venv-lean/bin/lean
+          --output lean-case-studies.json
+
+      - name: Retain LEAN provenance
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: lean-comparison-${{ github.sha }}
+          path: |
+            lean-environment.json
+            lean-native.json
+            lean-case-studies.json
+          if-no-files-found: error
+          retention-days: 90

.github/workflows/release.yml

@@ -24,10 +24,15 @@ jobs:
     name: Qualify Release Candidate
     uses: ./.github/workflows/ci.yml
 
+  private-comparisons:
+    name: Qualify Licensed and Container Comparisons
+    uses: ./.github/workflows/private-comparisons.yml
+    secrets: inherit
+
   publish:
     name: Publish to PyPI
     runs-on: ubuntu-latest
-    needs: [ecosystem-qualification, qualification]
+    needs: [ecosystem-qualification, private-comparisons, qualification]
     environment: pypi
     permissions:
       contents: read

.gitignore

@@ -45,6 +45,7 @@ src/ml4t/backtest/_version.py
 validation/.zipline/
 validation/CORRECTNESS_RESULTS.md
 validation/BENCHMARK_RESULTS.md
+validation/candidates/
 validation/trade_logs/
 validation/lean/workspace/
 validation/nautilus/.venv-nautilus/

README.md

@@ -126,10 +126,10 @@ config = BacktestConfig.from_preset("vectorbt")
 # Match Backtrader behavior (next-bar open fills, integer shares)
 config = BacktestConfig.from_preset("backtrader")
 
-# Match Zipline behavior (next-bar open fills, integer shares, per-share commission)
+# Match the documented Zipline comparison protocol (next-bar open, no default costs)
 config = BacktestConfig.from_preset("zipline")
 
-# Match QuantConnect LEAN behavior (same-bar close fills, integer shares)
+# Match the frozen LEAN daily US-equity protocol (next-session open, integer shares)
 config = BacktestConfig.from_preset("lean")
 
 # Conservative production settings (higher costs, cash buffer)
@@ -276,28 +276,35 @@ opt-in.
 
 ## Cross-Framework Validation
 
-Profiles configure framework-specific execution behavior. The generated table below reports only
-claims supported by the retained release-candidate evidence.
+The primary audit uses frozen data and targets from ETF allocation, CME futures, and crypto
+perpetual-funding case studies. Profiles configure framework-specific execution behavior, and only
+genuinely supported framework and asset combinations are required. The broad real-strategy gate
+currently passes 6 of 8 required pairs; the generated table reports the exact remaining gaps.
+
+The scenario matrix and 250-asset workload remain useful synthetic diagnostic and stress tests.
+They do not establish realistic strategy equivalence.
 
 <!-- parity-claims:start -->
 <!-- Generated by validation/generate_parity_claims.py. Do not edit by hand. -->
 
-Scenario claims use the retained release-candidate matrix. "Exact" appears only when every required scenario has zero canonical gap.
+Scenario claims use the retained accepted matrix. "Exact" means terminal values, ordered closed trades, and ordered fills match after 1e-8 quantization. Each record declares whether a surface is native, reconstructed, aggregate-only, input-only, or unavailable; the claim does not extend to unavailable order-lifecycle fields.
 
 | Profile | Pinned framework | Required scenarios | Evidence |
 |---|---|---:|---|
-| `vectorbt_strict` | [VectorBT Pro 2025.12.31](https://github.com/polakowo/vectorbt.pro) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
-| `vectorbt` | [VectorBT OSS 0.28.2](https://pypi.org/project/vectorbt/0.28.2/) | 15/15 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
-| `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
-| `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 15/15 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
-
-Large-scale claims are published only when a retained workload has zero canonical gap.
-
-| Profile | Pinned framework | Compared | Trade gap | Terminal value | Evidence |
-|---|---|---:|---:|---:|---|
-| `vectorbt_strict` | VectorBT Pro 2025.12.31 (`1305a1e19743`) | 225,844 trades | 0 | 685179.007330 | [large-scale evidence](https://github.com/ml4t/backtest/blob/main/validation/vectorbt_pro/large_scale_parity.json) |
-
-No large-scale claim is published for Backtrader, Zipline, VectorBT OSS, or LEAN without a passing retained artifact.
+| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
+| `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
+| `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 17/17 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
+| `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 16/16 exact | [scenario evidence](https://github.com/ml4t/backtest/blob/main/validation/CORRECTNESS_RESULTS.json) |
+
+The controlled scale workload contains 250 assets and 5,040 daily sessions (1,260,000 bars). Every row has zero canonical gap for target intents, native fills, closed trades reconstructed from those fills, and terminal state reconstructed from the fill ledger and final marks.
+
+| Profile | Current framework | Target intents | Native fills | Fill-derived closed trades | Terminal value | Evidence |
+|---|---|---:|---:|---:|---:|---|
+| `vectorbt_strict` | [VectorBT Pro 2026.6.27](https://github.com/polakowo/vectorbt.pro) | 427,790 | 390,369 | 188,549 | 716,785.408089 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) |
+| `vectorbt_oss_strict` | [VectorBT OSS 1.1.0](https://pypi.org/project/vectorbt/1.1.0/) | 427,790 | 390,369 | 188,549 | 716,785.408089 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) |
+| `backtrader_strict` | [Backtrader 1.9.78.123](https://pypi.org/project/backtrader/1.9.78.123/) | 427,790 | 343,813 | 182,019 | -9,166,273.555954 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) |
+| `zipline_strict` | [Zipline Reloaded 3.1.1](https://pypi.org/project/zipline-reloaded/3.1.1/) | 427,790 | 427,696 | 226,434 | 10,504,095.900000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) |
+| `lean` | [LEAN 18001](https://github.com/QuantConnect/Lean) | 427,790 | 361,297 | 191,297 | 184,538.130000 | [scale evidence](https://github.com/ml4t/backtest/blob/main/validation/LARGE_SCALE_RESULTS.json) |
 <!-- parity-claims:end -->
 
 See [validation/README.md](validation/README.md) for methodology and detailed results.
@@ -334,6 +341,17 @@ Workload definitions and expected checksums are retained in
 `validation/performance_baselines.json`. The project does not publish hardware-dependent runtime,
 throughput, memory, or cross-framework ratios as stable claims.
 
+Cross-framework performance evidence uses a common 50-asset, 252-session controlled workload.
+Each runner receives one isolated warm-up followed by ten isolated measurements. The retained
+artifact contains raw samples, whole-process wall time, process-tree peak RSS, deterministic 95%
+bootstrap intervals, output checksums, framework identities, and semantic disclosures for the
+idiomatic view. The results remain audit evidence pending a separate publication decision.
+
+The real-strategy performance artifact times only engine execution for correctness-passing pairs.
+Inputs, model inference, target construction, adapter preparation, extraction, and reporting are
+excluded. See `validation/REAL_STRATEGY_PERFORMANCE.json`; its ratios are dated audit measurements,
+not stable framework claims.
+
 ## Documentation
 
 - [Getting Started](docs/getting-started/quickstart.md) — your first backtest

docs/getting-started/quickstart.md

@@ -171,13 +171,13 @@ result = run_backtest(prices, strategy, config=config)
 Match the exact behavior of another backtesting framework:
 
 ```python
-# Backtrader-compatible: next-bar execution, integer shares, margin account
+# Backtrader-compatible: next-bar execution, integer shares, default cash checks
 result = run_backtest(prices, strategy, config="backtrader")
 
 # VectorBT-compatible: same-bar execution, fractional shares, no costs
 result = run_backtest(prices, strategy, config="vectorbt")
 
-# Zipline-compatible: next-bar execution, per-share commission
+# Zipline comparison protocol: next-bar open execution, no default costs
 result = run_backtest(prices, strategy, config="zipline")
 
 # Conservative production settings