Move coding-agent tracing into the Rust daemon

.github/scripts/mock-braintrust.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Minimal Braintrust API used by the real-session release smoke test."""
+
+import gzip
+import json
+import os
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+port = int(os.environ.get("MOCK_COLLECTOR_PORT", "53999"))
+summary_path = Path(os.environ["MOCK_COLLECTOR_OUT"])
+summary = {"logs3Requests": 0, "totalRows": 0}
+
+
+def save_summary() -> None:
+    summary_path.write_text(json.dumps(summary), encoding="utf-8")
+
+
+class Handler(BaseHTTPRequestHandler):
+    def log_message(self, format: str, *args: object) -> None:
+        print(f"mock-braintrust: {format % args}", flush=True)
+
+    def send_json(self, value: object) -> None:
+        body = json.dumps(value).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self) -> None:
+        if self.path == "/version":
+            self.send_json({"logs3_payload_max_bytes": None})
+        else:
+            self.send_json({})
+
+    def do_POST(self) -> None:
+        length = int(self.headers.get("Content-Length", "0"))
+        body = self.rfile.read(length)
+        if self.headers.get("Content-Encoding") == "gzip":
+            body = gzip.decompress(body)
+
+        if self.path == "/api/apikey/login":
+            base = f"http://127.0.0.1:{port}"
+            self.send_json(
+                {
+                    "org_info": [
+                        {
+                            "id": "smoke-org",
+                            "name": "smoke",
+                            "api_url": base,
+                            "proxy_url": base,
+                        }
+                    ]
+                }
+            )
+            return
+        if self.path == "/api/project/register":
+            self.send_json(
+                {
+                    "project": {
+                        "id": "00000000-0000-0000-0000-000000000000",
+                        "name": "smoke",
+                    }
+                }
+            )
+            return
+        if self.path in ("/logs3", "/logs3/overflow"):
+            try:
+                rows = json.loads(body or b"{}").get("rows", [])
+            except (json.JSONDecodeError, AttributeError):
+                rows = []
+            summary["logs3Requests"] += 1
+            summary["totalRows"] += len(rows)
+            save_summary()
+            print(
+                f"mock-braintrust: received {len(rows)} row(s), "
+                f"{summary['totalRows']} total",
+                flush=True,
+            )
+        self.send_json({})
+
+
+save_summary()
+ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever()

.github/workflows/_release.yml

@@ -43,10 +43,7 @@ concurrency:
 
 jobs:
   release:
-    # codex builds native macOS binaries that must be ad-hoc codesigned (only
-    # `codesign`, i.e. macOS, can do that), so it runs on a macOS runner. claude
-    # has no binaries, so it stays on cheaper Linux.
-    runs-on: ${{ inputs.plugin == 'codex' && 'macos-14' || 'ubuntu-24.04' }}
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
     steps:
       - name: Checkout monorepo
@@ -77,14 +74,6 @@ jobs:
           echo "tag=$tag" >> "$GITHUB_OUTPUT"
           echo "Releasing $tag -> ${{ inputs.dist_repo }} (record=${{ inputs.record }})"
 
-      - name: Set up Node (for building codex binaries)
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
-        with:
-          node-version: 22
-
-      - name: Enable pnpm
-        run: corepack enable
-
       - name: Bump plugin manifest versions
         run: python3 scripts/set-plugin-version.py "${{ inputs.plugin }}" "${{ steps.vars.outputs.version }}"
 

.github/workflows/ci.yml

@@ -1,24 +1,71 @@
-# CI: build every plugin and validate the built trees (`make test`).
 name: CI
 
 on:
   workflow_dispatch:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
 jobs:
-  test:
-    # Pin to a specific runner version so the workflow is reproducible.
+  plugins:
+    name: Plugin packages
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
-      - name: Checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
-
-      - name: Build and validate all plugins
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - name: Build and validate plugins
         run: make test
+
+  daemon:
+    name: Daemon (${{ matrix.os }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            agent_suffix: ""
+          - os: macos-latest
+            agent_suffix: ""
+          - os: windows-latest
+            agent_suffix: ".cmd"
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - name: Install Rust
+        run: |
+          rustup toolchain install stable --profile minimal
+          rustup default stable
+          rustup component add clippy rustfmt
+      - name: Install latest coding agents
+        run: npm install --prefix "${{ runner.temp }}/coding-agents" --no-save --no-package-lock --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest
+      - name: Report coding-agent versions
+        run: |
+          npm exec --prefix "${{ runner.temp }}/coding-agents" -- codex --version
+          npm exec --prefix "${{ runner.temp }}/coding-agents" -- claude --version
+      - name: Check formatting
+        if: runner.os == 'Linux'
+        run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check
+      - name: Build daemon
+        run: cargo build --manifest-path bt-daemon/Cargo.toml --all-features --locked
+      - name: Test daemon
+        run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked
+      - name: Test coding-agent integrations with deterministic inference
+        env:
+          BT_AGENT_INFERENCE_MODE: mock
+          BT_AGENT_INGEST_MODE: mock
+          CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }}
+          CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }}
+        run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_integration -- --ignored --nocapture --test-threads=1
+      - name: Lint daemon
+        run: cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features --locked -- -D warnings

.github/workflows/smoke-codex.yml

@@ -1,18 +1,8 @@
-# End-to-end smoke test for the codex plugin against a deployed distribution repo.
-# Installs Codex + the plugin from the marketplace on Linux + both macOS arches,
-# runs a real `codex exec` session with tracing pointed at a local mock Braintrust
-# collector, and asserts at least one trace row was reported. This catches the
-# per-platform failure modes a source build can't (code signing, cross-compiled
-# binaries, marketplace install).
-#
-# Two entry points:
-#   - workflow_call: run by _release.yml after a deploy, against the repo it just
-#     published to.
-#   - workflow_dispatch: run manually against any dist repo (default: the test repo).
-#
-# Requires the OPENAI_API_KEY secret for `codex exec`; if it is unset the smoke is
-# skipped (with a warning) rather than failing. PUBLISH_TOKEN is used to clone a
-# private/internal dist repo.
+# Post-deploy end-to-end smoke test for the Codex plugin. It installs the
+# deployed marketplace, runs a real Codex session through the daemon-capable
+# `bt` CLI, and verifies that the Rust daemon sends span rows to a local mock
+# Braintrust backend. Credentials and backend URLs are supplied to `bt`; the
+# shared daemon config contains behavior settings only.
 
 name: Smoke (codex)
 
@@ -39,18 +29,15 @@ permissions:
   contents: read
 
 env:
-  PLUGIN_DIR: src/plugins/codex/content/plugins/trace-codex
   MARKETPLACE: braintrust-codex-plugins
 
 jobs:
-  # Skip the (matrix) smoke cleanly when no OpenAI key is configured, instead of
-  # failing every release. Job-level `if` can't read secrets, so gate here.
   guard:
     runs-on: ubuntu-24.04
     outputs:
-      has_key: ${{ steps.c.outputs.has_key }}
+      has_key: ${{ steps.check.outputs.has_key }}
     steps:
-      - id: c
+      - id: check
         env:
           KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
@@ -82,29 +69,46 @@ jobs:
         with:
           node-version: 22
 
-      - name: Enable pnpm
-        run: corepack enable
-
-      - name: Install plugin dependencies (for the mock collector)
-        working-directory: ${{ env.PLUGIN_DIR }}
-        run: pnpm install
+      - name: Install Codex and bt CLIs
+        run: |
+          npm install -g @openai/codex
+          curl -fsSL https://bt.dev/cli/install.sh | sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
-      - name: Install Codex CLI
-        run: npm install -g @openai/codex
+      - name: Require daemon-capable bt
+        run: |
+          "$HOME/.local/bin/bt" daemon hook --help
 
-      - name: Install codex plugin from ${{ inputs.dist_repo }}
+      - name: Install plugin from ${{ inputs.dist_repo }}
         env:
-          # PUBLISH_TOKEN can read the (private/internal) dist repo; rewrite
-          # github.com clones to use it so `codex plugin marketplace add` (a git
-          # clone) authenticates.
           GH_TOKEN: ${{ secrets.PUBLISH_TOKEN }}
         run: |
           git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/"
           codex plugin marketplace add "${{ inputs.dist_repo }}"
           codex plugin add "trace-codex@${MARKETPLACE}"
 
-      - name: Run smoke test (${{ matrix.label }})
-        working-directory: ${{ env.PLUGIN_DIR }}
+      - name: Run real traced Codex session (${{ matrix.label }})
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        run: sh scripts/smoke-test.sh
+          CODEX_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          BRAINTRUST_API_KEY: smoke-key
+          BRAINTRUST_API_URL: http://127.0.0.1:53999
+          BRAINTRUST_APP_URL: http://127.0.0.1:53999
+          BT_DAEMON_CONFIG: ${{ runner.temp }}/bt-daemon-config.json
+          MOCK_COLLECTOR_OUT: ${{ runner.temp }}/mock-summary.json
+        run: |
+          printf '%s\n' '{"traceToBraintrust":true,"project":"trace-codex-smoke","flushOnTurnEnd":true}' > "$BT_DAEMON_CONFIG"
+          python3 .github/scripts/mock-braintrust.py > "${{ runner.temp }}/mock-collector.log" 2>&1 &
+          collector_pid=$!
+          trap 'kill "$collector_pid" 2>/dev/null || true' EXIT
+          for attempt in $(seq 1 50); do
+            curl -fsS http://127.0.0.1:53999/version >/dev/null && break
+            sleep 0.2
+          done
+          curl -fsS http://127.0.0.1:53999/version >/dev/null
+          codex exec \
+            --skip-git-repo-check \
+            --dangerously-bypass-hook-trust \
+            --sandbox read-only \
+            "say hi"
+          python3 -c 'import json, os; s=json.load(open(os.environ["MOCK_COLLECTOR_OUT"])); assert s["totalRows"] >= 1, s; print(s)'