From 7c59dba7eb250fcf39821d445a50ae12acc81e83 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 21:15:43 +0800 Subject: [PATCH 01/17] Add deterministic agent inference test harness Add separate OpenAI Responses and Anthropic Messages mock servers with closure-driven responses, errors, request capture, and protocol-valid SSE streams. Run real Codex and Claude Code sessions through deterministic tool and provider-error scenarios, then verify the daemon delivers the resulting traces to a mock Braintrust backend. Install the latest unpinned agents in dedicated CI so upstream compatibility breaks surface immediately. Signed-off-by: Stephen Belanger --- .github/workflows/ci.yml | 20 +- bt-daemon/Cargo.lock | 121 +++++++ bt-daemon/Cargo.toml | 4 + bt-daemon/tests/agent_e2e.rs | 330 ++++++++++++++++++ bt-daemon/tests/inference_mocks.rs | 100 ++++++ bt-daemon/tests/support/agent_process.rs | 176 ++++++++++ bt-daemon/tests/support/inference/README.md | 47 +++ .../tests/support/inference/anthropic.rs | 268 ++++++++++++++ bt-daemon/tests/support/inference/mod.rs | 94 +++++ bt-daemon/tests/support/inference/openai.rs | 232 ++++++++++++ bt-daemon/tests/support/mod.rs | 7 + bt-daemon/tests/support/server.rs | 59 ++++ bt-daemon/tests/support/trace_collector.rs | 108 ++++++ 13 files changed, 1565 insertions(+), 1 deletion(-) create mode 100644 bt-daemon/tests/agent_e2e.rs create mode 100644 bt-daemon/tests/inference_mocks.rs create mode 100644 bt-daemon/tests/support/agent_process.rs create mode 100644 bt-daemon/tests/support/inference/README.md create mode 100644 bt-daemon/tests/support/inference/anthropic.rs create mode 100644 bt-daemon/tests/support/inference/mod.rs create mode 100644 bt-daemon/tests/support/inference/openai.rs create mode 100644 bt-daemon/tests/support/mod.rs create mode 100644 bt-daemon/tests/support/server.rs create mode 100644 bt-daemon/tests/support/trace_collector.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95cbcf9..4967ed9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -51,3 +50,22 @@ jobs: run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked - name: Lint daemon run: cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features --locked -- -D warnings + + agent-e2e: + name: Latest agent E2E + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Install Rust + run: | + rustup toolchain install stable --profile minimal + rustup default stable + - name: Install latest Codex and Claude Code + run: npm install --global @openai/codex@latest @anthropic-ai/claude-code@latest + - name: Report agent versions + run: | + codex --version + claude --version + - name: Run real agents against deterministic inference + run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_e2e -- --ignored --nocapture --test-threads=1 diff --git a/bt-daemon/Cargo.lock b/bt-daemon/Cargo.lock index 04b317a..dfc514e 100644 --- a/bt-daemon/Cargo.lock +++ b/bt-daemon/Cargo.lock @@ -118,6 +118,58 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backoff" version = "0.4.0" @@ -212,10 +264,13 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "axum", "braintrust-sdk-rust", + "bytes", "chrono", "clap", "regex", + "reqwest", "serde", "serde_json", "sha2", @@ -226,6 +281,7 @@ dependencies = [ "tracing-subscriber", "uuid", "wiremock", + "zstd", ] [[package]] @@ -247,6 +303,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -987,6 +1045,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -1043,6 +1111,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" @@ -1134,6 +1208,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1536,6 +1616,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.21" @@ -1834,6 +1925,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1872,6 +1964,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2387,3 +2480,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/bt-daemon/Cargo.toml b/bt-daemon/Cargo.toml index 0a6d87b..b76b55a 100644 --- a/bt-daemon/Cargo.toml +++ b/bt-daemon/Cargo.toml @@ -34,5 +34,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = tr uuid = { version = "1", features = ["v4", "v5"] } [dev-dependencies] +axum = "0.8" +bytes = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "stream"] } tempfile = "3" wiremock = "0.6" +zstd = "0.13" diff --git a/bt-daemon/tests/agent_e2e.rs b/bt-daemon/tests/agent_e2e.rs new file mode 100644 index 0000000..fe258fa --- /dev/null +++ b/bt-daemon/tests/agent_e2e.rs @@ -0,0 +1,330 @@ +#![cfg(unix)] + +mod support; + +use axum::http::StatusCode; +use serde_json::json; +use std::path::{Path, PathBuf}; +use support::agent_process::AgentTestWorld; +use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; +use tokio::process::Command; +use uuid::Uuid; + +fn repository_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("repository root") + .to_path_buf() +} + +fn command_from_env(name: &str, fallback: &str) -> Command { + Command::new(std::env::var_os(name).unwrap_or_else(|| fallback.into())) +} + +fn output_text(output: &std::process::Output) -> String { + format!( + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the latest Codex CLI installed on PATH"] +async fn latest_codex_runs_through_mock_inference_and_emits_traces() { + let inference = OpenAiMock::start(|context, request| { + assert_eq!(request.model(), Some("mock-model")); + match context.request_index { + 0 => { + assert!( + request.contains_text("Run the deterministic command"), + "unexpected Codex request: {}", + request.body + ); + MockReply::response(OpenAiTurn::tool_call( + "call_mock_1", + "exec_command", + json!({"cmd":"printf CODEX_TOOL_OK","login":false}), + )) + } + 1 => { + assert!( + request.has_function_output("call_mock_1"), + "Codex did not return the tool result: {}", + request.body + ); + MockReply::response(OpenAiTurn::text("CODEX_MOCK_OK")) + } + 2 => MockReply::http_error( + StatusCode::BAD_REQUEST, + json!({ + "error": { + "type": "invalid_request_error", + "code": "mock_bad_request", + "message": "deterministic Codex inference failure" + } + }), + ), + index => panic!( + "unexpected Codex inference request {index}: {}", + request.body + ), + } + }) + .await; + let world = AgentTestWorld::start().await; + let codex_home = world.temp_path("codex-home"); + std::fs::create_dir_all(&codex_home).unwrap(); + + let marketplace = repository_root().join("src/plugins/codex/content"); + let mut add_marketplace = command_from_env("CODEX_BIN", "codex"); + add_marketplace + .arg("plugin") + .arg("marketplace") + .arg("add") + .arg(&marketplace) + .env("CODEX_HOME", &codex_home); + world.configure(&mut add_marketplace); + let output = world.output(&mut add_marketplace).await; + assert!(output.status.success(), "{}", output_text(&output)); + + let mut add_plugin = command_from_env("CODEX_BIN", "codex"); + add_plugin + .args(["plugin", "add", "trace-codex@braintrust-codex-plugins"]) + .env("CODEX_HOME", &codex_home); + world.configure(&mut add_plugin); + let output = world.output(&mut add_plugin).await; + assert!(output.status.success(), "{}", output_text(&output)); + + let provider = format!( + r#"model_providers.mock={{name="Mock",base_url="{}/v1",wire_api="responses",env_key="MOCK_API_KEY",request_max_retries=0,stream_max_retries=0,stream_idle_timeout_ms=5000}}"#, + inference.base_url() + ); + let chatgpt_base_url = format!(r#"chatgpt_base_url="{}/backend-api""#, inference.base_url()); + let mut codex = command_from_env("CODEX_BIN", "codex"); + codex + .args([ + "exec", + "--skip-git-repo-check", + "--dangerously-bypass-hook-trust", + "--sandbox", + "read-only", + "-c", + r#"model="mock-model""#, + "-c", + r#"model_provider="mock""#, + "-c", + r#"approval_policy="never""#, + "-c", + &provider, + "-c", + &chatgpt_base_url, + "Run the deterministic command, then return the deterministic marker.", + ]) + .current_dir(world.workspace()) + .env("CODEX_HOME", &codex_home) + .env("MOCK_API_KEY", "test-key"); + world.configure(&mut codex); + let output = world.output(&mut codex).await; + assert!(output.status.success(), "{}", output_text(&output)); + assert!( + output_text(&output).contains("CODEX_MOCK_OK"), + "{}", + output_text(&output) + ); + assert_eq!(inference.requests().len(), 2); + + let mut failing_codex = command_from_env("CODEX_BIN", "codex"); + failing_codex + .args([ + "exec", + "--skip-git-repo-check", + "--dangerously-bypass-hook-trust", + "--sandbox", + "read-only", + "-c", + r#"model="mock-model""#, + "-c", + r#"model_provider="mock""#, + "-c", + r#"approval_policy="never""#, + "-c", + &provider, + "-c", + &chatgpt_base_url, + "Trigger the deterministic inference error.", + ]) + .current_dir(world.workspace()) + .env("CODEX_HOME", &codex_home) + .env("MOCK_API_KEY", "test-key"); + world.configure(&mut failing_codex); + let failed = world.output(&mut failing_codex).await; + assert!(!failed.status.success(), "{}", output_text(&failed)); + assert!( + output_text(&failed).contains("deterministic Codex inference failure"), + "{}", + output_text(&failed) + ); + assert_eq!(inference.requests().len(), 3); + + let rows = world.wait_for_trace_rows().await; + let serialized = serde_json::to_string(&rows).unwrap(); + assert!( + serialized.contains("braintrust.plugin.codex"), + "{serialized}" + ); + assert!(serialized.contains("test_harness"), "{serialized}"); + assert!(serialized.contains("\"type\":\"tool\""), "{serialized}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the latest Claude Code CLI installed on PATH"] +async fn latest_claude_runs_through_mock_inference_and_emits_traces() { + let inference = AnthropicMock::start(|context, request| match context.request_index { + 0 => { + assert!( + request.contains_text("Run the deterministic command"), + "unexpected Claude request: {}", + request.body + ); + MockReply::response(AnthropicTurn::tool_use( + "toolu_mock_1", + "Bash", + json!({"command":"printf CLAUDE_TOOL_OK"}), + )) + } + 1 => { + assert!( + request.has_tool_result("toolu_mock_1"), + "Claude did not return the tool result: {}", + request.body + ); + MockReply::response(AnthropicTurn::text("CLAUDE_MOCK_OK")) + } + 2 => MockReply::http_error( + StatusCode::BAD_REQUEST, + json!({ + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "deterministic Claude inference failure" + } + }), + ), + index => panic!( + "unexpected Claude inference request {index}: {}", + request.body + ), + }) + .await; + let world = AgentTestWorld::start().await; + let claude_config = world.temp_path("claude-config"); + let home = world.temp_path("home"); + std::fs::create_dir_all(&claude_config).unwrap(); + std::fs::create_dir_all(&home).unwrap(); + + let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); + let session_id = Uuid::new_v4().to_string(); + let mut claude = command_from_env("CLAUDE_BIN", "claude"); + claude + .args([ + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + "--model", + "mock-model", + "--session-id", + &session_id, + "--plugin-dir", + ]) + .arg(plugin) + .arg("Run the deterministic command, then return the deterministic marker.") + .current_dir(world.workspace()) + .env("HOME", &home) + .env("CLAUDE_CONFIG_DIR", &claude_config) + .env("ANTHROPIC_BASE_URL", inference.base_url()) + .env("ANTHROPIC_API_KEY", "test-key") + .env("ANTHROPIC_AUTH_TOKEN", "test-key") + .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") + .env("ANTHROPIC_DEFAULT_SONNET_MODEL", "mock-model") + .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model") + .env("ANTHROPIC_MAX_RETRIES", "0") + .env("DISABLE_AUTOUPDATER", "1") + .env("DISABLE_TELEMETRY", "1") + .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); + world.configure(&mut claude); + let output = world.output(&mut claude).await; + assert!(output.status.success(), "{}", output_text(&output)); + assert!( + output_text(&output).contains("CLAUDE_MOCK_OK"), + "{}", + output_text(&output) + ); + assert!(!inference.requests().is_empty()); + + let failing_session_id = Uuid::new_v4().to_string(); + let mut failing_claude = command_from_env("CLAUDE_BIN", "claude"); + failing_claude + .args([ + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + "--model", + "mock-model", + "--session-id", + &failing_session_id, + "--plugin-dir", + ]) + .arg(repository_root().join("src/plugins/claude/content/plugins/trace-claude-code")) + .arg("Trigger the deterministic inference error.") + .current_dir(world.workspace()) + .env("HOME", &home) + .env("CLAUDE_CONFIG_DIR", &claude_config) + .env("ANTHROPIC_BASE_URL", inference.base_url()) + .env("ANTHROPIC_API_KEY", "test-key") + .env("ANTHROPIC_AUTH_TOKEN", "test-key") + .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") + .env("ANTHROPIC_DEFAULT_SONNET_MODEL", "mock-model") + .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model") + .env("ANTHROPIC_MAX_RETRIES", "0") + .env("DISABLE_AUTOUPDATER", "1") + .env("DISABLE_TELEMETRY", "1") + .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); + world.configure(&mut failing_claude); + let failed = world.output(&mut failing_claude).await; + assert!(!failed.status.success(), "{}", output_text(&failed)); + assert!( + output_text(&failed).contains("deterministic Claude inference failure"), + "{}", + output_text(&failed) + ); + assert_eq!(inference.requests().len(), 3); + + let rows = world.wait_for_trace_rows().await; + let serialized = serde_json::to_string(&rows).unwrap(); + assert!( + serialized.contains("\"source\":\"claude-code\""), + "{serialized}" + ); + assert!(serialized.contains("test_harness"), "{serialized}"); + assert!(serialized.contains("\"type\":\"tool\""), "{serialized}"); +} + +#[test] +fn request_helpers_recognize_tool_results() { + let openai = support::inference::OpenAiRequest { + body: json!({"input":[{"type":"function_call_output","call_id":"call-1"}]}), + }; + assert!(openai.has_function_output("call-1")); + + let anthropic = support::inference::AnthropicRequest { + body: json!({ + "messages":[{ + "content":[{"type":"tool_result","tool_use_id":"toolu-1"}] + }] + }), + }; + assert!(anthropic.has_tool_result("toolu-1")); +} diff --git a/bt-daemon/tests/inference_mocks.rs b/bt-daemon/tests/inference_mocks.rs new file mode 100644 index 0000000..9c658ca --- /dev/null +++ b/bt-daemon/tests/inference_mocks.rs @@ -0,0 +1,100 @@ +mod support; + +use axum::http::StatusCode; +use serde_json::json; +use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; + +#[tokio::test] +async fn openai_mock_streams_text_and_captures_requests() { + let mock = OpenAiMock::start(|context, request| { + assert_eq!(context.request_index, 0); + assert_eq!(request.model(), Some("mock-model")); + MockReply::response(OpenAiTurn::text("deterministic")) + }) + .await; + + let response = reqwest::Client::new() + .post(format!("{}/v1/responses", mock.base_url())) + .json(&json!({"model":"mock-model","input":[],"stream":true})) + .send() + .await + .unwrap(); + let body = response.text().await.unwrap(); + + assert!(body.contains("response.output_item.done")); + assert!(body.contains("deterministic")); + assert_eq!(mock.requests().len(), 1); +} + +#[tokio::test] +async fn openai_mock_injects_retryable_and_malformed_responses() { + let mock = OpenAiMock::start(|context, _request| match context.request_index { + 0 => MockReply::http_error( + StatusCode::TOO_MANY_REQUESTS, + json!({"error":{"type":"rate_limit_error","message":"deterministic limit"}}), + ), + _ => MockReply::raw_sse("event: response.output_item.done\ndata: not-json\n\n"), + }) + .await; + let client = reqwest::Client::new(); + + let limited = client + .post(format!("{}/v1/responses", mock.base_url())) + .json(&json!({"model":"mock-model","input":[],"stream":true})) + .send() + .await + .unwrap(); + assert_eq!(limited.status(), StatusCode::TOO_MANY_REQUESTS); + + let malformed = client + .post(format!("{}/v1/responses", mock.base_url())) + .json(&json!({"model":"mock-model","input":[],"stream":true})) + .send() + .await + .unwrap(); + assert!(malformed.text().await.unwrap().contains("not-json")); +} + +#[tokio::test] +async fn anthropic_mock_supports_tool_use_and_http_errors() { + let mock = AnthropicMock::start(|context, request| match context.request_index { + 0 => { + assert!(request.contains_text("run a command")); + MockReply::response(AnthropicTurn::tool_use( + "toolu_mock", + "Bash", + json!({"command":"printf hello"}), + )) + } + _ => MockReply::http_error( + StatusCode::TOO_MANY_REQUESTS, + json!({ + "type":"error", + "error":{"type":"rate_limit_error","message":"deterministic limit"} + }), + ), + }) + .await; + + let client = reqwest::Client::new(); + let first = client + .post(format!("{}/v1/messages", mock.base_url())) + .json(&json!({ + "model":"mock-model", + "messages":[{"role":"user","content":"run a command"}], + "stream":true + })) + .send() + .await + .unwrap(); + assert!(first.text().await.unwrap().contains("toolu_mock")); + + let second = client + .post(format!("{}/v1/messages", mock.base_url())) + .json(&json!({"model":"mock-model","messages":[],"stream":true})) + .send() + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(mock.requests().len(), 2); +} diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs new file mode 100644 index 0000000..5cf753e --- /dev/null +++ b/bt-daemon/tests/support/agent_process.rs @@ -0,0 +1,176 @@ +use crate::support::trace_collector::TraceCollector; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; +use tempfile::TempDir; +use tokio::process::{Child, Command}; + +pub struct AgentTestWorld { + root: TempDir, + collector: TraceCollector, + daemon: Child, + wrapper_dir: PathBuf, + socket: PathBuf, + data_dir: PathBuf, + config_path: PathBuf, +} + +impl AgentTestWorld { + pub async fn start() -> Self { + let root = tempfile::tempdir().expect("create agent test root"); + let collector = TraceCollector::start().await; + let wrapper_dir = root.path().join("bin"); + let data_dir = root.path().join("daemon"); + let socket = root.path().join("daemon.sock"); + let config_path = data_dir.join("config.json"); + std::fs::create_dir_all(&wrapper_dir).expect("create wrapper directory"); + std::fs::create_dir_all(&data_dir).expect("create daemon data directory"); + std::fs::write( + &config_path, + serde_json::to_vec_pretty(&json!({ + "traceToBraintrust": true, + "project": "agent-e2e", + "flushOnTurnEnd": true, + "additionalMetadata": {"test_harness": true} + })) + .unwrap(), + ) + .expect("write daemon config"); + + let daemon_binary = Path::new(env!("CARGO_BIN_EXE_bt-daemon")); + write_bt_wrapper(&wrapper_dir.join("bt"), daemon_binary); + + let mut command = Command::new(daemon_binary); + command + .arg("serve") + .arg("--socket") + .arg(&socket) + .arg("--data-dir") + .arg(&data_dir) + .arg("--idle-timeout-secs") + .arg("0") + .env("BRAINTRUST_API_URL", collector.base_url()) + .env("BRAINTRUST_APP_URL", collector.base_url()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let daemon = command.spawn().expect("start daemon"); + + wait_for_path(&socket).await; + Self { + root, + collector, + daemon, + wrapper_dir, + socket, + data_dir, + config_path, + } + } + + pub fn workspace(&self) -> PathBuf { + let workspace = self.root.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create agent workspace"); + workspace + } + + pub fn temp_path(&self, name: &str) -> PathBuf { + self.root.path().join(name) + } + + pub fn configure(&self, command: &mut Command) { + let path = std::env::var_os("PATH").unwrap_or_default(); + let mut entries = vec![self.wrapper_dir.clone()]; + entries.extend(std::env::split_paths(&path)); + let combined = std::env::join_paths(entries).expect("construct test PATH"); + command + .env("PATH", combined) + .env("BT_DAEMON_SOCKET", &self.socket) + .env("BT_DAEMON_DATA_DIR", &self.data_dir) + .env("BT_DAEMON_CONFIG", &self.config_path) + .env("BRAINTRUST_API_KEY", "test-key") + .env("BRAINTRUST_API_URL", self.collector.base_url()) + .env("BRAINTRUST_APP_URL", self.collector.base_url()) + .env("BRAINTRUST_PROJECT", "agent-e2e") + .env("BRAINTRUST_FLUSH_ON_TURN_END", "true") + .stdin(Stdio::null()); + } + + pub async fn output(&self, command: &mut Command) -> std::process::Output { + command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let child = command.spawn().expect("spawn agent command"); + tokio::time::timeout(Duration::from_secs(90), child.wait_with_output()) + .await + .expect("agent command timed out") + .expect("wait for agent command") + } + + pub async fn wait_for_trace_rows(&self) -> Vec { + for _ in 0..100 { + let rows = self.collector.rows(); + if !rows.is_empty() { + return rows; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!( + "daemon delivered no trace rows; {}; daemon files:\n{}", + self.collector.diagnostics(), + directory_contents(&self.data_dir) + ); + } +} + +impl Drop for AgentTestWorld { + fn drop(&mut self) { + let _ = self.daemon.start_kill(); + } +} + +fn write_bt_wrapper(path: &Path, daemon_binary: &Path) { + use std::os::unix::fs::PermissionsExt; + + let script = format!( + "#!/bin/sh\nif [ \"$1\" = daemon ]; then shift; fi\nexec '{}' \"$@\"\n", + daemon_binary.display() + ); + std::fs::write(path, script).expect("write bt test wrapper"); + let mut permissions = std::fs::metadata(path).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(path, permissions).expect("make bt wrapper executable"); +} + +async fn wait_for_path(path: &Path) { + for _ in 0..100 { + if path.exists() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("daemon endpoint was not created at {}", path.display()); +} + +fn directory_contents(root: &Path) -> String { + fn visit(path: &Path, output: &mut String) { + let Ok(entries) = std::fs::read_dir(path) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + visit(&path, output); + } else { + let body = std::fs::read_to_string(&path).unwrap_or_else(|_| "".into()); + output.push_str(&format!("{}:\n{}\n", path.display(), body)); + } + } + } + let mut output = String::new(); + visit(root, &mut output); + output +} diff --git a/bt-daemon/tests/support/inference/README.md b/bt-daemon/tests/support/inference/README.md new file mode 100644 index 0000000..6494551 --- /dev/null +++ b/bt-daemon/tests/support/inference/README.md @@ -0,0 +1,47 @@ +# Deterministic inference test support + +This directory contains two protocol-faithful, test-only inference servers: + +- `OpenAiMock` implements the OpenAI Responses API surface used by Codex. +- `AnthropicMock` implements the Anthropic Messages API surface used by + Claude Code. + +They intentionally share only generic HTTP lifecycle, request indexing, and +transport outcomes. Request and response types remain provider-specific so a +test cannot accidentally hide a wire-protocol incompatibility behind a common +model abstraction. + +Both mocks accept a thread-safe closure: + +```rust,ignore +let mock = OpenAiMock::start(|context, request| { + match context.request_index { + 0 => MockReply::response(OpenAiTurn::tool_call( + "call-1", + "exec_command", + json!({"cmd":"printf hello"}), + )), + 1 if request.has_function_output("call-1") => { + MockReply::response(OpenAiTurn::text("done")) + } + index => panic!("unexpected request {index}: {}", request.body), + } +}).await; +``` + +`MockReply` supports normal provider responses, arbitrary HTTP errors, and raw +response bodies for malformed or truncated stream tests. Typed turn builders +generate deterministic ids, token usage, and valid provider SSE sequences. +Every inference request is captured for later assertions. + +The modules do not depend on `bt-daemon`. The higher-level +`support::agent_process` harness owns daemon, plugin, agent-process, and trace +collector integration. This boundary is deliberate so the inference mocks and +generic server handle can later move into a reusable crate without carrying +Braintrust-specific concepts with them. + +`agent_e2e.rs` runs real Codex and Claude Code processes against these mocks. +The tests are ignored in the normal Rust suite because they require agent +executables, while the dedicated CI job installs the latest release of each +agent on every run. This is intentionally unpinned so upstream compatibility +breaks are visible immediately. diff --git a/bt-daemon/tests/support/inference/anthropic.rs b/bt-daemon/tests/support/inference/anthropic.rs new file mode 100644 index 0000000..7cdc8b7 --- /dev/null +++ b/bt-daemon/tests/support/inference/anthropic.rs @@ -0,0 +1,268 @@ +use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; +use crate::support::server::TestServer; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::{get, post}; +use axum::Router; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone)] +pub struct AnthropicRequest { + pub body: Value, +} + +impl AnthropicRequest { + pub fn model(&self) -> Option<&str> { + self.body["model"].as_str() + } + + pub fn contains_text(&self, text: &str) -> bool { + self.body.to_string().contains(text) + } + + pub fn has_tool_result(&self, tool_use_id: &str) -> bool { + self.body["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["content"].as_array().is_some_and(|blocks| { + blocks.iter().any(|block| { + block["type"] == "tool_result" && block["tool_use_id"] == tool_use_id + }) + }) + }) + }) + } +} + +#[derive(Debug, Clone)] +pub enum AnthropicTurn { + Text { + text: String, + input_tokens: u64, + output_tokens: u64, + }, + ToolUse { + tool_use_id: String, + name: String, + input: Value, + input_tokens: u64, + output_tokens: u64, + }, + Events(Vec), +} + +impl AnthropicTurn { + pub fn text(text: impl Into) -> Self { + Self::Text { + text: text.into(), + input_tokens: 10, + output_tokens: 5, + } + } + + pub fn tool_use(tool_use_id: impl Into, name: impl Into, input: Value) -> Self { + Self::ToolUse { + tool_use_id: tool_use_id.into(), + name: name.into(), + input, + input_tokens: 10, + output_tokens: 5, + } + } + + fn events(self, response_index: usize) -> Vec { + let message_id = format!("msg_mock_{response_index}"); + match self { + Self::Text { + text, + input_tokens, + output_tokens, + } => { + let mut events = message_start(&message_id, input_tokens); + events.extend([ + json!({ + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""} + }), + json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": text} + }), + json!({"type": "content_block_stop", "index": 0}), + message_delta("end_turn", output_tokens), + json!({"type": "message_stop"}), + ]); + events + } + Self::ToolUse { + tool_use_id, + name, + input, + input_tokens, + output_tokens, + } => { + let mut events = message_start(&message_id, input_tokens); + events.extend([ + json!({ + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": tool_use_id, + "name": name, + "input": {} + } + }), + json!({ + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": input.to_string() + } + }), + json!({"type": "content_block_stop", "index": 0}), + message_delta("tool_use", output_tokens), + json!({"type": "message_stop"}), + ]); + events + } + Self::Events(events) => events, + } + } +} + +fn message_start(id: &str, input_tokens: u64) -> Vec { + vec![json!({ + "type": "message_start", + "message": { + "id": id, + "type": "message", + "role": "assistant", + "content": [], + "model": "mock-model", + "stop_reason": null, + "stop_sequence": null, + "usage": { + "input_tokens": input_tokens, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 1 + } + } + })] +} + +fn message_delta(stop_reason: &str, output_tokens: u64) -> Value { + json!({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": null}, + "usage": {"output_tokens": output_tokens} + }) +} + +type Handler = + dyn Fn(RequestContext, AnthropicRequest) -> MockReply + Send + Sync + 'static; + +struct MockState { + handler: Arc, + requests: Mutex>, + next_index: AtomicUsize, +} + +pub struct AnthropicMock { + server: TestServer, + state: Arc, +} + +impl AnthropicMock { + pub async fn start(handler: H) -> Self + where + H: Fn(RequestContext, AnthropicRequest) -> MockReply + Send + Sync + 'static, + { + let state = Arc::new(MockState { + handler: Arc::new(handler), + requests: Mutex::new(Vec::new()), + next_index: AtomicUsize::new(0), + }); + let router = Router::new() + .route("/v1/models", get(models)) + .route("/v1/messages", post(messages)) + .route("/v1/messages/count_tokens", post(count_tokens)) + .with_state(Arc::clone(&state)); + Self { + server: TestServer::start(router).await, + state, + } + } + + pub fn base_url(&self) -> &str { + self.server.uri() + } + + pub fn requests(&self) -> Vec { + self.state.requests.lock().expect("request lock").clone() + } + + pub async fn shutdown(self) { + self.server.shutdown().await; + } +} + +async fn models() -> axum::Json { + axum::Json(json!({ + "data": [{ + "type": "model", + "id": "mock-model", + "display_name": "Mock model", + "created_at": "2026-01-01T00:00:00Z" + }], + "has_more": false, + "first_id": "mock-model", + "last_id": "mock-model" + })) +} + +async fn count_tokens() -> axum::Json { + axum::Json(json!({"input_tokens": 10})) +} + +async fn messages( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> axum::response::Response { + let body = match decode_json_body(&headers, &body) { + Ok(body) => body, + Err(error) => return json_response(StatusCode::BAD_REQUEST, json!({"error": error})), + }; + let request = AnthropicRequest { body }; + state + .requests + .lock() + .expect("request lock") + .push(request.clone()); + let index = state.next_index.fetch_add(1, Ordering::SeqCst); + match (state.handler)( + RequestContext { + request_index: index, + }, + request, + ) { + MockReply::Response(turn) => raw_response( + StatusCode::OK, + "text/event-stream", + sse(&turn.events(index)), + ), + MockReply::HttpError { status, body } => json_response(status, body), + MockReply::Raw { + status, + content_type, + body, + } => raw_response(status, content_type, body), + } +} diff --git a/bt-daemon/tests/support/inference/mod.rs b/bt-daemon/tests/support/inference/mod.rs new file mode 100644 index 0000000..4339036 --- /dev/null +++ b/bt-daemon/tests/support/inference/mod.rs @@ -0,0 +1,94 @@ +mod anthropic; +mod openai; + +#[allow(unused_imports)] +pub use anthropic::{AnthropicMock, AnthropicRequest, AnthropicTurn}; +#[allow(unused_imports)] +pub use openai::{OpenAiMock, OpenAiRequest, OpenAiTurn}; + +use axum::http::StatusCode; +use serde_json::Value; + +#[derive(Debug, Clone, Copy)] +pub struct RequestContext { + pub request_index: usize, +} + +/// A provider-neutral transport outcome. Protocol response bodies remain +/// provider-specific and are rendered by the OpenAI/Anthropic adapters. +#[derive(Debug, Clone)] +pub enum MockReply { + Response(T), + HttpError { + status: StatusCode, + body: Value, + }, + Raw { + status: StatusCode, + content_type: &'static str, + body: Vec, + }, +} + +impl MockReply { + pub fn response(value: T) -> Self { + Self::Response(value) + } + + pub fn http_error(status: StatusCode, body: Value) -> Self { + Self::HttpError { status, body } + } + + pub fn raw_sse(body: impl Into>) -> Self { + Self::Raw { + status: StatusCode::OK, + content_type: "text/event-stream", + body: body.into(), + } + } +} + +fn decode_json_body(headers: &axum::http::HeaderMap, body: &[u8]) -> Result { + let decoded = match headers + .get(axum::http::header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + { + Some(value) if value.split(',').any(|part| part.trim() == "zstd") => { + zstd::stream::decode_all(std::io::Cursor::new(body)) + .map_err(|error| format!("decode zstd request: {error}"))? + } + _ => body.to_vec(), + }; + serde_json::from_slice(&decoded).map_err(|error| format!("decode JSON request: {error}")) +} + +fn json_response(status: StatusCode, body: Value) -> axum::response::Response { + use axum::response::IntoResponse; + (status, axum::Json(body)).into_response() +} + +fn raw_response( + status: StatusCode, + content_type: &'static str, + body: Vec, +) -> axum::response::Response { + use axum::response::IntoResponse; + ( + status, + [(axum::http::header::CONTENT_TYPE, content_type)], + body, + ) + .into_response() +} + +fn sse(events: &[Value]) -> Vec { + use std::fmt::Write; + + let mut body = String::new(); + for event in events { + let kind = event["type"].as_str().expect("SSE event type"); + writeln!(&mut body, "event: {kind}").expect("write SSE event"); + writeln!(&mut body, "data: {event}\n").expect("write SSE data"); + } + body.into_bytes() +} diff --git a/bt-daemon/tests/support/inference/openai.rs b/bt-daemon/tests/support/inference/openai.rs new file mode 100644 index 0000000..54ee4a6 --- /dev/null +++ b/bt-daemon/tests/support/inference/openai.rs @@ -0,0 +1,232 @@ +use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; +use crate::support::server::TestServer; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::{get, post}; +use axum::Router; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone)] +pub struct OpenAiRequest { + pub body: Value, +} + +impl OpenAiRequest { + pub fn model(&self) -> Option<&str> { + self.body["model"].as_str() + } + + pub fn contains_text(&self, text: &str) -> bool { + self.body.to_string().contains(text) + } + + pub fn has_function_output(&self, call_id: &str) -> bool { + self.body["input"].as_array().is_some_and(|items| { + items + .iter() + .any(|item| item["type"] == "function_call_output" && item["call_id"] == call_id) + }) + } +} + +#[derive(Debug, Clone)] +pub enum OpenAiTurn { + Text { + text: String, + input_tokens: u64, + output_tokens: u64, + }, + ToolCall { + call_id: String, + name: String, + arguments: Value, + input_tokens: u64, + output_tokens: u64, + }, + Events(Vec), +} + +impl OpenAiTurn { + pub fn text(text: impl Into) -> Self { + Self::Text { + text: text.into(), + input_tokens: 10, + output_tokens: 5, + } + } + + pub fn tool_call( + call_id: impl Into, + name: impl Into, + arguments: Value, + ) -> Self { + Self::ToolCall { + call_id: call_id.into(), + name: name.into(), + arguments, + input_tokens: 10, + output_tokens: 5, + } + } + + fn events(self, response_index: usize) -> Vec { + let response_id = format!("resp_mock_{response_index}"); + let created = json!({ + "type": "response.created", + "response": {"id": response_id} + }); + match self { + Self::Text { + text, + input_tokens, + output_tokens, + } => vec![ + created, + json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "id": format!("msg_mock_{response_index}"), + "content": [{"type": "output_text", "text": text}] + } + }), + completed(&response_id, input_tokens, output_tokens), + ], + Self::ToolCall { + call_id, + name, + arguments, + input_tokens, + output_tokens, + } => vec![ + created, + json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "name": name, + "arguments": arguments.to_string() + } + }), + completed(&response_id, input_tokens, output_tokens), + ], + Self::Events(events) => events, + } + } +} + +fn completed(id: &str, input_tokens: u64, output_tokens: u64) -> Value { + json!({ + "type": "response.completed", + "response": { + "id": id, + "usage": { + "input_tokens": input_tokens, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": output_tokens, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": input_tokens + output_tokens + } + } + }) +} + +type Handler = + dyn Fn(RequestContext, OpenAiRequest) -> MockReply + Send + Sync + 'static; + +struct MockState { + handler: Arc, + requests: Mutex>, + next_index: AtomicUsize, +} + +pub struct OpenAiMock { + server: TestServer, + state: Arc, +} + +impl OpenAiMock { + pub async fn start(handler: H) -> Self + where + H: Fn(RequestContext, OpenAiRequest) -> MockReply + Send + Sync + 'static, + { + let state = Arc::new(MockState { + handler: Arc::new(handler), + requests: Mutex::new(Vec::new()), + next_index: AtomicUsize::new(0), + }); + let router = Router::new() + .route("/v1/models", get(models)) + .route("/v1/responses", post(responses)) + .route("/backend-api/plugins/featured", get(featured_plugins)) + .with_state(Arc::clone(&state)); + Self { + server: TestServer::start(router).await, + state, + } + } + + pub fn base_url(&self) -> &str { + self.server.uri() + } + + pub fn requests(&self) -> Vec { + self.state.requests.lock().expect("request lock").clone() + } + + pub async fn shutdown(self) { + self.server.shutdown().await; + } +} + +async fn models() -> axum::Json { + axum::Json(json!({ + "object": "list", + "data": [{"id": "mock-model", "object": "model", "owned_by": "mock"}] + })) +} + +async fn featured_plugins() -> axum::Json { + axum::Json(json!([])) +} + +async fn responses( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> axum::response::Response { + let body = match decode_json_body(&headers, &body) { + Ok(body) => body, + Err(error) => return json_response(StatusCode::BAD_REQUEST, json!({"error": error})), + }; + let request = OpenAiRequest { body }; + state + .requests + .lock() + .expect("request lock") + .push(request.clone()); + let index = state.next_index.fetch_add(1, Ordering::SeqCst); + match (state.handler)( + RequestContext { + request_index: index, + }, + request, + ) { + MockReply::Response(turn) => raw_response( + StatusCode::OK, + "text/event-stream", + sse(&turn.events(index)), + ), + MockReply::HttpError { status, body } => json_response(status, body), + MockReply::Raw { + status, + content_type, + body, + } => raw_response(status, content_type, body), + } +} diff --git a/bt-daemon/tests/support/mod.rs b/bt-daemon/tests/support/mod.rs new file mode 100644 index 0000000..b8b6f63 --- /dev/null +++ b/bt-daemon/tests/support/mod.rs @@ -0,0 +1,7 @@ +#![allow(dead_code)] + +#[cfg(unix)] +pub mod agent_process; +pub mod inference; +pub mod server; +pub mod trace_collector; diff --git a/bt-daemon/tests/support/server.rs b/bt-daemon/tests/support/server.rs new file mode 100644 index 0000000..235ce6b --- /dev/null +++ b/bt-daemon/tests/support/server.rs @@ -0,0 +1,59 @@ +use axum::Router; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +/// Lifecycle wrapper around an ephemeral HTTP server. +/// +/// This deliberately knows nothing about inference or Braintrust so the test +/// protocol adapters can later move into a standalone crate. +pub struct TestServer { + uri: String, + shutdown: Option>, + task: Option>>, +} + +impl TestServer { + pub async fn start(router: Router) -> Self { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral test server"); + let address = listener.local_addr().expect("read test server address"); + let (shutdown, shutdown_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await + }); + Self { + uri: format!("http://{address}"), + shutdown: Some(shutdown), + task: Some(task), + } + } + + pub fn uri(&self) -> &str { + &self.uri + } + + pub async fn shutdown(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + let _ = task.await; + } + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} diff --git a/bt-daemon/tests/support/trace_collector.rs b/bt-daemon/tests/support/trace_collector.rs new file mode 100644 index 0000000..feb9d2a --- /dev/null +++ b/bt-daemon/tests/support/trace_collector.rs @@ -0,0 +1,108 @@ +use crate::support::server::TestServer; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::HeaderMap; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct CollectorState { + rows: Mutex>, + registrations: AtomicUsize, + log_requests: AtomicUsize, +} + +pub struct TraceCollector { + server: TestServer, + state: Arc, +} + +impl TraceCollector { + pub async fn start() -> Self { + let state = Arc::new(CollectorState::default()); + let router = Router::new() + .route("/version", get(version)) + .route("/api/apikey/login", post(login)) + .route("/api/project/register", post(register_project)) + .route("/logs3", post(logs)) + .route("/logs3/overflow", post(logs)) + .with_state(Arc::clone(&state)); + Self { + server: TestServer::start(router).await, + state, + } + } + + pub fn base_url(&self) -> &str { + self.server.uri() + } + + pub fn rows(&self) -> Vec { + self.state.rows.lock().expect("trace row lock").clone() + } + + pub fn diagnostics(&self) -> String { + format!( + "project registrations: {}; log requests: {}; rows: {}", + self.state.registrations.load(Ordering::SeqCst), + self.state.log_requests.load(Ordering::SeqCst), + self.rows().len() + ) + } +} + +async fn version() -> Json { + Json(json!({"logs3_payload_max_bytes": null})) +} + +async fn login() -> Json { + Json(json!({ + "org_info": [{ + "id": "mock-org", + "name": "mock", + "api_url": "unused", + "proxy_url": "unused" + }] + })) +} + +async fn register_project(State(state): State>) -> Json { + state.registrations.fetch_add(1, Ordering::SeqCst); + Json(json!({ + "project": { + "id": "00000000-0000-0000-0000-000000000001", + "name": "agent-e2e" + } + })) +} + +async fn logs( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Json { + state.log_requests.fetch_add(1, Ordering::SeqCst); + let decoded = match headers + .get(axum::http::header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + { + Some(value) if value.split(',').any(|part| part.trim() == "gzip") => { + // The SDK currently sends uncompressed bodies in this path. Keep a + // clear failure if that changes so the collector can add decoding. + panic!("gzip-compressed Braintrust rows are not yet supported") + } + _ => body.to_vec(), + }; + let payload: Value = serde_json::from_slice(&decoded).expect("decode /logs3 body"); + if let Some(rows) = payload["rows"].as_array() { + state + .rows + .lock() + .expect("trace row lock") + .extend(rows.iter().cloned()); + } + Json(json!({})) +} From 2a02810edf3fa84ee16aaa24cb6257ddd981e945 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 22:43:32 +0800 Subject: [PATCH 02/17] Make agent integration tests core and cross-platform Run the latest real Codex and Claude Code integrations inside the existing Linux, macOS, and Windows daemon matrix instead of a separate Linux-only job. Support both deterministic mock inference with exact scenario assertions and live inference with stable trace-invariant assertions through the same agent runner. Keep the complete programmable inference endpoints self-contained and independent of daemon orchestration so they can later be extracted as a reusable mock-inference crate. Signed-off-by: Stephen Belanger --- .github/workflows/ci.yml | 31 +- .../{agent_e2e.rs => agent_integration.rs} | 340 ++++++++++++------ bt-daemon/tests/support/agent_process.rs | 79 +++- bt-daemon/tests/support/inference/README.md | 50 ++- .../tests/support/inference/anthropic.rs | 6 +- bt-daemon/tests/support/inference/mod.rs | 1 + bt-daemon/tests/support/inference/openai.rs | 6 +- .../tests/support/{ => inference}/server.rs | 24 +- bt-daemon/tests/support/mod.rs | 2 - bt-daemon/tests/support/trace_collector.rs | 46 ++- 10 files changed, 400 insertions(+), 185 deletions(-) rename bt-daemon/tests/{agent_e2e.rs => agent_integration.rs} (60%) rename bt-daemon/tests/support/{ => inference}/server.rs (67%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4967ed9..7f3b611 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: matrix: os: [ubuntu-24.04, macos-latest, windows-latest] runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Rust @@ -41,6 +41,12 @@ jobs: rustup toolchain install stable --profile minimal rustup default stable rustup component add clippy rustfmt + - name: Install latest coding agents + run: npm install --global @openai/codex@latest @anthropic-ai/claude-code@latest + - name: Report coding-agent versions + run: | + codex --version + claude --version - name: Check formatting if: runner.os == 'Linux' run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check @@ -48,24 +54,9 @@ jobs: run: cargo build --manifest-path bt-daemon/Cargo.toml --all-features --locked - name: Test daemon run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked + - name: Test coding-agent integrations with deterministic inference + env: + BT_AGENT_TEST_MODE: mock + run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_integration -- --ignored --nocapture --test-threads=1 - name: Lint daemon run: cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features --locked -- -D warnings - - agent-e2e: - name: Latest agent E2E - runs-on: ubuntu-24.04 - timeout-minutes: 20 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - name: Install Rust - run: | - rustup toolchain install stable --profile minimal - rustup default stable - - name: Install latest Codex and Claude Code - run: npm install --global @openai/codex@latest @anthropic-ai/claude-code@latest - - name: Report agent versions - run: | - codex --version - claude --version - - name: Run real agents against deterministic inference - run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_e2e -- --ignored --nocapture --test-threads=1 diff --git a/bt-daemon/tests/agent_e2e.rs b/bt-daemon/tests/agent_integration.rs similarity index 60% rename from bt-daemon/tests/agent_e2e.rs rename to bt-daemon/tests/agent_integration.rs index fe258fa..e24c21f 100644 --- a/bt-daemon/tests/agent_e2e.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -1,15 +1,32 @@ -#![cfg(unix)] - mod support; use axum::http::StatusCode; -use serde_json::json; +use serde_json::{json, Value}; use std::path::{Path, PathBuf}; use support::agent_process::AgentTestWorld; use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; use tokio::process::Command; use uuid::Uuid; +const TEST_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AgentTestMode { + Mock, + Live, +} + +impl AgentTestMode { + fn from_env() -> Self { + match std::env::var(TEST_MODE_ENV).as_deref() { + Ok("live") => Self::Live, + Ok("mock" | "deterministic") | Err(std::env::VarError::NotPresent) => Self::Mock, + Ok(value) => panic!("{TEST_MODE_ENV} must be `mock` or `live`, got {value:?}"), + Err(error) => panic!("could not read {TEST_MODE_ENV}: {error}"), + } + } +} + fn repository_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() @@ -18,7 +35,12 @@ fn repository_root() -> PathBuf { } fn command_from_env(name: &str, fallback: &str) -> Command { - Command::new(std::env::var_os(name).unwrap_or_else(|| fallback.into())) + if let Some(command) = std::env::var_os(name) { + return Command::new(command); + } + #[cfg(windows)] + let fallback = format!("{fallback}.cmd"); + Command::new(fallback) } fn output_text(output: &std::process::Output) -> String { @@ -29,9 +51,128 @@ fn output_text(output: &std::process::Output) -> String { ) } +fn codex_tool_command() -> &'static str { + #[cfg(unix)] + { + "printf CODEX_TOOL_OK" + } + #[cfg(windows)] + { + "Write-Output CODEX_TOOL_OK" + } +} + +fn configured_codex_home() -> Option { + std::env::var_os("CODEX_HOME") + .map(PathBuf::from) + .or_else(|| { + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .map(PathBuf::from) + .map(|home| home.join(".codex")) + }) +} + +fn seed_codex_live_auth(codex_home: &Path) { + if std::env::var_os("OPENAI_API_KEY").is_some() { + return; + } + let source = configured_codex_home() + .map(|home| home.join("auth.json")) + .filter(|path| path.is_file()) + .unwrap_or_else(|| { + panic!( + "{TEST_MODE_ENV}=live requires OPENAI_API_KEY or auth.json in the configured Codex home" + ) + }); + std::fs::copy(source, codex_home.join("auth.json")).expect("copy Codex live credentials"); +} + +async fn install_codex_plugin(world: &AgentTestWorld, codex_home: &Path) { + let marketplace = repository_root().join("src/plugins/codex/content"); + let mut add_marketplace = command_from_env("CODEX_BIN", "codex"); + add_marketplace + .arg("plugin") + .arg("marketplace") + .arg("add") + .arg(&marketplace) + .env("CODEX_HOME", codex_home); + world.configure(&mut add_marketplace); + let output = world.output(&mut add_marketplace).await; + assert!(output.status.success(), "{}", output_text(&output)); + + let mut add_plugin = command_from_env("CODEX_BIN", "codex"); + add_plugin + .args(["plugin", "add", "trace-codex@braintrust-codex-plugins"]) + .env("CODEX_HOME", codex_home); + world.configure(&mut add_plugin); + let output = world.output(&mut add_plugin).await; + assert!(output.status.success(), "{}", output_text(&output)); +} + +fn codex_command(world: &AgentTestWorld, codex_home: &Path) -> Command { + let mut command = command_from_env("CODEX_BIN", "codex"); + command + .args([ + "exec", + "--skip-git-repo-check", + "--dangerously-bypass-hook-trust", + "--sandbox", + "read-only", + "-c", + r#"approval_policy="never""#, + ]) + .current_dir(world.workspace()) + .env("CODEX_HOME", codex_home); + world.configure(&mut command); + command +} + +fn claude_command(world: &AgentTestWorld) -> Command { + let session_id = Uuid::new_v4().to_string(); + let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); + let mut command = command_from_env("CLAUDE_BIN", "claude"); + command + .args([ + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + "--session-id", + &session_id, + "--plugin-dir", + ]) + .arg(plugin) + .current_dir(world.workspace()) + .env("ANTHROPIC_MAX_RETRIES", "0") + .env("DISABLE_AUTOUPDATER", "1") + .env("DISABLE_TELEMETRY", "1") + .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); + world.configure(&mut command); + command +} + +async fn wait_for_trace_fragments(world: &AgentTestWorld, fragments: &[&str]) -> Vec { + world + .wait_for_trace_rows_matching(|rows| { + let serialized = serde_json::to_string(rows).expect("serialize trace rows"); + fragments + .iter() + .all(|fragment| serialized.contains(fragment)) + }) + .await +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "requires the latest Codex CLI installed on PATH"] -async fn latest_codex_runs_through_mock_inference_and_emits_traces() { +#[ignore = "requires the Codex CLI installed on PATH"] +async fn codex_session_emits_traces() { + match AgentTestMode::from_env() { + AgentTestMode::Mock => run_codex_mock().await, + AgentTestMode::Live => run_codex_live().await, + } +} + +async fn run_codex_mock() { let inference = OpenAiMock::start(|context, request| { assert_eq!(request.model(), Some("mock-model")); match context.request_index { @@ -44,7 +185,7 @@ async fn latest_codex_runs_through_mock_inference_and_emits_traces() { MockReply::response(OpenAiTurn::tool_call( "call_mock_1", "exec_command", - json!({"cmd":"printf CODEX_TOOL_OK","login":false}), + json!({"cmd":codex_tool_command(),"login":false}), )) } 1 => { @@ -75,56 +216,27 @@ async fn latest_codex_runs_through_mock_inference_and_emits_traces() { let world = AgentTestWorld::start().await; let codex_home = world.temp_path("codex-home"); std::fs::create_dir_all(&codex_home).unwrap(); - - let marketplace = repository_root().join("src/plugins/codex/content"); - let mut add_marketplace = command_from_env("CODEX_BIN", "codex"); - add_marketplace - .arg("plugin") - .arg("marketplace") - .arg("add") - .arg(&marketplace) - .env("CODEX_HOME", &codex_home); - world.configure(&mut add_marketplace); - let output = world.output(&mut add_marketplace).await; - assert!(output.status.success(), "{}", output_text(&output)); - - let mut add_plugin = command_from_env("CODEX_BIN", "codex"); - add_plugin - .args(["plugin", "add", "trace-codex@braintrust-codex-plugins"]) - .env("CODEX_HOME", &codex_home); - world.configure(&mut add_plugin); - let output = world.output(&mut add_plugin).await; - assert!(output.status.success(), "{}", output_text(&output)); + install_codex_plugin(&world, &codex_home).await; let provider = format!( r#"model_providers.mock={{name="Mock",base_url="{}/v1",wire_api="responses",env_key="MOCK_API_KEY",request_max_retries=0,stream_max_retries=0,stream_idle_timeout_ms=5000}}"#, inference.base_url() ); let chatgpt_base_url = format!(r#"chatgpt_base_url="{}/backend-api""#, inference.base_url()); - let mut codex = command_from_env("CODEX_BIN", "codex"); + let mut codex = codex_command(&world, &codex_home); codex .args([ - "exec", - "--skip-git-repo-check", - "--dangerously-bypass-hook-trust", - "--sandbox", - "read-only", "-c", r#"model="mock-model""#, "-c", r#"model_provider="mock""#, "-c", - r#"approval_policy="never""#, - "-c", &provider, "-c", &chatgpt_base_url, - "Run the deterministic command, then return the deterministic marker.", ]) - .current_dir(world.workspace()) - .env("CODEX_HOME", &codex_home) + .arg("Run the deterministic command, then return the deterministic marker.") .env("MOCK_API_KEY", "test-key"); - world.configure(&mut codex); let output = world.output(&mut codex).await; assert!(output.status.success(), "{}", output_text(&output)); assert!( @@ -134,30 +246,20 @@ async fn latest_codex_runs_through_mock_inference_and_emits_traces() { ); assert_eq!(inference.requests().len(), 2); - let mut failing_codex = command_from_env("CODEX_BIN", "codex"); + let mut failing_codex = codex_command(&world, &codex_home); failing_codex .args([ - "exec", - "--skip-git-repo-check", - "--dangerously-bypass-hook-trust", - "--sandbox", - "read-only", "-c", r#"model="mock-model""#, "-c", r#"model_provider="mock""#, "-c", - r#"approval_policy="never""#, - "-c", &provider, "-c", &chatgpt_base_url, - "Trigger the deterministic inference error.", ]) - .current_dir(world.workspace()) - .env("CODEX_HOME", &codex_home) + .arg("Trigger the deterministic inference error.") .env("MOCK_API_KEY", "test-key"); - world.configure(&mut failing_codex); let failed = world.output(&mut failing_codex).await; assert!(!failed.status.success(), "{}", output_text(&failed)); assert!( @@ -167,21 +269,48 @@ async fn latest_codex_runs_through_mock_inference_and_emits_traces() { ); assert_eq!(inference.requests().len(), 3); - let rows = world.wait_for_trace_rows().await; - let serialized = serde_json::to_string(&rows).unwrap(); - assert!( - serialized.contains("braintrust.plugin.codex"), - "{serialized}" - ); - assert!(serialized.contains("test_harness"), "{serialized}"); - assert!(serialized.contains("\"type\":\"tool\""), "{serialized}"); + let rows = wait_for_trace_fragments( + &world, + &[ + "braintrust.plugin.codex", + "test_harness", + r#""type":"tool""#, + "CODEX_TOOL_OK", + ], + ) + .await; + assert!(!rows.is_empty()); +} + +async fn run_codex_live() { + let world = AgentTestWorld::start().await; + let codex_home = world.temp_path("codex-home"); + std::fs::create_dir_all(&codex_home).unwrap(); + seed_codex_live_auth(&codex_home); + install_codex_plugin(&world, &codex_home).await; + + let mut codex = codex_command(&world, &codex_home); + codex.arg("Reply briefly to confirm this tracing integration test."); + let output = world.output(&mut codex).await; + assert!(output.status.success(), "{}", output_text(&output)); + + let rows = wait_for_trace_fragments(&world, &["braintrust.plugin.codex", "test_harness"]).await; + assert!(!rows.is_empty()); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "requires the latest Claude Code CLI installed on PATH"] -async fn latest_claude_runs_through_mock_inference_and_emits_traces() { +#[ignore = "requires the Claude Code CLI installed on PATH"] +async fn claude_session_emits_traces() { + match AgentTestMode::from_env() { + AgentTestMode::Mock => run_claude_mock().await, + AgentTestMode::Live => run_claude_live().await, + } +} + +async fn run_claude_mock() { let inference = AnthropicMock::start(|context, request| match context.request_index { 0 => { + assert_eq!(request.model(), Some("mock-model")); assert!( request.contains_text("Run the deterministic command"), "unexpected Claude request: {}", @@ -223,24 +352,10 @@ async fn latest_claude_runs_through_mock_inference_and_emits_traces() { std::fs::create_dir_all(&claude_config).unwrap(); std::fs::create_dir_all(&home).unwrap(); - let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); - let session_id = Uuid::new_v4().to_string(); - let mut claude = command_from_env("CLAUDE_BIN", "claude"); + let mut claude = claude_command(&world); claude - .args([ - "-p", - "--output-format", - "json", - "--dangerously-skip-permissions", - "--model", - "mock-model", - "--session-id", - &session_id, - "--plugin-dir", - ]) - .arg(plugin) + .args(["--model", "mock-model"]) .arg("Run the deterministic command, then return the deterministic marker.") - .current_dir(world.workspace()) .env("HOME", &home) .env("CLAUDE_CONFIG_DIR", &claude_config) .env("ANTHROPIC_BASE_URL", inference.base_url()) @@ -248,12 +363,7 @@ async fn latest_claude_runs_through_mock_inference_and_emits_traces() { .env("ANTHROPIC_AUTH_TOKEN", "test-key") .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") .env("ANTHROPIC_DEFAULT_SONNET_MODEL", "mock-model") - .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model") - .env("ANTHROPIC_MAX_RETRIES", "0") - .env("DISABLE_AUTOUPDATER", "1") - .env("DISABLE_TELEMETRY", "1") - .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); - world.configure(&mut claude); + .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model"); let output = world.output(&mut claude).await; assert!(output.status.success(), "{}", output_text(&output)); assert!( @@ -261,25 +371,12 @@ async fn latest_claude_runs_through_mock_inference_and_emits_traces() { "{}", output_text(&output) ); - assert!(!inference.requests().is_empty()); + assert_eq!(inference.requests().len(), 2); - let failing_session_id = Uuid::new_v4().to_string(); - let mut failing_claude = command_from_env("CLAUDE_BIN", "claude"); + let mut failing_claude = claude_command(&world); failing_claude - .args([ - "-p", - "--output-format", - "json", - "--dangerously-skip-permissions", - "--model", - "mock-model", - "--session-id", - &failing_session_id, - "--plugin-dir", - ]) - .arg(repository_root().join("src/plugins/claude/content/plugins/trace-claude-code")) + .args(["--model", "mock-model"]) .arg("Trigger the deterministic inference error.") - .current_dir(world.workspace()) .env("HOME", &home) .env("CLAUDE_CONFIG_DIR", &claude_config) .env("ANTHROPIC_BASE_URL", inference.base_url()) @@ -287,12 +384,7 @@ async fn latest_claude_runs_through_mock_inference_and_emits_traces() { .env("ANTHROPIC_AUTH_TOKEN", "test-key") .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") .env("ANTHROPIC_DEFAULT_SONNET_MODEL", "mock-model") - .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model") - .env("ANTHROPIC_MAX_RETRIES", "0") - .env("DISABLE_AUTOUPDATER", "1") - .env("DISABLE_TELEMETRY", "1") - .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); - world.configure(&mut failing_claude); + .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model"); let failed = world.output(&mut failing_claude).await; assert!(!failed.status.success(), "{}", output_text(&failed)); assert!( @@ -302,14 +394,29 @@ async fn latest_claude_runs_through_mock_inference_and_emits_traces() { ); assert_eq!(inference.requests().len(), 3); - let rows = world.wait_for_trace_rows().await; - let serialized = serde_json::to_string(&rows).unwrap(); - assert!( - serialized.contains("\"source\":\"claude-code\""), - "{serialized}" - ); - assert!(serialized.contains("test_harness"), "{serialized}"); - assert!(serialized.contains("\"type\":\"tool\""), "{serialized}"); + let rows = wait_for_trace_fragments( + &world, + &[ + r#""source":"claude-code""#, + "test_harness", + r#""type":"tool""#, + "CLAUDE_TOOL_OK", + ], + ) + .await; + assert!(!rows.is_empty()); +} + +async fn run_claude_live() { + let world = AgentTestWorld::start().await; + let mut claude = claude_command(&world); + claude.arg("Reply briefly to confirm this tracing integration test."); + let output = world.output(&mut claude).await; + assert!(output.status.success(), "{}", output_text(&output)); + + let rows = + wait_for_trace_fragments(&world, &[r#""source":"claude-code""#, "test_harness"]).await; + assert!(!rows.is_empty()); } #[test] @@ -328,3 +435,10 @@ fn request_helpers_recognize_tool_results() { }; assert!(anthropic.has_tool_result("toolu-1")); } + +#[test] +fn test_mode_defaults_to_mock_and_accepts_documented_values() { + // Parsing itself is intentionally tiny and covered indirectly by every + // agent integration run. Keep the enum shape explicit for future modes. + assert_ne!(AgentTestMode::Mock, AgentTestMode::Live); +} diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index 5cf753e..e35ad81 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -5,6 +5,8 @@ use std::process::Stdio; use std::time::Duration; use tempfile::TempDir; use tokio::process::{Child, Command}; +#[cfg(windows)] +use uuid::Uuid; pub struct AgentTestWorld { root: TempDir, @@ -22,7 +24,7 @@ impl AgentTestWorld { let collector = TraceCollector::start().await; let wrapper_dir = root.path().join("bin"); let data_dir = root.path().join("daemon"); - let socket = root.path().join("daemon.sock"); + let socket = test_endpoint(root.path()); let config_path = data_dir.join("config.json"); std::fs::create_dir_all(&wrapper_dir).expect("create wrapper directory"); std::fs::create_dir_all(&data_dir).expect("create daemon data directory"); @@ -39,7 +41,7 @@ impl AgentTestWorld { .expect("write daemon config"); let daemon_binary = Path::new(env!("CARGO_BIN_EXE_bt-daemon")); - write_bt_wrapper(&wrapper_dir.join("bt"), daemon_binary); + write_bt_wrapper(&wrapper_dir, daemon_binary); let mut command = Command::new(daemon_binary); command @@ -58,7 +60,7 @@ impl AgentTestWorld { .kill_on_drop(true); let daemon = command.spawn().expect("start daemon"); - wait_for_path(&socket).await; + wait_for_daemon(daemon_binary, &socket).await; Self { root, collector, @@ -111,9 +113,17 @@ impl AgentTestWorld { } pub async fn wait_for_trace_rows(&self) -> Vec { + self.wait_for_trace_rows_matching(|rows| !rows.is_empty()) + .await + } + + pub async fn wait_for_trace_rows_matching( + &self, + predicate: impl Fn(&[Value]) -> bool, + ) -> Vec { for _ in 0..100 { let rows = self.collector.rows(); - if !rows.is_empty() { + if predicate(&rows) { return rows; } tokio::time::sleep(Duration::from_millis(100)).await; @@ -132,27 +142,72 @@ impl Drop for AgentTestWorld { } } -fn write_bt_wrapper(path: &Path, daemon_binary: &Path) { +#[cfg(unix)] +fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { use std::os::unix::fs::PermissionsExt; + let path = directory.join("bt"); let script = format!( "#!/bin/sh\nif [ \"$1\" = daemon ]; then shift; fi\nexec '{}' \"$@\"\n", daemon_binary.display() ); - std::fs::write(path, script).expect("write bt test wrapper"); - let mut permissions = std::fs::metadata(path).unwrap().permissions(); + std::fs::write(&path, script).expect("write bt test wrapper"); + let mut permissions = std::fs::metadata(&path).unwrap().permissions(); permissions.set_mode(0o755); - std::fs::set_permissions(path, permissions).expect("make bt wrapper executable"); + std::fs::set_permissions(&path, permissions).expect("make bt wrapper executable"); +} + +#[cfg(windows)] +fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { + let powershell = directory.join("bt-wrapper.ps1"); + let script = format!( + "$forward = @($args)\n\ + if ($forward.Count -gt 0 -and $forward[0] -eq 'daemon') {{\n\ + if ($forward.Count -eq 1) {{ $forward = @() }} else {{ $forward = @($forward[1..($forward.Count - 1)]) }}\n\ + }}\n\ + & '{}' @forward\n\ + exit $LASTEXITCODE\n", + daemon_binary.display() + ); + std::fs::write(&powershell, script).expect("write bt PowerShell wrapper"); + std::fs::write( + directory.join("bt.cmd"), + "@echo off\r\npowershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File \"%~dp0bt-wrapper.ps1\" %*\r\n", + ) + .expect("write bt command wrapper"); } -async fn wait_for_path(path: &Path) { +#[cfg(unix)] +fn test_endpoint(root: &Path) -> PathBuf { + root.join("daemon.sock") +} + +#[cfg(windows)] +fn test_endpoint(_root: &Path) -> PathBuf { + PathBuf::from(format!( + r"\\.\pipe\braintrust-bt-daemon-test-{}", + Uuid::new_v4() + )) +} + +async fn wait_for_daemon(daemon_binary: &Path, endpoint: &Path) { for _ in 0..100 { - if path.exists() { - return; + let output = Command::new(daemon_binary) + .arg("status") + .arg("--socket") + .arg(endpoint) + .output() + .await; + if let Ok(output) = output { + if output.status.success() + && !String::from_utf8_lossy(&output.stdout).contains("not running") + { + return; + } } tokio::time::sleep(Duration::from_millis(50)).await; } - panic!("daemon endpoint was not created at {}", path.display()); + panic!("daemon endpoint was not ready at {}", endpoint.display()); } fn directory_contents(root: &Path) -> String { diff --git a/bt-daemon/tests/support/inference/README.md b/bt-daemon/tests/support/inference/README.md index 6494551..27a7cae 100644 --- a/bt-daemon/tests/support/inference/README.md +++ b/bt-daemon/tests/support/inference/README.md @@ -1,15 +1,19 @@ # Deterministic inference test support -This directory contains two protocol-faithful, test-only inference servers: +This directory is a self-contained mock-inference component with two +protocol-faithful servers: - `OpenAiMock` implements the OpenAI Responses API surface used by Codex. - `AnthropicMock` implements the Anthropic Messages API surface used by Claude Code. -They intentionally share only generic HTTP lifecycle, request indexing, and -transport outcomes. Request and response types remain provider-specific so a -test cannot accidentally hide a wire-protocol incompatibility behind a common -model abstraction. +Each public mock owns its HTTP listener and lifecycle. There is intentionally +no separately exposed generic server: the useful abstraction is a +programmatically controlled inference endpoint. The two providers share only +private transport mechanics, request indexing, and transport outcomes. +Request and response types remain provider-specific so a test cannot +accidentally hide a wire-protocol incompatibility behind a common model +abstraction. Both mocks accept a thread-safe closure: @@ -34,14 +38,28 @@ response bodies for malformed or truncated stream tests. Typed turn builders generate deterministic ids, token usage, and valid provider SSE sequences. Every inference request is captured for later assertions. -The modules do not depend on `bt-daemon`. The higher-level -`support::agent_process` harness owns daemon, plugin, agent-process, and trace -collector integration. This boundary is deliberate so the inference mocks and -generic server handle can later move into a reusable crate without carrying -Braintrust-specific concepts with them. - -`agent_e2e.rs` runs real Codex and Claude Code processes against these mocks. -The tests are ignored in the normal Rust suite because they require agent -executables, while the dedicated CI job installs the latest release of each -agent on every run. This is intentionally unpinned so upstream compatibility -breaks are visible immediately. +The component does not depend on `bt-daemon`, the coding-agent runner, or the +trace collector. The higher-level `support::agent_process` harness composes +with it only from the integration test. This boundary is deliberate so the +whole mock-inference component can later move into a reusable crate and serve +any client that can target an OpenAI Responses or Anthropic Messages endpoint. + +`agent_integration.rs` runs real Codex and Claude Code processes against these +mocks. +The tests are ignored in a plain Rust run because they require agent +executables. The core cross-platform CI matrix installs the latest release of +each agent and runs them in the default `mock` mode on every host. This is +intentionally unpinned so upstream compatibility breaks are visible +immediately. + +The same agent tests can run without mock inference: + +```console +BT_AGENT_TEST_MODE=live cargo test --manifest-path bt-daemon/Cargo.toml \ + --all-features --test agent_integration -- --ignored --test-threads=1 +``` + +Live mode uses the normal provider endpoint/model and the agent's normal login +or provider credentials. It validates only stable integration invariants such +as trace delivery and origin metadata. Mock mode additionally validates exact +request sequences, tool results, output content, and injected failures. diff --git a/bt-daemon/tests/support/inference/anthropic.rs b/bt-daemon/tests/support/inference/anthropic.rs index 7cdc8b7..371247f 100644 --- a/bt-daemon/tests/support/inference/anthropic.rs +++ b/bt-daemon/tests/support/inference/anthropic.rs @@ -1,5 +1,5 @@ +use super::server::InferenceServer; use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; -use crate::support::server::TestServer; use axum::body::Bytes; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; @@ -175,7 +175,7 @@ struct MockState { } pub struct AnthropicMock { - server: TestServer, + server: InferenceServer, state: Arc, } @@ -195,7 +195,7 @@ impl AnthropicMock { .route("/v1/messages/count_tokens", post(count_tokens)) .with_state(Arc::clone(&state)); Self { - server: TestServer::start(router).await, + server: InferenceServer::start(router).await, state, } } diff --git a/bt-daemon/tests/support/inference/mod.rs b/bt-daemon/tests/support/inference/mod.rs index 4339036..a06f7f1 100644 --- a/bt-daemon/tests/support/inference/mod.rs +++ b/bt-daemon/tests/support/inference/mod.rs @@ -1,5 +1,6 @@ mod anthropic; mod openai; +mod server; #[allow(unused_imports)] pub use anthropic::{AnthropicMock, AnthropicRequest, AnthropicTurn}; diff --git a/bt-daemon/tests/support/inference/openai.rs b/bt-daemon/tests/support/inference/openai.rs index 54ee4a6..5e5e764 100644 --- a/bt-daemon/tests/support/inference/openai.rs +++ b/bt-daemon/tests/support/inference/openai.rs @@ -1,5 +1,5 @@ +use super::server::InferenceServer; use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; -use crate::support::server::TestServer; use axum::body::Bytes; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; @@ -146,7 +146,7 @@ struct MockState { } pub struct OpenAiMock { - server: TestServer, + server: InferenceServer, state: Arc, } @@ -166,7 +166,7 @@ impl OpenAiMock { .route("/backend-api/plugins/featured", get(featured_plugins)) .with_state(Arc::clone(&state)); Self { - server: TestServer::start(router).await, + server: InferenceServer::start(router).await, state, } } diff --git a/bt-daemon/tests/support/server.rs b/bt-daemon/tests/support/inference/server.rs similarity index 67% rename from bt-daemon/tests/support/server.rs rename to bt-daemon/tests/support/inference/server.rs index 235ce6b..43a3406 100644 --- a/bt-daemon/tests/support/server.rs +++ b/bt-daemon/tests/support/inference/server.rs @@ -2,22 +2,22 @@ use axum::Router; use tokio::net::TcpListener; use tokio::sync::oneshot; -/// Lifecycle wrapper around an ephemeral HTTP server. -/// -/// This deliberately knows nothing about inference or Braintrust so the test -/// protocol adapters can later move into a standalone crate. -pub struct TestServer { +/// HTTP lifecycle owned by the mock-inference component. It stays private: +/// consumers start an OpenAI or Anthropic mock, not a generic test server. +pub(super) struct InferenceServer { uri: String, shutdown: Option>, task: Option>>, } -impl TestServer { - pub async fn start(router: Router) -> Self { +impl InferenceServer { + pub(super) async fn start(router: Router) -> Self { let listener = TcpListener::bind("127.0.0.1:0") .await - .expect("bind ephemeral test server"); - let address = listener.local_addr().expect("read test server address"); + .expect("bind mock inference server"); + let address = listener + .local_addr() + .expect("read mock inference server address"); let (shutdown, shutdown_rx) = oneshot::channel(); let task = tokio::spawn(async move { axum::serve(listener, router) @@ -33,11 +33,11 @@ impl TestServer { } } - pub fn uri(&self) -> &str { + pub(super) fn uri(&self) -> &str { &self.uri } - pub async fn shutdown(mut self) { + pub(super) async fn shutdown(mut self) { if let Some(shutdown) = self.shutdown.take() { let _ = shutdown.send(()); } @@ -47,7 +47,7 @@ impl TestServer { } } -impl Drop for TestServer { +impl Drop for InferenceServer { fn drop(&mut self) { if let Some(shutdown) = self.shutdown.take() { let _ = shutdown.send(()); diff --git a/bt-daemon/tests/support/mod.rs b/bt-daemon/tests/support/mod.rs index b8b6f63..990e8f9 100644 --- a/bt-daemon/tests/support/mod.rs +++ b/bt-daemon/tests/support/mod.rs @@ -1,7 +1,5 @@ #![allow(dead_code)] -#[cfg(unix)] pub mod agent_process; pub mod inference; -pub mod server; pub mod trace_collector; diff --git a/bt-daemon/tests/support/trace_collector.rs b/bt-daemon/tests/support/trace_collector.rs index feb9d2a..f676a7f 100644 --- a/bt-daemon/tests/support/trace_collector.rs +++ b/bt-daemon/tests/support/trace_collector.rs @@ -1,4 +1,3 @@ -use crate::support::server::TestServer; use axum::body::Bytes; use axum::extract::State; use axum::http::HeaderMap; @@ -7,6 +6,45 @@ use axum::{Json, Router}; use serde_json::{json, Value}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +struct CollectorServer { + uri: String, + shutdown: Option>, + task: tokio::task::JoinHandle>, +} + +impl CollectorServer { + async fn start(router: Router) -> Self { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind trace collector"); + let address = listener.local_addr().expect("read trace collector address"); + let (shutdown, shutdown_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await + }); + Self { + uri: format!("http://{address}"), + shutdown: Some(shutdown), + task, + } + } +} + +impl Drop for CollectorServer { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + self.task.abort(); + } +} #[derive(Default)] struct CollectorState { @@ -16,7 +54,7 @@ struct CollectorState { } pub struct TraceCollector { - server: TestServer, + server: CollectorServer, state: Arc, } @@ -31,13 +69,13 @@ impl TraceCollector { .route("/logs3/overflow", post(logs)) .with_state(Arc::clone(&state)); Self { - server: TestServer::start(router).await, + server: CollectorServer::start(router).await, state, } } pub fn base_url(&self) -> &str { - self.server.uri() + &self.server.uri } pub fn rows(&self) -> Vec { From 874645ed1761494318788646c4e560d6f9ee89f8 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 22:51:52 +0800 Subject: [PATCH 03/17] Separate protocol routers from test hosting Have inference and ingest own only their protocol-specific Axum routers and captured state, while the upper-level tests host both through one generic server container. Expose an extensionless Windows test wrapper for hooks launched through Git Bash and make native command hooks prefer executable or command wrappers explicitly. Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 24 ++++--- bt-daemon/tests/inference_mocks.rs | 29 ++++----- bt-daemon/tests/support/README.md | 17 +++++ bt-daemon/tests/support/agent_process.rs | 28 ++++++--- bt-daemon/tests/support/inference/README.md | 29 ++++----- .../tests/support/inference/anthropic.rs | 24 +++---- bt-daemon/tests/support/inference/mod.rs | 1 - bt-daemon/tests/support/inference/openai.rs | 24 +++---- .../support/{trace_collector.rs => ingest.rs} | 62 +++---------------- bt-daemon/tests/support/mod.rs | 3 +- .../tests/support/{inference => }/server.rs | 21 +++---- .../trace-claude-code/bin/claude-hook.cmd | 4 +- .../plugins/trace-codex/bin/codex-hook.cmd | 4 +- 13 files changed, 122 insertions(+), 148 deletions(-) create mode 100644 bt-daemon/tests/support/README.md rename bt-daemon/tests/support/{trace_collector.rs => ingest.rs} (65%) rename bt-daemon/tests/support/{inference => }/server.rs (67%) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index e24c21f..7fe5416 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -5,6 +5,7 @@ use serde_json::{json, Value}; use std::path::{Path, PathBuf}; use support::agent_process::AgentTestWorld; use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; +use support::server::TestServer; use tokio::process::Command; use uuid::Uuid; @@ -173,7 +174,7 @@ async fn codex_session_emits_traces() { } async fn run_codex_mock() { - let inference = OpenAiMock::start(|context, request| { + let inference = OpenAiMock::new(|context, request| { assert_eq!(request.model(), Some("mock-model")); match context.request_index { 0 => { @@ -211,8 +212,8 @@ async fn run_codex_mock() { request.body ), } - }) - .await; + }); + let inference_server = TestServer::start(inference.router()).await; let world = AgentTestWorld::start().await; let codex_home = world.temp_path("codex-home"); std::fs::create_dir_all(&codex_home).unwrap(); @@ -220,9 +221,12 @@ async fn run_codex_mock() { let provider = format!( r#"model_providers.mock={{name="Mock",base_url="{}/v1",wire_api="responses",env_key="MOCK_API_KEY",request_max_retries=0,stream_max_retries=0,stream_idle_timeout_ms=5000}}"#, - inference.base_url() + inference_server.uri() + ); + let chatgpt_base_url = format!( + r#"chatgpt_base_url="{}/backend-api""#, + inference_server.uri() ); - let chatgpt_base_url = format!(r#"chatgpt_base_url="{}/backend-api""#, inference.base_url()); let mut codex = codex_command(&world, &codex_home); codex .args([ @@ -308,7 +312,7 @@ async fn claude_session_emits_traces() { } async fn run_claude_mock() { - let inference = AnthropicMock::start(|context, request| match context.request_index { + let inference = AnthropicMock::new(|context, request| match context.request_index { 0 => { assert_eq!(request.model(), Some("mock-model")); assert!( @@ -344,8 +348,8 @@ async fn run_claude_mock() { "unexpected Claude inference request {index}: {}", request.body ), - }) - .await; + }); + let inference_server = TestServer::start(inference.router()).await; let world = AgentTestWorld::start().await; let claude_config = world.temp_path("claude-config"); let home = world.temp_path("home"); @@ -358,7 +362,7 @@ async fn run_claude_mock() { .arg("Run the deterministic command, then return the deterministic marker.") .env("HOME", &home) .env("CLAUDE_CONFIG_DIR", &claude_config) - .env("ANTHROPIC_BASE_URL", inference.base_url()) + .env("ANTHROPIC_BASE_URL", inference_server.uri()) .env("ANTHROPIC_API_KEY", "test-key") .env("ANTHROPIC_AUTH_TOKEN", "test-key") .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") @@ -379,7 +383,7 @@ async fn run_claude_mock() { .arg("Trigger the deterministic inference error.") .env("HOME", &home) .env("CLAUDE_CONFIG_DIR", &claude_config) - .env("ANTHROPIC_BASE_URL", inference.base_url()) + .env("ANTHROPIC_BASE_URL", inference_server.uri()) .env("ANTHROPIC_API_KEY", "test-key") .env("ANTHROPIC_AUTH_TOKEN", "test-key") .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") diff --git a/bt-daemon/tests/inference_mocks.rs b/bt-daemon/tests/inference_mocks.rs index 9c658ca..03f3990 100644 --- a/bt-daemon/tests/inference_mocks.rs +++ b/bt-daemon/tests/inference_mocks.rs @@ -3,18 +3,19 @@ mod support; use axum::http::StatusCode; use serde_json::json; use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; +use support::server::TestServer; #[tokio::test] async fn openai_mock_streams_text_and_captures_requests() { - let mock = OpenAiMock::start(|context, request| { + let mock = OpenAiMock::new(|context, request| { assert_eq!(context.request_index, 0); assert_eq!(request.model(), Some("mock-model")); MockReply::response(OpenAiTurn::text("deterministic")) - }) - .await; + }); + let server = TestServer::start(mock.router()).await; let response = reqwest::Client::new() - .post(format!("{}/v1/responses", mock.base_url())) + .post(format!("{}/v1/responses", server.uri())) .json(&json!({"model":"mock-model","input":[],"stream":true})) .send() .await @@ -28,18 +29,18 @@ async fn openai_mock_streams_text_and_captures_requests() { #[tokio::test] async fn openai_mock_injects_retryable_and_malformed_responses() { - let mock = OpenAiMock::start(|context, _request| match context.request_index { + let mock = OpenAiMock::new(|context, _request| match context.request_index { 0 => MockReply::http_error( StatusCode::TOO_MANY_REQUESTS, json!({"error":{"type":"rate_limit_error","message":"deterministic limit"}}), ), _ => MockReply::raw_sse("event: response.output_item.done\ndata: not-json\n\n"), - }) - .await; + }); + let server = TestServer::start(mock.router()).await; let client = reqwest::Client::new(); let limited = client - .post(format!("{}/v1/responses", mock.base_url())) + .post(format!("{}/v1/responses", server.uri())) .json(&json!({"model":"mock-model","input":[],"stream":true})) .send() .await @@ -47,7 +48,7 @@ async fn openai_mock_injects_retryable_and_malformed_responses() { assert_eq!(limited.status(), StatusCode::TOO_MANY_REQUESTS); let malformed = client - .post(format!("{}/v1/responses", mock.base_url())) + .post(format!("{}/v1/responses", server.uri())) .json(&json!({"model":"mock-model","input":[],"stream":true})) .send() .await @@ -57,7 +58,7 @@ async fn openai_mock_injects_retryable_and_malformed_responses() { #[tokio::test] async fn anthropic_mock_supports_tool_use_and_http_errors() { - let mock = AnthropicMock::start(|context, request| match context.request_index { + let mock = AnthropicMock::new(|context, request| match context.request_index { 0 => { assert!(request.contains_text("run a command")); MockReply::response(AnthropicTurn::tool_use( @@ -73,12 +74,12 @@ async fn anthropic_mock_supports_tool_use_and_http_errors() { "error":{"type":"rate_limit_error","message":"deterministic limit"} }), ), - }) - .await; + }); + let server = TestServer::start(mock.router()).await; let client = reqwest::Client::new(); let first = client - .post(format!("{}/v1/messages", mock.base_url())) + .post(format!("{}/v1/messages", server.uri())) .json(&json!({ "model":"mock-model", "messages":[{"role":"user","content":"run a command"}], @@ -90,7 +91,7 @@ async fn anthropic_mock_supports_tool_use_and_http_errors() { assert!(first.text().await.unwrap().contains("toolu_mock")); let second = client - .post(format!("{}/v1/messages", mock.base_url())) + .post(format!("{}/v1/messages", server.uri())) .json(&json!({"model":"mock-model","messages":[],"stream":true})) .send() .await diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md new file mode 100644 index 0000000..cfdef64 --- /dev/null +++ b/bt-daemon/tests/support/README.md @@ -0,0 +1,17 @@ +# Agent integration test architecture + +The test infrastructure has three independent layers: + +- `server` is a generic container that binds any Axum `Router` to an + ephemeral address and owns its lifecycle. +- `inference` contains OpenAI Responses and Anthropic Messages protocol logic, + programmable scenarios, and captured inference requests. Each mock exports + an Axum router and can be hosted or embedded by any caller. +- `ingest` contains the mock Braintrust API and captured trace rows. It also + exports an Axum router and has no dependency on the server container. + +`agent_process` is the Braintrust-specific orchestration layer. It hosts the +ingest router, starts the daemon, and configures coding-agent processes. The +integration test separately hosts an inference router when running in +deterministic mode. This keeps both protocol mocks usable without the coding +agent runner and keeps the generic server unaware of either protocol. diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index e35ad81..78ceb27 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -1,4 +1,5 @@ -use crate::support::trace_collector::TraceCollector; +use crate::support::ingest::IngestMock; +use crate::support::server::TestServer; use serde_json::{json, Value}; use std::path::{Path, PathBuf}; use std::process::Stdio; @@ -10,7 +11,8 @@ use uuid::Uuid; pub struct AgentTestWorld { root: TempDir, - collector: TraceCollector, + collector: IngestMock, + collector_server: TestServer, daemon: Child, wrapper_dir: PathBuf, socket: PathBuf, @@ -21,7 +23,8 @@ pub struct AgentTestWorld { impl AgentTestWorld { pub async fn start() -> Self { let root = tempfile::tempdir().expect("create agent test root"); - let collector = TraceCollector::start().await; + let collector = IngestMock::new(); + let collector_server = TestServer::start(collector.router()).await; let wrapper_dir = root.path().join("bin"); let data_dir = root.path().join("daemon"); let socket = test_endpoint(root.path()); @@ -52,8 +55,8 @@ impl AgentTestWorld { .arg(&data_dir) .arg("--idle-timeout-secs") .arg("0") - .env("BRAINTRUST_API_URL", collector.base_url()) - .env("BRAINTRUST_APP_URL", collector.base_url()) + .env("BRAINTRUST_API_URL", collector_server.uri()) + .env("BRAINTRUST_APP_URL", collector_server.uri()) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::piped()) @@ -64,6 +67,7 @@ impl AgentTestWorld { Self { root, collector, + collector_server, daemon, wrapper_dir, socket, @@ -93,8 +97,8 @@ impl AgentTestWorld { .env("BT_DAEMON_DATA_DIR", &self.data_dir) .env("BT_DAEMON_CONFIG", &self.config_path) .env("BRAINTRUST_API_KEY", "test-key") - .env("BRAINTRUST_API_URL", self.collector.base_url()) - .env("BRAINTRUST_APP_URL", self.collector.base_url()) + .env("BRAINTRUST_API_URL", self.collector_server.uri()) + .env("BRAINTRUST_APP_URL", self.collector_server.uri()) .env("BRAINTRUST_PROJECT", "agent-e2e") .env("BRAINTRUST_FLUSH_ON_TURN_END", "true") .stdin(Stdio::null()); @@ -175,6 +179,16 @@ fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { "@echo off\r\npowershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File \"%~dp0bt-wrapper.ps1\" %*\r\n", ) .expect("write bt command wrapper"); + + // Claude Code, and some Codex releases, launch the portable `command` + // hook through Git Bash even on Windows. Git Bash does not resolve + // PATHEXT, so expose an extensionless shim in addition to bt.cmd. + let shell_binary = daemon_binary.to_string_lossy().replace('\\', "/"); + let shell = format!( + "#!/bin/sh\nif [ \"$1\" = daemon ]; then shift; fi\nexec '{}' \"$@\"\n", + shell_binary + ); + std::fs::write(directory.join("bt"), shell).expect("write bt Git Bash wrapper"); } #[cfg(unix)] diff --git a/bt-daemon/tests/support/inference/README.md b/bt-daemon/tests/support/inference/README.md index 27a7cae..da11de1 100644 --- a/bt-daemon/tests/support/inference/README.md +++ b/bt-daemon/tests/support/inference/README.md @@ -7,18 +7,17 @@ protocol-faithful servers: - `AnthropicMock` implements the Anthropic Messages API surface used by Claude Code. -Each public mock owns its HTTP listener and lifecycle. There is intentionally -no separately exposed generic server: the useful abstraction is a -programmatically controlled inference endpoint. The two providers share only -private transport mechanics, request indexing, and transport outcomes. -Request and response types remain provider-specific so a test cannot -accidentally hide a wire-protocol incompatibility behind a common model -abstraction. +Each public mock owns its protocol routes, scenario closure, and captured +requests, and exports an Axum `Router`. Callers can bind that router with the +shared ephemeral test server or embed it in another Axum application. The two +providers share request indexing and transport outcomes. Request and response +types remain provider-specific so a test cannot accidentally hide a +wire-protocol incompatibility behind a common model abstraction. Both mocks accept a thread-safe closure: ```rust,ignore -let mock = OpenAiMock::start(|context, request| { +let mock = OpenAiMock::new(|context, request| { match context.request_index { 0 => MockReply::response(OpenAiTurn::tool_call( "call-1", @@ -30,7 +29,8 @@ let mock = OpenAiMock::start(|context, request| { } index => panic!("unexpected request {index}: {}", request.body), } -}).await; +}); +let server = TestServer::start(mock.router()).await; ``` `MockReply` supports normal provider responses, arbitrary HTTP errors, and raw @@ -38,11 +38,12 @@ response bodies for malformed or truncated stream tests. Typed turn builders generate deterministic ids, token usage, and valid provider SSE sequences. Every inference request is captured for later assertions. -The component does not depend on `bt-daemon`, the coding-agent runner, or the -trace collector. The higher-level `support::agent_process` harness composes -with it only from the integration test. This boundary is deliberate so the -whole mock-inference component can later move into a reusable crate and serve -any client that can target an OpenAI Responses or Anthropic Messages endpoint. +The component does not depend on `bt-daemon`, the coding-agent runner, the +ingest mock, or a particular listener implementation. The higher-level +`support::agent_process` harness composes with it only from the integration +test. This boundary is deliberate so the whole mock-inference component can +later move into a reusable crate and serve any client that can target an +OpenAI Responses or Anthropic Messages endpoint. `agent_integration.rs` runs real Codex and Claude Code processes against these mocks. diff --git a/bt-daemon/tests/support/inference/anthropic.rs b/bt-daemon/tests/support/inference/anthropic.rs index 371247f..59662b1 100644 --- a/bt-daemon/tests/support/inference/anthropic.rs +++ b/bt-daemon/tests/support/inference/anthropic.rs @@ -1,4 +1,3 @@ -use super::server::InferenceServer; use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; use axum::body::Bytes; use axum::extract::State; @@ -175,12 +174,11 @@ struct MockState { } pub struct AnthropicMock { - server: InferenceServer, state: Arc, } impl AnthropicMock { - pub async fn start(handler: H) -> Self + pub fn new(handler: H) -> Self where H: Fn(RequestContext, AnthropicRequest) -> MockReply + Send + Sync + 'static, { @@ -189,28 +187,20 @@ impl AnthropicMock { requests: Mutex::new(Vec::new()), next_index: AtomicUsize::new(0), }); - let router = Router::new() + Self { state } + } + + pub fn router(&self) -> Router { + Router::new() .route("/v1/models", get(models)) .route("/v1/messages", post(messages)) .route("/v1/messages/count_tokens", post(count_tokens)) - .with_state(Arc::clone(&state)); - Self { - server: InferenceServer::start(router).await, - state, - } - } - - pub fn base_url(&self) -> &str { - self.server.uri() + .with_state(Arc::clone(&self.state)) } pub fn requests(&self) -> Vec { self.state.requests.lock().expect("request lock").clone() } - - pub async fn shutdown(self) { - self.server.shutdown().await; - } } async fn models() -> axum::Json { diff --git a/bt-daemon/tests/support/inference/mod.rs b/bt-daemon/tests/support/inference/mod.rs index a06f7f1..4339036 100644 --- a/bt-daemon/tests/support/inference/mod.rs +++ b/bt-daemon/tests/support/inference/mod.rs @@ -1,6 +1,5 @@ mod anthropic; mod openai; -mod server; #[allow(unused_imports)] pub use anthropic::{AnthropicMock, AnthropicRequest, AnthropicTurn}; diff --git a/bt-daemon/tests/support/inference/openai.rs b/bt-daemon/tests/support/inference/openai.rs index 5e5e764..96edf0e 100644 --- a/bt-daemon/tests/support/inference/openai.rs +++ b/bt-daemon/tests/support/inference/openai.rs @@ -1,4 +1,3 @@ -use super::server::InferenceServer; use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; use axum::body::Bytes; use axum::extract::State; @@ -146,12 +145,11 @@ struct MockState { } pub struct OpenAiMock { - server: InferenceServer, state: Arc, } impl OpenAiMock { - pub async fn start(handler: H) -> Self + pub fn new(handler: H) -> Self where H: Fn(RequestContext, OpenAiRequest) -> MockReply + Send + Sync + 'static, { @@ -160,28 +158,20 @@ impl OpenAiMock { requests: Mutex::new(Vec::new()), next_index: AtomicUsize::new(0), }); - let router = Router::new() + Self { state } + } + + pub fn router(&self) -> Router { + Router::new() .route("/v1/models", get(models)) .route("/v1/responses", post(responses)) .route("/backend-api/plugins/featured", get(featured_plugins)) - .with_state(Arc::clone(&state)); - Self { - server: InferenceServer::start(router).await, - state, - } - } - - pub fn base_url(&self) -> &str { - self.server.uri() + .with_state(Arc::clone(&self.state)) } pub fn requests(&self) -> Vec { self.state.requests.lock().expect("request lock").clone() } - - pub async fn shutdown(self) { - self.server.shutdown().await; - } } async fn models() -> axum::Json { diff --git a/bt-daemon/tests/support/trace_collector.rs b/bt-daemon/tests/support/ingest.rs similarity index 65% rename from bt-daemon/tests/support/trace_collector.rs rename to bt-daemon/tests/support/ingest.rs index f676a7f..8175415 100644 --- a/bt-daemon/tests/support/trace_collector.rs +++ b/bt-daemon/tests/support/ingest.rs @@ -6,45 +6,6 @@ use axum::{Json, Router}; use serde_json::{json, Value}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; -use tokio::net::TcpListener; -use tokio::sync::oneshot; - -struct CollectorServer { - uri: String, - shutdown: Option>, - task: tokio::task::JoinHandle>, -} - -impl CollectorServer { - async fn start(router: Router) -> Self { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind trace collector"); - let address = listener.local_addr().expect("read trace collector address"); - let (shutdown, shutdown_rx) = oneshot::channel(); - let task = tokio::spawn(async move { - axum::serve(listener, router) - .with_graceful_shutdown(async move { - let _ = shutdown_rx.await; - }) - .await - }); - Self { - uri: format!("http://{address}"), - shutdown: Some(shutdown), - task, - } - } -} - -impl Drop for CollectorServer { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - self.task.abort(); - } -} #[derive(Default)] struct CollectorState { @@ -53,29 +14,24 @@ struct CollectorState { log_requests: AtomicUsize, } -pub struct TraceCollector { - server: CollectorServer, +pub struct IngestMock { state: Arc, } -impl TraceCollector { - pub async fn start() -> Self { +impl IngestMock { + pub fn new() -> Self { let state = Arc::new(CollectorState::default()); - let router = Router::new() + Self { state } + } + + pub fn router(&self) -> Router { + Router::new() .route("/version", get(version)) .route("/api/apikey/login", post(login)) .route("/api/project/register", post(register_project)) .route("/logs3", post(logs)) .route("/logs3/overflow", post(logs)) - .with_state(Arc::clone(&state)); - Self { - server: CollectorServer::start(router).await, - state, - } - } - - pub fn base_url(&self) -> &str { - &self.server.uri + .with_state(Arc::clone(&self.state)) } pub fn rows(&self) -> Vec { diff --git a/bt-daemon/tests/support/mod.rs b/bt-daemon/tests/support/mod.rs index 990e8f9..22b646b 100644 --- a/bt-daemon/tests/support/mod.rs +++ b/bt-daemon/tests/support/mod.rs @@ -2,4 +2,5 @@ pub mod agent_process; pub mod inference; -pub mod trace_collector; +pub mod ingest; +pub mod server; diff --git a/bt-daemon/tests/support/inference/server.rs b/bt-daemon/tests/support/server.rs similarity index 67% rename from bt-daemon/tests/support/inference/server.rs rename to bt-daemon/tests/support/server.rs index 43a3406..fec25d9 100644 --- a/bt-daemon/tests/support/inference/server.rs +++ b/bt-daemon/tests/support/server.rs @@ -2,22 +2,19 @@ use axum::Router; use tokio::net::TcpListener; use tokio::sync::oneshot; -/// HTTP lifecycle owned by the mock-inference component. It stays private: -/// consumers start an OpenAI or Anthropic mock, not a generic test server. -pub(super) struct InferenceServer { +/// Lifecycle wrapper for any ephemeral Axum test service. +pub struct TestServer { uri: String, shutdown: Option>, task: Option>>, } -impl InferenceServer { - pub(super) async fn start(router: Router) -> Self { +impl TestServer { + pub async fn start(router: Router) -> Self { let listener = TcpListener::bind("127.0.0.1:0") .await - .expect("bind mock inference server"); - let address = listener - .local_addr() - .expect("read mock inference server address"); + .expect("bind ephemeral test server"); + let address = listener.local_addr().expect("read test server address"); let (shutdown, shutdown_rx) = oneshot::channel(); let task = tokio::spawn(async move { axum::serve(listener, router) @@ -33,11 +30,11 @@ impl InferenceServer { } } - pub(super) fn uri(&self) -> &str { + pub fn uri(&self) -> &str { &self.uri } - pub(super) async fn shutdown(mut self) { + pub async fn shutdown(mut self) { if let Some(shutdown) = self.shutdown.take() { let _ = shutdown.send(()); } @@ -47,7 +44,7 @@ impl InferenceServer { } } -impl Drop for InferenceServer { +impl Drop for TestServer { fn drop(&mut self) { if let Some(shutdown) = self.shutdown.take() { let _ = shutdown.send(()); diff --git a/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd b/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd index 9dc9173..bb9ebb8 100644 --- a/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd +++ b/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd @@ -4,7 +4,9 @@ REM Invokes: bt daemon hook --source claude-code setlocal EnableExtensions DisableDelayedExpansion set "BT_HOOK_BIN=" -for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +for /f "delims=" %%B in ('where bt.exe 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt.cmd 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" if not defined BT_HOOK_BIN if exist "%USERPROFILE%\.local\bin\bt.exe" set "BT_HOOK_BIN=%USERPROFILE%\.local\bin\bt.exe" if not defined BT_HOOK_BIN ( echo trace-claude-code: bt CLI is unavailable; tracing disabled for this event.>&2 diff --git a/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd b/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd index 60f192d..ff544e1 100644 --- a/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd +++ b/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd @@ -4,7 +4,9 @@ REM Invokes: bt daemon hook --source codex setlocal EnableExtensions DisableDelayedExpansion set "BT_HOOK_BIN=" -for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +for /f "delims=" %%B in ('where bt.exe 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt.cmd 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" if not defined BT_HOOK_BIN if exist "%USERPROFILE%\.local\bin\bt.exe" set "BT_HOOK_BIN=%USERPROFILE%\.local\bin\bt.exe" if not defined BT_HOOK_BIN ( echo trace-codex: bt CLI is unavailable; tracing disabled for this event.>&2 From 15ce7b09d6297df0af1c25694b7b34498f84044b Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 22:59:39 +0800 Subject: [PATCH 04/17] Add ordered ingest trace scenarios Add named row-shape expectations over the captured ingest stream, matched as an ordered subsequence independently of HTTP batching and unrelated update rows. Use the same scenario mechanism for deterministic deep trace assertions and live invariant-only assertions, with a focused router and ordering test. Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 62 ++++++++++++------------ bt-daemon/tests/ingest_mock.rs | 37 ++++++++++++++ bt-daemon/tests/support/README.md | 4 +- bt-daemon/tests/support/agent_process.rs | 18 ++++++- bt-daemon/tests/support/ingest.rs | 58 ++++++++++++++++++++++ 5 files changed, 145 insertions(+), 34 deletions(-) create mode 100644 bt-daemon/tests/ingest_mock.rs diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 7fe5416..4d30572 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -5,6 +5,7 @@ use serde_json::{json, Value}; use std::path::{Path, PathBuf}; use support::agent_process::AgentTestWorld; use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; +use support::ingest::IngestScenario; use support::server::TestServer; use tokio::process::Command; use uuid::Uuid; @@ -153,15 +154,11 @@ fn claude_command(world: &AgentTestWorld) -> Command { command } -async fn wait_for_trace_fragments(world: &AgentTestWorld, fragments: &[&str]) -> Vec { - world - .wait_for_trace_rows_matching(|rows| { - let serialized = serde_json::to_string(rows).expect("serialize trace rows"); - fragments - .iter() - .all(|fragment| serialized.contains(fragment)) - }) - .await +fn row_contains(row: &Value, fragments: &[&str]) -> bool { + let serialized = serde_json::to_string(row).expect("serialize trace row"); + fragments + .iter() + .all(|fragment| serialized.contains(fragment)) } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -273,16 +270,14 @@ async fn run_codex_mock() { ); assert_eq!(inference.requests().len(), 3); - let rows = wait_for_trace_fragments( - &world, - &[ - "braintrust.plugin.codex", - "test_harness", - r#""type":"tool""#, - "CODEX_TOOL_OK", - ], - ) - .await; + let scenario = IngestScenario::new() + .expect("Codex trace origin", |row| { + row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + }) + .expect("Codex tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) + }); + let rows = world.wait_for_ingest_scenario(&scenario).await; assert!(!rows.is_empty()); } @@ -298,7 +293,10 @@ async fn run_codex_live() { let output = world.output(&mut codex).await; assert!(output.status.success(), "{}", output_text(&output)); - let rows = wait_for_trace_fragments(&world, &["braintrust.plugin.codex", "test_harness"]).await; + let scenario = IngestScenario::new().expect("Codex trace origin", |row| { + row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + }); + let rows = world.wait_for_ingest_scenario(&scenario).await; assert!(!rows.is_empty()); } @@ -398,16 +396,14 @@ async fn run_claude_mock() { ); assert_eq!(inference.requests().len(), 3); - let rows = wait_for_trace_fragments( - &world, - &[ - r#""source":"claude-code""#, - "test_harness", - r#""type":"tool""#, - "CLAUDE_TOOL_OK", - ], - ) - .await; + let scenario = IngestScenario::new() + .expect("Claude trace source", |row| { + row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + }) + .expect("Claude tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) + }); + let rows = world.wait_for_ingest_scenario(&scenario).await; assert!(!rows.is_empty()); } @@ -418,8 +414,10 @@ async fn run_claude_live() { let output = world.output(&mut claude).await; assert!(output.status.success(), "{}", output_text(&output)); - let rows = - wait_for_trace_fragments(&world, &[r#""source":"claude-code""#, "test_harness"]).await; + let scenario = IngestScenario::new().expect("Claude trace source", |row| { + row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + }); + let rows = world.wait_for_ingest_scenario(&scenario).await; assert!(!rows.is_empty()); } diff --git a/bt-daemon/tests/ingest_mock.rs b/bt-daemon/tests/ingest_mock.rs new file mode 100644 index 0000000..9d9430b --- /dev/null +++ b/bt-daemon/tests/ingest_mock.rs @@ -0,0 +1,37 @@ +mod support; + +use serde_json::json; +use support::ingest::{IngestMock, IngestScenario}; +use support::server::TestServer; + +#[tokio::test] +async fn ingest_router_captures_rows_and_matches_ordered_shapes() { + let ingest = IngestMock::new(); + let server = TestServer::start(ingest.router()).await; + + let response = reqwest::Client::new() + .post(format!("{}/logs3", server.uri())) + .json(&json!({ + "rows": [ + {"span_attributes":{"type":"task"},"metadata":{"source":"codex"}}, + {"span_attributes":{"type":"llm"}}, + {"span_attributes":{"type":"tool"},"output":"deterministic"} + ] + })) + .send() + .await + .unwrap(); + assert!(response.status().is_success()); + + let scenario = IngestScenario::new() + .expect("root task", |row| row["span_attributes"]["type"] == "task") + .expect("tool result", |row| { + row["span_attributes"]["type"] == "tool" && row["output"] == "deterministic" + }); + assert_eq!(ingest.evaluate(&scenario).unwrap().len(), 3); + + let reversed = IngestScenario::new() + .expect("tool first", |row| row["span_attributes"]["type"] == "tool") + .expect("task later", |row| row["span_attributes"]["type"] == "task"); + assert!(ingest.evaluate(&reversed).is_err()); +} diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index cfdef64..e8943b0 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -8,7 +8,9 @@ The test infrastructure has three independent layers: programmable scenarios, and captured inference requests. Each mock exports an Axum router and can be hosted or embedded by any caller. - `ingest` contains the mock Braintrust API and captured trace rows. It also - exports an Axum router and has no dependency on the server container. + exports an Axum router and has no dependency on the server container. Its + scenario builder matches named row shapes as an ordered subsequence, + independent of HTTP batching and unrelated SDK update rows. `agent_process` is the Braintrust-specific orchestration layer. It hosts the ingest router, starts the daemon, and configures coding-agent processes. The diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index 78ceb27..b2cdde4 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -1,4 +1,4 @@ -use crate::support::ingest::IngestMock; +use crate::support::ingest::{IngestMock, IngestScenario}; use crate::support::server::TestServer; use serde_json::{json, Value}; use std::path::{Path, PathBuf}; @@ -138,6 +138,22 @@ impl AgentTestWorld { directory_contents(&self.data_dir) ); } + + pub async fn wait_for_ingest_scenario(&self, scenario: &IngestScenario) -> Vec { + let mut last_error = String::new(); + for _ in 0..100 { + match self.collector.evaluate(scenario) { + Ok(rows) => return rows, + Err(error) => last_error = error, + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!( + "ingest scenario did not complete: {last_error}; {}; daemon files:\n{}", + self.collector.diagnostics(), + directory_contents(&self.data_dir) + ); + } } impl Drop for AgentTestWorld { diff --git a/bt-daemon/tests/support/ingest.rs b/bt-daemon/tests/support/ingest.rs index 8175415..d018c69 100644 --- a/bt-daemon/tests/support/ingest.rs +++ b/bt-daemon/tests/support/ingest.rs @@ -7,6 +7,58 @@ use serde_json::{json, Value}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +type RowMatcher = dyn Fn(&Value) -> bool + Send + Sync + 'static; + +struct ExpectedRow { + name: String, + matcher: Arc, +} + +#[derive(Default)] +pub struct IngestScenario { + expected: Vec, +} + +impl IngestScenario { + pub fn new() -> Self { + Self::default() + } + + /// Require a row shape after all previously declared shapes. Unrelated + /// rows are ignored, so matching is independent of HTTP batching and + /// SDK-generated update rows. + pub fn expect( + mut self, + name: impl Into, + matcher: impl Fn(&Value) -> bool + Send + Sync + 'static, + ) -> Self { + self.expected.push(ExpectedRow { + name: name.into(), + matcher: Arc::new(matcher), + }); + self + } + + pub fn evaluate(&self, rows: &[Value]) -> Result<(), String> { + let mut cursor = 0; + for (matched, expected) in self.expected.iter().enumerate() { + let Some(offset) = rows[cursor..] + .iter() + .position(|row| (expected.matcher)(row)) + else { + return Err(format!( + "missing ingest shape {:?} after matching {} of {} shapes", + expected.name, + matched, + self.expected.len() + )); + }; + cursor += offset + 1; + } + Ok(()) + } +} + #[derive(Default)] struct CollectorState { rows: Mutex>, @@ -46,6 +98,12 @@ impl IngestMock { self.rows().len() ) } + + pub fn evaluate(&self, scenario: &IngestScenario) -> Result, String> { + let rows = self.rows(); + scenario.evaluate(&rows)?; + Ok(rows) + } } async fn version() -> Json { From 7c8e28ffb5c14861c858e5098d209c18afca7853 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:04:50 +0800 Subject: [PATCH 05/17] Isolate coding-agent install cache in CI Keep both agents unpinned at latest while using a per-runner temporary npm cache and skipping audit/funding requests, avoiding hosted Windows global-cache stalls before the test suite. Signed-off-by: Stephen Belanger --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f3b611..369a92e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: rustup default stable rustup component add clippy rustfmt - name: Install latest coding agents - run: npm install --global @openai/codex@latest @anthropic-ai/claude-code@latest + run: npm install --global --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest - name: Report coding-agent versions run: | codex --version From 683feb90d0deeb4f679121d8ad3e0b387423bf76 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:07:52 +0800 Subject: [PATCH 06/17] Install CI agents into an isolated prefix Install the latest Codex and Claude Code packages into a runner-temporary npm prefix on every platform and pass their exact executable paths into the shared tests, avoiding Windows global-install state. Signed-off-by: Stephen Belanger --- .github/workflows/ci.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 369a92e..8c95fd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,18 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-latest, windows-latest] + include: + - os: ubuntu-24.04 + agent_suffix: "" + - os: macos-latest + agent_suffix: "" + - os: windows-latest + agent_suffix: ".cmd" runs-on: ${{ matrix.os }} timeout-minutes: 30 + env: + CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }} + CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Rust @@ -42,11 +51,11 @@ jobs: rustup default stable rustup component add clippy rustfmt - name: Install latest coding agents - run: npm install --global --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest + run: npm install --prefix "${{ runner.temp }}/coding-agents" --no-save --no-package-lock --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest - name: Report coding-agent versions run: | - codex --version - claude --version + npm exec --prefix "${{ runner.temp }}/coding-agents" -- codex --version + npm exec --prefix "${{ runner.temp }}/coding-agents" -- claude --version - name: Check formatting if: runner.os == 'Linux' run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check From d58ab354c036dcca17a1a30941cba92546e68b28 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:08:49 +0800 Subject: [PATCH 07/17] Scope agent executable paths to the test step Resolve runner-temporary executable paths only after the matrix runner exists, avoiding workflow validation failure while retaining exact cross-platform agent paths. Signed-off-by: Stephen Belanger --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c95fd1..a03163d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,9 +40,6 @@ jobs: agent_suffix: ".cmd" runs-on: ${{ matrix.os }} timeout-minutes: 30 - env: - CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }} - CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Rust @@ -66,6 +63,8 @@ jobs: - name: Test coding-agent integrations with deterministic inference env: BT_AGENT_TEST_MODE: mock + CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }} + CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }} run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_integration -- --ignored --nocapture --test-threads=1 - name: Lint daemon run: cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features --locked -- -D warnings From 32110f3b1072e139418511b6d338bff943163030 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:14:59 +0800 Subject: [PATCH 08/17] Expose Windows Codex hook diagnostics Print the real Codex process output on Windows so hook-selection failures are visible in the cross-platform integration job. Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 4d30572..17d9d45 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -239,6 +239,8 @@ async fn run_codex_mock() { .arg("Run the deterministic command, then return the deterministic marker.") .env("MOCK_API_KEY", "test-key"); let output = world.output(&mut codex).await; + #[cfg(windows)] + eprintln!("Codex deterministic session:\n{}", output_text(&output)); assert!(output.status.success(), "{}", output_text(&output)); assert!( output_text(&output).contains("CODEX_MOCK_OK"), @@ -262,6 +264,11 @@ async fn run_codex_mock() { .arg("Trigger the deterministic inference error.") .env("MOCK_API_KEY", "test-key"); let failed = world.output(&mut failing_codex).await; + #[cfg(windows)] + eprintln!( + "Codex deterministic error session:\n{}", + output_text(&failed) + ); assert!(!failed.status.success(), "{}", output_text(&failed)); assert!( output_text(&failed).contains("deterministic Codex inference failure"), From c9831f36df27ef662b9aa12de23befacf9ca7bff Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:24:44 +0800 Subject: [PATCH 09/17] Extract reusable coding agent test adapters Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 338 +++++--------------- bt-daemon/tests/support/README.md | 18 +- bt-daemon/tests/support/agents/claude.rs | 108 +++++++ bt-daemon/tests/support/agents/codex.rs | 146 +++++++++ bt-daemon/tests/support/agents/mod.rs | 98 ++++++ bt-daemon/tests/support/inference/openai.rs | 9 + bt-daemon/tests/support/mod.rs | 1 + 7 files changed, 463 insertions(+), 255 deletions(-) create mode 100644 bt-daemon/tests/support/agents/claude.rs create mode 100644 bt-daemon/tests/support/agents/codex.rs create mode 100644 bt-daemon/tests/support/agents/mod.rs diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 17d9d45..99aada7 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -2,13 +2,14 @@ mod support; use axum::http::StatusCode; use serde_json::{json, Value}; -use std::path::{Path, PathBuf}; use support::agent_process::AgentTestWorld; -use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; +use support::agents::{ClaudeAgent, ClaudeRun, CodexAgent, CodexRun}; +use support::inference::{ + AnthropicMock, AnthropicRequest, AnthropicTurn, MockReply, OpenAiMock, OpenAiRequest, + OpenAiTurn, +}; use support::ingest::IngestScenario; use support::server::TestServer; -use tokio::process::Command; -use uuid::Uuid; const TEST_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; @@ -29,28 +30,23 @@ impl AgentTestMode { } } -fn repository_root() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("repository root") - .to_path_buf() -} - -fn command_from_env(name: &str, fallback: &str) -> Command { - if let Some(command) = std::env::var_os(name) { - return Command::new(command); +fn codex_tool_call(request: &OpenAiRequest) -> OpenAiTurn { + let names = request.tool_names(); + if names.contains(&"exec_command") { + return OpenAiTurn::tool_call( + "call_mock_1", + "exec_command", + json!({"cmd":codex_tool_command(),"login":false}), + ); } - #[cfg(windows)] - let fallback = format!("{fallback}.cmd"); - Command::new(fallback) -} - -fn output_text(output: &std::process::Output) -> String { - format!( - "stdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ) + if names.contains(&"shell") { + return OpenAiTurn::tool_call( + "call_mock_1", + "shell", + json!({"command":codex_tool_command()}), + ); + } + panic!("Codex offered no supported shell tool; offered tools: {names:?}"); } fn codex_tool_command() -> &'static str { @@ -64,96 +60,6 @@ fn codex_tool_command() -> &'static str { } } -fn configured_codex_home() -> Option { - std::env::var_os("CODEX_HOME") - .map(PathBuf::from) - .or_else(|| { - std::env::var_os("HOME") - .or_else(|| std::env::var_os("USERPROFILE")) - .map(PathBuf::from) - .map(|home| home.join(".codex")) - }) -} - -fn seed_codex_live_auth(codex_home: &Path) { - if std::env::var_os("OPENAI_API_KEY").is_some() { - return; - } - let source = configured_codex_home() - .map(|home| home.join("auth.json")) - .filter(|path| path.is_file()) - .unwrap_or_else(|| { - panic!( - "{TEST_MODE_ENV}=live requires OPENAI_API_KEY or auth.json in the configured Codex home" - ) - }); - std::fs::copy(source, codex_home.join("auth.json")).expect("copy Codex live credentials"); -} - -async fn install_codex_plugin(world: &AgentTestWorld, codex_home: &Path) { - let marketplace = repository_root().join("src/plugins/codex/content"); - let mut add_marketplace = command_from_env("CODEX_BIN", "codex"); - add_marketplace - .arg("plugin") - .arg("marketplace") - .arg("add") - .arg(&marketplace) - .env("CODEX_HOME", codex_home); - world.configure(&mut add_marketplace); - let output = world.output(&mut add_marketplace).await; - assert!(output.status.success(), "{}", output_text(&output)); - - let mut add_plugin = command_from_env("CODEX_BIN", "codex"); - add_plugin - .args(["plugin", "add", "trace-codex@braintrust-codex-plugins"]) - .env("CODEX_HOME", codex_home); - world.configure(&mut add_plugin); - let output = world.output(&mut add_plugin).await; - assert!(output.status.success(), "{}", output_text(&output)); -} - -fn codex_command(world: &AgentTestWorld, codex_home: &Path) -> Command { - let mut command = command_from_env("CODEX_BIN", "codex"); - command - .args([ - "exec", - "--skip-git-repo-check", - "--dangerously-bypass-hook-trust", - "--sandbox", - "read-only", - "-c", - r#"approval_policy="never""#, - ]) - .current_dir(world.workspace()) - .env("CODEX_HOME", codex_home); - world.configure(&mut command); - command -} - -fn claude_command(world: &AgentTestWorld) -> Command { - let session_id = Uuid::new_v4().to_string(); - let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); - let mut command = command_from_env("CLAUDE_BIN", "claude"); - command - .args([ - "-p", - "--output-format", - "json", - "--dangerously-skip-permissions", - "--session-id", - &session_id, - "--plugin-dir", - ]) - .arg(plugin) - .current_dir(world.workspace()) - .env("ANTHROPIC_MAX_RETRIES", "0") - .env("DISABLE_AUTOUPDATER", "1") - .env("DISABLE_TELEMETRY", "1") - .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); - world.configure(&mut command); - command -} - fn row_contains(row: &Value, fragments: &[&str]) -> bool { let serialized = serde_json::to_string(row).expect("serialize trace row"); fragments @@ -180,11 +86,7 @@ async fn run_codex_mock() { "unexpected Codex request: {}", request.body ); - MockReply::response(OpenAiTurn::tool_call( - "call_mock_1", - "exec_command", - json!({"cmd":codex_tool_command(),"login":false}), - )) + MockReply::response(codex_tool_call(&request)) } 1 => { assert!( @@ -212,69 +114,26 @@ async fn run_codex_mock() { }); let inference_server = TestServer::start(inference.router()).await; let world = AgentTestWorld::start().await; - let codex_home = world.temp_path("codex-home"); - std::fs::create_dir_all(&codex_home).unwrap(); - install_codex_plugin(&world, &codex_home).await; - - let provider = format!( - r#"model_providers.mock={{name="Mock",base_url="{}/v1",wire_api="responses",env_key="MOCK_API_KEY",request_max_retries=0,stream_max_retries=0,stream_idle_timeout_ms=5000}}"#, - inference_server.uri() - ); - let chatgpt_base_url = format!( - r#"chatgpt_base_url="{}/backend-api""#, - inference_server.uri() - ); - let mut codex = codex_command(&world, &codex_home); - codex - .args([ - "-c", - r#"model="mock-model""#, - "-c", - r#"model_provider="mock""#, - "-c", - &provider, - "-c", - &chatgpt_base_url, - ]) - .arg("Run the deterministic command, then return the deterministic marker.") - .env("MOCK_API_KEY", "test-key"); - let output = world.output(&mut codex).await; - #[cfg(windows)] - eprintln!("Codex deterministic session:\n{}", output_text(&output)); - assert!(output.status.success(), "{}", output_text(&output)); - assert!( - output_text(&output).contains("CODEX_MOCK_OK"), - "{}", - output_text(&output) - ); + let codex = CodexAgent::install(&world).await; + + let output = codex + .run(CodexRun::mock( + "Run the deterministic command, then return the deterministic marker.", + inference_server.uri(), + )) + .await; + output.assert_success(); + output.assert_contains("CODEX_MOCK_OK"); assert_eq!(inference.requests().len(), 2); - let mut failing_codex = codex_command(&world, &codex_home); - failing_codex - .args([ - "-c", - r#"model="mock-model""#, - "-c", - r#"model_provider="mock""#, - "-c", - &provider, - "-c", - &chatgpt_base_url, - ]) - .arg("Trigger the deterministic inference error.") - .env("MOCK_API_KEY", "test-key"); - let failed = world.output(&mut failing_codex).await; - #[cfg(windows)] - eprintln!( - "Codex deterministic error session:\n{}", - output_text(&failed) - ); - assert!(!failed.status.success(), "{}", output_text(&failed)); - assert!( - output_text(&failed).contains("deterministic Codex inference failure"), - "{}", - output_text(&failed) - ); + let failed = codex + .run(CodexRun::mock( + "Trigger the deterministic inference error.", + inference_server.uri(), + )) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Codex inference failure"); assert_eq!(inference.requests().len(), 3); let scenario = IngestScenario::new() @@ -284,27 +143,25 @@ async fn run_codex_mock() { .expect("Codex tool output", |row| { row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) }); - let rows = world.wait_for_ingest_scenario(&scenario).await; - assert!(!rows.is_empty()); + assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); } async fn run_codex_live() { let world = AgentTestWorld::start().await; - let codex_home = world.temp_path("codex-home"); - std::fs::create_dir_all(&codex_home).unwrap(); - seed_codex_live_auth(&codex_home); - install_codex_plugin(&world, &codex_home).await; + let codex = CodexAgent::install(&world).await; + codex.seed_live_auth(); - let mut codex = codex_command(&world, &codex_home); - codex.arg("Reply briefly to confirm this tracing integration test."); - let output = world.output(&mut codex).await; - assert!(output.status.success(), "{}", output_text(&output)); + codex + .run(CodexRun::live( + "Reply briefly to confirm this tracing integration test.", + )) + .await + .assert_success(); let scenario = IngestScenario::new().expect("Codex trace origin", |row| { row_contains(row, &["braintrust.plugin.codex", "test_harness"]) }); - let rows = world.wait_for_ingest_scenario(&scenario).await; - assert!(!rows.is_empty()); + assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -356,51 +213,26 @@ async fn run_claude_mock() { }); let inference_server = TestServer::start(inference.router()).await; let world = AgentTestWorld::start().await; - let claude_config = world.temp_path("claude-config"); - let home = world.temp_path("home"); - std::fs::create_dir_all(&claude_config).unwrap(); - std::fs::create_dir_all(&home).unwrap(); - - let mut claude = claude_command(&world); - claude - .args(["--model", "mock-model"]) - .arg("Run the deterministic command, then return the deterministic marker.") - .env("HOME", &home) - .env("CLAUDE_CONFIG_DIR", &claude_config) - .env("ANTHROPIC_BASE_URL", inference_server.uri()) - .env("ANTHROPIC_API_KEY", "test-key") - .env("ANTHROPIC_AUTH_TOKEN", "test-key") - .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") - .env("ANTHROPIC_DEFAULT_SONNET_MODEL", "mock-model") - .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model"); - let output = world.output(&mut claude).await; - assert!(output.status.success(), "{}", output_text(&output)); - assert!( - output_text(&output).contains("CLAUDE_MOCK_OK"), - "{}", - output_text(&output) - ); + let claude = ClaudeAgent::new(&world); + + let output = claude + .run(ClaudeRun::mock( + "Run the deterministic command, then return the deterministic marker.", + inference_server.uri(), + )) + .await; + output.assert_success(); + output.assert_contains("CLAUDE_MOCK_OK"); assert_eq!(inference.requests().len(), 2); - let mut failing_claude = claude_command(&world); - failing_claude - .args(["--model", "mock-model"]) - .arg("Trigger the deterministic inference error.") - .env("HOME", &home) - .env("CLAUDE_CONFIG_DIR", &claude_config) - .env("ANTHROPIC_BASE_URL", inference_server.uri()) - .env("ANTHROPIC_API_KEY", "test-key") - .env("ANTHROPIC_AUTH_TOKEN", "test-key") - .env("ANTHROPIC_DEFAULT_OPUS_MODEL", "mock-model") - .env("ANTHROPIC_DEFAULT_SONNET_MODEL", "mock-model") - .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", "mock-model"); - let failed = world.output(&mut failing_claude).await; - assert!(!failed.status.success(), "{}", output_text(&failed)); - assert!( - output_text(&failed).contains("deterministic Claude inference failure"), - "{}", - output_text(&failed) - ); + let failed = claude + .run(ClaudeRun::mock( + "Trigger the deterministic inference error.", + inference_server.uri(), + )) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Claude inference failure"); assert_eq!(inference.requests().len(), 3); let scenario = IngestScenario::new() @@ -410,32 +242,38 @@ async fn run_claude_mock() { .expect("Claude tool output", |row| { row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) }); - let rows = world.wait_for_ingest_scenario(&scenario).await; - assert!(!rows.is_empty()); + assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); } async fn run_claude_live() { let world = AgentTestWorld::start().await; - let mut claude = claude_command(&world); - claude.arg("Reply briefly to confirm this tracing integration test."); - let output = world.output(&mut claude).await; - assert!(output.status.success(), "{}", output_text(&output)); + let claude = ClaudeAgent::new(&world); + + claude + .run(ClaudeRun::live( + "Reply briefly to confirm this tracing integration test.", + )) + .await + .assert_success(); let scenario = IngestScenario::new().expect("Claude trace source", |row| { row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) }); - let rows = world.wait_for_ingest_scenario(&scenario).await; - assert!(!rows.is_empty()); + assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); } #[test] -fn request_helpers_recognize_tool_results() { - let openai = support::inference::OpenAiRequest { - body: json!({"input":[{"type":"function_call_output","call_id":"call-1"}]}), +fn request_helpers_recognize_tool_results_and_advertised_tools() { + let openai = OpenAiRequest { + body: json!({ + "input":[{"type":"function_call_output","call_id":"call-1"}], + "tools":[{"type":"function","name":"shell"}] + }), }; assert!(openai.has_function_output("call-1")); + assert_eq!(openai.tool_names(), vec!["shell"]); - let anthropic = support::inference::AnthropicRequest { + let anthropic = AnthropicRequest { body: json!({ "messages":[{ "content":[{"type":"tool_result","tool_use_id":"toolu-1"}] @@ -446,8 +284,6 @@ fn request_helpers_recognize_tool_results() { } #[test] -fn test_mode_defaults_to_mock_and_accepts_documented_values() { - // Parsing itself is intentionally tiny and covered indirectly by every - // agent integration run. Keep the enum shape explicit for future modes. +fn test_modes_remain_distinct() { assert_ne!(AgentTestMode::Mock, AgentTestMode::Live); } diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index e8943b0..33338cf 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -13,7 +13,17 @@ The test infrastructure has three independent layers: independent of HTTP batching and unrelated SDK update rows. `agent_process` is the Braintrust-specific orchestration layer. It hosts the -ingest router, starts the daemon, and configures coding-agent processes. The -integration test separately hosts an inference router when running in -deterministic mode. This keeps both protocol mocks usable without the coding -agent runner and keeps the generic server unaware of either protocol. +ingest router, starts the daemon, and provides the environment shared by agent +processes. + +`agents` contains reusable adapters for real coding-agent CLIs. Each adapter +owns agent installation, isolated configuration, standard invocation flags, +mock-inference routing, and process output. Runs remain configurable with +additional arguments and environment variables so scenarios can add inputs +such as attachment paths without duplicating CLI setup. + +The integration test composes those pieces: it hosts an inference router, +starts the daemon world, runs an agent, and evaluates the ingest scenario. This +keeps both protocol mocks usable without coding agents, keeps the generic +server unaware of either protocol, and lets new end-to-end scenarios focus on +model behavior and expected trace shapes. diff --git a/bt-daemon/tests/support/agents/claude.rs b/bt-daemon/tests/support/agents/claude.rs new file mode 100644 index 0000000..e369a5c --- /dev/null +++ b/bt-daemon/tests/support/agents/claude.rs @@ -0,0 +1,108 @@ +use super::{command_from_env, repository_root, AgentOutput, ProcessOptions}; +use crate::support::agent_process::AgentTestWorld; +use std::ffi::OsString; +use std::path::PathBuf; +use uuid::Uuid; + +pub struct ClaudeAgent<'a> { + world: &'a AgentTestWorld, + isolated_home: PathBuf, + isolated_config: PathBuf, +} + +pub struct ClaudeRun { + prompt: OsString, + inference: Option, + options: ProcessOptions, +} + +struct ClaudeInference { + base_url: String, + model: String, + api_key: String, +} + +impl ClaudeRun { + pub fn live(prompt: impl Into) -> Self { + Self { + prompt: prompt.into(), + inference: None, + options: ProcessOptions::default(), + } + } + + pub fn mock(prompt: impl Into, base_url: impl Into) -> Self { + Self { + prompt: prompt.into(), + inference: Some(ClaudeInference { + base_url: base_url.into(), + model: "mock-model".into(), + api_key: "test-key".into(), + }), + options: ProcessOptions::default(), + } + } + + pub fn arg(mut self, value: impl Into) -> Self { + self.options.arg(value); + self + } + + pub fn env(mut self, key: impl Into, value: impl Into) -> Self { + self.options.env(key, value); + self + } +} + +impl<'a> ClaudeAgent<'a> { + pub fn new(world: &'a AgentTestWorld) -> Self { + let isolated_home = world.temp_path("claude-home"); + let isolated_config = world.temp_path("claude-config"); + std::fs::create_dir_all(&isolated_home).expect("create Claude home"); + std::fs::create_dir_all(&isolated_config).expect("create Claude config"); + Self { + world, + isolated_home, + isolated_config, + } + } + + pub async fn run(&self, run: ClaudeRun) -> AgentOutput { + let session_id = Uuid::new_v4().to_string(); + let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); + let mut command = command_from_env("CLAUDE_BIN", "claude"); + command + .args([ + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + "--session-id", + &session_id, + "--plugin-dir", + ]) + .arg(plugin) + .current_dir(self.world.workspace()) + .env("ANTHROPIC_MAX_RETRIES", "0") + .env("DISABLE_AUTOUPDATER", "1") + .env("DISABLE_TELEMETRY", "1") + .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); + self.world.configure(&mut command); + + if let Some(inference) = &run.inference { + command + .args(["--model", &inference.model]) + .env("HOME", &self.isolated_home) + .env("CLAUDE_CONFIG_DIR", &self.isolated_config) + .env("ANTHROPIC_BASE_URL", &inference.base_url) + .env("ANTHROPIC_API_KEY", &inference.api_key) + .env("ANTHROPIC_AUTH_TOKEN", &inference.api_key) + .env("ANTHROPIC_DEFAULT_OPUS_MODEL", &inference.model) + .env("ANTHROPIC_DEFAULT_SONNET_MODEL", &inference.model) + .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", &inference.model); + } + run.options.apply(&mut command); + command.arg(run.prompt); + self.world.output(&mut command).await.into() + } +} diff --git a/bt-daemon/tests/support/agents/codex.rs b/bt-daemon/tests/support/agents/codex.rs new file mode 100644 index 0000000..41837ce --- /dev/null +++ b/bt-daemon/tests/support/agents/codex.rs @@ -0,0 +1,146 @@ +use super::{command_from_env, configured_home, repository_root, AgentOutput, ProcessOptions}; +use crate::support::agent_process::AgentTestWorld; +use std::ffi::OsString; +use std::path::PathBuf; +use tokio::process::Command; + +const TEST_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; + +pub struct CodexAgent<'a> { + world: &'a AgentTestWorld, + home: PathBuf, +} + +pub struct CodexRun { + prompt: OsString, + inference: Option, + options: ProcessOptions, +} + +struct CodexInference { + base_url: String, + model: String, + api_key: String, +} + +impl CodexRun { + pub fn live(prompt: impl Into) -> Self { + Self { + prompt: prompt.into(), + inference: None, + options: ProcessOptions::default(), + } + } + + pub fn mock(prompt: impl Into, base_url: impl Into) -> Self { + Self { + prompt: prompt.into(), + inference: Some(CodexInference { + base_url: base_url.into(), + model: "mock-model".into(), + api_key: "test-key".into(), + }), + options: ProcessOptions::default(), + } + } + + pub fn arg(mut self, value: impl Into) -> Self { + self.options.arg(value); + self + } + + pub fn env(mut self, key: impl Into, value: impl Into) -> Self { + self.options.env(key, value); + self + } +} + +impl<'a> CodexAgent<'a> { + pub async fn install(world: &'a AgentTestWorld) -> Self { + let home = world.temp_path("codex-home"); + std::fs::create_dir_all(&home).expect("create Codex home"); + + let marketplace = repository_root().join("src/plugins/codex/content"); + let mut add_marketplace = command_from_env("CODEX_BIN", "codex"); + add_marketplace + .arg("plugin") + .arg("marketplace") + .arg("add") + .arg(&marketplace) + .env("CODEX_HOME", &home); + world.configure(&mut add_marketplace); + AgentOutput::from(world.output(&mut add_marketplace).await).assert_success(); + + let mut add_plugin = command_from_env("CODEX_BIN", "codex"); + add_plugin + .args(["plugin", "add", "trace-codex@braintrust-codex-plugins"]) + .env("CODEX_HOME", &home); + world.configure(&mut add_plugin); + AgentOutput::from(world.output(&mut add_plugin).await).assert_success(); + + Self { world, home } + } + + pub fn seed_live_auth(&self) { + if std::env::var_os("OPENAI_API_KEY").is_some() { + return; + } + let source = configured_home("CODEX_HOME", ".codex") + .map(|home| home.join("auth.json")) + .filter(|path| path.is_file()) + .unwrap_or_else(|| { + panic!( + "{TEST_MODE_ENV}=live requires OPENAI_API_KEY or auth.json in the configured Codex home" + ) + }); + std::fs::copy(source, self.home.join("auth.json")).expect("copy Codex live credentials"); + } + + pub async fn run(&self, run: CodexRun) -> AgentOutput { + let mut command = self.command(); + if let Some(inference) = &run.inference { + configure_mock_inference(&mut command, inference); + } + run.options.apply(&mut command); + command.arg(run.prompt); + self.world.output(&mut command).await.into() + } + + fn command(&self) -> Command { + let mut command = command_from_env("CODEX_BIN", "codex"); + command + .args([ + "exec", + "--skip-git-repo-check", + "--dangerously-bypass-hook-trust", + "--sandbox", + "read-only", + "-c", + r#"approval_policy="never""#, + ]) + .current_dir(self.world.workspace()) + .env("CODEX_HOME", &self.home); + self.world.configure(&mut command); + command + } +} + +fn configure_mock_inference(command: &mut Command, inference: &CodexInference) { + let provider = format!( + r#"model_providers.mock={{name="Mock",base_url="{}/v1",wire_api="responses",env_key="MOCK_API_KEY",request_max_retries=0,stream_max_retries=0,stream_idle_timeout_ms=5000}}"#, + inference.base_url + ); + let chatgpt_base_url = format!(r#"chatgpt_base_url="{}/backend-api""#, inference.base_url); + command + .args([ + "-c", + &format!(r#"model="{}""#, inference.model), + "-c", + r#"model_provider="mock""#, + "-c", + &provider, + "-c", + &chatgpt_base_url, + ]) + .env("MOCK_API_KEY", &inference.api_key); +} diff --git a/bt-daemon/tests/support/agents/mod.rs b/bt-daemon/tests/support/agents/mod.rs new file mode 100644 index 0000000..9a83fdb --- /dev/null +++ b/bt-daemon/tests/support/agents/mod.rs @@ -0,0 +1,98 @@ +mod claude; +mod codex; + +#[allow(unused_imports)] +pub use claude::{ClaudeAgent, ClaudeRun}; +#[allow(unused_imports)] +pub use codex::{CodexAgent, CodexRun}; + +use std::ffi::OsString; +use std::path::PathBuf; +use tokio::process::Command; + +pub struct AgentOutput { + output: std::process::Output, +} + +impl AgentOutput { + pub fn success(&self) -> bool { + self.output.status.success() + } + + pub fn text(&self) -> String { + format!( + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&self.output.stdout), + String::from_utf8_lossy(&self.output.stderr) + ) + } + + pub fn assert_success(&self) { + assert!(self.success(), "{}", self.text()); + } + + pub fn assert_failure(&self) { + assert!(!self.success(), "{}", self.text()); + } + + pub fn assert_contains(&self, expected: &str) { + assert!( + self.text().contains(expected), + "agent output did not contain {expected:?}:\n{}", + self.text() + ); + } +} + +impl From for AgentOutput { + fn from(output: std::process::Output) -> Self { + Self { output } + } +} + +#[derive(Default)] +struct ProcessOptions { + args: Vec, + env: Vec<(OsString, OsString)>, +} + +impl ProcessOptions { + fn arg(&mut self, value: impl Into) { + self.args.push(value.into()); + } + + fn env(&mut self, key: impl Into, value: impl Into) { + self.env.push((key.into(), value.into())); + } + + fn apply(&self, command: &mut Command) { + command + .args(&self.args) + .envs(self.env.iter().map(|(k, v)| (k, v))); + } +} + +fn command_from_env(name: &str, fallback: &str) -> Command { + if let Some(command) = std::env::var_os(name) { + return Command::new(command); + } + #[cfg(windows)] + let fallback = format!("{fallback}.cmd"); + Command::new(fallback) +} + +fn repository_root() -> PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("repository root") + .to_path_buf() +} + +fn configured_home(config_env: &str, directory: &str) -> Option { + std::env::var_os(config_env).map(PathBuf::from).or_else(|| { + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .map(PathBuf::from) + .map(|home| home.join(directory)) + }) +} diff --git a/bt-daemon/tests/support/inference/openai.rs b/bt-daemon/tests/support/inference/openai.rs index 96edf0e..92e3ec4 100644 --- a/bt-daemon/tests/support/inference/openai.rs +++ b/bt-daemon/tests/support/inference/openai.rs @@ -29,6 +29,15 @@ impl OpenAiRequest { .any(|item| item["type"] == "function_call_output" && item["call_id"] == call_id) }) } + + pub fn tool_names(&self) -> Vec<&str> { + self.body["tools"] + .as_array() + .into_iter() + .flatten() + .filter_map(|tool| tool["name"].as_str()) + .collect() + } } #[derive(Debug, Clone)] diff --git a/bt-daemon/tests/support/mod.rs b/bt-daemon/tests/support/mod.rs index 22b646b..12d793e 100644 --- a/bt-daemon/tests/support/mod.rs +++ b/bt-daemon/tests/support/mod.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] pub mod agent_process; +pub mod agents; pub mod inference; pub mod ingest; pub mod server; From 1406ebae7e0d8c9c51fd8acb4867b38eeeb3d1f8 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:34:27 +0800 Subject: [PATCH 10/17] Support Codex shell command tool in tests Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 99aada7..7d848fb 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -46,6 +46,13 @@ fn codex_tool_call(request: &OpenAiRequest) -> OpenAiTurn { json!({"command":codex_tool_command()}), ); } + if names.contains(&"shell_command") { + return OpenAiTurn::tool_call( + "call_mock_1", + "shell_command", + json!({"command":codex_tool_command()}), + ); + } panic!("Codex offered no supported shell tool; offered tools: {names:?}"); } @@ -267,11 +274,20 @@ fn request_helpers_recognize_tool_results_and_advertised_tools() { let openai = OpenAiRequest { body: json!({ "input":[{"type":"function_call_output","call_id":"call-1"}], - "tools":[{"type":"function","name":"shell"}] + "tools":[{"type":"function","name":"shell_command"}] }), }; assert!(openai.has_function_output("call-1")); - assert_eq!(openai.tool_names(), vec!["shell"]); + assert_eq!(openai.tool_names(), vec!["shell_command"]); + match codex_tool_call(&openai) { + OpenAiTurn::ToolCall { + name, arguments, .. + } => { + assert_eq!(name, "shell_command"); + assert!(arguments["command"].is_string()); + } + _ => panic!("expected a Codex tool call"), + } let anthropic = AnthropicRequest { body: json!({ From cd67940a6096374a7779530e00de191093f88cca Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:40:53 +0800 Subject: [PATCH 11/17] Return from Windows bt hook preflight Signed-off-by: Stephen Belanger --- .../content/plugins/trace-claude-code/bin/claude-hook.cmd | 2 +- src/plugins/claude/validate.sh | 3 +++ src/plugins/codex/content/plugins/trace-codex/Makefile | 1 + .../codex/content/plugins/trace-codex/bin/codex-hook.cmd | 2 +- src/plugins/codex/validate.sh | 3 +++ 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd b/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd index bb9ebb8..cb1b13f 100644 --- a/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd +++ b/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd @@ -13,7 +13,7 @@ if not defined BT_HOOK_BIN ( exit /b 0 ) -"%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 +call "%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 if errorlevel 1 ( echo trace-claude-code: a daemon-capable bt CLI is unavailable; tracing disabled for this event.>&2 exit /b 0 diff --git a/src/plugins/claude/validate.sh b/src/plugins/claude/validate.sh index 8a4b4e5..c83eca6 100755 --- a/src/plugins/claude/validate.sh +++ b/src/plugins/claude/validate.sh @@ -46,6 +46,9 @@ done grep -q "'daemon','hook','--source','claude-code'" \ "$TARGET_DIR/plugins/trace-claude-code/bin/claude-hook.cmd" \ || fail "Claude Windows hook does not invoke bt daemon" +grep -Fq 'call "%BT_HOOK_BIN%" daemon hook --help' \ + "$TARGET_DIR/plugins/trace-claude-code/bin/claude-hook.cmd" \ + || fail "Claude Windows hook does not return from bt.cmd compatibility check" grep -q 'daemon hook --source claude-code' \ "$TARGET_DIR/plugins/trace-claude-code/bin/claude-hook.sh" \ || fail "Claude Unix hook does not invoke bt daemon" diff --git a/src/plugins/codex/content/plugins/trace-codex/Makefile b/src/plugins/codex/content/plugins/trace-codex/Makefile index 047b716..f808099 100644 --- a/src/plugins/codex/content/plugins/trace-codex/Makefile +++ b/src/plugins/codex/content/plugins/trace-codex/Makefile @@ -4,5 +4,6 @@ test: @sh -n bin/codex-hook.sh @grep -q "'daemon','hook','--source','codex'" bin/codex-hook.cmd + @grep -Fq 'call "%BT_HOOK_BIN%" daemon hook --help' bin/codex-hook.cmd @jq empty hooks/hooks.json .codex-plugin/plugin.json @echo "trace-codex shim OK" diff --git a/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd b/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd index ff544e1..2996eef 100644 --- a/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd +++ b/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd @@ -13,7 +13,7 @@ if not defined BT_HOOK_BIN ( exit /b 0 ) -"%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 +call "%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 if errorlevel 1 ( echo trace-codex: a daemon-capable bt CLI is unavailable; tracing disabled for this event.>&2 exit /b 0 diff --git a/src/plugins/codex/validate.sh b/src/plugins/codex/validate.sh index 577dceb..e2ef355 100755 --- a/src/plugins/codex/validate.sh +++ b/src/plugins/codex/validate.sh @@ -46,6 +46,9 @@ done grep -q "'daemon','hook','--source','codex'" \ "$TARGET_DIR/plugins/trace-codex/bin/codex-hook.cmd" \ || fail "Codex Windows hook does not invoke bt daemon" +grep -Fq 'call "%BT_HOOK_BIN%" daemon hook --help' \ + "$TARGET_DIR/plugins/trace-codex/bin/codex-hook.cmd" \ + || fail "Codex Windows hook does not return from bt.cmd compatibility check" grep -q 'daemon hook --source codex' \ "$TARGET_DIR/plugins/trace-codex/bin/codex-hook.sh" \ || fail "Codex Unix hook does not invoke bt daemon" From 99a13aff54e2e1d7864db6409df5094f58ab4989 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 29 Jul 2026 23:54:52 +0800 Subject: [PATCH 12/17] Decouple agent adapters from test world Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 58 +++++++++++++++--------- bt-daemon/tests/support/README.md | 10 ++-- bt-daemon/tests/support/agents/claude.rs | 16 +++---- bt-daemon/tests/support/agents/codex.rs | 21 ++++----- 4 files changed, 59 insertions(+), 46 deletions(-) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 7d848fb..ef12efe 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -124,20 +124,26 @@ async fn run_codex_mock() { let codex = CodexAgent::install(&world).await; let output = codex - .run(CodexRun::mock( - "Run the deterministic command, then return the deterministic marker.", - inference_server.uri(), - )) + .run( + &world, + CodexRun::mock( + "Run the deterministic command, then return the deterministic marker.", + inference_server.uri(), + ), + ) .await; output.assert_success(); output.assert_contains("CODEX_MOCK_OK"); assert_eq!(inference.requests().len(), 2); let failed = codex - .run(CodexRun::mock( - "Trigger the deterministic inference error.", - inference_server.uri(), - )) + .run( + &world, + CodexRun::mock( + "Trigger the deterministic inference error.", + inference_server.uri(), + ), + ) .await; failed.assert_failure(); failed.assert_contains("deterministic Codex inference failure"); @@ -159,9 +165,10 @@ async fn run_codex_live() { codex.seed_live_auth(); codex - .run(CodexRun::live( - "Reply briefly to confirm this tracing integration test.", - )) + .run( + &world, + CodexRun::live("Reply briefly to confirm this tracing integration test."), + ) .await .assert_success(); @@ -223,20 +230,26 @@ async fn run_claude_mock() { let claude = ClaudeAgent::new(&world); let output = claude - .run(ClaudeRun::mock( - "Run the deterministic command, then return the deterministic marker.", - inference_server.uri(), - )) + .run( + &world, + ClaudeRun::mock( + "Run the deterministic command, then return the deterministic marker.", + inference_server.uri(), + ), + ) .await; output.assert_success(); output.assert_contains("CLAUDE_MOCK_OK"); assert_eq!(inference.requests().len(), 2); let failed = claude - .run(ClaudeRun::mock( - "Trigger the deterministic inference error.", - inference_server.uri(), - )) + .run( + &world, + ClaudeRun::mock( + "Trigger the deterministic inference error.", + inference_server.uri(), + ), + ) .await; failed.assert_failure(); failed.assert_contains("deterministic Claude inference failure"); @@ -257,9 +270,10 @@ async fn run_claude_live() { let claude = ClaudeAgent::new(&world); claude - .run(ClaudeRun::live( - "Reply briefly to confirm this tracing integration test.", - )) + .run( + &world, + ClaudeRun::live("Reply briefly to confirm this tracing integration test."), + ) .await .assert_success(); diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index 33338cf..7d1ccc3 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -17,10 +17,12 @@ ingest router, starts the daemon, and provides the environment shared by agent processes. `agents` contains reusable adapters for real coding-agent CLIs. Each adapter -owns agent installation, isolated configuration, standard invocation flags, -mock-inference routing, and process output. Runs remain configurable with -additional arguments and environment variables so scenarios can add inputs -such as attachment paths without duplicating CLI setup. +owns only agent installation and isolated configuration state. The daemon +world is passed to each run as its execution context, avoiding any lifetime or +ownership coupling between the two layers. Adapters provide standard +invocation flags, mock-inference routing, and process output. Runs remain +configurable with additional arguments and environment variables so scenarios +can add inputs such as attachment paths without duplicating CLI setup. The integration test composes those pieces: it hosts an inference router, starts the daemon world, runs an agent, and evaluates the ingest scenario. This diff --git a/bt-daemon/tests/support/agents/claude.rs b/bt-daemon/tests/support/agents/claude.rs index e369a5c..c395691 100644 --- a/bt-daemon/tests/support/agents/claude.rs +++ b/bt-daemon/tests/support/agents/claude.rs @@ -4,8 +4,7 @@ use std::ffi::OsString; use std::path::PathBuf; use uuid::Uuid; -pub struct ClaudeAgent<'a> { - world: &'a AgentTestWorld, +pub struct ClaudeAgent { isolated_home: PathBuf, isolated_config: PathBuf, } @@ -54,20 +53,19 @@ impl ClaudeRun { } } -impl<'a> ClaudeAgent<'a> { - pub fn new(world: &'a AgentTestWorld) -> Self { +impl ClaudeAgent { + pub fn new(world: &AgentTestWorld) -> Self { let isolated_home = world.temp_path("claude-home"); let isolated_config = world.temp_path("claude-config"); std::fs::create_dir_all(&isolated_home).expect("create Claude home"); std::fs::create_dir_all(&isolated_config).expect("create Claude config"); Self { - world, isolated_home, isolated_config, } } - pub async fn run(&self, run: ClaudeRun) -> AgentOutput { + pub async fn run(&self, world: &AgentTestWorld, run: ClaudeRun) -> AgentOutput { let session_id = Uuid::new_v4().to_string(); let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); let mut command = command_from_env("CLAUDE_BIN", "claude"); @@ -82,12 +80,12 @@ impl<'a> ClaudeAgent<'a> { "--plugin-dir", ]) .arg(plugin) - .current_dir(self.world.workspace()) + .current_dir(world.workspace()) .env("ANTHROPIC_MAX_RETRIES", "0") .env("DISABLE_AUTOUPDATER", "1") .env("DISABLE_TELEMETRY", "1") .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); - self.world.configure(&mut command); + world.configure(&mut command); if let Some(inference) = &run.inference { command @@ -103,6 +101,6 @@ impl<'a> ClaudeAgent<'a> { } run.options.apply(&mut command); command.arg(run.prompt); - self.world.output(&mut command).await.into() + world.output(&mut command).await.into() } } diff --git a/bt-daemon/tests/support/agents/codex.rs b/bt-daemon/tests/support/agents/codex.rs index 41837ce..da0a9ae 100644 --- a/bt-daemon/tests/support/agents/codex.rs +++ b/bt-daemon/tests/support/agents/codex.rs @@ -6,8 +6,7 @@ use tokio::process::Command; const TEST_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; -pub struct CodexAgent<'a> { - world: &'a AgentTestWorld, +pub struct CodexAgent { home: PathBuf, } @@ -55,8 +54,8 @@ impl CodexRun { } } -impl<'a> CodexAgent<'a> { - pub async fn install(world: &'a AgentTestWorld) -> Self { +impl CodexAgent { + pub async fn install(world: &AgentTestWorld) -> Self { let home = world.temp_path("codex-home"); std::fs::create_dir_all(&home).expect("create Codex home"); @@ -78,7 +77,7 @@ impl<'a> CodexAgent<'a> { world.configure(&mut add_plugin); AgentOutput::from(world.output(&mut add_plugin).await).assert_success(); - Self { world, home } + Self { home } } pub fn seed_live_auth(&self) { @@ -96,17 +95,17 @@ impl<'a> CodexAgent<'a> { std::fs::copy(source, self.home.join("auth.json")).expect("copy Codex live credentials"); } - pub async fn run(&self, run: CodexRun) -> AgentOutput { - let mut command = self.command(); + pub async fn run(&self, world: &AgentTestWorld, run: CodexRun) -> AgentOutput { + let mut command = self.command(world); if let Some(inference) = &run.inference { configure_mock_inference(&mut command, inference); } run.options.apply(&mut command); command.arg(run.prompt); - self.world.output(&mut command).await.into() + world.output(&mut command).await.into() } - fn command(&self) -> Command { + fn command(&self, world: &AgentTestWorld) -> Command { let mut command = command_from_env("CODEX_BIN", "codex"); command .args([ @@ -118,9 +117,9 @@ impl<'a> CodexAgent<'a> { "-c", r#"approval_policy="never""#, ]) - .current_dir(self.world.workspace()) + .current_dir(world.workspace()) .env("CODEX_HOME", &self.home); - self.world.configure(&mut command); + world.configure(&mut command); command } } From 33b9a3953bef7983b920f08aeb0fc657112280fa Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 30 Jul 2026 00:09:09 +0800 Subject: [PATCH 13/17] Unify live and deterministic agent scenarios Signed-off-by: Stephen Belanger --- .github/workflows/ci.yml | 3 +- bt-daemon/tests/agent_integration.rs | 169 ++++++-------------- bt-daemon/tests/support/README.md | 17 ++ bt-daemon/tests/support/agent_process.rs | 131 ++++++++++++++- bt-daemon/tests/support/agents/claude.rs | 29 ++-- bt-daemon/tests/support/agents/codex.rs | 39 +++-- bt-daemon/tests/support/inference/README.md | 24 ++- 7 files changed, 247 insertions(+), 165 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a03163d..b99cd73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,8 @@ jobs: run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked - name: Test coding-agent integrations with deterministic inference env: - BT_AGENT_TEST_MODE: mock + BT_AGENT_INFERENCE_MODE: mock + BT_AGENT_INGEST_MODE: mock CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }} CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }} run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_integration -- --ignored --nocapture --test-threads=1 diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index ef12efe..ce2ca31 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -11,25 +11,6 @@ use support::inference::{ use support::ingest::IngestScenario; use support::server::TestServer; -const TEST_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum AgentTestMode { - Mock, - Live, -} - -impl AgentTestMode { - fn from_env() -> Self { - match std::env::var(TEST_MODE_ENV).as_deref() { - Ok("live") => Self::Live, - Ok("mock" | "deterministic") | Err(std::env::VarError::NotPresent) => Self::Mock, - Ok(value) => panic!("{TEST_MODE_ENV} must be `mock` or `live`, got {value:?}"), - Err(error) => panic!("could not read {TEST_MODE_ENV}: {error}"), - } - } -} - fn codex_tool_call(request: &OpenAiRequest) -> OpenAiTurn { let names = request.tool_names(); if names.contains(&"exec_command") { @@ -77,19 +58,12 @@ fn row_contains(row: &Value, fragments: &[&str]) -> bool { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "requires the Codex CLI installed on PATH"] async fn codex_session_emits_traces() { - match AgentTestMode::from_env() { - AgentTestMode::Mock => run_codex_mock().await, - AgentTestMode::Live => run_codex_live().await, - } -} - -async fn run_codex_mock() { let inference = OpenAiMock::new(|context, request| { assert_eq!(request.model(), Some("mock-model")); match context.request_index { 0 => { assert!( - request.contains_text("Run the deterministic command"), + request.contains_text("CODEX_TOOL_OK"), "unexpected Codex request: {}", request.body ); @@ -126,73 +100,48 @@ async fn run_codex_mock() { let output = codex .run( &world, - CodexRun::mock( - "Run the deterministic command, then return the deterministic marker.", - inference_server.uri(), - ), + CodexRun::new("Run the command `printf CODEX_TOOL_OK` and then reply briefly.") + .mock_inference(inference_server.uri()), ) .await; output.assert_success(); - output.assert_contains("CODEX_MOCK_OK"); - assert_eq!(inference.requests().len(), 2); - - let failed = codex - .run( - &world, - CodexRun::mock( - "Trigger the deterministic inference error.", - inference_server.uri(), - ), - ) - .await; - failed.assert_failure(); - failed.assert_contains("deterministic Codex inference failure"); - assert_eq!(inference.requests().len(), 3); + world.when_mock_inference(|| { + output.assert_contains("CODEX_MOCK_OK"); + assert_eq!(inference.requests().len(), 2); + }); - let scenario = IngestScenario::new() - .expect("Codex trace origin", |row| { - row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + world + .when_mock_inference_async(|| async { + let failed = codex + .run( + &world, + CodexRun::new("Trigger the deterministic inference error.") + .mock_inference(inference_server.uri()), + ) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Codex inference failure"); + assert_eq!(inference.requests().len(), 3); }) - .expect("Codex tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) - }); - assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); -} - -async fn run_codex_live() { - let world = AgentTestWorld::start().await; - let codex = CodexAgent::install(&world).await; - codex.seed_live_auth(); - - codex - .run( - &world, - CodexRun::live("Reply briefly to confirm this tracing integration test."), - ) - .await - .assert_success(); + .await; let scenario = IngestScenario::new().expect("Codex trace origin", |row| { row_contains(row, &["braintrust.plugin.codex", "test_harness"]) }); - assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); + let scenario = world.expect_mock_inference(scenario, "Codex tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) + }); + world.wait_for_ingest_scenario(&scenario).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "requires the Claude Code CLI installed on PATH"] async fn claude_session_emits_traces() { - match AgentTestMode::from_env() { - AgentTestMode::Mock => run_claude_mock().await, - AgentTestMode::Live => run_claude_live().await, - } -} - -async fn run_claude_mock() { let inference = AnthropicMock::new(|context, request| match context.request_index { 0 => { assert_eq!(request.model(), Some("mock-model")); assert!( - request.contains_text("Run the deterministic command"), + request.contains_text("CLAUDE_TOOL_OK"), "unexpected Claude request: {}", request.body ); @@ -232,55 +181,38 @@ async fn run_claude_mock() { let output = claude .run( &world, - ClaudeRun::mock( - "Run the deterministic command, then return the deterministic marker.", - inference_server.uri(), - ), + ClaudeRun::new("Run the command `printf CLAUDE_TOOL_OK` and then reply briefly.") + .mock_inference(inference_server.uri()), ) .await; output.assert_success(); - output.assert_contains("CLAUDE_MOCK_OK"); - assert_eq!(inference.requests().len(), 2); - - let failed = claude - .run( - &world, - ClaudeRun::mock( - "Trigger the deterministic inference error.", - inference_server.uri(), - ), - ) - .await; - failed.assert_failure(); - failed.assert_contains("deterministic Claude inference failure"); - assert_eq!(inference.requests().len(), 3); + world.when_mock_inference(|| { + output.assert_contains("CLAUDE_MOCK_OK"); + assert_eq!(inference.requests().len(), 2); + }); - let scenario = IngestScenario::new() - .expect("Claude trace source", |row| { - row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + world + .when_mock_inference_async(|| async { + let failed = claude + .run( + &world, + ClaudeRun::new("Trigger the deterministic inference error.") + .mock_inference(inference_server.uri()), + ) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Claude inference failure"); + assert_eq!(inference.requests().len(), 3); }) - .expect("Claude tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) - }); - assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); -} - -async fn run_claude_live() { - let world = AgentTestWorld::start().await; - let claude = ClaudeAgent::new(&world); - - claude - .run( - &world, - ClaudeRun::live("Reply briefly to confirm this tracing integration test."), - ) - .await - .assert_success(); + .await; let scenario = IngestScenario::new().expect("Claude trace source", |row| { row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) }); - assert!(!world.wait_for_ingest_scenario(&scenario).await.is_empty()); + let scenario = world.expect_mock_inference(scenario, "Claude tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) + }); + world.wait_for_ingest_scenario(&scenario).await; } #[test] @@ -312,8 +244,3 @@ fn request_helpers_recognize_tool_results_and_advertised_tools() { }; assert!(anthropic.has_tool_result("toolu-1")); } - -#[test] -fn test_modes_remain_distinct() { - assert_ne!(AgentTestMode::Mock, AgentTestMode::Live); -} diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index 7d1ccc3..9d66381 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -29,3 +29,20 @@ starts the daemon world, runs an agent, and evaluates the ingest scenario. This keeps both protocol mocks usable without coding agents, keeps the generic server unaware of either protocol, and lets new end-to-end scenarios focus on model behavior and expected trace shapes. + +The world controls inference and ingest independently: + +- `BT_AGENT_INFERENCE_MODE=mock|live` selects deterministic mock inference or + the agent's normal provider. +- `BT_AGENT_INGEST_MODE=mock|live` selects captured local ingest or the normal + Braintrust backend. +- `BT_AGENT_TEST_MODE=mock|live` remains a shorthand that supplies the default + for both settings when the more specific variable is absent. + +This allows deterministic inference to drive real Braintrust ingest without +paying for model inference. Tests always assert stable process behavior, +provider request and response details only when inference is mocked, and +captured row shapes only when ingest is mocked. Live ingest instead waits for +the daemon to report emitted spans and fails on sink errors. Mock inference can +add stricter row expectations to the same scenario without requiring a second +test body. diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index b2cdde4..54c020c 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -1,6 +1,8 @@ use crate::support::ingest::{IngestMock, IngestScenario}; use crate::support::server::TestServer; +use bt_daemon::{run_status, StatusArgs}; use serde_json::{json, Value}; +use std::future::Future; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; @@ -9,7 +11,34 @@ use tokio::process::{Child, Command}; #[cfg(windows)] use uuid::Uuid; +const LEGACY_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; +const INFERENCE_MODE_ENV: &str = "BT_AGENT_INFERENCE_MODE"; +const INGEST_MODE_ENV: &str = "BT_AGENT_INGEST_MODE"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TestBackendMode { + Mock, + Live, +} + +impl TestBackendMode { + fn from_env(name: &str) -> Self { + let value = std::env::var(name).or_else(|error| match error { + std::env::VarError::NotPresent => std::env::var(LEGACY_MODE_ENV), + _ => Err(error), + }); + match value.as_deref() { + Ok("live") => Self::Live, + Ok("mock" | "deterministic") | Err(std::env::VarError::NotPresent) => Self::Mock, + Ok(value) => panic!("{name} must be `mock` or `live`, got {value:?}"), + Err(error) => panic!("could not read {name}: {error}"), + } + } +} + pub struct AgentTestWorld { + inference_mode: TestBackendMode, + ingest_mode: TestBackendMode, root: TempDir, collector: IngestMock, collector_server: TestServer, @@ -22,6 +51,8 @@ pub struct AgentTestWorld { impl AgentTestWorld { pub async fn start() -> Self { + let inference_mode = TestBackendMode::from_env(INFERENCE_MODE_ENV); + let ingest_mode = TestBackendMode::from_env(INGEST_MODE_ENV); let root = tempfile::tempdir().expect("create agent test root"); let collector = IngestMock::new(); let collector_server = TestServer::start(collector.router()).await; @@ -55,16 +86,21 @@ impl AgentTestWorld { .arg(&data_dir) .arg("--idle-timeout-secs") .arg("0") - .env("BRAINTRUST_API_URL", collector_server.uri()) - .env("BRAINTRUST_APP_URL", collector_server.uri()) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::piped()) .kill_on_drop(true); + if ingest_mode == TestBackendMode::Mock { + command + .env("BRAINTRUST_API_URL", collector_server.uri()) + .env("BRAINTRUST_APP_URL", collector_server.uri()); + } let daemon = command.spawn().expect("start daemon"); wait_for_daemon(daemon_binary, &socket).await; Self { + inference_mode, + ingest_mode, root, collector, collector_server, @@ -76,6 +112,47 @@ impl AgentTestWorld { } } + pub fn uses_mock_inference(&self) -> bool { + self.inference_mode == TestBackendMode::Mock + } + + pub fn uses_live_inference(&self) -> bool { + self.inference_mode == TestBackendMode::Live + } + + pub fn uses_mock_ingest(&self) -> bool { + self.ingest_mode == TestBackendMode::Mock + } + + pub fn when_mock_inference(&self, assertion: impl FnOnce()) { + if self.uses_mock_inference() { + assertion(); + } + } + + pub async fn when_mock_inference_async(&self, operation: F) + where + F: FnOnce() -> Fut, + Fut: Future, + { + if self.uses_mock_inference() { + operation().await; + } + } + + pub fn expect_mock_inference( + &self, + scenario: IngestScenario, + name: impl Into, + matcher: impl Fn(&Value) -> bool + Send + Sync + 'static, + ) -> IngestScenario { + if self.uses_mock_inference() { + scenario.expect(name, matcher) + } else { + scenario + } + } + pub fn workspace(&self) -> PathBuf { let workspace = self.root.path().join("workspace"); std::fs::create_dir_all(&workspace).expect("create agent workspace"); @@ -96,12 +173,15 @@ impl AgentTestWorld { .env("BT_DAEMON_SOCKET", &self.socket) .env("BT_DAEMON_DATA_DIR", &self.data_dir) .env("BT_DAEMON_CONFIG", &self.config_path) - .env("BRAINTRUST_API_KEY", "test-key") - .env("BRAINTRUST_API_URL", self.collector_server.uri()) - .env("BRAINTRUST_APP_URL", self.collector_server.uri()) - .env("BRAINTRUST_PROJECT", "agent-e2e") .env("BRAINTRUST_FLUSH_ON_TURN_END", "true") .stdin(Stdio::null()); + if self.uses_mock_ingest() { + command + .env("BRAINTRUST_API_KEY", "test-key") + .env("BRAINTRUST_API_URL", self.collector_server.uri()) + .env("BRAINTRUST_APP_URL", self.collector_server.uri()) + .env("BRAINTRUST_PROJECT", "agent-e2e"); + } } pub async fn output(&self, command: &mut Command) -> std::process::Output { @@ -140,6 +220,9 @@ impl AgentTestWorld { } pub async fn wait_for_ingest_scenario(&self, scenario: &IngestScenario) -> Vec { + if !self.uses_mock_ingest() { + return self.wait_for_live_ingest().await; + } let mut last_error = String::new(); for _ in 0..100 { match self.collector.evaluate(scenario) { @@ -154,6 +237,42 @@ impl AgentTestWorld { directory_contents(&self.data_dir) ); } + + async fn wait_for_live_ingest(&self) -> Vec { + let mut last_status = String::new(); + for _ in 0..100 { + match run_status(StatusArgs { + socket: Some(self.socket.clone()), + session_id: None, + }) + .await + { + Ok(Some(status)) => { + last_status = format!("{:?}", status.sessions); + let emitted = status + .sessions + .iter() + .any(|session| session.spans_emitted > 0); + let errors = status + .sessions + .iter() + .filter_map(|session| session.last_error.as_deref()) + .collect::>(); + assert!( + errors.is_empty(), + "live ingest reported daemon sink errors: {errors:?}" + ); + if emitted { + return Vec::new(); + } + } + Ok(None) => last_status = "daemon not running".into(), + Err(error) => last_status = error.to_string(), + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("live ingest emitted no spans; last daemon status: {last_status}"); + } } impl Drop for AgentTestWorld { diff --git a/bt-daemon/tests/support/agents/claude.rs b/bt-daemon/tests/support/agents/claude.rs index c395691..7fb32ec 100644 --- a/bt-daemon/tests/support/agents/claude.rs +++ b/bt-daemon/tests/support/agents/claude.rs @@ -11,7 +11,7 @@ pub struct ClaudeAgent { pub struct ClaudeRun { prompt: OsString, - inference: Option, + mock_inference: Option, options: ProcessOptions, } @@ -22,24 +22,21 @@ struct ClaudeInference { } impl ClaudeRun { - pub fn live(prompt: impl Into) -> Self { + pub fn new(prompt: impl Into) -> Self { Self { prompt: prompt.into(), - inference: None, + mock_inference: None, options: ProcessOptions::default(), } } - pub fn mock(prompt: impl Into, base_url: impl Into) -> Self { - Self { - prompt: prompt.into(), - inference: Some(ClaudeInference { - base_url: base_url.into(), - model: "mock-model".into(), - api_key: "test-key".into(), - }), - options: ProcessOptions::default(), - } + pub fn mock_inference(mut self, base_url: impl Into) -> Self { + self.mock_inference = Some(ClaudeInference { + base_url: base_url.into(), + model: "mock-model".into(), + api_key: "test-key".into(), + }); + self } pub fn arg(mut self, value: impl Into) -> Self { @@ -87,7 +84,11 @@ impl ClaudeAgent { .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); world.configure(&mut command); - if let Some(inference) = &run.inference { + if world.uses_mock_inference() { + let inference = run + .mock_inference + .as_ref() + .expect("mock Claude runs require a mock inference endpoint"); command .args(["--model", &inference.model]) .env("HOME", &self.isolated_home) diff --git a/bt-daemon/tests/support/agents/codex.rs b/bt-daemon/tests/support/agents/codex.rs index da0a9ae..be33da0 100644 --- a/bt-daemon/tests/support/agents/codex.rs +++ b/bt-daemon/tests/support/agents/codex.rs @@ -4,7 +4,7 @@ use std::ffi::OsString; use std::path::PathBuf; use tokio::process::Command; -const TEST_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; +const INFERENCE_MODE_ENV: &str = "BT_AGENT_INFERENCE_MODE"; pub struct CodexAgent { home: PathBuf, @@ -12,7 +12,7 @@ pub struct CodexAgent { pub struct CodexRun { prompt: OsString, - inference: Option, + mock_inference: Option, options: ProcessOptions, } @@ -23,24 +23,21 @@ struct CodexInference { } impl CodexRun { - pub fn live(prompt: impl Into) -> Self { + pub fn new(prompt: impl Into) -> Self { Self { prompt: prompt.into(), - inference: None, + mock_inference: None, options: ProcessOptions::default(), } } - pub fn mock(prompt: impl Into, base_url: impl Into) -> Self { - Self { - prompt: prompt.into(), - inference: Some(CodexInference { - base_url: base_url.into(), - model: "mock-model".into(), - api_key: "test-key".into(), - }), - options: ProcessOptions::default(), - } + pub fn mock_inference(mut self, base_url: impl Into) -> Self { + self.mock_inference = Some(CodexInference { + base_url: base_url.into(), + model: "mock-model".into(), + api_key: "test-key".into(), + }); + self } pub fn arg(mut self, value: impl Into) -> Self { @@ -77,7 +74,11 @@ impl CodexAgent { world.configure(&mut add_plugin); AgentOutput::from(world.output(&mut add_plugin).await).assert_success(); - Self { home } + let agent = Self { home }; + if world.uses_live_inference() { + agent.seed_live_auth(); + } + agent } pub fn seed_live_auth(&self) { @@ -89,7 +90,7 @@ impl CodexAgent { .filter(|path| path.is_file()) .unwrap_or_else(|| { panic!( - "{TEST_MODE_ENV}=live requires OPENAI_API_KEY or auth.json in the configured Codex home" + "{INFERENCE_MODE_ENV}=live requires OPENAI_API_KEY or auth.json in the configured Codex home" ) }); std::fs::copy(source, self.home.join("auth.json")).expect("copy Codex live credentials"); @@ -97,7 +98,11 @@ impl CodexAgent { pub async fn run(&self, world: &AgentTestWorld, run: CodexRun) -> AgentOutput { let mut command = self.command(world); - if let Some(inference) = &run.inference { + if world.uses_mock_inference() { + let inference = run + .mock_inference + .as_ref() + .expect("mock Codex runs require a mock inference endpoint"); configure_mock_inference(&mut command, inference); } run.options.apply(&mut command); diff --git a/bt-daemon/tests/support/inference/README.md b/bt-daemon/tests/support/inference/README.md index da11de1..92567d3 100644 --- a/bt-daemon/tests/support/inference/README.md +++ b/bt-daemon/tests/support/inference/README.md @@ -53,14 +53,26 @@ each agent and runs them in the default `mock` mode on every host. This is intentionally unpinned so upstream compatibility breaks are visible immediately. -The same agent tests can run without mock inference: +The same agent tests can run without mock inference while continuing to use +captured local ingest: ```console -BT_AGENT_TEST_MODE=live cargo test --manifest-path bt-daemon/Cargo.toml \ +BT_AGENT_INFERENCE_MODE=live BT_AGENT_INGEST_MODE=mock \ + cargo test --manifest-path bt-daemon/Cargo.toml \ --all-features --test agent_integration -- --ignored --test-threads=1 ``` -Live mode uses the normal provider endpoint/model and the agent's normal login -or provider credentials. It validates only stable integration invariants such -as trace delivery and origin metadata. Mock mode additionally validates exact -request sequences, tool results, output content, and injected failures. +Live inference uses the normal provider endpoint/model and the agent's normal +login or provider credentials. It validates only stable integration invariants +such as trace delivery and origin metadata. Mock inference additionally +validates exact request sequences, tool results, output content, and injected +failures. + +Inference and ingest selection are independent. To drive deterministic model +behavior while reporting traces to the normal Braintrust backend: + +```console +BT_AGENT_INFERENCE_MODE=mock BT_AGENT_INGEST_MODE=live \ + cargo test --manifest-path bt-daemon/Cargo.toml \ + --all-features --test agent_integration -- --ignored --test-threads=1 +``` From 8d22f349e94395b460fcbd456b3a002d7f227457 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 30 Jul 2026 00:13:17 +0800 Subject: [PATCH 14/17] Remove unreleased test mode alias Signed-off-by: Stephen Belanger --- bt-daemon/tests/support/README.md | 2 -- bt-daemon/tests/support/agent_process.rs | 7 +------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index 9d66381..0979186 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -36,8 +36,6 @@ The world controls inference and ingest independently: the agent's normal provider. - `BT_AGENT_INGEST_MODE=mock|live` selects captured local ingest or the normal Braintrust backend. -- `BT_AGENT_TEST_MODE=mock|live` remains a shorthand that supplies the default - for both settings when the more specific variable is absent. This allows deterministic inference to drive real Braintrust ingest without paying for model inference. Tests always assert stable process behavior, diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index 54c020c..5c86f60 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -11,7 +11,6 @@ use tokio::process::{Child, Command}; #[cfg(windows)] use uuid::Uuid; -const LEGACY_MODE_ENV: &str = "BT_AGENT_TEST_MODE"; const INFERENCE_MODE_ENV: &str = "BT_AGENT_INFERENCE_MODE"; const INGEST_MODE_ENV: &str = "BT_AGENT_INGEST_MODE"; @@ -23,11 +22,7 @@ pub enum TestBackendMode { impl TestBackendMode { fn from_env(name: &str) -> Self { - let value = std::env::var(name).or_else(|error| match error { - std::env::VarError::NotPresent => std::env::var(LEGACY_MODE_ENV), - _ => Err(error), - }); - match value.as_deref() { + match std::env::var(name).as_deref() { Ok("live") => Self::Live, Ok("mock" | "deterministic") | Err(std::env::VarError::NotPresent) => Self::Mock, Ok(value) => panic!("{name} must be `mock` or `live`, got {value:?}"), From 0e63f2fe5ce282ce5d72d92c1b2856d040806285 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 30 Jul 2026 00:36:21 +0800 Subject: [PATCH 15/17] Layer deterministic trace expectations over baseline checks Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 26 +++++++++-------- bt-daemon/tests/ingest_mock.rs | 32 +++++++++++++++++++-- bt-daemon/tests/support/README.md | 17 +++++++---- bt-daemon/tests/support/agent_process.rs | 18 +++--------- bt-daemon/tests/support/ingest.rs | 36 ++++++++++++++++++++---- 5 files changed, 90 insertions(+), 39 deletions(-) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index ce2ca31..5323596 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -125,12 +125,13 @@ async fn codex_session_emits_traces() { }) .await; - let scenario = IngestScenario::new().expect("Codex trace origin", |row| { - row_contains(row, &["braintrust.plugin.codex", "test_harness"]) - }); - let scenario = world.expect_mock_inference(scenario, "Codex tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) - }); + let scenario = IngestScenario::new() + .expect("Codex trace origin", |row| { + row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + }) + .expect_strict("Codex tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) + }); world.wait_for_ingest_scenario(&scenario).await; } @@ -206,12 +207,13 @@ async fn claude_session_emits_traces() { }) .await; - let scenario = IngestScenario::new().expect("Claude trace source", |row| { - row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) - }); - let scenario = world.expect_mock_inference(scenario, "Claude tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) - }); + let scenario = IngestScenario::new() + .expect("Claude trace source", |row| { + row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + }) + .expect_strict("Claude tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) + }); world.wait_for_ingest_scenario(&scenario).await; } diff --git a/bt-daemon/tests/ingest_mock.rs b/bt-daemon/tests/ingest_mock.rs index 9d9430b..7e07f43 100644 --- a/bt-daemon/tests/ingest_mock.rs +++ b/bt-daemon/tests/ingest_mock.rs @@ -28,10 +28,38 @@ async fn ingest_router_captures_rows_and_matches_ordered_shapes() { .expect("tool result", |row| { row["span_attributes"]["type"] == "tool" && row["output"] == "deterministic" }); - assert_eq!(ingest.evaluate(&scenario).unwrap().len(), 3); + assert_eq!(ingest.evaluate(&scenario, true).unwrap().len(), 3); let reversed = IngestScenario::new() .expect("tool first", |row| row["span_attributes"]["type"] == "tool") .expect("task later", |row| row["span_attributes"]["type"] == "task"); - assert!(ingest.evaluate(&reversed).is_err()); + assert!(ingest.evaluate(&reversed, true).is_err()); +} + +#[tokio::test] +async fn strict_shapes_are_additive_to_always_active_baseline_shapes() { + let ingest = IngestMock::new(); + let server = TestServer::start(ingest.router()).await; + + reqwest::Client::new() + .post(format!("{}/logs3", server.uri())) + .json(&json!({ + "rows": [ + {"span_attributes":{"type":"task"},"metadata":{"source":"codex"}} + ] + })) + .send() + .await + .unwrap(); + + let scenario = IngestScenario::new() + .expect("baseline root task", |row| { + row["span_attributes"]["type"] == "task" + }) + .expect_strict("deterministic tool result", |row| { + row["span_attributes"]["type"] == "tool" + }); + + assert!(ingest.evaluate(&scenario, false).is_ok()); + assert!(ingest.evaluate(&scenario, true).is_err()); } diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index 0979186..a7e6437 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -38,9 +38,14 @@ The world controls inference and ingest independently: Braintrust backend. This allows deterministic inference to drive real Braintrust ingest without -paying for model inference. Tests always assert stable process behavior, -provider request and response details only when inference is mocked, and -captured row shapes only when ingest is mocked. Live ingest instead waits for -the daemon to report emitted spans and fails on sink errors. Mock inference can -add stricter row expectations to the same scenario without requiring a second -test body. +paying for model inference. Every test always asserts stable process behavior +and successful trace delivery. With mock ingest, `IngestScenario::expect` +declares baseline row shapes that are evaluated with both live and mock +inference. `IngestScenario::expect_strict` adds deterministic row shapes that +are evaluated in addition to the baseline when inference is mocked. With live +ingest, where captured rows are not locally inspectable, the equivalent +baseline is that the daemon reports emitted spans and no sink errors. + +Provider request sequences, exact model output, and injected provider failures +are additional mock-inference assertions; they do not replace the baseline +assertions. diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index 5c86f60..d9e28fd 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -135,19 +135,6 @@ impl AgentTestWorld { } } - pub fn expect_mock_inference( - &self, - scenario: IngestScenario, - name: impl Into, - matcher: impl Fn(&Value) -> bool + Send + Sync + 'static, - ) -> IngestScenario { - if self.uses_mock_inference() { - scenario.expect(name, matcher) - } else { - scenario - } - } - pub fn workspace(&self) -> PathBuf { let workspace = self.root.path().join("workspace"); std::fs::create_dir_all(&workspace).expect("create agent workspace"); @@ -220,7 +207,10 @@ impl AgentTestWorld { } let mut last_error = String::new(); for _ in 0..100 { - match self.collector.evaluate(scenario) { + match self + .collector + .evaluate(scenario, self.uses_mock_inference()) + { Ok(rows) => return rows, Err(error) => last_error = error, } diff --git a/bt-daemon/tests/support/ingest.rs b/bt-daemon/tests/support/ingest.rs index d018c69..6b659c4 100644 --- a/bt-daemon/tests/support/ingest.rs +++ b/bt-daemon/tests/support/ingest.rs @@ -12,6 +12,7 @@ type RowMatcher = dyn Fn(&Value) -> bool + Send + Sync + 'static; struct ExpectedRow { name: String, matcher: Arc, + strict: bool, } #[derive(Default)] @@ -35,13 +36,34 @@ impl IngestScenario { self.expected.push(ExpectedRow { name: name.into(), matcher: Arc::new(matcher), + strict: false, }); self } - pub fn evaluate(&self, rows: &[Value]) -> Result<(), String> { + /// Require an additional row shape when deterministic inference is in use. + /// Baseline expectations declared with [`Self::expect`] are always active. + pub fn expect_strict( + mut self, + name: impl Into, + matcher: impl Fn(&Value) -> bool + Send + Sync + 'static, + ) -> Self { + self.expected.push(ExpectedRow { + name: name.into(), + matcher: Arc::new(matcher), + strict: true, + }); + self + } + + pub fn evaluate(&self, rows: &[Value], include_strict: bool) -> Result<(), String> { + let active_expected = self + .expected + .iter() + .filter(|expected| include_strict || !expected.strict) + .collect::>(); let mut cursor = 0; - for (matched, expected) in self.expected.iter().enumerate() { + for (matched, expected) in active_expected.iter().enumerate() { let Some(offset) = rows[cursor..] .iter() .position(|row| (expected.matcher)(row)) @@ -50,7 +72,7 @@ impl IngestScenario { "missing ingest shape {:?} after matching {} of {} shapes", expected.name, matched, - self.expected.len() + active_expected.len() )); }; cursor += offset + 1; @@ -99,9 +121,13 @@ impl IngestMock { ) } - pub fn evaluate(&self, scenario: &IngestScenario) -> Result, String> { + pub fn evaluate( + &self, + scenario: &IngestScenario, + include_strict: bool, + ) -> Result, String> { let rows = self.rows(); - scenario.evaluate(&rows)?; + scenario.evaluate(&rows, include_strict)?; Ok(rows) } } From f2a62aa6c9c8b0c25c6f8a9a4ad64bd77ff465a7 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 30 Jul 2026 00:44:44 +0800 Subject: [PATCH 16/17] Separate live assertions from mock scenarios Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 50 +++++++++++++++++------- bt-daemon/tests/ingest_mock.rs | 32 +-------------- bt-daemon/tests/support/README.md | 21 +++++----- bt-daemon/tests/support/agent_process.rs | 28 +++++++++---- bt-daemon/tests/support/ingest.rs | 36 +++-------------- 5 files changed, 74 insertions(+), 93 deletions(-) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 5323596..698856f 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -125,14 +125,25 @@ async fn codex_session_emits_traces() { }) .await; - let scenario = IngestScenario::new() - .expect("Codex trace origin", |row| { - row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + let rows = world.wait_for_trace_delivery().await; + if world.uses_mock_ingest() { + assert!( + rows.iter() + .any(|row| { row_contains(row, &["braintrust.plugin.codex", "test_harness"]) }), + "Codex trace origin metadata was not emitted" + ); + } + world + .assert_mock_ingest_scenario(|| { + IngestScenario::new() + .expect("Codex trace origin", |row| { + row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + }) + .expect("Codex tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) + }) }) - .expect_strict("Codex tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) - }); - world.wait_for_ingest_scenario(&scenario).await; + .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -207,14 +218,25 @@ async fn claude_session_emits_traces() { }) .await; - let scenario = IngestScenario::new() - .expect("Claude trace source", |row| { - row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + let rows = world.wait_for_trace_delivery().await; + if world.uses_mock_ingest() { + assert!( + rows.iter() + .any(|row| { row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) }), + "Claude trace source metadata was not emitted" + ); + } + world + .assert_mock_ingest_scenario(|| { + IngestScenario::new() + .expect("Claude trace source", |row| { + row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + }) + .expect("Claude tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) + }) }) - .expect_strict("Claude tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) - }); - world.wait_for_ingest_scenario(&scenario).await; + .await; } #[test] diff --git a/bt-daemon/tests/ingest_mock.rs b/bt-daemon/tests/ingest_mock.rs index 7e07f43..9d9430b 100644 --- a/bt-daemon/tests/ingest_mock.rs +++ b/bt-daemon/tests/ingest_mock.rs @@ -28,38 +28,10 @@ async fn ingest_router_captures_rows_and_matches_ordered_shapes() { .expect("tool result", |row| { row["span_attributes"]["type"] == "tool" && row["output"] == "deterministic" }); - assert_eq!(ingest.evaluate(&scenario, true).unwrap().len(), 3); + assert_eq!(ingest.evaluate(&scenario).unwrap().len(), 3); let reversed = IngestScenario::new() .expect("tool first", |row| row["span_attributes"]["type"] == "tool") .expect("task later", |row| row["span_attributes"]["type"] == "task"); - assert!(ingest.evaluate(&reversed, true).is_err()); -} - -#[tokio::test] -async fn strict_shapes_are_additive_to_always_active_baseline_shapes() { - let ingest = IngestMock::new(); - let server = TestServer::start(ingest.router()).await; - - reqwest::Client::new() - .post(format!("{}/logs3", server.uri())) - .json(&json!({ - "rows": [ - {"span_attributes":{"type":"task"},"metadata":{"source":"codex"}} - ] - })) - .send() - .await - .unwrap(); - - let scenario = IngestScenario::new() - .expect("baseline root task", |row| { - row["span_attributes"]["type"] == "task" - }) - .expect_strict("deterministic tool result", |row| { - row["span_attributes"]["type"] == "tool" - }); - - assert!(ingest.evaluate(&scenario, false).is_ok()); - assert!(ingest.evaluate(&scenario, true).is_err()); + assert!(ingest.evaluate(&reversed).is_err()); } diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md index a7e6437..744c1ff 100644 --- a/bt-daemon/tests/support/README.md +++ b/bt-daemon/tests/support/README.md @@ -38,14 +38,13 @@ The world controls inference and ingest independently: Braintrust backend. This allows deterministic inference to drive real Braintrust ingest without -paying for model inference. Every test always asserts stable process behavior -and successful trace delivery. With mock ingest, `IngestScenario::expect` -declares baseline row shapes that are evaluated with both live and mock -inference. `IngestScenario::expect_strict` adds deterministic row shapes that -are evaluated in addition to the baseline when inference is mocked. With live -ingest, where captured rows are not locally inspectable, the equivalent -baseline is that the daemon reports emitted spans and no sink errors. - -Provider request sequences, exact model output, and injected provider failures -are additional mock-inference assertions; they do not replace the baseline -assertions. +paying for model inference. Every test uses ordinary assertions for stable +process behavior and trace delivery regardless of mode. When ingest is mocked, +the captured rows are also available for ordinary assertions over stable +metadata. With live ingest, the daemon must report emitted spans and no sink +errors. + +`IngestScenario` is exclusively for the additional deterministic expectations +when both inference and ingest are mocked. Provider request sequences, exact +model output, injected provider failures, and ordered trace shapes are layered +on top of the always-run assertions. diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index d9e28fd..a866652 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -201,16 +201,30 @@ impl AgentTestWorld { ); } - pub async fn wait_for_ingest_scenario(&self, scenario: &IngestScenario) -> Vec { - if !self.uses_mock_ingest() { - return self.wait_for_live_ingest().await; + /// Wait for the stable trace-delivery invariant in every backend mode. + /// Mock ingest returns captured rows for ordinary assertions; live ingest + /// verifies daemon emission and sink health. + pub async fn wait_for_trace_delivery(&self) -> Vec { + if self.uses_mock_ingest() { + self.wait_for_trace_rows().await + } else { + self.wait_for_live_ingest().await } + } + + /// Evaluate deterministic trace shapes only when both inference and ingest + /// are mocked. The closure keeps the mock scenario out of live-mode setup. + pub async fn assert_mock_ingest_scenario( + &self, + scenario: impl FnOnce() -> IngestScenario, + ) -> Vec { + if !self.uses_mock_inference() || !self.uses_mock_ingest() { + return Vec::new(); + } + let scenario = scenario(); let mut last_error = String::new(); for _ in 0..100 { - match self - .collector - .evaluate(scenario, self.uses_mock_inference()) - { + match self.collector.evaluate(&scenario) { Ok(rows) => return rows, Err(error) => last_error = error, } diff --git a/bt-daemon/tests/support/ingest.rs b/bt-daemon/tests/support/ingest.rs index 6b659c4..d018c69 100644 --- a/bt-daemon/tests/support/ingest.rs +++ b/bt-daemon/tests/support/ingest.rs @@ -12,7 +12,6 @@ type RowMatcher = dyn Fn(&Value) -> bool + Send + Sync + 'static; struct ExpectedRow { name: String, matcher: Arc, - strict: bool, } #[derive(Default)] @@ -36,34 +35,13 @@ impl IngestScenario { self.expected.push(ExpectedRow { name: name.into(), matcher: Arc::new(matcher), - strict: false, }); self } - /// Require an additional row shape when deterministic inference is in use. - /// Baseline expectations declared with [`Self::expect`] are always active. - pub fn expect_strict( - mut self, - name: impl Into, - matcher: impl Fn(&Value) -> bool + Send + Sync + 'static, - ) -> Self { - self.expected.push(ExpectedRow { - name: name.into(), - matcher: Arc::new(matcher), - strict: true, - }); - self - } - - pub fn evaluate(&self, rows: &[Value], include_strict: bool) -> Result<(), String> { - let active_expected = self - .expected - .iter() - .filter(|expected| include_strict || !expected.strict) - .collect::>(); + pub fn evaluate(&self, rows: &[Value]) -> Result<(), String> { let mut cursor = 0; - for (matched, expected) in active_expected.iter().enumerate() { + for (matched, expected) in self.expected.iter().enumerate() { let Some(offset) = rows[cursor..] .iter() .position(|row| (expected.matcher)(row)) @@ -72,7 +50,7 @@ impl IngestScenario { "missing ingest shape {:?} after matching {} of {} shapes", expected.name, matched, - active_expected.len() + self.expected.len() )); }; cursor += offset + 1; @@ -121,13 +99,9 @@ impl IngestMock { ) } - pub fn evaluate( - &self, - scenario: &IngestScenario, - include_strict: bool, - ) -> Result, String> { + pub fn evaluate(&self, scenario: &IngestScenario) -> Result, String> { let rows = self.rows(); - scenario.evaluate(&rows, include_strict)?; + scenario.evaluate(&rows)?; Ok(rows) } } From d3af285e3e04cc5e1644fd21bb61e45254ea8b56 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 30 Jul 2026 00:56:36 +0800 Subject: [PATCH 17/17] Use direct mode checks in agent tests Signed-off-by: Stephen Belanger --- bt-daemon/tests/agent_integration.rs | 98 +++++++++++------------- bt-daemon/tests/support/agent_process.rs | 34 ++------ 2 files changed, 50 insertions(+), 82 deletions(-) diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs index 698856f..e5d48e4 100644 --- a/bt-daemon/tests/agent_integration.rs +++ b/bt-daemon/tests/agent_integration.rs @@ -105,25 +105,21 @@ async fn codex_session_emits_traces() { ) .await; output.assert_success(); - world.when_mock_inference(|| { + if world.uses_mock_inference() { output.assert_contains("CODEX_MOCK_OK"); assert_eq!(inference.requests().len(), 2); - }); - world - .when_mock_inference_async(|| async { - let failed = codex - .run( - &world, - CodexRun::new("Trigger the deterministic inference error.") - .mock_inference(inference_server.uri()), - ) - .await; - failed.assert_failure(); - failed.assert_contains("deterministic Codex inference failure"); - assert_eq!(inference.requests().len(), 3); - }) - .await; + let failed = codex + .run( + &world, + CodexRun::new("Trigger the deterministic inference error.") + .mock_inference(inference_server.uri()), + ) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Codex inference failure"); + assert_eq!(inference.requests().len(), 3); + } let rows = world.wait_for_trace_delivery().await; if world.uses_mock_ingest() { @@ -133,17 +129,16 @@ async fn codex_session_emits_traces() { "Codex trace origin metadata was not emitted" ); } - world - .assert_mock_ingest_scenario(|| { - IngestScenario::new() - .expect("Codex trace origin", |row| { - row_contains(row, &["braintrust.plugin.codex", "test_harness"]) - }) - .expect("Codex tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) - }) - }) - .await; + if world.uses_mock_inference() && world.uses_mock_ingest() { + let scenario = IngestScenario::new() + .expect("Codex trace origin", |row| { + row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + }) + .expect("Codex tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) + }); + world.wait_for_mock_ingest_scenario(&scenario).await; + } } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -198,25 +193,21 @@ async fn claude_session_emits_traces() { ) .await; output.assert_success(); - world.when_mock_inference(|| { + if world.uses_mock_inference() { output.assert_contains("CLAUDE_MOCK_OK"); assert_eq!(inference.requests().len(), 2); - }); - world - .when_mock_inference_async(|| async { - let failed = claude - .run( - &world, - ClaudeRun::new("Trigger the deterministic inference error.") - .mock_inference(inference_server.uri()), - ) - .await; - failed.assert_failure(); - failed.assert_contains("deterministic Claude inference failure"); - assert_eq!(inference.requests().len(), 3); - }) - .await; + let failed = claude + .run( + &world, + ClaudeRun::new("Trigger the deterministic inference error.") + .mock_inference(inference_server.uri()), + ) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Claude inference failure"); + assert_eq!(inference.requests().len(), 3); + } let rows = world.wait_for_trace_delivery().await; if world.uses_mock_ingest() { @@ -226,17 +217,16 @@ async fn claude_session_emits_traces() { "Claude trace source metadata was not emitted" ); } - world - .assert_mock_ingest_scenario(|| { - IngestScenario::new() - .expect("Claude trace source", |row| { - row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) - }) - .expect("Claude tool output", |row| { - row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) - }) - }) - .await; + if world.uses_mock_inference() && world.uses_mock_ingest() { + let scenario = IngestScenario::new() + .expect("Claude trace source", |row| { + row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + }) + .expect("Claude tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) + }); + world.wait_for_mock_ingest_scenario(&scenario).await; + } } #[test] diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs index a866652..1f3ff2b 100644 --- a/bt-daemon/tests/support/agent_process.rs +++ b/bt-daemon/tests/support/agent_process.rs @@ -2,7 +2,6 @@ use crate::support::ingest::{IngestMock, IngestScenario}; use crate::support::server::TestServer; use bt_daemon::{run_status, StatusArgs}; use serde_json::{json, Value}; -use std::future::Future; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; @@ -119,22 +118,6 @@ impl AgentTestWorld { self.ingest_mode == TestBackendMode::Mock } - pub fn when_mock_inference(&self, assertion: impl FnOnce()) { - if self.uses_mock_inference() { - assertion(); - } - } - - pub async fn when_mock_inference_async(&self, operation: F) - where - F: FnOnce() -> Fut, - Fut: Future, - { - if self.uses_mock_inference() { - operation().await; - } - } - pub fn workspace(&self) -> PathBuf { let workspace = self.root.path().join("workspace"); std::fs::create_dir_all(&workspace).expect("create agent workspace"); @@ -212,19 +195,14 @@ impl AgentTestWorld { } } - /// Evaluate deterministic trace shapes only when both inference and ingest - /// are mocked. The closure keeps the mock scenario out of live-mode setup. - pub async fn assert_mock_ingest_scenario( - &self, - scenario: impl FnOnce() -> IngestScenario, - ) -> Vec { - if !self.uses_mock_inference() || !self.uses_mock_ingest() { - return Vec::new(); - } - let scenario = scenario(); + pub async fn wait_for_mock_ingest_scenario(&self, scenario: &IngestScenario) -> Vec { + assert!( + self.uses_mock_ingest(), + "ingest scenarios require mock ingest" + ); let mut last_error = String::new(); for _ in 0..100 { - match self.collector.evaluate(&scenario) { + match self.collector.evaluate(scenario) { Ok(rows) => return rows, Err(error) => last_error = error, }