diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95cbcf9..b99cd73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -32,9 +31,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-latest, windows-latest] + include: + - os: ubuntu-24.04 + agent_suffix: "" + - os: macos-latest + agent_suffix: "" + - os: windows-latest + agent_suffix: ".cmd" runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Rust @@ -42,6 +47,12 @@ jobs: rustup toolchain install stable --profile minimal rustup default stable rustup component add clippy rustfmt + - name: Install latest coding agents + run: npm install --prefix "${{ runner.temp }}/coding-agents" --no-save --no-package-lock --no-audit --no-fund --cache "${{ runner.temp }}/npm-cache" @openai/codex@latest @anthropic-ai/claude-code@latest + - name: Report coding-agent versions + run: | + npm exec --prefix "${{ runner.temp }}/coding-agents" -- codex --version + npm exec --prefix "${{ runner.temp }}/coding-agents" -- claude --version - name: Check formatting if: runner.os == 'Linux' run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check @@ -49,5 +60,12 @@ jobs: run: cargo build --manifest-path bt-daemon/Cargo.toml --all-features --locked - name: Test daemon run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked + - name: Test coding-agent integrations with deterministic inference + env: + BT_AGENT_INFERENCE_MODE: mock + BT_AGENT_INGEST_MODE: mock + CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex${{ matrix.agent_suffix }} + CLAUDE_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/claude${{ matrix.agent_suffix }} + run: cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked --test agent_integration -- --ignored --nocapture --test-threads=1 - name: Lint daemon run: cargo clippy --manifest-path bt-daemon/Cargo.toml --all-targets --all-features --locked -- -D warnings diff --git a/bt-daemon/Cargo.lock b/bt-daemon/Cargo.lock index 04b317a..dfc514e 100644 --- a/bt-daemon/Cargo.lock +++ b/bt-daemon/Cargo.lock @@ -118,6 +118,58 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "backoff" version = "0.4.0" @@ -212,10 +264,13 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "axum", "braintrust-sdk-rust", + "bytes", "chrono", "clap", "regex", + "reqwest", "serde", "serde_json", "sha2", @@ -226,6 +281,7 @@ dependencies = [ "tracing-subscriber", "uuid", "wiremock", + "zstd", ] [[package]] @@ -247,6 +303,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -987,6 +1045,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -1043,6 +1111,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.3" @@ -1134,6 +1208,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1536,6 +1616,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.21" @@ -1834,6 +1925,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1872,6 +1964,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2387,3 +2480,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/bt-daemon/Cargo.toml b/bt-daemon/Cargo.toml index 0a6d87b..b76b55a 100644 --- a/bt-daemon/Cargo.toml +++ b/bt-daemon/Cargo.toml @@ -34,5 +34,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = tr uuid = { version = "1", features = ["v4", "v5"] } [dev-dependencies] +axum = "0.8" +bytes = "1" +reqwest = { version = "0.12", default-features = false, features = ["json", "stream"] } tempfile = "3" wiremock = "0.6" +zstd = "0.13" diff --git a/bt-daemon/tests/agent_integration.rs b/bt-daemon/tests/agent_integration.rs new file mode 100644 index 0000000..e5d48e4 --- /dev/null +++ b/bt-daemon/tests/agent_integration.rs @@ -0,0 +1,260 @@ +mod support; + +use axum::http::StatusCode; +use serde_json::{json, Value}; +use support::agent_process::AgentTestWorld; +use support::agents::{ClaudeAgent, ClaudeRun, CodexAgent, CodexRun}; +use support::inference::{ + AnthropicMock, AnthropicRequest, AnthropicTurn, MockReply, OpenAiMock, OpenAiRequest, + OpenAiTurn, +}; +use support::ingest::IngestScenario; +use support::server::TestServer; + +fn codex_tool_call(request: &OpenAiRequest) -> OpenAiTurn { + let names = request.tool_names(); + if names.contains(&"exec_command") { + return OpenAiTurn::tool_call( + "call_mock_1", + "exec_command", + json!({"cmd":codex_tool_command(),"login":false}), + ); + } + if names.contains(&"shell") { + return OpenAiTurn::tool_call( + "call_mock_1", + "shell", + json!({"command":codex_tool_command()}), + ); + } + if names.contains(&"shell_command") { + return OpenAiTurn::tool_call( + "call_mock_1", + "shell_command", + json!({"command":codex_tool_command()}), + ); + } + panic!("Codex offered no supported shell tool; offered tools: {names:?}"); +} + +fn codex_tool_command() -> &'static str { + #[cfg(unix)] + { + "printf CODEX_TOOL_OK" + } + #[cfg(windows)] + { + "Write-Output CODEX_TOOL_OK" + } +} + +fn row_contains(row: &Value, fragments: &[&str]) -> bool { + let serialized = serde_json::to_string(row).expect("serialize trace row"); + fragments + .iter() + .all(|fragment| serialized.contains(fragment)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the Codex CLI installed on PATH"] +async fn codex_session_emits_traces() { + let inference = OpenAiMock::new(|context, request| { + assert_eq!(request.model(), Some("mock-model")); + match context.request_index { + 0 => { + assert!( + request.contains_text("CODEX_TOOL_OK"), + "unexpected Codex request: {}", + request.body + ); + MockReply::response(codex_tool_call(&request)) + } + 1 => { + assert!( + request.has_function_output("call_mock_1"), + "Codex did not return the tool result: {}", + request.body + ); + MockReply::response(OpenAiTurn::text("CODEX_MOCK_OK")) + } + 2 => MockReply::http_error( + StatusCode::BAD_REQUEST, + json!({ + "error": { + "type": "invalid_request_error", + "code": "mock_bad_request", + "message": "deterministic Codex inference failure" + } + }), + ), + index => panic!( + "unexpected Codex inference request {index}: {}", + request.body + ), + } + }); + let inference_server = TestServer::start(inference.router()).await; + let world = AgentTestWorld::start().await; + let codex = CodexAgent::install(&world).await; + + let output = codex + .run( + &world, + CodexRun::new("Run the command `printf CODEX_TOOL_OK` and then reply briefly.") + .mock_inference(inference_server.uri()), + ) + .await; + output.assert_success(); + if world.uses_mock_inference() { + output.assert_contains("CODEX_MOCK_OK"); + assert_eq!(inference.requests().len(), 2); + + let failed = codex + .run( + &world, + CodexRun::new("Trigger the deterministic inference error.") + .mock_inference(inference_server.uri()), + ) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Codex inference failure"); + assert_eq!(inference.requests().len(), 3); + } + + let rows = world.wait_for_trace_delivery().await; + if world.uses_mock_ingest() { + assert!( + rows.iter() + .any(|row| { row_contains(row, &["braintrust.plugin.codex", "test_harness"]) }), + "Codex trace origin metadata was not emitted" + ); + } + if world.uses_mock_inference() && world.uses_mock_ingest() { + let scenario = IngestScenario::new() + .expect("Codex trace origin", |row| { + row_contains(row, &["braintrust.plugin.codex", "test_harness"]) + }) + .expect("Codex tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CODEX_TOOL_OK"]) + }); + world.wait_for_mock_ingest_scenario(&scenario).await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires the Claude Code CLI installed on PATH"] +async fn claude_session_emits_traces() { + let inference = AnthropicMock::new(|context, request| match context.request_index { + 0 => { + assert_eq!(request.model(), Some("mock-model")); + assert!( + request.contains_text("CLAUDE_TOOL_OK"), + "unexpected Claude request: {}", + request.body + ); + MockReply::response(AnthropicTurn::tool_use( + "toolu_mock_1", + "Bash", + json!({"command":"printf CLAUDE_TOOL_OK"}), + )) + } + 1 => { + assert!( + request.has_tool_result("toolu_mock_1"), + "Claude did not return the tool result: {}", + request.body + ); + MockReply::response(AnthropicTurn::text("CLAUDE_MOCK_OK")) + } + 2 => MockReply::http_error( + StatusCode::BAD_REQUEST, + json!({ + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "deterministic Claude inference failure" + } + }), + ), + index => panic!( + "unexpected Claude inference request {index}: {}", + request.body + ), + }); + let inference_server = TestServer::start(inference.router()).await; + let world = AgentTestWorld::start().await; + let claude = ClaudeAgent::new(&world); + + let output = claude + .run( + &world, + ClaudeRun::new("Run the command `printf CLAUDE_TOOL_OK` and then reply briefly.") + .mock_inference(inference_server.uri()), + ) + .await; + output.assert_success(); + if world.uses_mock_inference() { + output.assert_contains("CLAUDE_MOCK_OK"); + assert_eq!(inference.requests().len(), 2); + + let failed = claude + .run( + &world, + ClaudeRun::new("Trigger the deterministic inference error.") + .mock_inference(inference_server.uri()), + ) + .await; + failed.assert_failure(); + failed.assert_contains("deterministic Claude inference failure"); + assert_eq!(inference.requests().len(), 3); + } + + let rows = world.wait_for_trace_delivery().await; + if world.uses_mock_ingest() { + assert!( + rows.iter() + .any(|row| { row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) }), + "Claude trace source metadata was not emitted" + ); + } + if world.uses_mock_inference() && world.uses_mock_ingest() { + let scenario = IngestScenario::new() + .expect("Claude trace source", |row| { + row_contains(row, &[r#""source":"claude-code""#, "test_harness"]) + }) + .expect("Claude tool output", |row| { + row_contains(row, &[r#""type":"tool""#, "CLAUDE_TOOL_OK"]) + }); + world.wait_for_mock_ingest_scenario(&scenario).await; + } +} + +#[test] +fn request_helpers_recognize_tool_results_and_advertised_tools() { + let openai = OpenAiRequest { + body: json!({ + "input":[{"type":"function_call_output","call_id":"call-1"}], + "tools":[{"type":"function","name":"shell_command"}] + }), + }; + assert!(openai.has_function_output("call-1")); + assert_eq!(openai.tool_names(), vec!["shell_command"]); + match codex_tool_call(&openai) { + OpenAiTurn::ToolCall { + name, arguments, .. + } => { + assert_eq!(name, "shell_command"); + assert!(arguments["command"].is_string()); + } + _ => panic!("expected a Codex tool call"), + } + + let anthropic = AnthropicRequest { + body: json!({ + "messages":[{ + "content":[{"type":"tool_result","tool_use_id":"toolu-1"}] + }] + }), + }; + assert!(anthropic.has_tool_result("toolu-1")); +} diff --git a/bt-daemon/tests/inference_mocks.rs b/bt-daemon/tests/inference_mocks.rs new file mode 100644 index 0000000..03f3990 --- /dev/null +++ b/bt-daemon/tests/inference_mocks.rs @@ -0,0 +1,101 @@ +mod support; + +use axum::http::StatusCode; +use serde_json::json; +use support::inference::{AnthropicMock, AnthropicTurn, MockReply, OpenAiMock, OpenAiTurn}; +use support::server::TestServer; + +#[tokio::test] +async fn openai_mock_streams_text_and_captures_requests() { + let mock = OpenAiMock::new(|context, request| { + assert_eq!(context.request_index, 0); + assert_eq!(request.model(), Some("mock-model")); + MockReply::response(OpenAiTurn::text("deterministic")) + }); + let server = TestServer::start(mock.router()).await; + + let response = reqwest::Client::new() + .post(format!("{}/v1/responses", server.uri())) + .json(&json!({"model":"mock-model","input":[],"stream":true})) + .send() + .await + .unwrap(); + let body = response.text().await.unwrap(); + + assert!(body.contains("response.output_item.done")); + assert!(body.contains("deterministic")); + assert_eq!(mock.requests().len(), 1); +} + +#[tokio::test] +async fn openai_mock_injects_retryable_and_malformed_responses() { + let mock = OpenAiMock::new(|context, _request| match context.request_index { + 0 => MockReply::http_error( + StatusCode::TOO_MANY_REQUESTS, + json!({"error":{"type":"rate_limit_error","message":"deterministic limit"}}), + ), + _ => MockReply::raw_sse("event: response.output_item.done\ndata: not-json\n\n"), + }); + let server = TestServer::start(mock.router()).await; + let client = reqwest::Client::new(); + + let limited = client + .post(format!("{}/v1/responses", server.uri())) + .json(&json!({"model":"mock-model","input":[],"stream":true})) + .send() + .await + .unwrap(); + assert_eq!(limited.status(), StatusCode::TOO_MANY_REQUESTS); + + let malformed = client + .post(format!("{}/v1/responses", server.uri())) + .json(&json!({"model":"mock-model","input":[],"stream":true})) + .send() + .await + .unwrap(); + assert!(malformed.text().await.unwrap().contains("not-json")); +} + +#[tokio::test] +async fn anthropic_mock_supports_tool_use_and_http_errors() { + let mock = AnthropicMock::new(|context, request| match context.request_index { + 0 => { + assert!(request.contains_text("run a command")); + MockReply::response(AnthropicTurn::tool_use( + "toolu_mock", + "Bash", + json!({"command":"printf hello"}), + )) + } + _ => MockReply::http_error( + StatusCode::TOO_MANY_REQUESTS, + json!({ + "type":"error", + "error":{"type":"rate_limit_error","message":"deterministic limit"} + }), + ), + }); + let server = TestServer::start(mock.router()).await; + + let client = reqwest::Client::new(); + let first = client + .post(format!("{}/v1/messages", server.uri())) + .json(&json!({ + "model":"mock-model", + "messages":[{"role":"user","content":"run a command"}], + "stream":true + })) + .send() + .await + .unwrap(); + assert!(first.text().await.unwrap().contains("toolu_mock")); + + let second = client + .post(format!("{}/v1/messages", server.uri())) + .json(&json!({"model":"mock-model","messages":[],"stream":true})) + .send() + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(mock.requests().len(), 2); +} diff --git a/bt-daemon/tests/ingest_mock.rs b/bt-daemon/tests/ingest_mock.rs new file mode 100644 index 0000000..9d9430b --- /dev/null +++ b/bt-daemon/tests/ingest_mock.rs @@ -0,0 +1,37 @@ +mod support; + +use serde_json::json; +use support::ingest::{IngestMock, IngestScenario}; +use support::server::TestServer; + +#[tokio::test] +async fn ingest_router_captures_rows_and_matches_ordered_shapes() { + let ingest = IngestMock::new(); + let server = TestServer::start(ingest.router()).await; + + let response = reqwest::Client::new() + .post(format!("{}/logs3", server.uri())) + .json(&json!({ + "rows": [ + {"span_attributes":{"type":"task"},"metadata":{"source":"codex"}}, + {"span_attributes":{"type":"llm"}}, + {"span_attributes":{"type":"tool"},"output":"deterministic"} + ] + })) + .send() + .await + .unwrap(); + assert!(response.status().is_success()); + + let scenario = IngestScenario::new() + .expect("root task", |row| row["span_attributes"]["type"] == "task") + .expect("tool result", |row| { + row["span_attributes"]["type"] == "tool" && row["output"] == "deterministic" + }); + assert_eq!(ingest.evaluate(&scenario).unwrap().len(), 3); + + let reversed = IngestScenario::new() + .expect("tool first", |row| row["span_attributes"]["type"] == "tool") + .expect("task later", |row| row["span_attributes"]["type"] == "task"); + assert!(ingest.evaluate(&reversed).is_err()); +} diff --git a/bt-daemon/tests/support/README.md b/bt-daemon/tests/support/README.md new file mode 100644 index 0000000..744c1ff --- /dev/null +++ b/bt-daemon/tests/support/README.md @@ -0,0 +1,50 @@ +# Agent integration test architecture + +The test infrastructure has three independent layers: + +- `server` is a generic container that binds any Axum `Router` to an + ephemeral address and owns its lifecycle. +- `inference` contains OpenAI Responses and Anthropic Messages protocol logic, + programmable scenarios, and captured inference requests. Each mock exports + an Axum router and can be hosted or embedded by any caller. +- `ingest` contains the mock Braintrust API and captured trace rows. It also + exports an Axum router and has no dependency on the server container. Its + scenario builder matches named row shapes as an ordered subsequence, + independent of HTTP batching and unrelated SDK update rows. + +`agent_process` is the Braintrust-specific orchestration layer. It hosts the +ingest router, starts the daemon, and provides the environment shared by agent +processes. + +`agents` contains reusable adapters for real coding-agent CLIs. Each adapter +owns only agent installation and isolated configuration state. The daemon +world is passed to each run as its execution context, avoiding any lifetime or +ownership coupling between the two layers. Adapters provide standard +invocation flags, mock-inference routing, and process output. Runs remain +configurable with additional arguments and environment variables so scenarios +can add inputs such as attachment paths without duplicating CLI setup. + +The integration test composes those pieces: it hosts an inference router, +starts the daemon world, runs an agent, and evaluates the ingest scenario. This +keeps both protocol mocks usable without coding agents, keeps the generic +server unaware of either protocol, and lets new end-to-end scenarios focus on +model behavior and expected trace shapes. + +The world controls inference and ingest independently: + +- `BT_AGENT_INFERENCE_MODE=mock|live` selects deterministic mock inference or + the agent's normal provider. +- `BT_AGENT_INGEST_MODE=mock|live` selects captured local ingest or the normal + Braintrust backend. + +This allows deterministic inference to drive real Braintrust ingest without +paying for model inference. Every test uses ordinary assertions for stable +process behavior and trace delivery regardless of mode. When ingest is mocked, +the captured rows are also available for ordinary assertions over stable +metadata. With live ingest, the daemon must report emitted spans and no sink +errors. + +`IngestScenario` is exclusively for the additional deterministic expectations +when both inference and ingest are mocked. Provider request sequences, exact +model output, injected provider failures, and ordered trace shapes are layered +on top of the always-run assertions. diff --git a/bt-daemon/tests/support/agent_process.rs b/bt-daemon/tests/support/agent_process.rs new file mode 100644 index 0000000..1f3ff2b --- /dev/null +++ b/bt-daemon/tests/support/agent_process.rs @@ -0,0 +1,357 @@ +use crate::support::ingest::{IngestMock, IngestScenario}; +use crate::support::server::TestServer; +use bt_daemon::{run_status, StatusArgs}; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; +use tempfile::TempDir; +use tokio::process::{Child, Command}; +#[cfg(windows)] +use uuid::Uuid; + +const INFERENCE_MODE_ENV: &str = "BT_AGENT_INFERENCE_MODE"; +const INGEST_MODE_ENV: &str = "BT_AGENT_INGEST_MODE"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TestBackendMode { + Mock, + Live, +} + +impl TestBackendMode { + fn from_env(name: &str) -> Self { + match std::env::var(name).as_deref() { + Ok("live") => Self::Live, + Ok("mock" | "deterministic") | Err(std::env::VarError::NotPresent) => Self::Mock, + Ok(value) => panic!("{name} must be `mock` or `live`, got {value:?}"), + Err(error) => panic!("could not read {name}: {error}"), + } + } +} + +pub struct AgentTestWorld { + inference_mode: TestBackendMode, + ingest_mode: TestBackendMode, + root: TempDir, + collector: IngestMock, + collector_server: TestServer, + daemon: Child, + wrapper_dir: PathBuf, + socket: PathBuf, + data_dir: PathBuf, + config_path: PathBuf, +} + +impl AgentTestWorld { + pub async fn start() -> Self { + let inference_mode = TestBackendMode::from_env(INFERENCE_MODE_ENV); + let ingest_mode = TestBackendMode::from_env(INGEST_MODE_ENV); + let root = tempfile::tempdir().expect("create agent test root"); + let collector = IngestMock::new(); + let collector_server = TestServer::start(collector.router()).await; + let wrapper_dir = root.path().join("bin"); + let data_dir = root.path().join("daemon"); + let socket = test_endpoint(root.path()); + let config_path = data_dir.join("config.json"); + std::fs::create_dir_all(&wrapper_dir).expect("create wrapper directory"); + std::fs::create_dir_all(&data_dir).expect("create daemon data directory"); + std::fs::write( + &config_path, + serde_json::to_vec_pretty(&json!({ + "traceToBraintrust": true, + "project": "agent-e2e", + "flushOnTurnEnd": true, + "additionalMetadata": {"test_harness": true} + })) + .unwrap(), + ) + .expect("write daemon config"); + + let daemon_binary = Path::new(env!("CARGO_BIN_EXE_bt-daemon")); + write_bt_wrapper(&wrapper_dir, daemon_binary); + + let mut command = Command::new(daemon_binary); + command + .arg("serve") + .arg("--socket") + .arg(&socket) + .arg("--data-dir") + .arg(&data_dir) + .arg("--idle-timeout-secs") + .arg("0") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + if ingest_mode == TestBackendMode::Mock { + command + .env("BRAINTRUST_API_URL", collector_server.uri()) + .env("BRAINTRUST_APP_URL", collector_server.uri()); + } + let daemon = command.spawn().expect("start daemon"); + + wait_for_daemon(daemon_binary, &socket).await; + Self { + inference_mode, + ingest_mode, + root, + collector, + collector_server, + daemon, + wrapper_dir, + socket, + data_dir, + config_path, + } + } + + pub fn uses_mock_inference(&self) -> bool { + self.inference_mode == TestBackendMode::Mock + } + + pub fn uses_live_inference(&self) -> bool { + self.inference_mode == TestBackendMode::Live + } + + pub fn uses_mock_ingest(&self) -> bool { + self.ingest_mode == TestBackendMode::Mock + } + + pub fn workspace(&self) -> PathBuf { + let workspace = self.root.path().join("workspace"); + std::fs::create_dir_all(&workspace).expect("create agent workspace"); + workspace + } + + pub fn temp_path(&self, name: &str) -> PathBuf { + self.root.path().join(name) + } + + pub fn configure(&self, command: &mut Command) { + let path = std::env::var_os("PATH").unwrap_or_default(); + let mut entries = vec![self.wrapper_dir.clone()]; + entries.extend(std::env::split_paths(&path)); + let combined = std::env::join_paths(entries).expect("construct test PATH"); + command + .env("PATH", combined) + .env("BT_DAEMON_SOCKET", &self.socket) + .env("BT_DAEMON_DATA_DIR", &self.data_dir) + .env("BT_DAEMON_CONFIG", &self.config_path) + .env("BRAINTRUST_FLUSH_ON_TURN_END", "true") + .stdin(Stdio::null()); + if self.uses_mock_ingest() { + command + .env("BRAINTRUST_API_KEY", "test-key") + .env("BRAINTRUST_API_URL", self.collector_server.uri()) + .env("BRAINTRUST_APP_URL", self.collector_server.uri()) + .env("BRAINTRUST_PROJECT", "agent-e2e"); + } + } + + pub async fn output(&self, command: &mut Command) -> std::process::Output { + command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let child = command.spawn().expect("spawn agent command"); + tokio::time::timeout(Duration::from_secs(90), child.wait_with_output()) + .await + .expect("agent command timed out") + .expect("wait for agent command") + } + + pub async fn wait_for_trace_rows(&self) -> Vec { + self.wait_for_trace_rows_matching(|rows| !rows.is_empty()) + .await + } + + pub async fn wait_for_trace_rows_matching( + &self, + predicate: impl Fn(&[Value]) -> bool, + ) -> Vec { + for _ in 0..100 { + let rows = self.collector.rows(); + if predicate(&rows) { + return rows; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!( + "daemon delivered no trace rows; {}; daemon files:\n{}", + self.collector.diagnostics(), + directory_contents(&self.data_dir) + ); + } + + /// Wait for the stable trace-delivery invariant in every backend mode. + /// Mock ingest returns captured rows for ordinary assertions; live ingest + /// verifies daemon emission and sink health. + pub async fn wait_for_trace_delivery(&self) -> Vec { + if self.uses_mock_ingest() { + self.wait_for_trace_rows().await + } else { + self.wait_for_live_ingest().await + } + } + + pub async fn wait_for_mock_ingest_scenario(&self, scenario: &IngestScenario) -> Vec { + assert!( + self.uses_mock_ingest(), + "ingest scenarios require mock ingest" + ); + let mut last_error = String::new(); + for _ in 0..100 { + match self.collector.evaluate(scenario) { + Ok(rows) => return rows, + Err(error) => last_error = error, + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!( + "ingest scenario did not complete: {last_error}; {}; daemon files:\n{}", + self.collector.diagnostics(), + directory_contents(&self.data_dir) + ); + } + + async fn wait_for_live_ingest(&self) -> Vec { + let mut last_status = String::new(); + for _ in 0..100 { + match run_status(StatusArgs { + socket: Some(self.socket.clone()), + session_id: None, + }) + .await + { + Ok(Some(status)) => { + last_status = format!("{:?}", status.sessions); + let emitted = status + .sessions + .iter() + .any(|session| session.spans_emitted > 0); + let errors = status + .sessions + .iter() + .filter_map(|session| session.last_error.as_deref()) + .collect::>(); + assert!( + errors.is_empty(), + "live ingest reported daemon sink errors: {errors:?}" + ); + if emitted { + return Vec::new(); + } + } + Ok(None) => last_status = "daemon not running".into(), + Err(error) => last_status = error.to_string(), + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("live ingest emitted no spans; last daemon status: {last_status}"); + } +} + +impl Drop for AgentTestWorld { + fn drop(&mut self) { + let _ = self.daemon.start_kill(); + } +} + +#[cfg(unix)] +fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { + use std::os::unix::fs::PermissionsExt; + + let path = directory.join("bt"); + let script = format!( + "#!/bin/sh\nif [ \"$1\" = daemon ]; then shift; fi\nexec '{}' \"$@\"\n", + daemon_binary.display() + ); + std::fs::write(&path, script).expect("write bt test wrapper"); + let mut permissions = std::fs::metadata(&path).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&path, permissions).expect("make bt wrapper executable"); +} + +#[cfg(windows)] +fn write_bt_wrapper(directory: &Path, daemon_binary: &Path) { + let powershell = directory.join("bt-wrapper.ps1"); + let script = format!( + "$forward = @($args)\n\ + if ($forward.Count -gt 0 -and $forward[0] -eq 'daemon') {{\n\ + if ($forward.Count -eq 1) {{ $forward = @() }} else {{ $forward = @($forward[1..($forward.Count - 1)]) }}\n\ + }}\n\ + & '{}' @forward\n\ + exit $LASTEXITCODE\n", + daemon_binary.display() + ); + std::fs::write(&powershell, script).expect("write bt PowerShell wrapper"); + std::fs::write( + directory.join("bt.cmd"), + "@echo off\r\npowershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File \"%~dp0bt-wrapper.ps1\" %*\r\n", + ) + .expect("write bt command wrapper"); + + // Claude Code, and some Codex releases, launch the portable `command` + // hook through Git Bash even on Windows. Git Bash does not resolve + // PATHEXT, so expose an extensionless shim in addition to bt.cmd. + let shell_binary = daemon_binary.to_string_lossy().replace('\\', "/"); + let shell = format!( + "#!/bin/sh\nif [ \"$1\" = daemon ]; then shift; fi\nexec '{}' \"$@\"\n", + shell_binary + ); + std::fs::write(directory.join("bt"), shell).expect("write bt Git Bash wrapper"); +} + +#[cfg(unix)] +fn test_endpoint(root: &Path) -> PathBuf { + root.join("daemon.sock") +} + +#[cfg(windows)] +fn test_endpoint(_root: &Path) -> PathBuf { + PathBuf::from(format!( + r"\\.\pipe\braintrust-bt-daemon-test-{}", + Uuid::new_v4() + )) +} + +async fn wait_for_daemon(daemon_binary: &Path, endpoint: &Path) { + for _ in 0..100 { + let output = Command::new(daemon_binary) + .arg("status") + .arg("--socket") + .arg(endpoint) + .output() + .await; + if let Ok(output) = output { + if output.status.success() + && !String::from_utf8_lossy(&output.stdout).contains("not running") + { + return; + } + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("daemon endpoint was not ready at {}", endpoint.display()); +} + +fn directory_contents(root: &Path) -> String { + fn visit(path: &Path, output: &mut String) { + let Ok(entries) = std::fs::read_dir(path) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + visit(&path, output); + } else { + let body = std::fs::read_to_string(&path).unwrap_or_else(|_| "".into()); + output.push_str(&format!("{}:\n{}\n", path.display(), body)); + } + } + } + let mut output = String::new(); + visit(root, &mut output); + output +} diff --git a/bt-daemon/tests/support/agents/claude.rs b/bt-daemon/tests/support/agents/claude.rs new file mode 100644 index 0000000..7fb32ec --- /dev/null +++ b/bt-daemon/tests/support/agents/claude.rs @@ -0,0 +1,107 @@ +use super::{command_from_env, repository_root, AgentOutput, ProcessOptions}; +use crate::support::agent_process::AgentTestWorld; +use std::ffi::OsString; +use std::path::PathBuf; +use uuid::Uuid; + +pub struct ClaudeAgent { + isolated_home: PathBuf, + isolated_config: PathBuf, +} + +pub struct ClaudeRun { + prompt: OsString, + mock_inference: Option, + options: ProcessOptions, +} + +struct ClaudeInference { + base_url: String, + model: String, + api_key: String, +} + +impl ClaudeRun { + pub fn new(prompt: impl Into) -> Self { + Self { + prompt: prompt.into(), + mock_inference: None, + options: ProcessOptions::default(), + } + } + + pub fn mock_inference(mut self, base_url: impl Into) -> Self { + self.mock_inference = Some(ClaudeInference { + base_url: base_url.into(), + model: "mock-model".into(), + api_key: "test-key".into(), + }); + self + } + + pub fn arg(mut self, value: impl Into) -> Self { + self.options.arg(value); + self + } + + pub fn env(mut self, key: impl Into, value: impl Into) -> Self { + self.options.env(key, value); + self + } +} + +impl ClaudeAgent { + pub fn new(world: &AgentTestWorld) -> Self { + let isolated_home = world.temp_path("claude-home"); + let isolated_config = world.temp_path("claude-config"); + std::fs::create_dir_all(&isolated_home).expect("create Claude home"); + std::fs::create_dir_all(&isolated_config).expect("create Claude config"); + Self { + isolated_home, + isolated_config, + } + } + + pub async fn run(&self, world: &AgentTestWorld, run: ClaudeRun) -> AgentOutput { + let session_id = Uuid::new_v4().to_string(); + let plugin = repository_root().join("src/plugins/claude/content/plugins/trace-claude-code"); + let mut command = command_from_env("CLAUDE_BIN", "claude"); + command + .args([ + "-p", + "--output-format", + "json", + "--dangerously-skip-permissions", + "--session-id", + &session_id, + "--plugin-dir", + ]) + .arg(plugin) + .current_dir(world.workspace()) + .env("ANTHROPIC_MAX_RETRIES", "0") + .env("DISABLE_AUTOUPDATER", "1") + .env("DISABLE_TELEMETRY", "1") + .env("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1"); + world.configure(&mut command); + + if world.uses_mock_inference() { + let inference = run + .mock_inference + .as_ref() + .expect("mock Claude runs require a mock inference endpoint"); + command + .args(["--model", &inference.model]) + .env("HOME", &self.isolated_home) + .env("CLAUDE_CONFIG_DIR", &self.isolated_config) + .env("ANTHROPIC_BASE_URL", &inference.base_url) + .env("ANTHROPIC_API_KEY", &inference.api_key) + .env("ANTHROPIC_AUTH_TOKEN", &inference.api_key) + .env("ANTHROPIC_DEFAULT_OPUS_MODEL", &inference.model) + .env("ANTHROPIC_DEFAULT_SONNET_MODEL", &inference.model) + .env("ANTHROPIC_DEFAULT_HAIKU_MODEL", &inference.model); + } + run.options.apply(&mut command); + command.arg(run.prompt); + world.output(&mut command).await.into() + } +} diff --git a/bt-daemon/tests/support/agents/codex.rs b/bt-daemon/tests/support/agents/codex.rs new file mode 100644 index 0000000..be33da0 --- /dev/null +++ b/bt-daemon/tests/support/agents/codex.rs @@ -0,0 +1,150 @@ +use super::{command_from_env, configured_home, repository_root, AgentOutput, ProcessOptions}; +use crate::support::agent_process::AgentTestWorld; +use std::ffi::OsString; +use std::path::PathBuf; +use tokio::process::Command; + +const INFERENCE_MODE_ENV: &str = "BT_AGENT_INFERENCE_MODE"; + +pub struct CodexAgent { + home: PathBuf, +} + +pub struct CodexRun { + prompt: OsString, + mock_inference: Option, + options: ProcessOptions, +} + +struct CodexInference { + base_url: String, + model: String, + api_key: String, +} + +impl CodexRun { + pub fn new(prompt: impl Into) -> Self { + Self { + prompt: prompt.into(), + mock_inference: None, + options: ProcessOptions::default(), + } + } + + pub fn mock_inference(mut self, base_url: impl Into) -> Self { + self.mock_inference = Some(CodexInference { + base_url: base_url.into(), + model: "mock-model".into(), + api_key: "test-key".into(), + }); + self + } + + pub fn arg(mut self, value: impl Into) -> Self { + self.options.arg(value); + self + } + + pub fn env(mut self, key: impl Into, value: impl Into) -> Self { + self.options.env(key, value); + self + } +} + +impl CodexAgent { + pub async fn install(world: &AgentTestWorld) -> Self { + let home = world.temp_path("codex-home"); + std::fs::create_dir_all(&home).expect("create Codex home"); + + let marketplace = repository_root().join("src/plugins/codex/content"); + let mut add_marketplace = command_from_env("CODEX_BIN", "codex"); + add_marketplace + .arg("plugin") + .arg("marketplace") + .arg("add") + .arg(&marketplace) + .env("CODEX_HOME", &home); + world.configure(&mut add_marketplace); + AgentOutput::from(world.output(&mut add_marketplace).await).assert_success(); + + let mut add_plugin = command_from_env("CODEX_BIN", "codex"); + add_plugin + .args(["plugin", "add", "trace-codex@braintrust-codex-plugins"]) + .env("CODEX_HOME", &home); + world.configure(&mut add_plugin); + AgentOutput::from(world.output(&mut add_plugin).await).assert_success(); + + let agent = Self { home }; + if world.uses_live_inference() { + agent.seed_live_auth(); + } + agent + } + + pub fn seed_live_auth(&self) { + if std::env::var_os("OPENAI_API_KEY").is_some() { + return; + } + let source = configured_home("CODEX_HOME", ".codex") + .map(|home| home.join("auth.json")) + .filter(|path| path.is_file()) + .unwrap_or_else(|| { + panic!( + "{INFERENCE_MODE_ENV}=live requires OPENAI_API_KEY or auth.json in the configured Codex home" + ) + }); + std::fs::copy(source, self.home.join("auth.json")).expect("copy Codex live credentials"); + } + + pub async fn run(&self, world: &AgentTestWorld, run: CodexRun) -> AgentOutput { + let mut command = self.command(world); + if world.uses_mock_inference() { + let inference = run + .mock_inference + .as_ref() + .expect("mock Codex runs require a mock inference endpoint"); + configure_mock_inference(&mut command, inference); + } + run.options.apply(&mut command); + command.arg(run.prompt); + world.output(&mut command).await.into() + } + + fn command(&self, world: &AgentTestWorld) -> Command { + let mut command = command_from_env("CODEX_BIN", "codex"); + command + .args([ + "exec", + "--skip-git-repo-check", + "--dangerously-bypass-hook-trust", + "--sandbox", + "read-only", + "-c", + r#"approval_policy="never""#, + ]) + .current_dir(world.workspace()) + .env("CODEX_HOME", &self.home); + world.configure(&mut command); + command + } +} + +fn configure_mock_inference(command: &mut Command, inference: &CodexInference) { + let provider = format!( + r#"model_providers.mock={{name="Mock",base_url="{}/v1",wire_api="responses",env_key="MOCK_API_KEY",request_max_retries=0,stream_max_retries=0,stream_idle_timeout_ms=5000}}"#, + inference.base_url + ); + let chatgpt_base_url = format!(r#"chatgpt_base_url="{}/backend-api""#, inference.base_url); + command + .args([ + "-c", + &format!(r#"model="{}""#, inference.model), + "-c", + r#"model_provider="mock""#, + "-c", + &provider, + "-c", + &chatgpt_base_url, + ]) + .env("MOCK_API_KEY", &inference.api_key); +} diff --git a/bt-daemon/tests/support/agents/mod.rs b/bt-daemon/tests/support/agents/mod.rs new file mode 100644 index 0000000..9a83fdb --- /dev/null +++ b/bt-daemon/tests/support/agents/mod.rs @@ -0,0 +1,98 @@ +mod claude; +mod codex; + +#[allow(unused_imports)] +pub use claude::{ClaudeAgent, ClaudeRun}; +#[allow(unused_imports)] +pub use codex::{CodexAgent, CodexRun}; + +use std::ffi::OsString; +use std::path::PathBuf; +use tokio::process::Command; + +pub struct AgentOutput { + output: std::process::Output, +} + +impl AgentOutput { + pub fn success(&self) -> bool { + self.output.status.success() + } + + pub fn text(&self) -> String { + format!( + "stdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&self.output.stdout), + String::from_utf8_lossy(&self.output.stderr) + ) + } + + pub fn assert_success(&self) { + assert!(self.success(), "{}", self.text()); + } + + pub fn assert_failure(&self) { + assert!(!self.success(), "{}", self.text()); + } + + pub fn assert_contains(&self, expected: &str) { + assert!( + self.text().contains(expected), + "agent output did not contain {expected:?}:\n{}", + self.text() + ); + } +} + +impl From for AgentOutput { + fn from(output: std::process::Output) -> Self { + Self { output } + } +} + +#[derive(Default)] +struct ProcessOptions { + args: Vec, + env: Vec<(OsString, OsString)>, +} + +impl ProcessOptions { + fn arg(&mut self, value: impl Into) { + self.args.push(value.into()); + } + + fn env(&mut self, key: impl Into, value: impl Into) { + self.env.push((key.into(), value.into())); + } + + fn apply(&self, command: &mut Command) { + command + .args(&self.args) + .envs(self.env.iter().map(|(k, v)| (k, v))); + } +} + +fn command_from_env(name: &str, fallback: &str) -> Command { + if let Some(command) = std::env::var_os(name) { + return Command::new(command); + } + #[cfg(windows)] + let fallback = format!("{fallback}.cmd"); + Command::new(fallback) +} + +fn repository_root() -> PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("repository root") + .to_path_buf() +} + +fn configured_home(config_env: &str, directory: &str) -> Option { + std::env::var_os(config_env).map(PathBuf::from).or_else(|| { + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .map(PathBuf::from) + .map(|home| home.join(directory)) + }) +} diff --git a/bt-daemon/tests/support/inference/README.md b/bt-daemon/tests/support/inference/README.md new file mode 100644 index 0000000..92567d3 --- /dev/null +++ b/bt-daemon/tests/support/inference/README.md @@ -0,0 +1,78 @@ +# Deterministic inference test support + +This directory is a self-contained mock-inference component with two +protocol-faithful servers: + +- `OpenAiMock` implements the OpenAI Responses API surface used by Codex. +- `AnthropicMock` implements the Anthropic Messages API surface used by + Claude Code. + +Each public mock owns its protocol routes, scenario closure, and captured +requests, and exports an Axum `Router`. Callers can bind that router with the +shared ephemeral test server or embed it in another Axum application. The two +providers share request indexing and transport outcomes. Request and response +types remain provider-specific so a test cannot accidentally hide a +wire-protocol incompatibility behind a common model abstraction. + +Both mocks accept a thread-safe closure: + +```rust,ignore +let mock = OpenAiMock::new(|context, request| { + match context.request_index { + 0 => MockReply::response(OpenAiTurn::tool_call( + "call-1", + "exec_command", + json!({"cmd":"printf hello"}), + )), + 1 if request.has_function_output("call-1") => { + MockReply::response(OpenAiTurn::text("done")) + } + index => panic!("unexpected request {index}: {}", request.body), + } +}); +let server = TestServer::start(mock.router()).await; +``` + +`MockReply` supports normal provider responses, arbitrary HTTP errors, and raw +response bodies for malformed or truncated stream tests. Typed turn builders +generate deterministic ids, token usage, and valid provider SSE sequences. +Every inference request is captured for later assertions. + +The component does not depend on `bt-daemon`, the coding-agent runner, the +ingest mock, or a particular listener implementation. The higher-level +`support::agent_process` harness composes with it only from the integration +test. This boundary is deliberate so the whole mock-inference component can +later move into a reusable crate and serve any client that can target an +OpenAI Responses or Anthropic Messages endpoint. + +`agent_integration.rs` runs real Codex and Claude Code processes against these +mocks. +The tests are ignored in a plain Rust run because they require agent +executables. The core cross-platform CI matrix installs the latest release of +each agent and runs them in the default `mock` mode on every host. This is +intentionally unpinned so upstream compatibility breaks are visible +immediately. + +The same agent tests can run without mock inference while continuing to use +captured local ingest: + +```console +BT_AGENT_INFERENCE_MODE=live BT_AGENT_INGEST_MODE=mock \ + cargo test --manifest-path bt-daemon/Cargo.toml \ + --all-features --test agent_integration -- --ignored --test-threads=1 +``` + +Live inference uses the normal provider endpoint/model and the agent's normal +login or provider credentials. It validates only stable integration invariants +such as trace delivery and origin metadata. Mock inference additionally +validates exact request sequences, tool results, output content, and injected +failures. + +Inference and ingest selection are independent. To drive deterministic model +behavior while reporting traces to the normal Braintrust backend: + +```console +BT_AGENT_INFERENCE_MODE=mock BT_AGENT_INGEST_MODE=live \ + cargo test --manifest-path bt-daemon/Cargo.toml \ + --all-features --test agent_integration -- --ignored --test-threads=1 +``` diff --git a/bt-daemon/tests/support/inference/anthropic.rs b/bt-daemon/tests/support/inference/anthropic.rs new file mode 100644 index 0000000..59662b1 --- /dev/null +++ b/bt-daemon/tests/support/inference/anthropic.rs @@ -0,0 +1,258 @@ +use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::{get, post}; +use axum::Router; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone)] +pub struct AnthropicRequest { + pub body: Value, +} + +impl AnthropicRequest { + pub fn model(&self) -> Option<&str> { + self.body["model"].as_str() + } + + pub fn contains_text(&self, text: &str) -> bool { + self.body.to_string().contains(text) + } + + pub fn has_tool_result(&self, tool_use_id: &str) -> bool { + self.body["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["content"].as_array().is_some_and(|blocks| { + blocks.iter().any(|block| { + block["type"] == "tool_result" && block["tool_use_id"] == tool_use_id + }) + }) + }) + }) + } +} + +#[derive(Debug, Clone)] +pub enum AnthropicTurn { + Text { + text: String, + input_tokens: u64, + output_tokens: u64, + }, + ToolUse { + tool_use_id: String, + name: String, + input: Value, + input_tokens: u64, + output_tokens: u64, + }, + Events(Vec), +} + +impl AnthropicTurn { + pub fn text(text: impl Into) -> Self { + Self::Text { + text: text.into(), + input_tokens: 10, + output_tokens: 5, + } + } + + pub fn tool_use(tool_use_id: impl Into, name: impl Into, input: Value) -> Self { + Self::ToolUse { + tool_use_id: tool_use_id.into(), + name: name.into(), + input, + input_tokens: 10, + output_tokens: 5, + } + } + + fn events(self, response_index: usize) -> Vec { + let message_id = format!("msg_mock_{response_index}"); + match self { + Self::Text { + text, + input_tokens, + output_tokens, + } => { + let mut events = message_start(&message_id, input_tokens); + events.extend([ + json!({ + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""} + }), + json!({ + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": text} + }), + json!({"type": "content_block_stop", "index": 0}), + message_delta("end_turn", output_tokens), + json!({"type": "message_stop"}), + ]); + events + } + Self::ToolUse { + tool_use_id, + name, + input, + input_tokens, + output_tokens, + } => { + let mut events = message_start(&message_id, input_tokens); + events.extend([ + json!({ + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": tool_use_id, + "name": name, + "input": {} + } + }), + json!({ + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": input.to_string() + } + }), + json!({"type": "content_block_stop", "index": 0}), + message_delta("tool_use", output_tokens), + json!({"type": "message_stop"}), + ]); + events + } + Self::Events(events) => events, + } + } +} + +fn message_start(id: &str, input_tokens: u64) -> Vec { + vec![json!({ + "type": "message_start", + "message": { + "id": id, + "type": "message", + "role": "assistant", + "content": [], + "model": "mock-model", + "stop_reason": null, + "stop_sequence": null, + "usage": { + "input_tokens": input_tokens, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 1 + } + } + })] +} + +fn message_delta(stop_reason: &str, output_tokens: u64) -> Value { + json!({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": null}, + "usage": {"output_tokens": output_tokens} + }) +} + +type Handler = + dyn Fn(RequestContext, AnthropicRequest) -> MockReply + Send + Sync + 'static; + +struct MockState { + handler: Arc, + requests: Mutex>, + next_index: AtomicUsize, +} + +pub struct AnthropicMock { + state: Arc, +} + +impl AnthropicMock { + pub fn new(handler: H) -> Self + where + H: Fn(RequestContext, AnthropicRequest) -> MockReply + Send + Sync + 'static, + { + let state = Arc::new(MockState { + handler: Arc::new(handler), + requests: Mutex::new(Vec::new()), + next_index: AtomicUsize::new(0), + }); + Self { state } + } + + pub fn router(&self) -> Router { + Router::new() + .route("/v1/models", get(models)) + .route("/v1/messages", post(messages)) + .route("/v1/messages/count_tokens", post(count_tokens)) + .with_state(Arc::clone(&self.state)) + } + + pub fn requests(&self) -> Vec { + self.state.requests.lock().expect("request lock").clone() + } +} + +async fn models() -> axum::Json { + axum::Json(json!({ + "data": [{ + "type": "model", + "id": "mock-model", + "display_name": "Mock model", + "created_at": "2026-01-01T00:00:00Z" + }], + "has_more": false, + "first_id": "mock-model", + "last_id": "mock-model" + })) +} + +async fn count_tokens() -> axum::Json { + axum::Json(json!({"input_tokens": 10})) +} + +async fn messages( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> axum::response::Response { + let body = match decode_json_body(&headers, &body) { + Ok(body) => body, + Err(error) => return json_response(StatusCode::BAD_REQUEST, json!({"error": error})), + }; + let request = AnthropicRequest { body }; + state + .requests + .lock() + .expect("request lock") + .push(request.clone()); + let index = state.next_index.fetch_add(1, Ordering::SeqCst); + match (state.handler)( + RequestContext { + request_index: index, + }, + request, + ) { + MockReply::Response(turn) => raw_response( + StatusCode::OK, + "text/event-stream", + sse(&turn.events(index)), + ), + MockReply::HttpError { status, body } => json_response(status, body), + MockReply::Raw { + status, + content_type, + body, + } => raw_response(status, content_type, body), + } +} diff --git a/bt-daemon/tests/support/inference/mod.rs b/bt-daemon/tests/support/inference/mod.rs new file mode 100644 index 0000000..4339036 --- /dev/null +++ b/bt-daemon/tests/support/inference/mod.rs @@ -0,0 +1,94 @@ +mod anthropic; +mod openai; + +#[allow(unused_imports)] +pub use anthropic::{AnthropicMock, AnthropicRequest, AnthropicTurn}; +#[allow(unused_imports)] +pub use openai::{OpenAiMock, OpenAiRequest, OpenAiTurn}; + +use axum::http::StatusCode; +use serde_json::Value; + +#[derive(Debug, Clone, Copy)] +pub struct RequestContext { + pub request_index: usize, +} + +/// A provider-neutral transport outcome. Protocol response bodies remain +/// provider-specific and are rendered by the OpenAI/Anthropic adapters. +#[derive(Debug, Clone)] +pub enum MockReply { + Response(T), + HttpError { + status: StatusCode, + body: Value, + }, + Raw { + status: StatusCode, + content_type: &'static str, + body: Vec, + }, +} + +impl MockReply { + pub fn response(value: T) -> Self { + Self::Response(value) + } + + pub fn http_error(status: StatusCode, body: Value) -> Self { + Self::HttpError { status, body } + } + + pub fn raw_sse(body: impl Into>) -> Self { + Self::Raw { + status: StatusCode::OK, + content_type: "text/event-stream", + body: body.into(), + } + } +} + +fn decode_json_body(headers: &axum::http::HeaderMap, body: &[u8]) -> Result { + let decoded = match headers + .get(axum::http::header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + { + Some(value) if value.split(',').any(|part| part.trim() == "zstd") => { + zstd::stream::decode_all(std::io::Cursor::new(body)) + .map_err(|error| format!("decode zstd request: {error}"))? + } + _ => body.to_vec(), + }; + serde_json::from_slice(&decoded).map_err(|error| format!("decode JSON request: {error}")) +} + +fn json_response(status: StatusCode, body: Value) -> axum::response::Response { + use axum::response::IntoResponse; + (status, axum::Json(body)).into_response() +} + +fn raw_response( + status: StatusCode, + content_type: &'static str, + body: Vec, +) -> axum::response::Response { + use axum::response::IntoResponse; + ( + status, + [(axum::http::header::CONTENT_TYPE, content_type)], + body, + ) + .into_response() +} + +fn sse(events: &[Value]) -> Vec { + use std::fmt::Write; + + let mut body = String::new(); + for event in events { + let kind = event["type"].as_str().expect("SSE event type"); + writeln!(&mut body, "event: {kind}").expect("write SSE event"); + writeln!(&mut body, "data: {event}\n").expect("write SSE data"); + } + body.into_bytes() +} diff --git a/bt-daemon/tests/support/inference/openai.rs b/bt-daemon/tests/support/inference/openai.rs new file mode 100644 index 0000000..92e3ec4 --- /dev/null +++ b/bt-daemon/tests/support/inference/openai.rs @@ -0,0 +1,231 @@ +use super::{decode_json_body, json_response, raw_response, sse, MockReply, RequestContext}; +use axum::body::Bytes; +use axum::extract::State; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::{get, post}; +use axum::Router; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone)] +pub struct OpenAiRequest { + pub body: Value, +} + +impl OpenAiRequest { + pub fn model(&self) -> Option<&str> { + self.body["model"].as_str() + } + + pub fn contains_text(&self, text: &str) -> bool { + self.body.to_string().contains(text) + } + + pub fn has_function_output(&self, call_id: &str) -> bool { + self.body["input"].as_array().is_some_and(|items| { + items + .iter() + .any(|item| item["type"] == "function_call_output" && item["call_id"] == call_id) + }) + } + + pub fn tool_names(&self) -> Vec<&str> { + self.body["tools"] + .as_array() + .into_iter() + .flatten() + .filter_map(|tool| tool["name"].as_str()) + .collect() + } +} + +#[derive(Debug, Clone)] +pub enum OpenAiTurn { + Text { + text: String, + input_tokens: u64, + output_tokens: u64, + }, + ToolCall { + call_id: String, + name: String, + arguments: Value, + input_tokens: u64, + output_tokens: u64, + }, + Events(Vec), +} + +impl OpenAiTurn { + pub fn text(text: impl Into) -> Self { + Self::Text { + text: text.into(), + input_tokens: 10, + output_tokens: 5, + } + } + + pub fn tool_call( + call_id: impl Into, + name: impl Into, + arguments: Value, + ) -> Self { + Self::ToolCall { + call_id: call_id.into(), + name: name.into(), + arguments, + input_tokens: 10, + output_tokens: 5, + } + } + + fn events(self, response_index: usize) -> Vec { + let response_id = format!("resp_mock_{response_index}"); + let created = json!({ + "type": "response.created", + "response": {"id": response_id} + }); + match self { + Self::Text { + text, + input_tokens, + output_tokens, + } => vec![ + created, + json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "id": format!("msg_mock_{response_index}"), + "content": [{"type": "output_text", "text": text}] + } + }), + completed(&response_id, input_tokens, output_tokens), + ], + Self::ToolCall { + call_id, + name, + arguments, + input_tokens, + output_tokens, + } => vec![ + created, + json!({ + "type": "response.output_item.done", + "item": { + "type": "function_call", + "call_id": call_id, + "name": name, + "arguments": arguments.to_string() + } + }), + completed(&response_id, input_tokens, output_tokens), + ], + Self::Events(events) => events, + } + } +} + +fn completed(id: &str, input_tokens: u64, output_tokens: u64) -> Value { + json!({ + "type": "response.completed", + "response": { + "id": id, + "usage": { + "input_tokens": input_tokens, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": output_tokens, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": input_tokens + output_tokens + } + } + }) +} + +type Handler = + dyn Fn(RequestContext, OpenAiRequest) -> MockReply + Send + Sync + 'static; + +struct MockState { + handler: Arc, + requests: Mutex>, + next_index: AtomicUsize, +} + +pub struct OpenAiMock { + state: Arc, +} + +impl OpenAiMock { + pub fn new(handler: H) -> Self + where + H: Fn(RequestContext, OpenAiRequest) -> MockReply + Send + Sync + 'static, + { + let state = Arc::new(MockState { + handler: Arc::new(handler), + requests: Mutex::new(Vec::new()), + next_index: AtomicUsize::new(0), + }); + Self { state } + } + + pub fn router(&self) -> Router { + Router::new() + .route("/v1/models", get(models)) + .route("/v1/responses", post(responses)) + .route("/backend-api/plugins/featured", get(featured_plugins)) + .with_state(Arc::clone(&self.state)) + } + + pub fn requests(&self) -> Vec { + self.state.requests.lock().expect("request lock").clone() + } +} + +async fn models() -> axum::Json { + axum::Json(json!({ + "object": "list", + "data": [{"id": "mock-model", "object": "model", "owned_by": "mock"}] + })) +} + +async fn featured_plugins() -> axum::Json { + axum::Json(json!([])) +} + +async fn responses( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> axum::response::Response { + let body = match decode_json_body(&headers, &body) { + Ok(body) => body, + Err(error) => return json_response(StatusCode::BAD_REQUEST, json!({"error": error})), + }; + let request = OpenAiRequest { body }; + state + .requests + .lock() + .expect("request lock") + .push(request.clone()); + let index = state.next_index.fetch_add(1, Ordering::SeqCst); + match (state.handler)( + RequestContext { + request_index: index, + }, + request, + ) { + MockReply::Response(turn) => raw_response( + StatusCode::OK, + "text/event-stream", + sse(&turn.events(index)), + ), + MockReply::HttpError { status, body } => json_response(status, body), + MockReply::Raw { + status, + content_type, + body, + } => raw_response(status, content_type, body), + } +} diff --git a/bt-daemon/tests/support/ingest.rs b/bt-daemon/tests/support/ingest.rs new file mode 100644 index 0000000..d018c69 --- /dev/null +++ b/bt-daemon/tests/support/ingest.rs @@ -0,0 +1,160 @@ +use axum::body::Bytes; +use axum::extract::State; +use axum::http::HeaderMap; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde_json::{json, Value}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +type RowMatcher = dyn Fn(&Value) -> bool + Send + Sync + 'static; + +struct ExpectedRow { + name: String, + matcher: Arc, +} + +#[derive(Default)] +pub struct IngestScenario { + expected: Vec, +} + +impl IngestScenario { + pub fn new() -> Self { + Self::default() + } + + /// Require a row shape after all previously declared shapes. Unrelated + /// rows are ignored, so matching is independent of HTTP batching and + /// SDK-generated update rows. + pub fn expect( + mut self, + name: impl Into, + matcher: impl Fn(&Value) -> bool + Send + Sync + 'static, + ) -> Self { + self.expected.push(ExpectedRow { + name: name.into(), + matcher: Arc::new(matcher), + }); + self + } + + pub fn evaluate(&self, rows: &[Value]) -> Result<(), String> { + let mut cursor = 0; + for (matched, expected) in self.expected.iter().enumerate() { + let Some(offset) = rows[cursor..] + .iter() + .position(|row| (expected.matcher)(row)) + else { + return Err(format!( + "missing ingest shape {:?} after matching {} of {} shapes", + expected.name, + matched, + self.expected.len() + )); + }; + cursor += offset + 1; + } + Ok(()) + } +} + +#[derive(Default)] +struct CollectorState { + rows: Mutex>, + registrations: AtomicUsize, + log_requests: AtomicUsize, +} + +pub struct IngestMock { + state: Arc, +} + +impl IngestMock { + pub fn new() -> Self { + let state = Arc::new(CollectorState::default()); + Self { state } + } + + pub fn router(&self) -> Router { + Router::new() + .route("/version", get(version)) + .route("/api/apikey/login", post(login)) + .route("/api/project/register", post(register_project)) + .route("/logs3", post(logs)) + .route("/logs3/overflow", post(logs)) + .with_state(Arc::clone(&self.state)) + } + + pub fn rows(&self) -> Vec { + self.state.rows.lock().expect("trace row lock").clone() + } + + pub fn diagnostics(&self) -> String { + format!( + "project registrations: {}; log requests: {}; rows: {}", + self.state.registrations.load(Ordering::SeqCst), + self.state.log_requests.load(Ordering::SeqCst), + self.rows().len() + ) + } + + pub fn evaluate(&self, scenario: &IngestScenario) -> Result, String> { + let rows = self.rows(); + scenario.evaluate(&rows)?; + Ok(rows) + } +} + +async fn version() -> Json { + Json(json!({"logs3_payload_max_bytes": null})) +} + +async fn login() -> Json { + Json(json!({ + "org_info": [{ + "id": "mock-org", + "name": "mock", + "api_url": "unused", + "proxy_url": "unused" + }] + })) +} + +async fn register_project(State(state): State>) -> Json { + state.registrations.fetch_add(1, Ordering::SeqCst); + Json(json!({ + "project": { + "id": "00000000-0000-0000-0000-000000000001", + "name": "agent-e2e" + } + })) +} + +async fn logs( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> Json { + state.log_requests.fetch_add(1, Ordering::SeqCst); + let decoded = match headers + .get(axum::http::header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + { + Some(value) if value.split(',').any(|part| part.trim() == "gzip") => { + // The SDK currently sends uncompressed bodies in this path. Keep a + // clear failure if that changes so the collector can add decoding. + panic!("gzip-compressed Braintrust rows are not yet supported") + } + _ => body.to_vec(), + }; + let payload: Value = serde_json::from_slice(&decoded).expect("decode /logs3 body"); + if let Some(rows) = payload["rows"].as_array() { + state + .rows + .lock() + .expect("trace row lock") + .extend(rows.iter().cloned()); + } + Json(json!({})) +} diff --git a/bt-daemon/tests/support/mod.rs b/bt-daemon/tests/support/mod.rs new file mode 100644 index 0000000..12d793e --- /dev/null +++ b/bt-daemon/tests/support/mod.rs @@ -0,0 +1,7 @@ +#![allow(dead_code)] + +pub mod agent_process; +pub mod agents; +pub mod inference; +pub mod ingest; +pub mod server; diff --git a/bt-daemon/tests/support/server.rs b/bt-daemon/tests/support/server.rs new file mode 100644 index 0000000..fec25d9 --- /dev/null +++ b/bt-daemon/tests/support/server.rs @@ -0,0 +1,56 @@ +use axum::Router; +use tokio::net::TcpListener; +use tokio::sync::oneshot; + +/// Lifecycle wrapper for any ephemeral Axum test service. +pub struct TestServer { + uri: String, + shutdown: Option>, + task: Option>>, +} + +impl TestServer { + pub async fn start(router: Router) -> Self { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral test server"); + let address = listener.local_addr().expect("read test server address"); + let (shutdown, shutdown_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }) + .await + }); + Self { + uri: format!("http://{address}"), + shutdown: Some(shutdown), + task: Some(task), + } + } + + pub fn uri(&self) -> &str { + &self.uri + } + + pub async fn shutdown(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + let _ = task.await; + } + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} diff --git a/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd b/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd index 9dc9173..cb1b13f 100644 --- a/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd +++ b/src/plugins/claude/content/plugins/trace-claude-code/bin/claude-hook.cmd @@ -4,14 +4,16 @@ REM Invokes: bt daemon hook --source claude-code setlocal EnableExtensions DisableDelayedExpansion set "BT_HOOK_BIN=" -for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +for /f "delims=" %%B in ('where bt.exe 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt.cmd 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" if not defined BT_HOOK_BIN if exist "%USERPROFILE%\.local\bin\bt.exe" set "BT_HOOK_BIN=%USERPROFILE%\.local\bin\bt.exe" if not defined BT_HOOK_BIN ( echo trace-claude-code: bt CLI is unavailable; tracing disabled for this event.>&2 exit /b 0 ) -"%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 +call "%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 if errorlevel 1 ( echo trace-claude-code: a daemon-capable bt CLI is unavailable; tracing disabled for this event.>&2 exit /b 0 diff --git a/src/plugins/claude/validate.sh b/src/plugins/claude/validate.sh index 8a4b4e5..c83eca6 100755 --- a/src/plugins/claude/validate.sh +++ b/src/plugins/claude/validate.sh @@ -46,6 +46,9 @@ done grep -q "'daemon','hook','--source','claude-code'" \ "$TARGET_DIR/plugins/trace-claude-code/bin/claude-hook.cmd" \ || fail "Claude Windows hook does not invoke bt daemon" +grep -Fq 'call "%BT_HOOK_BIN%" daemon hook --help' \ + "$TARGET_DIR/plugins/trace-claude-code/bin/claude-hook.cmd" \ + || fail "Claude Windows hook does not return from bt.cmd compatibility check" grep -q 'daemon hook --source claude-code' \ "$TARGET_DIR/plugins/trace-claude-code/bin/claude-hook.sh" \ || fail "Claude Unix hook does not invoke bt daemon" diff --git a/src/plugins/codex/content/plugins/trace-codex/Makefile b/src/plugins/codex/content/plugins/trace-codex/Makefile index 047b716..f808099 100644 --- a/src/plugins/codex/content/plugins/trace-codex/Makefile +++ b/src/plugins/codex/content/plugins/trace-codex/Makefile @@ -4,5 +4,6 @@ test: @sh -n bin/codex-hook.sh @grep -q "'daemon','hook','--source','codex'" bin/codex-hook.cmd + @grep -Fq 'call "%BT_HOOK_BIN%" daemon hook --help' bin/codex-hook.cmd @jq empty hooks/hooks.json .codex-plugin/plugin.json @echo "trace-codex shim OK" diff --git a/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd b/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd index 60f192d..2996eef 100644 --- a/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd +++ b/src/plugins/codex/content/plugins/trace-codex/bin/codex-hook.cmd @@ -4,14 +4,16 @@ REM Invokes: bt daemon hook --source codex setlocal EnableExtensions DisableDelayedExpansion set "BT_HOOK_BIN=" -for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +for /f "delims=" %%B in ('where bt.exe 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt.cmd 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" +if not defined BT_HOOK_BIN for /f "delims=" %%B in ('where bt 2^>nul') do if not defined BT_HOOK_BIN set "BT_HOOK_BIN=%%B" if not defined BT_HOOK_BIN if exist "%USERPROFILE%\.local\bin\bt.exe" set "BT_HOOK_BIN=%USERPROFILE%\.local\bin\bt.exe" if not defined BT_HOOK_BIN ( echo trace-codex: bt CLI is unavailable; tracing disabled for this event.>&2 exit /b 0 ) -"%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 +call "%BT_HOOK_BIN%" daemon hook --help >nul 2>&1 if errorlevel 1 ( echo trace-codex: a daemon-capable bt CLI is unavailable; tracing disabled for this event.>&2 exit /b 0 diff --git a/src/plugins/codex/validate.sh b/src/plugins/codex/validate.sh index 577dceb..e2ef355 100755 --- a/src/plugins/codex/validate.sh +++ b/src/plugins/codex/validate.sh @@ -46,6 +46,9 @@ done grep -q "'daemon','hook','--source','codex'" \ "$TARGET_DIR/plugins/trace-codex/bin/codex-hook.cmd" \ || fail "Codex Windows hook does not invoke bt daemon" +grep -Fq 'call "%BT_HOOK_BIN%" daemon hook --help' \ + "$TARGET_DIR/plugins/trace-codex/bin/codex-hook.cmd" \ + || fail "Codex Windows hook does not return from bt.cmd compatibility check" grep -q 'daemon hook --source codex' \ "$TARGET_DIR/plugins/trace-codex/bin/codex-hook.sh" \ || fail "Codex Unix hook does not invoke bt daemon"