feat(harbor): add native Braintrust evaluation plugin

examples/README.md

@@ -69,6 +69,7 @@ Unless noted otherwise, every example below uses `braintrust.auto_instrument()`.
 | `dspy/` | DSPy `ReAct` agent with two tools (LiteLLM token metrics propagate) |
 | `evals/` | The `Eval` framework — does **not** use `auto_instrument()` |
 | `google_genai/` | Google GenAI `generate_content` against Gemini |
+| `harbor/` | Native Harbor evaluation plugin — does **not** use `auto_instrument()` |
 | `langchain/` | LangChain `prompt | model` chain — global handler installed by `auto_instrument()` |
 | `langsmith/` | Migration helper for projects coming from LangSmith — uses `setup_langsmith()` |
 | `litellm/` | LiteLLM `completion` |

examples/harbor/.env.example

@@ -0,0 +1,3 @@
+BRAINTRUST_API_KEY=
+OPENAI_API_KEY=
+HARBOR_BRAINTRUST_PROJECT=example-harbor

examples/harbor/.gitignore

@@ -0,0 +1 @@
+jobs/

examples/harbor/README.md

@@ -0,0 +1,60 @@
+# Harbor + Braintrust
+
+Runs a small, self-contained [Harbor](https://harborframework.com/) evaluation and uses Harbor's native Braintrust job plugin to sync the result. Braintrust receives a managed dataset, an experiment row for the final trial, verifier rewards, and the Harbor lifecycle and ATIF trace.
+
+The plugin is discovered automatically through Harbor's `braintrust` entry point. The Braintrust API key remains in the host process; it is not passed into the task container.
+
+## Setup
+
+Install the example's dependencies:
+
+```bash
+uv sync
+```
+
+The command below reads credentials from the repository's root `.env`. It requires:
+
+```dotenv
+BRAINTRUST_API_KEY=...
+OPENAI_API_KEY=...
+```
+
+Alternatively, copy `.env.example` to `.env` in this directory and change `--env-file ../../.env` below to `--env-file .env`.
+
+## Run
+
+Docker must be running. From this directory, run:
+
+```bash
+uv run harbor run \
+  --path task \
+  --agent terminus-2 \
+  --model openai/gpt-4.1-mini \
+  --job-name braintrust-harbor-example \
+  --jobs-dir jobs \
+  --env-file ../../.env \
+  --plugin braintrust \
+  --plugin-kwarg project_name=example-harbor \
+  --yes
+```
+
+The agent solves the task in `task/`, and Harbor's verifier emits a normalized `reward` plus an `answer_length` metric. The plugin creates `jobs/braintrust-harbor-example/braintrust-sync.json` after synchronization.
+
+Harbor also accepts plugin options through `HARBOR_BRAINTRUST_*` variables. For example, setting this in `.env` removes the need for the `project_name` plugin argument:
+
+```dotenv
+HARBOR_BRAINTRUST_PROJECT=example-harbor
+```
+
+Then omit `--plugin-kwarg project_name=example-harbor` from the command.
+
+## Backfill an existing job
+
+To synchronize the persisted job again without rerunning the agent or verifier:
+
+```bash
+uv run --env-file ../../.env python backfill.py jobs/braintrust-harbor-example \
+  --project example-harbor
+```
+
+Backfill uses the same deterministic dataset, experiment, and span identities, so it reconciles the existing Braintrust data instead of creating duplicate rows.

examples/harbor/backfill.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Backfill a persisted Harbor job into Braintrust."""
+
+import argparse
+import asyncio
+from pathlib import Path
+
+from braintrust.integrations.harbor import backfill_job
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("job_dir", type=Path, help="Persisted Harbor job directory")
+    parser.add_argument("--project", help="Braintrust project name (otherwise read from the environment)")
+    args = parser.parse_args()
+
+    options = {"project_name": args.project} if args.project else {}
+    asyncio.run(backfill_job(args.job_dir, **options))
+
+
+if __name__ == "__main__":
+    main()

examples/harbor/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "braintrust-harbor-example"
+version = "0.1.0"
+description = "Run a Harbor evaluation and sync it to Braintrust"
+requires-python = ">=3.12"
+dependencies = [
+    "braintrust",
+    "harbor==0.20.0",
+]
+
+[tool.uv.sources]
+braintrust = { path = "../../py", editable = true }

examples/harbor/task/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:24.04
+
+WORKDIR /app

examples/harbor/task/instruction.md

@@ -0,0 +1 @@
+Calculate 17 × 6. Create `/app/answer.txt` containing only the decimal result and a trailing newline.

examples/harbor/task/task.toml

@@ -0,0 +1,24 @@
+schema_version = "1.3"
+artifacts = []
+
+[metadata]
+category = "arithmetic"
+
+[verifier]
+timeout_sec = 60.0
+collect = []
+
+[verifier.env]
+
+[agent]
+timeout_sec = 300.0
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 300.0
+os = "linux"
+mcp_servers = []
+
+[environment.env]
+
+[solution.env]

examples/harbor/task/tests/test.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+answer="$(tr -d '[:space:]' < /app/answer.txt 2>/dev/null || true)"
+if [ "$answer" = "102" ]; then
+  reward=1
+else
+  reward=0
+fi
+
+printf '{"reward":%s,"answer_length":%s}\n' "$reward" "${#answer}" > /logs/verifier/reward.json

py/noxfile.py

@@ -626,6 +626,19 @@ def test_temporal(session, version):
     _run_tests(session, f"{INTEGRATION_DIR}/temporal")
 
 
+HARBOR_VERSIONS = _get_matrix_versions("harbor")
+
+
+@nox.session()
+@nox.parametrize("version", HARBOR_VERSIONS, ids=HARBOR_VERSIONS)
+def test_harbor(session, version):
+    if Version(platform.python_version()) < Version("3.12"):
+        session.skip("Harbor requires Python 3.12+")
+    _install_test_deps(session)
+    _install_matrix_dep(session, "harbor", version)
+    _run_tests(session, f"{INTEGRATION_DIR}/harbor", version=version)
+
+
 PYTEST_VERSIONS = _get_matrix_versions("pytest-matrix")
 
 

py/pyproject.toml

@@ -42,6 +42,9 @@ braintrust = "braintrust.cli.__main__:main"
 [project.entry-points.pytest11]
 braintrust = "braintrust.wrappers.pytest_plugin.plugin"
 
+[project.entry-points."harbor.plugins"]
+braintrust = "braintrust.integrations.harbor:HarborPlugin"
+
 [project.optional-dependencies]
 cli = ["boto3", "python-dotenv", "uv", "starlette", "uvicorn"]
 # TODO: remove the doc extra in the next major version.
@@ -458,6 +461,9 @@ latest = "temporalio==1.31.0"
 "1.20.0" = "temporalio==1.20.0"
 "1.19.0" = "temporalio==1.19.0"
 
+[tool.braintrust.matrix.harbor]
+latest = "harbor==0.20.0"
+
 [tool.braintrust.matrix.pytest-matrix]
 # Canonical pytest pin. The matching entry in [dependency-groups].test is
 # kept in sync by py/scripts/sync-pytest-pin.py (enforced by pre-commit).
@@ -505,6 +511,7 @@ crewai = ["crewai"]
 dspy = ["dspy"]
 google_genai = ["google-genai"]
 huggingface_hub = ["huggingface-hub"]
+harbor = ["harbor"]
 instructor = ["instructor"]
 langchain = ["langchain-core", "deepagents"]
 litellm = ["litellm"]
@@ -527,6 +534,7 @@ cohere = "cohere"
 autoevals = "autoevals"
 braintrust-core = "braintrust_core"
 boto3 = "boto3"
+harbor = "harbor"
 botocore = "botocore"
 crewai = "crewai"
 dspy = "dspy"

py/src/braintrust/integrations/harbor/__init__.py

@@ -0,0 +1,10 @@
+"""Braintrust's native Harbor job plugin.
+
+Harbor is optional. Importing this module does not import Harbor; the package is
+only required when Harbor constructs the plugin or backfill reads Harbor models.
+"""
+
+from .plugin import HarborPlugin, backfill_job
+
+
+__all__ = ["HarborPlugin", "backfill_job"]