From f19fa01293a3c68f12e50889a2db738a0cdad23a Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 30 Jul 2026 18:36:22 -0400 Subject: [PATCH 1/5] plan docs --- docs/harbor-braintrust-plugin-design.md | 651 +++++++++++++ docs/harbor-framework-research.md | 1165 +++++++++++++++++++++++ 2 files changed, 1816 insertions(+) create mode 100644 docs/harbor-braintrust-plugin-design.md create mode 100644 docs/harbor-framework-research.md diff --git a/docs/harbor-braintrust-plugin-design.md b/docs/harbor-braintrust-plugin-design.md new file mode 100644 index 000000000..0f2531b8b --- /dev/null +++ b/docs/harbor-braintrust-plugin-design.md @@ -0,0 +1,651 @@ +# Design: Braintrust Plugin for Harbor + +Status: proposal +Harbor: [`harbor==0.20.0` at `e76f7e3`](https://github.com/harbor-framework/harbor/tree/e76f7e32f5644fb9f648cd23151aac5c67492ea0) +Background: [`harbor-framework-research.md`](harbor-framework-research.md) +Contract: [`braintrustdata/braintrust-spec`](https://github.com/braintrustdata/braintrust-spec/tree/main/skills/instrumentation-spec), especially its [instrumentation guide](https://github.com/braintrustdata/braintrust-spec/blob/main/skills/instrumentation-spec/references/instrumentation-guide.md) and [eval-span spec](https://github.com/braintrustdata/braintrust-spec/blob/main/skills/instrumentation-spec/references/features/eval-spans.md) + +## Summary + +Build a native Harbor job plugin, not an OTLP adapter. Harbor remains responsible for execution, retries, sandboxing, and verification; Braintrust provides datasets, experiments, comparisons, and trace analysis. + +```text +Harbor dataset/task selection → Braintrust Dataset +Harbor eval group → Braintrust Experiment +Harbor final trial → Braintrust root eval span +Harbor verifier rewards → Braintrust scores or metrics +Harbor verifier labels → Braintrust classifications +Harbor lifecycle + ATIF → child spans +Harbor JobResult → final reconciliation + job summary +``` + +### Key decisions + +1. Construct eval traces with Braintrust SDK primitives; do not call `braintrust.Eval()`. Harbor already owns the eval loop. +2. Within each Harbor job, create one experiment per dataset and semantic system variant. Resume and backfill update those same experiments rather than creating new ones. +3. Sync the exact resolved task set to a Braintrust dataset by default. +4. Treat Harbor rewards as authoritative. Only values known to be normalized and higher-is-better become scores; other numeric values become metrics. Always retain raw rewards. +5. Make the trial the root `eval` span. Import lifecycle and conforming ATIF spans beneath its canonical `task` child. +6. Route hook events through explicit job and per-trial state machines. Reconcile from the final `JobResult`; retried failures must not become experiment rows. +7. Use deterministic IDs and names so resume and backfill converge without duplicates. +8. Keep Braintrust credentials on the host. ATIF import does not require sandbox credentials. +9. Isolate plugin failures by default; `strict=True` is opt-in. +10. Preserve user metadata under a plugin-owned namespace after normalization and redaction. + +## Scope + +The plugin should: + +- log every final Harbor trial as one Braintrust eval row; +- support multi-dataset, multi-agent, multi-model, repeated-attempt, resume, and regrade jobs; +- preserve all verifier rewards without clamping or silent coercion; +- associate rows with stable task records and dataset versions; +- import useful lifecycle and ATIF detail when available; +- support online use and offline backfill from a Harbor job directory; +- avoid blocking Harbor's event loop or changing benchmark behavior. + +Version 1 does not need to: + +- replace Harbor's verifier or aggregate metric system; +- reproduce arbitrary Harbor `metric.py` or pass@k logic in Braintrust summaries; +- inject credentials or tracing SDKs into agent containers; +- upload arbitrary workspace artifacts; +- deduplicate separately instrumented native agent traces; +- depend on Harbor's MLflow-specific OTel uploader. + +## Data model + +| Harbor | Braintrust | Notes | +|---|---|---| +| Job | Shared metadata + optional project-log summary | A job may create several experiments. | +| Resolved dataset/task set | Dataset | Scope is the exact logical task selection. | +| Task | Dataset record | Deterministic ID and canonical task input. | +| Eval group/variant | Experiment | Partition by dataset and semantic agent config. | +| Final trial | Root `eval` span named `eval` | One experiment row. | +| Trial execution | Direct child `task` span named `task` | Canonical eval wrapper. | +| Setup, agent, verifier phases | Nested `task` spans | Use Harbor's recorded timestamps. | +| Identifiable model call | `llm` span | Only if it satisfies the instrumentation contract. | +| Tool execution | `tool` span | Correlate call arguments with its observation. | +| Ambiguous ATIF step/subagent | `task` span/tree | Never mislabel partial data as an LLM/tool call. | +| Normalized reward | Direct score child with `purpose="scorer"` | Score must be in `[0, 1]` and higher-is-better. | +| Arbitrary numeric reward | Eval-root metric | Preserve original value in metadata too. | +| Structured verifier label | Root classification + classifier child | Do not coerce labels to numbers. | +| Exception | Error on root and canonical task | Omit output on both. | +| Regrade | New experiment | Link the source experiment as base when exact. | + +## Identity and partitioning + +### Experiment partitions + +A Harbor job may combine datasets and system variants. Mixing all rows into one experiment would blend score summaries and weaken comparisons. Partition each job by: + +```text +(dataset identity, normalized agent semantic config, resolved skill digests) +``` + +The semantic config includes agent name/import path, model, kwargs, MCP configuration, resume behavior, non-secret environment values, sensitive-value templates, and resolved skill digests. It excludes execution controls such as concurrency, logging, output paths, and retry policy. + +Prefer the resolved Harbor lock over raw config because it includes resolved task and skill content. Never include secrets in a fingerprint or name. + +```text +partition_key = sha256(dataset_key + normalized_semantic_config + skill_digests) +experiment = · @ · · +``` + +Include the Harbor job ID in the deterministic internal name so a reused display name cannot update an unrelated experiment. Initialize with `update=True` to support resume and backfill. + +### Stable IDs + +```text +dataset record ID = UUIDv5(plugin namespace, dataset scope + logical task key) +eval root ID = Harbor TrialResult.id +child span ID = UUIDv5(trial ID, semantic child path) +``` + +Examples of semantic paths: + +```text +task/environment_setup +task/agent_execution/turn/0/llm/2 +task/agent_execution/tool/call_123 +task/verification +scorer/reward +``` + +IDs must depend on semantics, not import time or traversal order where a stable source identifier exists. + +## Datasets + +Dataset sync is enabled by default because it enables task-level comparison, provenance, and reruns. + +### Modes + +| Mode | Behavior | +|---|---| +| `sync` | Upsert a managed dataset and associate its version with each experiment. Default. | +| `none` | Log stable inputs without creating a dataset. | +| `existing` | Map to a user-provided dataset. Deferred until record matching is specified. | + +### Dataset scope and records + +Use one dataset per Harbor source and exact logical task selection: + +```text +harbor · · tasks- +``` + +Hash logical keys, not content. A content change then creates a new version of the same logical dataset; a different selected subset uses a different dataset and cannot prune another run's records. + +Choose a logical task key in this order: + +1. published package identity; +2. Git repository plus relative path; +3. Harbor task name plus source; +4. normalized local relative identity. + +Do not expose absolute paths. + +The record input contains task-authored semantics, not run-specific agent settings: + +```json +{"task": "terminal-bench/example", "instruction": "..."} +``` + +For multi-step tasks: + +```json +{"task": "org/task", "steps": [{"name": "build", "instruction": "..."}]} +``` + +Harbor has no universal expected output. Keep `expected` explicitly `null` unless a configured adapter exposes a safe expected value. Never use solution or verifier implementation files as expected output. + +Record metadata should include stable source identity, task version/digest, schema version, resource requirements, and normalized user task metadata. Exclude secrets, absolute paths, full Compose files, solutions, verifiers, and arbitrary artifacts. + +### Sync sequence + +1. Initialize the dataset with `use_output=False`. +2. Upsert all resolved records with deterministic IDs. +3. Flush and fetch rows to resolve dataset version and record transaction metadata. +4. Initialize each experiment with its dataset. +5. Add dataset origin to each root eval span using the same shape as Braintrust's eval runner. + +Hide origin plumbing behind an adapter and test it against the supported Braintrust SDK version. + +## Eval trace + +### Required shape + +```text +eval [eval] +├── task [task] +│ ├── environment_setup [task] +│ ├── agent_setup [task] +│ ├── agent_execution [task] +│ │ └── ordered ATIF task/llm/tool tree +│ └── verification [task] +├── reward [score, purpose=scorer] +└── category [classifier, purpose=scorer] +``` + +For multi-step trials, add `step:` task spans under the canonical task. Keep score/classifier spans direct children of the eval root. + +Create spans with explicit parent objects, `set_current=False`, and recorded timestamps. Harbor runs trials concurrently, so do not rely on context-manager parenting. + +Every span should carry Braintrust span-origin context identifying the Harbor integration and plugin version. Use the reserved `braintrust.plugin.harbor` identity only when Braintrust owns the plugin. + +### Root and canonical task + +The root and direct `task` child must have identical: + +- canonical dataset input; +- expected value, including explicit JSON null; +- bounded final output, or the same error with output omitted. + +Choose output in this order: + +1. a standardized answer from agent metadata; +2. the final non-copied ATIF agent message; +3. per-step final messages; +4. a small completion/status object. + +Do not put a full trajectory or artifact manifest in output. Put actual run-specific instructions on `agent_execution.input` and in metadata, not in dataset input. + +If `TrialResult.exception_info` exists, log `: ` as the root and task error. Tracebacks are opt-in metadata or attachments. A missing reward without an exception is not a zero; record it as an unevaluated warning. + +### Metadata + +Use a collision-safe namespace: + +```json +{ + "harbor": { + "job_id": "...", + "trial_id": "...", + "task_name": "...", + "agent": "...", + "model": "...", + "attempt_index": 0, + "retry_index": 0, + "raw_rewards": {}, + "trajectory": {"present": true, "schema_version": "ATIF-v1.7"}, + "custom": {"job": {}, "task": {}, "trial": {}} + } +} +``` + +Preserve explicitly supplied user metadata at the narrowest matching scope: + +- job metadata → experiment and eval root; +- task metadata → dataset record and eval root; +- trial metadata → eval root; +- permitted ATIF root extras → eval-root trajectory metadata. + +Normalize JSON consistently across live sync and backfill. Preserve keys, nesting, types, and explicit nulls, but apply secret-key filtering, regex redaction, path filtering, and depth/size limits. Record dropped paths as warnings. Never merge user fields into plugin-owned keys or copy arbitrary metadata to conforming LLM/tool leaves. + +## Rewards and classifications + +Harbor rewards are arbitrary numbers. Braintrust scores are normalized, higher-is-better values in `[0, 1]`; Braintrust metrics may be arbitrary numbers. Range alone does not establish score semantics. + +### Reward classification + +For each reward key, apply this precedence: + +1. Exact `reward_rules` entry. +2. Configured `score_keys` or `metric_keys` glob. Reject overlaps at initialization. +3. The conventional key `reward` is a score only when it is in `[0, 1]`. +4. An invalid configured score follows `invalid_score_policy` (`metric` by default). +5. Every other numeric reward is a metric, even if its value happens to be in `[0, 1]`. +6. Preserve the complete original dictionary in `metadata.harbor.raw_rewards`. + +Never clamp by default or infer direction from names. Exact rules may define explicit normalization: + +```json +{ + "correctness": {"type": "score", "direction": "maximize"}, + "error_rate": { + "type": "score", + "direction": "minimize", + "min": 0, + "max": 1, + "score_name": "error_rate_score" + }, + "runtime_sec": {"type": "metric"} +} +``` + +For finite configured bounds: + +```text +maximize = (value - min) / (max - min) +minimize = (max - value) / (max - min) +``` + +Keep a transformed source value as `harbor_reward.raw.` metric as well as in raw metadata. If a Harbor reward name collides with a Braintrust standard metric but has different semantics, emit it as `harbor_reward.`. Read reserved metric names from the SDK/backend source of truth rather than duplicating a snapshot here. + +For every score, create a direct child span with: + +```text +name = score name +type = score +purpose = scorer +scores = {: } +``` + +Do not duplicate scores on the eval root. Eval metrics stay on the root. Do not invent a `success` score; users may configure one explicitly. + +### Classifications + +Structured categorical verifier outcomes become Braintrust classifications, never numeric surrogates. Each item has a string `id`, optional `label`, and optional JSON metadata. Preserve source order and duplicates. + +Create one direct `classifier` child per source classifier with `purpose="scorer"`, then log the grouped, non-empty classification dictionary on the root. Omit the root field when no valid classifications exist. + +Validate atomically per source classifier: one malformed item fails that classifier only. Log its span error and add an eval-root warning; continue syncing numeric rewards and other classifiers. + +Version 1 should only read labels from known Harbor adapters or explicit `classifier_rules` pointing to documented JSON paths. Do not infer labels from arbitrary metadata or files. + +### Reward details and aggregates + +If `reward-details.json` exists, put a bounded summary in the scorer output and optionally attach the complete JSON. Do not create criterion-level or judge-LLM spans until Rewardkit has a stable, tested mapping. + +Harbor's job metrics, custom `metric.py`, and pass@k remain authoritative aggregate results. Do not create a synthetic eval row for aggregates. Optionally write one project-log trace, `harbor.job.summary`, containing exact Harbor aggregates and links to partition experiments. + +## ATIF import + +Default to host-side ATIF import. It works across built-in agents, supports backfill, and keeps credentials out of sandboxes. + +Modes: + +| Mode | Behavior | +|---|---| +| `atif` | Import conforming ATIF detail. Default. | +| `summary` | Lifecycle and aggregate trajectory metadata only. | +| `native` | Skip ATIF because the agent is instrumented elsewhere. | + +Do not attempt native/ATIF deduplication in version 1. + +### Conformance gate + +Imported `llm` and `tool` leaves are Braintrust instrumentation and must satisfy the linked instrumentation guide. This document does not duplicate that contract. + +An ATIF step may be an `llm` span only when it represents exactly one model call and the converter can provide required identity, canonical input/output, timing, and tool configuration. Otherwise emit a `task` summary with a warning. In particular: + +- `llm_call_count == 0` is deterministic work; +- unknown or multiple calls are not one LLM call unless ATIF exposes each call; +- a streaming call without measurable time-to-first-token is downgraded; +- redaction that removes required payload content also downgrades the leaf. + +A tool span requires call arguments and a correlated result or error. Preserve call IDs across model output, tool spans, and tool-result messages. Preserve execution order: model → tool → model. Subagents become nested task trees. + +Normalize non-Anthropic/Google providers to OpenAI Chat Completions payloads. Preserve Anthropic or Google native payloads only with the matching provider identity. Use provider operation names for spans, not model names. Tool definitions belong in LLM metadata, not the message list. + +Never copy arbitrary ATIF/provider fields to leaf metadata. Required `model` and `provider`, request controls, canonical tool fields, approved prompt provenance, and allowed metrics are sufficient. Keep unsupported values in eval-root reconciliation metadata. + +### Usage, cost, and timing + +Leaf LLM spans are authoritative calls. Normalize only metrics allowed by both the Braintrust backend and instrumentation guide. Important mappings include: + +```text +input/prompt tokens → prompt_tokens +output/completion tokens → completion_tokens +total tokens → tokens +cache reads → prompt_cached_tokens +cache writes → prompt_cache_creation_tokens +reasoning tokens → completion_reasoning_tokens +first-token milliseconds → time_to_first_token (seconds) +per-call cost → estimated_cost +``` + +Do not emit aliases together or add subset counts to totals. Omit unknown values rather than fabricating zero. Counts must be non-negative integers; costs and durations must be finite and non-negative. + +Use explicit span timestamps rather than duplicating duration metrics. Exact aggregate usage may appear on `agent_execution`; otherwise keep it in eval-root Harbor metadata. Keep Harbor totals there for reconciliation even when leaf detail exists. + +For missing ATIF timestamps, use valid values within the agent phase, clamp outliers, and interpolate missing values monotonically. Record repairs on the eval root, not on conforming leaves. Never emit epoch timestamps as a fallback. + +### Content policy + +| Mode | Captured | +|---|---| +| `metadata` | Structure, timing, usage, and scores; detailed leaves become task summaries. | +| `messages` | Canonical model messages and tool inputs/results. Default. | +| `full` | `messages` plus fields explicitly allowed by the instrumentation contract. | + +All modes support byte limits, sensitive-key filtering, regex redaction, and reasoning exclusion. A policy must not leave an `llm` or `tool` label on a materially incomplete payload. + +Convert inline media to Braintrust attachment references in place. If conversion/upload fails, preserve the original payload unless privacy policy requires removal; in that case downgrade the leaf rather than partially rewriting it. + +## Hook state machines, retries, and resume + +Use a small reducer-based state machine rather than independent hook callbacks mutating shared dictionaries. Maintain one job machine and one trial machine per stable logical trial identity. Use `trial_name` only if Harbor guarantees that it is unique within the job; otherwise derive the identity from the resolved trial plan. + +### Job machine + +```text +NEW → INITIALIZING → ACTIVE → RECONCILING → CLOSED +any nonterminal state ──unrecoverable plugin error──→ DISABLED | FAILED +``` + +Unrecoverable initialization or dispatcher failures transition to `DISABLED` by default and preserve diagnostics. A single malformed event or effect records a warning and leaves the machine active. In strict mode, unrecoverable failures transition to `FAILED` and raise where Harbor permits. `on_job_end` first moves the machine to `RECONCILING`, stops accepting live events, drains in-flight effects, and then closes it. + +### Trial machine + +`n_attempts` creates separate logical trial machines and experiment rows. Execution retries are successive attempts within one machine and produce only one final row. + +```text +PENDING ──START──→ ACTIVE(phase=started, retry=n) +ACTIVE ──phase event──→ ACTIVE(next phase) +ACTIVE ──END, retry predicted──→ WAITING_RETRY +ACTIVE ──END, otherwise────────→ FINAL_CANDIDATE +ACTIVE ──CANCEL────────────────→ CANCELLED +WAITING_RETRY or FINAL_CANDIDATE ──START──→ ACTIVE(attempt=n+1) + +any nonterminal state ──final JobResult contains trial──→ FINALIZING → SYNCED +any nonterminal state ──final JobResult omits trial─────→ OMITTED +``` + +The nested active phases are monotonic: + +```text +started → environment → agent → agent_done → verification +``` + +A failure may skip phases, so `END` and `CANCEL` are legal from any active phase. Duplicate events are no-ops. A backward or otherwise illegal transition records a warning instead of raising unless strict mode is enabled. + +Harbor 0.20 emits `END` before the queue decides whether to retry. The reducer may use public retry config to classify the next state, but `FINAL_CANDIDATE` is deliberately nonterminal: a later `START` invalidates the candidate and begins the next attempt. + +Do not upload candidate eval rows to the main experiment by default. Stage their result references, then dispatch authoritative final-result events from `JobResult.trial_results` during reconciliation. This guarantees that Braintrust receives exactly Harbor's retained rows. Optional retry-attempt traces may be emitted to project logs because they cannot affect experiment counts. + +Implement transitions as a pure function: + +```text +(state, event) → (new_state, effects) +``` + +Effects include staging a result, recording retry/cancellation diagnostics, logging an operational attempt, syncing a final row, and closing an omitted trial. Serialize transitions per logical trial with an `asyncio.Lock`, then run I/O through an ordered per-trial effect queue outside that lock. Different trials may progress concurrently. + +At reconciliation, sync all final results, including trials loaded by `harbor job resume`. Set root `metrics.retries` from the machine's completed execution attempts when known; intentional `n_attempts` remain separate rows. Persist terminal state, retry count, and warnings in `braintrust-sync.json` so backfill can resume safely. + +Harbor should eventually add `retry_index` and `is_final_attempt` to trial events, or emit a dedicated final-attempt event. That would allow safe live row sync without changing the state-machine contract. + +## Regrades + +A regrade creates new experiments and preserves source output/trajectory. Link each partition to its exact source experiment as a Braintrust base using, in order: + +1. explicit base experiment ID/mapping; +2. source IDs from `braintrust-sync.json`; +3. deterministic source experiment identity. + +Never use fuzzy name matching. If no exact source exists, continue without a base and record a warning. Mark recorded source cost as not incurred by the regrade. + +## Attachments, privacy, and failure handling + +Default attachment policy: + +- include bounded `reward-details.json`; +- include an artifact-manifest summary in metadata; +- exclude all other trial files. + +Optional modes may include trajectory, redacted config/lock, or allowlisted artifacts. Enforce per-file and total limits. Attach values in semantically named output fields using Braintrust's attachment representation; do not invent attachment metadata fields. Read soon-to-be-deleted files into memory before constructing attachments. + +Host-only credential rules: + +- read Braintrust credentials from the Harbor host; +- never inject or persist them in agent/task config, locks, results, metadata, or manifests; +- redact secret-like keys and user-configured patterns; +- do not upload absolute paths, source, shell output, reasoning, or artifacts outside the selected content policy. + +Default failures are isolated: + +| Failure | Behavior | +|---|---| +| Initialization/auth | Disable sync with a clear error; raise only in strict mode. | +| Hook callback | Catch, record, continue. | +| Dataset sync | Fall back to unassociated experiment rows when possible. | +| Malformed ATIF | Keep the eval/lifecycle trace without detailed leaves. | +| Auxiliary attachment | Omit it and record a warning. | +| Flush | Record a local error for later backfill. | + +Write `/braintrust-sync.json` with plugin version, job/project identity, experiment and dataset IDs, synced trial IDs, errors, and completion state. Never include credentials. This manifest supports diagnosis, exact regrade linking, and idempotent backfill. + +## Plugin API and packaging + +Suggested constructor: + +```python +BraintrustPlugin( + project_name=None, + project_id=None, + experiment_prefix=None, + base_experiment_name=None, + base_experiment_id=None, + dataset_mode="sync", + dataset_name=None, + trajectory_mode="atif", + content_mode="messages", + include_custom_metadata=True, + max_custom_metadata_bytes=100_000, + score_keys=None, + metric_keys=None, + reward_rules=None, + classifier_rules=None, + invalid_score_policy="metric", + include_tracebacks=False, + attachments="verifier-details", + artifact_include=None, + max_attachment_bytes=5_000_000, + max_total_attachment_bytes=20_000_000, + max_content_bytes=20_000, + log_job_summary=True, + log_retry_attempts=False, + strict=False, +) +``` + +Constructor options should have `HARBOR_BRAINTRUST_*` environment fallbacks, with precedence `explicit option > environment > default`. Standard Braintrust variables continue to configure API key, organization, and app URL. Complex rule values should use JSON because Harbor only supports `--plugin-kwarg` when one plugin is supplied. Validate mutually exclusive identifiers, overlapping reward rules, globs, bounds, byte limits, and mode combinations before performing network I/O. + +Ship the integration in the Braintrust distribution under `py/src/braintrust/integrations/harbor/` and register `braintrust.integrations.harbor:BraintrustPlugin` in the `harbor.plugins` entry-point group. Harbor stays optional: importing Braintrust without Harbor installed must continue to work, and the integration must not raise Braintrust's minimum Python version. Test supported Harbor releases through a dedicated, explicitly pinned version matrix. + +Keep lifecycle orchestration, compatibility access, state reduction, dataset/eval conversion, ATIF conversion, reward handling, attachments, and identity generation behind clear internal boundaries. The internal file layout is an implementation choice rather than part of this design. + +Online sync and backfill must share the same normalization, partitioning, ID generation, reward classification, ATIF conversion, and persistence core. They should differ only in how Harbor events and final results are obtained. Keep the public surface minimal until the backfill and extension use cases establish which helpers need to be supported APIs. + +### Harbor compatibility boundary + +Prefer public job identity/config/directory, hook registration, event fields, and final `JobResult`. Isolate Harbor-version compatibility behind one internal boundary. Until Harbor exposes resolved plans publicly, that boundary may feature-detect read-only access to: + +- `job._task_configs`; +- `job._task_download_results`; +- `job._trial_configs`. + +Do not mutate private fields or depend on queue, metrics, progress, existing-trial, or lock internals. Prefer persisted `lock.json` once available and pin every tested Harbor version. + +## Lifecycle + +### `on_job_start` + +1. Dispatch job `INITIALIZE`. +2. Validate config/auth and read resolved tasks, lock, and custom metadata. +3. Normalize metadata, build partitions, sync datasets, and initialize experiments. +4. Register all trial hooks with one thin dispatcher. +5. Dispatch job `READY`; optionally start a project-log job trace. + +Keep blocking SDK and filesystem work off Harbor's event loop. Preserve per-trial effect ordering while allowing independent trials to make progress concurrently. + +### Hook dispatcher + +Subscribe to `START`, `ENVIRONMENT_START`, `AGENT_START`, `AGENT_END`, `VERIFICATION_START`, `END`, and `CANCEL`. Each callback only converts Harbor data to a typed state-machine event and dispatches it. It must not contain retry, synchronization, or transition logic. + +The dispatcher catches callback errors because Harbor awaits hooks. State transitions remain cheap and synchronous; generated I/O effects run outside the per-trial lock. + +### `on_job_end` + +1. Dispatch job `RECONCILE` and drain in-flight hook effects. +2. Dispatch one authoritative `FINAL_RESULT` event for every `JobResult.trial_results` entry. +3. Mark remaining nonterminal trial machines `OMITTED` and drain final effects. +4. Verify expected root IDs and required metadata. +5. Write the optional aggregate summary, flush, and persist machine state. +6. Dispatch job `CLOSE` and report collected errors. + +Harbor currently swallows finalizer errors, so fully strict end-of-job failure may require an upstream API change. + +## Testing strategy + +Follow [`docs/vcr-testing.md`](vcr-testing.md): real SDKs and recorded traffic are the default; mocks are a narrow exception. The primary test should exercise a real Harbor result through the real Braintrust SDK, not a hand-built `TrialResult` through a fake logger. + +### Coverage layers + +| Layer | Input | Braintrust side | Purpose | +|---|---|---|---| +| Pure contracts | Real Harbor/Pydantic values or JSON fixtures | No network | Reducers, IDs, reward rules, normalization, redaction. | +| Replay integration | Recorded Harbor job + hook stream | VCR-replayed Braintrust HTTP | Primary CI coverage. | +| Real Harbor smoke | Tiny real Harbor job, preferably local Docker | VCR-replayed Braintrust HTTP | Plugin discovery and actual hook ordering. | +| Live backend | Same recorded jobs | Real Braintrust project | Small opt-in round-trip suite. | + +#### 1. Pure contract tests + +Use table-driven tests for deterministic code: partitioning, IDs, metadata, rewards, classifications, ATIF conversion, timestamp repair, attachment policy, and state transitions. Construct typed events and run the reducer directly; do not mock Harbor callback methods or Braintrust clients. + +Pure tests are appropriate here because there is no external protocol to record. Prefer loading real serialized Harbor models and ATIF documents over `MagicMock`, ad hoc objects, or invented provider responses. + +#### 2. Recorded Harbor inputs + Braintrust VCR + +This is the main test path. Capture small, sanitized outputs from real Harbor runs: resolved config and lock data, final results, trajectories, verifier details, and an ordered event journal ending at the final `JobResult` boundary. Version recordings when Harbor's serialized models or event behavior differ across the supported matrix. + +The event journal is the equivalent of a transport cassette: it replays a non-HTTP protocol captured from the real system. Replay it through the actual hook dispatcher and state machines; do not replace it with a fake Harbor job. + +Use the repository's VCR marker and shared recording infrastructure for sync and backfill tests. Let the real Braintrust SDK perform authentication/project lookup, dataset upserts/fetches, experiment creation, span/attachment uploads, and summary queries. Select HTTP recordings by the Harbor version under test. + +Use deterministic job, dataset, experiment, record, and span IDs so request bodies replay reliably. Match write requests on body as well as method/path where practical; canonicalize only fields proven to be volatile. After flush, query the experiment through the Braintrust API and assert the persisted rows, origins, hierarchy, scores, metrics, classifications, and attachments. In-memory span assertions may supplement this round trip, but should not replace it. + +Follow the repository record modes: + +- local: `once`; +- CI: `none`; +- intentional focused re-record: `--vcr-record=all -k `. + +The existing VCR config removes Braintrust authentication headers. Add Harbor-specific request-body redaction and scan recorded jobs/cassettes for secrets, absolute paths, and user content before commit; header filtering alone is insufficient. + +#### 3. Real Harbor smoke tests + +Replay proves deterministic behavior but not that the plugin still attaches to Harbor correctly. Run at least one tiny real Harbor job using the installed Harbor package and entry point. Prefer a real built-in deterministic path such as an oracle task over a fake job object. Assert the observed hook sequence, final `JobResult`, sync manifest, and VCR-backed Braintrust result. Because live-run IDs and timestamps vary, this smoke test may use normal endpoint matching plus local payload assertions; keep strict write-body matching in the primary recorded-job tests. + +Keep Docker-dependent smoke coverage in a dedicated Linux CI job if it is not portable across the normal SDK matrix. A second opt-in recording job can use a real built-in model agent to refresh the ATIF fixture; provider calls made inside Harbor's sandbox are not intercepted by host-side VCR. + +#### 4. Live Braintrust tests + +VCR cannot detect every backend contract change because it replays old responses. Keep a small opt-in suite that uploads one recorded successful job and verifies dataset version/origin links, experiment rows, attachments, summaries, base-experiment linkage, resume, and backfill idempotency through public Braintrust APIs. Use deterministic names in a dedicated test project and clean up when supported. + +### Required scenarios + +Keep recordings few but semantically dense: + +1. successful single-step trial with LLM → tool → LLM ATIF, multiple rewards, metadata, and an attachment; +2. exception and verifier-disabled/missing-trajectory results; +3. retry followed by success, cancellation, duplicate hook delivery, and final reconciliation; +4. multiple dataset/agent partitions and intentional `n_attempts`; +5. multi-step and subagent trajectories; +6. regrade with exact base-experiment resolution; +7. malformed ATIF/classification data that degrades locally without losing the eval row. + +Pure transition tests should enumerate skipped phases, illegal/backward events, duplicates, concurrent trials, retries, cancellation, `FINAL_RESULT`, and `OMITTED`. The recorded retry scenario remains the primary lifecycle proof. + +### Mock policy + +Do not use mocks/fakes as primary coverage for: + +- Harbor hook ordering or retry behavior; +- `TrialResult`, `JobResult`, or ATIF response shape; +- Braintrust dataset/experiment APIs; +- emitted span shape derived from a real Harbor result; +- attachment upload or resume/backfill behavior. + +Narrow monkeypatching is acceptable only for failures that cannot be recorded safely or deterministically, such as a disk write failure, task cancellation at an exact await point, or transport interruption. Keep those tests supplemental and patch the smallest effect boundary, not the Harbor or Braintrust object graph. + +### Version-matrix and recording wiring + +Add Harbor to the dependency version matrix, register its versioned recordings with the repository's shared cassette infrastructure, and add a dedicated parametrized nox session. Harbor requires Python 3.12+, so CI should schedule that session only on supported interpreters. + +Use the nox session for playback and intentional focused re-recording so the selected Harbor version also selects compatible recordings. CI must replay with `record_mode="none"`; a missing recording is a failure, not permission to fall back to a fake. + +## Delivery + +1. **Eval core:** plugin entry point, config, partitioned experiments, dataset sync, final eval rows, rewards/classifications, lifecycle spans, reconciliation, manifest, and recorded/VCR coverage. +2. **ATIF:** conforming model/tool/subagent conversion, content policy, timestamp repair, and attachment handling. +3. **Advanced Harbor semantics:** multi-step detail, regrade bases, retry operational traces, exact job summaries, and selected artifacts. +4. **Upstream improvements:** public resolved plans, final-attempt event metadata, plugin-scoped YAML config, health reporting, and generic trace-parent propagation. + +## Expected user experience + +```bash +pip install harbor braintrust +export BRAINTRUST_API_KEY=... +export HARBOR_BRAINTRUST_PROJECT=agent-benchmarks + +harbor run \ + -d terminal-bench/terminal-bench-2@latest \ + -a claude-code \ + -m anthropic/claude-sonnet-4-6 \ + -n 32 \ + --plugin braintrust +``` + +Braintrust should show the exact selected dataset, one experiment per system variant, one row per final trial, faithful scores/metrics/classifications, nested lifecycle and ATIF traces, useful usage/error slices, and stable links back to Harbor. Resume and backfill should update the same objects rather than duplicate them. diff --git a/docs/harbor-framework-research.md b/docs/harbor-framework-research.md new file mode 100644 index 000000000..e42ba3556 --- /dev/null +++ b/docs/harbor-framework-research.md @@ -0,0 +1,1165 @@ +# Harbor Framework Research: Evals, Tracing, and Configuration + +Research date: 2026-07-29 +Upstream repository: [`harbor-framework/harbor`](https://github.com/harbor-framework/harbor) +Inspected commit: [`e76f7e32f5644fb9f648cd23151aac5c67492ea0`](https://github.com/harbor-framework/harbor/tree/e76f7e32f5644fb9f648cd23151aac5c67492ea0) +Inspected package version: `harbor==0.20.0` + +## Executive summary + +Harbor is primarily a **sandboxed agent evaluation harness**, not an LLM tracing SDK. Its central abstraction is: + +> A job expands tasks, agents, models, and attempts into trials; each trial runs an agent in a container, runs a verifier, emits one or more numeric rewards, and preserves the execution record on disk. + +The core data flow is: + +```text +dataset(s) / task(s) + + +agent configuration(s) + + +runtime environment configuration + ↓ +JobConfig → resolved JobPlan/JobLock + ↓ expands attempts × tasks × agents +TrialConfig[] + ↓ concurrent execution +sandbox setup → agent setup → agent run → artifact collection → verifier + ↓ +trajectory.json + reward.json/reward.txt + result.json + lock.json + ↓ +job-level metrics, pass@k, viewer, Hub, trace exporters, plugins +``` + +The most important conclusions are: + +1. **Evals are task/verifier based.** A task contains an instruction, a container environment, and a test script. The verifier script decides the reward by writing `/logs/verifier/reward.json` or `reward.txt`. +2. **Trials are the unit of execution and scoring.** A job is a matrix of trials, normally `n_attempts × tasks × agents`. +3. **Configuration has three distinct layers:** task-owned TOML, run/job YAML or JSON, and CLI overrides. Resolved config and content-addressed locks are persisted for reproducibility. +4. **Tracing is file-first.** Integrated agents write an Agent Trajectory Interchange Format (ATIF) file at `agent/trajectory.json`. Harbor later visualizes or exports that file. +5. **ATIF is richer than a basic chat log.** It covers reasoning, tool calls, observations, token/cost metrics, multimodal content, continuations, copied context, and embedded or referenced subagent trajectories. +6. **Harbor has two separate trace export paths:** + - ATIF → Hugging Face conversational datasets for SFT. + - ATIF → OpenTelemetry/OpenInference spans through the separate `harbor-atif2otel` package. +7. **The current OTel uploader is not backend-neutral in practice.** Conversion is generic, but direct upload is hard-wired to MLflow APIs and headers. +8. **The LangSmith plugin is the best model for a Braintrust integration.** It synchronizes datasets/examples, represents trial lifecycle phases, records usage, and publishes rewards as feedback. +9. **A Braintrust integration should be a native Harbor job plugin plus ATIF ingestion**, rather than only pointing Harbor's current OTel uploader at Braintrust. + +## 1. What Harbor is + +Harbor describes itself as a framework for evaluating and optimizing agents and language models in sandboxed environments. It supports built-in and custom agents, local Docker and cloud sandbox providers, task/dataset publishing, RL rollout generation, SFT export, and a local/hosted results viewer. + +Relevant upstream sources: + +- [README](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/README.md) +- [Core concepts](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/core-concepts.mdx) +- [`JobConfig`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/job/config.py) +- [`TrialConfig`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/trial/config.py) + +### Core concepts + +| Concept | Meaning in Harbor | +|---|---| +| **Task** | One instruction, sandbox definition, and verifier/test implementation. | +| **Dataset** | A collection of tasks, optionally with custom aggregate metrics. | +| **Agent** | A built-in or custom implementation that operates in the sandbox. | +| **Environment** | A local or remote container runtime implementing `BaseEnvironment`. | +| **Trial** | One agent attempt on one task. Conceptually, a rollout that produces a reward. | +| **Job** | A collection of trials, often spanning datasets, agents, models, and repeated attempts. | +| **Reward** | Per-trial numeric verifier output, potentially multi-dimensional. | +| **Metric** | Job/dataset-level aggregation over trial reward dictionaries. | +| **Trajectory** | The agent interaction history, normally stored as ATIF JSON. | + +Harbor is deliberately agent- and environment-agnostic. Built-in factories lazily resolve agent names and environment provider types, while custom implementations can be supplied as `module.path:ClassName` import paths. + +## 2. How an evaluation works + +### 2.1 Define or select tasks + +A local task typically has this shape: + +```text +my-task/ +├── instruction.md +├── task.toml +├── environment/ +│ ├── Dockerfile +│ └── ... +├── solution/ # optional; used by the oracle agent +│ ├── solve.sh +│ └── ... +└── tests/ + ├── test.sh + └── ... +``` + +The task definition owns the benchmark semantics: + +- `instruction.md` is sent to the agent. +- `environment/` defines or contributes files to the sandbox. +- `task.toml` defines metadata, timeouts, resources, users, networking, MCP servers, verifier isolation, and artifacts. +- `tests/test.sh` performs grading and must write a reward file. +- `solution/solve.sh`, when present, lets the `oracle` agent sanity-check the task. + +Published and Git-hosted datasets resolve to the same task model. Harbor supports: + +- local task/dataset paths; +- package references such as `org/name@ref`; +- legacy registry entries; +- Git repositories, optionally pinned to a ref; +- task include/exclude glob filters and a post-filter `n_tasks` limit. + +Sources: + +- [Task structure](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/tasks/index.mdx) +- [Datasets](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/datasets/index.mdx) +- [`DatasetConfig`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/job/config.py) + +### 2.2 Build the job + +The normal entry point is: + +```bash +harbor run -d "org/dataset@ref" -a claude-code -m anthropic/claude-sonnet-4-6 +``` + +or: + +```bash +harbor run -c job.yaml +``` + +A resolved job does the following before execution: + +1. Resolves agent skills to local cached directories. +2. Resolves datasets into concrete task configurations. +3. Validates environment resource policies. +4. Resolves dataset and job metrics. +5. Downloads/caches package or Git tasks. +6. Builds trial configs. +7. Builds a content-addressed `JobLock`. + +The trial expansion is explicit in `JobPlan.build_trial_configs()`: + +```text +for each attempt + for each task + for each agent configuration + create TrialConfig +``` + +Therefore: + +```text +trial count = n_attempts × number of resolved tasks × number of agent configs +``` + +Multiple `--model` values become multiple agent configurations when an agent is selected on the CLI. + +Source: [`JobPlan`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/job_plan.py) + +### 2.3 Execute trials concurrently + +`n_concurrent_trials` is the global trial limit, defaulting to 4. Each agent config can also declare `n_concurrent`, optionally sharing a named `concurrency_group` with other agent configs. A per-agent limit cannot exceed the global trial limit. + +Retries use exponential backoff and can include or exclude exception types. By default, semantic failures such as agent/verifier timeouts, missing reward files, authentication failures, model-not-found failures, and API usage limits are excluded from retry. + +A single-step trial follows this lifecycle: + +```text +initialize config/result/lock + ↓ +start agent environment + ↓ +run environment healthcheck + ↓ +upload skills + ↓ +install/setup agent + ↓ +run agent + ↓ +download/synchronize agent logs and trajectory + ↓ +collect artifacts + ↓ +run verifier in shared or separate environment + ↓ +parse rewards + ↓ +stop environment, scrub known secrets, write result.json +``` + +Harbor emits hook events around this lifecycle: + +- `START` +- `ENVIRONMENT_START` +- `AGENT_START` +- `AGENT_END` +- `VERIFICATION_START` +- `END` +- `CANCEL` + +Those hooks power job plugins such as LangSmith, OTel export, and Harbor Hub upload. + +Sources: + +- [`Trial.run()` and lifecycle](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/trial/trial.py) +- [`SingleStepTrial`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/trial/single_step.py) +- [Trial hook model](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/trial/hooks.py) + +### 2.4 Grade the trial + +The default verifier copies the task's test directory into `/tests`, executes its OS-appropriate test script, downloads `/logs/verifier`, and parses: + +1. `/logs/verifier/reward.json`, if present; otherwise +2. `/logs/verifier/reward.txt`. + +`reward.txt` becomes: + +```json +{"reward": 1.0} +``` + +`reward.json` can expose multiple numeric dimensions: + +```json +{ + "correctness": 1.0, + "quality": 0.8, + "efficiency": 0.6, + "reward": 0.82 +} +``` + +The Pydantic result type restricts rewards to `dict[str, float | int]`. + +A minimal verifier is: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +if pytest -q /tests/test_solution.py; then + printf '1\n' > /logs/verifier/reward.txt +else + printf '0\n' > /logs/verifier/reward.txt +fi +``` + +Sources: + +- [Verifier implementation](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/verifier/verifier.py) +- [`VerifierResult`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/verifier/result.py) + +### 2.5 Shared versus separate verification + +By default, verification is **shared**: tests execute in the agent's container and can see its workspace and installed tools. + +A task can instead define a **separate verifier environment**. This is useful for: + +- hiding proprietary grading logic; +- reducing agent tampering; +- using a clean grading image; +- grading on a different OS; +- regrading a recorded trial later. + +In separate mode, Harbor collects declared artifacts from the agent environment and re-materializes them at their original absolute paths in the verifier environment. The verifier image is built from `tests/` and must contain its own `/tests/test.sh` or Windows equivalent. + +Example: + +```toml +schema_version = "1.4" +artifacts = ["/app/output.json", "/logs/agent/trajectory.json"] + +[environment] +docker_image = "python:3.12-slim" +network_mode = "no-network" + +[agent] +timeout_sec = 900 + +[verifier] +environment_mode = "separate" +timeout_sec = 120 + +[verifier.environment] +docker_image = "my-org/private-grader:latest" +network_mode = "no-network" +``` + +Important behavior: + +- Merely declaring `[verifier.environment]` implies separate mode. +- `environment_mode = "separate"` without a verifier-specific environment uses a fresh copy of the top-level task environment. +- `environment_mode = "shared"` plus `[verifier.environment]` is invalid. +- `/logs/agent` is not implicitly transferred; declare the trajectory as an artifact if the verifier grades the agent's process. +- Sidecar collection can snapshot databases or service logs before teardown. +- For tamper-sensitive sidecar evidence, single-step or final-step separate verification has the strongest isolation. + +### 2.6 Regrade without rerunning the agent + +`harbor job regrade` and `harbor trial regrade` fork existing execution records and run a new verifier over their recorded artifacts. The source trial is not modified, and no agent credentials or new model cost are required. + +```bash +harbor job regrade jobs/old-job -p ./updated-task -e docker +``` + +Regrade requires: + +- a completed, currently single-step source trial; +- a separate-mode new verifier; +- all newly declared artifacts to exist successfully in the old artifact manifest. + +The derived config and lock record source provenance. This is a strong primitive for verifier iteration, regression analysis, and keeping agent execution fixed while testing scoring changes. + +Source: [Regrade documentation](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/run-jobs/regrade.mdx) + +## 3. Rewards, metrics, and eval result aggregation + +### 3.1 Reward versus metric + +Harbor uses these terms distinctly: + +- A **reward** is emitted by one trial's verifier. +- A **metric** aggregates reward dictionaries across trials in an agent/model/dataset group. + +Built-in metric types are: + +- `mean` +- `sum` +- `min` +- `max` +- `uv-script` + +The default is mean. Missing rewards are treated as zero by the built-in dictionary aggregation. For a single reward key, the output key is the aggregation name, such as `{"mean": 0.81}`. For multiple reward keys, each dimension is aggregated independently and missing dimensions are zero-filled. + +Harbor also computes `pass@k` grouped by agent/model/dataset. + +### 3.2 Custom dataset metrics + +A dataset can ship a `metric.py`. Harbor invokes it through `uv` with: + +```text +metric.py -i rewards.jsonl -o metric.json +``` + +The input has one reward dictionary or `null` per line. The script writes one JSON object containing aggregate metric names and numeric values. + +This means a dataset can own benchmark-specific aggregation rather than forcing all benchmarks into mean reward. + +Sources: + +- [Metrics documentation](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/datasets/metrics.mdx) +- [Built-in aggregation](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/metrics/base.py) +- [`UvScript`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/metrics/uv_script.py) + +### 3.3 Rewardkit + +Rewardkit is a separate package in the Harbor workspace that standardizes richer verifiers. A `tests/` tree can mix: + +- programmatic Python criteria; +- LLM-as-a-judge criteria; +- agent-as-a-judge criteria; +- built-in workspace, command, structured-data, image, and trajectory checks. + +Each directory maps to a reward dimension. Criteria have weights, and judge rubrics can aggregate with `weighted_mean`, `all_pass`, `any_pass`, `threshold`, or `required_pass`. A root `reward.toml` can add an overall `reward` while retaining the component dimensions. + +Rewardkit writes: + +```text +reward.json # numeric dimensions consumed by Harbor +reward-details.json # criterion-level scores, reasoning, and errors +``` + +It can grade the process by loading an ATIF trajectory via `atif-trajectory`. This is the direct bridge between tracing and evals inside Harbor: **the trajectory itself can be an input to the verifier**. + +Sources: + +- [Rewardkit overview](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/rewardkit/index.mdx) +- [Judge configuration](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/rewardkit/judge-criteria.mdx) + +## 4. Configuration model + +Harbor configuration is easiest to understand as three layers. + +### Layer A: task-owned `task.toml` + +The task author controls benchmark semantics and requirements: + +- package metadata and arbitrary metadata; +- agent and verifier base timeouts/users; +- agent/verifier network policies; +- environment image/build/resources/OS; +- environment variables; +- healthchecks; +- MCP servers; +- artifacts and collect hooks; +- shared/separate verifier behavior; +- multi-step task definitions and reward strategy. + +This layer travels with the task and should remain stable across evaluators. + +### Layer B: run-owned job or trial config + +A `JobConfig` chooses how to execute tasks: + +- job name/output directory; +- attempts and concurrency; +- retries; +- agent(s), model(s), kwargs, environment variables, skills, and MCP config; +- runtime environment provider and provider kwargs; +- runtime resource enforcement/overrides; +- verifier override/import path and environment variables; +- dataset/task sources and filters; +- job-level artifacts and metrics; +- timeout multipliers. + +A `TrialConfig` is the corresponding single-task/single-agent execution unit. + +### Layer C: CLI overrides + +CLI flags mutate or replace fields loaded from `-c`. Explicit CLI values generally win. Important details from the implementation: + +- `--agent` replaces the config's full `agents` list. +- Multiple `--model` values create multiple agent configs when `--agent` is supplied. +- Agent kwargs/env/skills can be merged into existing agents when no replacement agent is supplied. +- A CLI path/task/dataset source replaces configured task/dataset sources. +- `--artifact` replaces job-level artifacts rather than extending them. +- verifier/environment kwargs and env mappings update existing mappings. +- `--install-only` implies disabled verification. +- `--load-trajectory` is reserved but currently rejected as unimplemented. + +Source: [`harbor run` config resolution](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/cli/jobs.py) + +### 4.1 Representative current job config + +```yaml +job_name: harbor-research-run +jobs_dir: jobs +n_attempts: 2 +n_concurrent_trials: 8 +timeout_multiplier: 1.0 + +retry: + max_retries: 2 + min_wait_sec: 1 + max_wait_sec: 30 + +environment: + type: docker + force_build: false + delete: true + cpu_enforcement_policy: auto + memory_enforcement_policy: auto + +verifier: + disable: false + env: + REWARDKIT_JUDGE: anthropic/claude-sonnet-4-6 + +agents: + - name: claude-code + model_name: anthropic/claude-sonnet-4-6 + n_concurrent: 4 + skills: [] + env: + ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY}" + kwargs: {} + +datasets: + - name: terminal-bench/terminal-bench-2 + ref: latest + task_names: + - "*" + n_tasks: 10 + +metrics: + - type: mean + +artifacts: + - /app/output.json +``` + +Run it with: + +```bash +harbor run -c job.yaml +``` + +Generate a schema-valid starting point instead of hand-authoring every field: + +```bash +harbor job init --full --config-output job.yaml +harbor trial init --full --config-output trial.yaml +``` + +The generated config is round-trip validated through the same Pydantic model used by execution. + +### 4.2 Timeouts and resources + +Task timeouts are base values. Run config can apply: + +- one global `timeout_multiplier`; +- phase-specific multipliers for agent execution, verification, agent setup, and environment build; +- agent/verifier override and maximum timeout values. + +Task CPU and memory declarations are interpreted through runtime policies: + +- `auto` +- `limit` +- `request` +- `guarantee` +- `ignore` + +Runtime config can also override CPU, memory, storage, GPU, and TPU values. Provider capability validation happens before trials run when Harbor knows the provider's capabilities. + +### 4.3 Network policy + +Network policy has a layered model of its own: + +1. Environment baseline: `public`, `no-network`, or `allowlist`. +2. Optional agent/verifier phase override. +3. Runtime host merges from CLI/run config. + +A phase override that differs from the baseline requires the environment provider to support dynamic policy changes. A separate verifier environment can avoid dynamic switching by having its own baseline. + +### 4.4 Environment variables and secrets + +Task env values support `${VAR}` and `${VAR:-default}` templates. Harbor asks for approval before exposing host environment values unless auto-confirmed. + +Sensitive agent/verifier environment values are templatized when configs are serialized. At trial finalization, Harbor also scans textual output files and replaces resolved known secret values with `[REDACTED]`. This is useful defense in depth, but it is not a complete secret scanner: binary, unreadable, unknown, or indirectly leaked values may remain. + +### 4.5 Reproducibility artifacts + +Each run persists both requested and resolved state: + +- `config.json`: replayable configuration; +- `lock.json`: resolved, content-addressed inputs; +- `result.json`: outcomes, timings, usage, errors, and rewards. + +Locks include: + +- Harbor package/version/Git metadata; +- task content digest and source identity; +- agent config; +- skill names, sources, Git commits, and digests; +- environment and verifier config; +- extra instruction and compose-file digests; +- concurrency/retry behavior; +- source-trial provenance for regrades. + +Task equality is content-digest based, not merely path based. This is a notably strong reproducibility design. + +Source: [Job/trial lock models](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/job/lock.py) + +## 5. Result and artifact layout + +A completed single-step job looks approximately like: + +```text +jobs// +├── config.json +├── lock.json +├── result.json +├── job.log +├── / +│ ├── config.json +│ ├── lock.json +│ ├── result.json +│ ├── trial.log +│ ├── agent/ +│ │ ├── trajectory.json +│ │ ├── recording.cast +│ │ └── ... +│ ├── verifier/ +│ │ ├── reward.json or reward.txt +│ │ ├── reward-details.json +│ │ ├── test-stdout.txt +│ │ └── test-stderr.txt +│ └── artifacts/ +│ ├── manifest.json +│ └── ... +└── ... +``` + +Multi-step trials move agent/verifier/artifact outputs under `steps//`. + +Artifacts are collected from: + +- `/logs/artifacts` by convention, without explicit config; +- arbitrary declared paths; +- Docker Compose sidecars; +- collect-hook output generated just before teardown. + +Collection is best-effort and writes status into `manifest.json`; collection failure does not itself fail a trial. Separate verification and regrade rely on the artifact record, so failed/skipped artifacts can still make those operations impossible. + +Harbor's local viewer (`harbor view jobs`) can inspect job/trial results, trajectories, timing, token usage, verifier output, rewards, and artifacts. Harbor Hub provides hosted storage, comparison, sharing, and leaderboard workflows. + +Sources: + +- [Run evals/results](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/run-jobs/run-evals.mdx) +- [Artifact collection](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/run-jobs/results-and-artifacts.mdx) +- [Hub](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/hub/index.mdx) + +## 6. Tracing and trajectories + +### 6.1 Harbor tracing is post-run trajectory capture + +Harbor does not primarily wrap model SDK calls at the harness level. Instead, each integrated agent is responsible for producing or converting its native logs into: + +```text +/agent/trajectory.json +``` + +The common format is ATIF. After the run, Harbor can: + +- render the trajectory in its viewer; +- derive aggregate token/cost fields for `TrialResult`; +- grade the trajectory with Rewardkit; +- turn it into an SFT dataset; +- convert it to OTel spans. + +This architecture has an important consequence: **trace completeness depends on the agent adapter**. If an agent does not emit ATIF, Harbor still runs and scores it, but standard trajectory visualization/export is unavailable. + +### 6.2 Agent Trajectory Interchange Format (ATIF) + +At the inspected commit, the active version is `ATIF-v1.7`. + +A trajectory root contains: + +- `schema_version`; +- optional run-scoped `session_id`; +- optional document-scoped `trajectory_id`; +- agent name/version/default model and optional tool definitions; +- ordered `steps`; +- optional notes and arbitrary `extra` metadata; +- optional aggregate `final_metrics`; +- optional continuation reference; +- optional embedded subagent trajectories. + +A step can contain: + +- sequential `step_id`; +- timestamp; +- `system`, `user`, or `agent` source; +- model name; +- message, including text/image content parts; +- reasoning content/effort; +- structured tool calls; +- observations correlated by tool-call ID; +- prompt/completion/cache tokens and cost; +- prompt/completion token IDs and log probabilities; +- an LLM call count; +- copied-context marking; +- extensible metadata. + +ATIF v1.7 distinguishes: + +- `session_id`: logical run identity, which may be shared; +- `trajectory_id`: unique trajectory-document identity used to resolve embedded subagent references. + +It also defines: + +- `llm_call_count = 0` for deterministic non-LLM dispatch; +- `llm_call_count > 1` for an aggregated multi-inference step; +- `is_copied_context = true` so SFT consumers can exclude duplicated context; +- a `context_management` convention for compaction/pruning boundaries; +- embedded or external-file subagent trajectories. + +Sources: + +- [ATIF documentation](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/agents/trajectory-format.mdx) +- [ATIF RFC](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/rfcs/0001-trajectory-format.md) +- [Pydantic models](https://github.com/harbor-framework/harbor/tree/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/trajectories) + +### 6.3 ATIF validation + +Harbor provides Pydantic models and a validator that checks, among other things: + +- required fields and types; +- sequential step IDs starting at 1; +- ISO timestamps; +- agent-only fields appearing on valid sources; +- tool call and observation references; +- embedded subagent identity rules. + +```bash +python -m harbor.utils.trajectory_validator trajectory.json +``` + +### 6.4 Hugging Face/SFT export + +The default `harbor traces export` format converts ATIF into conversational rows in a Hugging Face `datasets.Dataset`: + +```bash +harbor traces export \ + --path jobs/my-job \ + --recursive \ + --episodes last \ + --filter success \ + --sharegpt \ + --instruction-metadata \ + --verifier-metadata +``` + +Rows can include: + +- OpenAI-style conversations; +- optional ShareGPT conversations; +- agent/model/provider; +- date, task, trial, run, and episode identity; +- result bucket; +- tool definitions; +- optional instruction and verifier output; +- subagent trace source. + +Exports can be pushed to Hugging Face Hub or produced programmatically and written as Parquet. Multimodal input is rejected by the text-only path rather than silently losing images. + +Source: [SFT export](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/content/docs/training-workflows/sft.mdx) and [`traces_utils.py`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/utils/traces_utils.py) + +### 6.5 OpenTelemetry export + +`harbor-atif2otel` is a separate package that converts ATIF into OTel protobuf `ResourceSpans` with OpenInference-style attributes. + +Install and export: + +```bash +pip install harbor-atif2otel + +harbor traces export \ + --path jobs/my-job \ + --format otel \ + --output traces.jsonl \ + --encoding json +``` + +The mapping is: + +| ATIF | OTel/OpenInference | +|---|---| +| trajectory | root `AGENT` span | +| multi-turn conversational turn | nested `AGENT` span | +| agent step | `LLM` span | +| tool call | `TOOL` span, sibling of the LLM span | +| context-management system step | `CHAIN` span | +| subagent | nested agent span tree | + +Selected attributes include: + +- `openinference.span.kind`; +- `session.id` and `trajectory.id`; +- `agent.name` and `agent.version`; +- `llm.model_name`; +- prompt/completion/cache token counts; +- total and per-step cost; +- `tool.name`; +- serialized input/output; +- reasoning content. + +Conversion behavior worth noting: + +- trace and span IDs are deterministic hashes; +- copied-context steps are filtered; +- `llm_call_count = 0` emits tools without an LLM span; +- large string attributes are truncated; +- image content becomes textual metadata rather than image payloads; +- missing timestamps become Unix nanoseconds `0`; +- all generated spans currently receive OTel `STATUS_CODE_OK`. + +Sources: + +- [`harbor-atif2otel` README](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/packages/harbor-atif2otel/README.md) +- [Converter](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/packages/harbor-atif2otel/src/harbor_atif2otel/convert.py) +- [Export orchestration](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/packages/harbor-atif2otel/src/harbor_atif2otel/export.py) + +### 6.6 Streaming and batch OTel plugin + +The package registers the `atif2otel` entry point under the `harbor.plugins` group. It supports: + +- streaming each completed trial in `on_trial_ended`; +- batch export after the job ends; +- file output, endpoint upload, or both. + +```bash +harbor run ... \ + --plugin atif2otel \ + --plugin-kwarg output_dir=./otel-traces \ + --plugin-kwarg encoding=json +``` + +The plugin intentionally catches export/upload failures and logs warnings so observability failure does not fail the eval job. + +## 7. Job plugins and external observability + +A plugin implements: + +```python +class JobPlugin(Protocol): + async def on_job_start(self, job: Job) -> None: ... + async def on_job_end(self, job_result: JobResult) -> None: ... +``` + +During `on_job_start`, the plugin can subscribe to the job's trial lifecycle hooks. Plugins are discovered via Python entry points in the `harbor.plugins` group or loaded with a full `module:Class` path. + +```bash +harbor plugins list +harbor run ... --plugin package.module:PluginClass +``` + +Current CLI behavior: + +- `--plugin` is repeatable. +- `--plugin-kwarg` requires exactly one plugin because kwargs are not scoped per repeated plugin. +- Plugins in job YAML/JSON are deprecated and ignored; plugins should be supplied on the CLI. +- Plugin `on_job_end` failures are logged rather than re-raised. + +Sources: + +- [Plugin protocol](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/models/job/plugin.py) +- [Plugin attachment](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/cli/job_plugins.py) +- [Entry-point registry](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/src/harbor/cli/plugin_registry.py) + +### LangSmith plugin behavior + +The `harbor-langsmith` package is more than trace upload. It maps Harbor's eval model into LangSmith: + +1. Optionally creates or finds a LangSmith dataset. +2. Upserts one example per Harbor task, including instruction and task identity. +3. Creates or reuses an experiment/session. +4. Creates one root chain run per trial. +5. Creates child phase runs for environment, agent, and verifier phases. +6. Emits a synthetic LLM child with token usage so LangSmith rolls usage up correctly. +7. Ends runs with rewards, output, cost, and errors. +8. Publishes every reward dimension as feedback. +9. Makes parent context available to in-process custom agents so their native LangSmith traces can nest beneath the Harbor trial. + +This plugin is the clearest upstream precedent for a full Braintrust integration. + +Source: [`LangSmithPlugin`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/packages/harbor-langsmith/src/harbor_langsmith/plugin.py) + +## 8. Braintrust integration analysis + +The Harbor repository has no Braintrust plugin at the inspected commit. Braintrust appears only as an OTel reference in the ATIF RFC. + +### 8.1 Why the existing Harbor OTel endpoint flag is not enough + +Although the CLI says it can upload to an OTLP endpoint, it constructs `MlflowProtobufUploader`. That uploader: + +1. calls MLflow experiment search/create REST APIs; +2. POSTs spans to `/v1/traces`; +3. adds `x-mlflow-experiment-id` and `X-Mlflow-Workspace` headers. + +Braintrust's OTel endpoint is instead: + +```text +https://api.braintrust.dev/otel/v1/traces +``` + +with headers such as: + +```text +Authorization: Bearer +x-bt-parent: project_name: +``` + +Therefore, simply setting Harbor's `--endpoint` or `OTEL_EXPORTER_OTLP_ENDPOINT` to Braintrust is expected to fail during the MLflow experiment lookup/create flow, even though the serialized trace payload itself is OTLP protobuf. + +Relevant code: + +- Harbor's [MLflow uploader](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/packages/harbor-atif2otel/src/harbor_atif2otel/uploaders/mlflow_protobuf.py) +- Braintrust's local [`OtelExporter`](../py/src/braintrust/otel/__init__.py) + +### 8.2 Trace-only integration option + +A narrow integration could implement `harbor_atif2otel.uploaders.base.Uploader` and POST `ExportTraceServiceRequest` bytes directly to Braintrust with the correct endpoint and headers. + +Advantages: + +- minimal Harbor changes; +- preserves ATIF's nested LLM/tool/subagent structure; +- uses the existing OpenInference attributes; +- can support both backfill and streaming. + +Limitations in the current ATIF-to-OTel export: + +- `result.json` is read only for success/failure filtering, not attached to spans; +- verifier reward dimensions are absent; +- task, dataset, job, attempt, and artifact metadata are absent; +- failures still produce `STATUS_CODE_OK` spans; +- trials with no ATIF have no exported trace; +- multi-reward filtering assumes a key named `reward` and treats its absence as zero; +- the exported root is the ATIF agent trajectory, not a complete environment/agent/verifier lifecycle trace. + +A trace-only uploader is useful, but it would not provide a complete Braintrust eval experience. + +### 8.3 Recommended integration: `harbor-braintrust` job plugin + +The stronger design is a dedicated package registered as: + +```toml +[project.entry-points."harbor.plugins"] +braintrust = "harbor_braintrust:BraintrustPlugin" +``` + +Conceptual CLI: + +```bash +pip install harbor-braintrust +export BRAINTRUST_API_KEY=... + +harbor run ... \ + --plugin braintrust \ + --plugin-kwarg project_name=harbor-evals \ + --plugin-kwarg experiment_name=my-run +``` + +Recommended mapping: + +| Harbor | Braintrust | +|---|---| +| Job | Experiment/run grouping | +| Dataset | Braintrust dataset, optionally synchronized | +| Task | Dataset record/example | +| Trial | Root eval span | +| Instruction | Span/eval input | +| Final agent response and metadata | Span output | +| Environment, agent, verifier phases | Child spans | +| ATIF agent/LLM/tool/subagent tree | Nested child spans imported after the agent run | +| `verifier_result.rewards` | Scores, one per reward key | +| Exception | Error/status metadata | +| token/cache/cost totals | Metrics | +| config and lock | Metadata/provenance, with size/redaction controls | +| task/artifact digests | Reproducibility metadata | + +The implementation can follow `LangSmithPlugin`: + +1. Initialize the target project/experiment during `on_job_start`. +2. Optionally synchronize Harbor tasks into a Braintrust dataset. +3. Subscribe to trial start, environment start, agent start/end, verification start, end, and cancel hooks. +4. Create stable trial identity from Harbor's job/trial UUIDs. +5. Log phase timing and runtime configuration. +6. At trial end, log all reward dimensions as scores and attach errors and usage. +7. Parse `agent/trajectory.json` and attach its detailed child spans. +8. Flush on job end without making observability failure invalidate the Harbor run by default; offer `fail_fast` for CI. + +### 8.4 Hybrid design details + +A production-grade plugin should address several details not solved by the generic converter today: + +- **Parenting:** allow ATIF conversion to accept an existing trace ID and parent span ID, or map ATIF directly into Braintrust child spans. +- **Lifecycle coverage:** preserve Harbor phase spans even when no ATIF is available. +- **Status:** derive root status from `exception_info`, verifier availability, and configured success semantics. +- **Scores:** attach every numeric reward key, not only a conventional `reward` key. +- **Inputs/outputs:** read task instruction and best available final agent response; do not assume ATIF always has one simple textual final answer. +- **Metadata size:** avoid blindly attaching the entire Pydantic config, raw verifier logs, or large artifacts. +- **Secrets:** rely on Harbor's serialized/redacted config and add a plugin-side metadata allowlist. +- **Idempotency:** use stable IDs so resumed jobs and plugin retries do not duplicate records. +- **Regrades:** preserve source-trial provenance and distinguish newly computed scores from original execution cost. +- **Multi-step trials:** emit one child span and score set per step, then the configured trial-level reward strategy. +- **Subagents/continuations:** support both ATIF v1.7 embedded references and external continuation files. +- **Custom agents:** accept any valid ATIF producer rather than checking only Harbor's built-in agent enum. + +### 8.5 Suggested implementation sequence + +1. Build a standalone ATIF → Braintrust/OTLP uploader PoC using checked-in Harbor golden trajectories. +2. Add trial result enrichment: task/job metadata, rewards, errors, and status. +3. Implement a Harbor `BraintrustPlugin` with lifecycle spans and score logging. +4. Add optional dataset synchronization. +5. Add streaming and batch/backfill modes. +6. Validate single-step, multi-step, failed, cancelled, retry, regrade, missing-trajectory, and multi-reward cases. +7. Upstream generic improvements to `harbor-atif2otel` where possible, especially pluggable endpoint upload and parent/status/resource metadata. + +## 9. Important gaps and documentation/code mismatches + +These findings are based on the inspected commit and may change quickly. + +### 9.1 OTel conversion is generic; direct upload is MLflow-specific + +The package README describes “any OTel-compatible backend,” but the bundled direct uploader and CLI endpoint path are specifically MLflow-aware. File export is backend-neutral; endpoint upload is not. + +### 9.2 OTel output omits eval outcomes + +The exporter reads `result.json` for filtering but does not place rewards, task identity, exceptions, or Harbor lifecycle timing onto spans. It also marks every converted span OK. An observability backend receives an agent trace, not a complete eval record. + +### 9.3 Multi-reward success filtering assumes `reward` + +OTel filtering looks for `verifier_result.rewards["reward"]`, defaulting to zero. A valid multi-dimensional verifier with only `correctness` and `quality` will be classified as failure for filtering purposes. + +### 9.4 Custom ATIF agents may fail Hugging Face export + +The SFT documentation says any ATIF-producing agent is supported, but the exporter currently coerces the agent name into the built-in `AgentName` enum and checks `AgentFactory.SUPPORTS_ATIF`. A custom import-path agent that emits valid ATIF may fail before export unless registered as a built-in name. + +### 9.5 Some SFT documentation still describes legacy episode files + +The SFT page describes rows as `agent/episode-*` plus `debug.json`/`response.json`, while the current utility says ATIF is preferred and discovers trials through `agent/trajectory.json`. This appears to be partially stale documentation from an older trace layout. + +### 9.6 Plugin config documentation has drift + +`harbor-langsmith` says plugin kwargs can come from job config, but current `JobConfig` migration explicitly removes and ignores a `plugins` key and tells users to use CLI `--plugin`. Treat CLI plugin configuration as authoritative. + +### 9.7 Existing example config uses deprecated `orchestrator` + +`examples/configs/features/job.yaml` still has an `orchestrator` block. The Pydantic model migrates it to top-level `n_concurrent_trials`, `quiet`, and `retry` with a deprecation warning. New configs should use top-level fields. + +### 9.8 Separate verification is recommended but not yet the default + +The regrade docs say separate mode is recommended and may become the default. Current resolution remains shared when neither `environment_mode` nor `[verifier.environment]` is set. + +### 9.9 OTel command naming is inconsistent in one package note + +One sequence document says `harbor trace export`; the actual Typer command is `harbor traces export`. + +### 9.10 Timestamps are optional in ATIF but important in OTel + +The converter maps missing timestamps to zero. Agents that omit timestamps can therefore generate technically serialized but poorly timed traces. + +## 10. How the documentation website works + +The live docs source is in the repository's `docs/` directory. It is a separate web app built with: + +- Next.js 16 App Router; +- React 19; +- Fumadocs MDX/core/UI; +- Tailwind CSS 4; +- Bun; +- Vercel deployment. + +The content flow is: + +```text +docs/content/docs/**/*.mdx + + meta.json navigation files + ↓ fumadocs-mdx postinstall generation + docs/.source/ + ↓ Fumadocs loader, base URL /docs +Next.js dynamic [[...slug]] route + ↓ +rendered docs + TOC + generated metadata/OG routes +``` + +Key mechanics: + +- `docs/source.config.ts` defines the MDX collection and stores processed Markdown. +- `docs/src/lib/source.ts` loads content at `/docs`, applies the Lucide icon plugin, and exposes text extraction. +- `docs/src/app/docs/[[...slug]]/page.tsx` resolves a page, renders MDX, generates static params, and generates page metadata. +- `meta.json` files control ordering and nested navigation. +- `/llms-full.txt` concatenates processed Markdown for every page into an LLM-friendly endpoint. +- `next.config.mjs` contains redirects from old documentation routes and sends `/registry` to Harbor Hub. +- Vercel deploys `main`; a GitHub workflow creates docs previews from the `docs/` working directory. +- `docs-mintlify/` also exists, but the current documented and deployed app is the Next.js/Fumadocs app under `docs/`. + +Relevant files: + +- [`docs/package.json`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/package.json) +- [`source.config.ts`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/source.config.ts) +- [`source.ts`](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/src/lib/source.ts) +- [Dynamic docs page](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/src/app/docs/%5B%5B...slug%5D%5D/page.tsx) +- [`llms-full.txt` route](https://github.com/harbor-framework/harbor/blob/e76f7e32f5644fb9f648cd23151aac5c67492ea0/docs/src/app/llms-full.txt/route.ts) + +Local docs development: + +```bash +cd docs +bun install +bun dev +``` + +## 11. Practical command reference + +### Install and inspect + +```bash +uv tool install harbor +harbor --help +harbor run --help +harbor dataset list +harbor plugins list +``` + +### Run one local task or dataset + +```bash +harbor run -p ./my-task -a claude-code -m anthropic/claude-sonnet-4-6 +harbor run -p ./my-dataset -a claude-code -m anthropic/claude-sonnet-4-6 -n 8 +``` + +### Run a published dataset + +```bash +harbor run -d "org/dataset@latest" -a claude-code -m anthropic/claude-sonnet-4-6 +``` + +### Generate configuration + +```bash +harbor job init --full --config-output job.yaml +harbor trial init --full --config-output trial.yaml +harbor run -c job.yaml --print-config +``` + +### Inspect results + +```bash +harbor view jobs +``` + +### Regrade + +```bash +harbor job regrade jobs/ -p ./updated-task +harbor trial regrade jobs// -p ./updated-task +``` + +### Export SFT conversations + +```bash +harbor traces export \ + -p jobs/ \ + --episodes last \ + --filter success \ + --sharegpt +``` + +### Export OTel files + +```bash +pip install harbor-atif2otel +harbor traces export \ + -p jobs/ \ + --format otel \ + --encoding json \ + --output harbor-traces.jsonl +``` + +### Stream or batch through a plugin + +```bash +harbor run ... \ + --plugin atif2otel \ + --plugin-kwarg output_dir=./otel-traces \ + --plugin-kwarg mode=batch +``` + +## 12. Bottom line + +Harbor's strongest ideas are: + +- tasks package execution and grading together; +- trials are reproducible, content-locked rollouts; +- rewards are intentionally simple numeric dictionaries; +- custom metrics and Rewardkit handle richer scoring; +- artifacts make isolated verification and regrading possible; +- ATIF gives diverse agents a common post-run trajectory format; +- plugins are the extension point for hosted eval/observability systems. + +For Braintrust, the most valuable integration is not merely “OTLP export.” It is a **Harbor-aware eval plugin** that combines: + +1. Harbor job/task/trial identity and reproducibility metadata; +2. complete lifecycle timing; +3. verifier rewards as Braintrust scores; +4. token/cost/error data; +5. ATIF's detailed LLM/tool/subagent trace tree; +6. dataset synchronization and regrade provenance. + +That hybrid would preserve both sides of Harbor's model: **the eval record** and **the agent trajectory**. From ae72a770e97b747c6a3dd5f9ce25df5064ae9be0 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 30 Jul 2026 19:24:07 -0400 Subject: [PATCH 2/5] feat(harbor): add native evaluation plugin Register HarborPlugin through the harbor.plugins entry point. Users install harbor and braintrust, configure standard Braintrust credentials plus optional HARBOR_BRAINTRUST_* settings, and select it with `--plugin braintrust`. The public Python API also exposes HarborPlugin and backfill_job for explicit construction and offline synchronization. Sync resolved tasks into Braintrust datasets, partition experiments by semantic agent configuration, and reconcile each retained Harbor trial into an eval trace with lifecycle spans, rewards, classifications, ATIF LLM/tool detail, errors, usage, attachments, and provenance metadata. Deterministic identities and braintrust-sync.json make resume and backfill idempotent. Add the pinned Harbor 0.20 test session, pure contract coverage using real Harbor models, and a VCR-backed round trip through the real Braintrust SDK. --- docs/harbor-braintrust-plugin-design.md | 4 +- py/noxfile.py | 13 + py/pyproject.toml | 8 + .../integrations/harbor/__init__.py | 10 + py/src/braintrust/integrations/harbor/atif.py | 412 ++++++++ ...tif_import_round_trips_with_real_sdks.yaml | 643 ++++++++++++ .../braintrust/integrations/harbor/compat.py | 263 +++++ .../braintrust/integrations/harbor/config.py | 202 ++++ .../integrations/harbor/identity.py | 214 ++++ .../braintrust/integrations/harbor/plugin.py | 938 ++++++++++++++++++ .../braintrust/integrations/harbor/rewards.py | 152 +++ .../braintrust/integrations/harbor/state.py | 220 ++++ .../integrations/harbor/test_harbor.py | 363 +++++++ 13 files changed, 3440 insertions(+), 2 deletions(-) create mode 100644 py/src/braintrust/integrations/harbor/__init__.py create mode 100644 py/src/braintrust/integrations/harbor/atif.py create mode 100644 py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml create mode 100644 py/src/braintrust/integrations/harbor/compat.py create mode 100644 py/src/braintrust/integrations/harbor/config.py create mode 100644 py/src/braintrust/integrations/harbor/identity.py create mode 100644 py/src/braintrust/integrations/harbor/plugin.py create mode 100644 py/src/braintrust/integrations/harbor/rewards.py create mode 100644 py/src/braintrust/integrations/harbor/state.py create mode 100644 py/src/braintrust/integrations/harbor/test_harbor.py diff --git a/docs/harbor-braintrust-plugin-design.md b/docs/harbor-braintrust-plugin-design.md index 0f2531b8b..0d2b878ce 100644 --- a/docs/harbor-braintrust-plugin-design.md +++ b/docs/harbor-braintrust-plugin-design.md @@ -471,7 +471,7 @@ Write `/braintrust-sync.json` with plugin version, job/project identity Suggested constructor: ```python -BraintrustPlugin( +HarborPlugin( project_name=None, project_id=None, experiment_prefix=None, @@ -502,7 +502,7 @@ BraintrustPlugin( Constructor options should have `HARBOR_BRAINTRUST_*` environment fallbacks, with precedence `explicit option > environment > default`. Standard Braintrust variables continue to configure API key, organization, and app URL. Complex rule values should use JSON because Harbor only supports `--plugin-kwarg` when one plugin is supplied. Validate mutually exclusive identifiers, overlapping reward rules, globs, bounds, byte limits, and mode combinations before performing network I/O. -Ship the integration in the Braintrust distribution under `py/src/braintrust/integrations/harbor/` and register `braintrust.integrations.harbor:BraintrustPlugin` in the `harbor.plugins` entry-point group. Harbor stays optional: importing Braintrust without Harbor installed must continue to work, and the integration must not raise Braintrust's minimum Python version. Test supported Harbor releases through a dedicated, explicitly pinned version matrix. +Ship the integration in the Braintrust distribution under `py/src/braintrust/integrations/harbor/` and register `braintrust.integrations.harbor:HarborPlugin` in the `harbor.plugins` entry-point group. Harbor stays optional: importing Braintrust without Harbor installed must continue to work, and the integration must not raise Braintrust's minimum Python version. Test supported Harbor releases through a dedicated, explicitly pinned version matrix. Keep lifecycle orchestration, compatibility access, state reduction, dataset/eval conversion, ATIF conversion, reward handling, attachments, and identity generation behind clear internal boundaries. The internal file layout is an implementation choice rather than part of this design. diff --git a/py/noxfile.py b/py/noxfile.py index 798b25226..40772b579 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -626,6 +626,19 @@ def test_temporal(session, version): _run_tests(session, f"{INTEGRATION_DIR}/temporal") +HARBOR_VERSIONS = _get_matrix_versions("harbor") + + +@nox.session() +@nox.parametrize("version", HARBOR_VERSIONS, ids=HARBOR_VERSIONS) +def test_harbor(session, version): + if Version(platform.python_version()) < Version("3.12"): + session.skip("Harbor requires Python 3.12+") + _install_test_deps(session) + _install_matrix_dep(session, "harbor", version) + _run_tests(session, f"{INTEGRATION_DIR}/harbor", version=version) + + PYTEST_VERSIONS = _get_matrix_versions("pytest-matrix") diff --git a/py/pyproject.toml b/py/pyproject.toml index 281d7a01d..396c59768 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -42,6 +42,9 @@ braintrust = "braintrust.cli.__main__:main" [project.entry-points.pytest11] braintrust = "braintrust.wrappers.pytest_plugin.plugin" +[project.entry-points."harbor.plugins"] +braintrust = "braintrust.integrations.harbor:HarborPlugin" + [project.optional-dependencies] cli = ["boto3", "python-dotenv", "uv", "starlette", "uvicorn"] # TODO: remove the doc extra in the next major version. @@ -458,6 +461,9 @@ latest = "temporalio==1.31.0" "1.20.0" = "temporalio==1.20.0" "1.19.0" = "temporalio==1.19.0" +[tool.braintrust.matrix.harbor] +latest = "harbor==0.20.0" + [tool.braintrust.matrix.pytest-matrix] # Canonical pytest pin. The matching entry in [dependency-groups].test is # kept in sync by py/scripts/sync-pytest-pin.py (enforced by pre-commit). @@ -505,6 +511,7 @@ crewai = ["crewai"] dspy = ["dspy"] google_genai = ["google-genai"] huggingface_hub = ["huggingface-hub"] +harbor = ["harbor"] instructor = ["instructor"] langchain = ["langchain-core", "deepagents"] litellm = ["litellm"] @@ -527,6 +534,7 @@ cohere = "cohere" autoevals = "autoevals" braintrust-core = "braintrust_core" boto3 = "boto3" +harbor = "harbor" botocore = "botocore" crewai = "crewai" dspy = "dspy" diff --git a/py/src/braintrust/integrations/harbor/__init__.py b/py/src/braintrust/integrations/harbor/__init__.py new file mode 100644 index 000000000..54a75f716 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/__init__.py @@ -0,0 +1,10 @@ +"""Braintrust's native Harbor job plugin. + +Harbor is optional. Importing this module does not import Harbor; the package is +only required when Harbor constructs the plugin or backfill reads Harbor models. +""" + +from .plugin import HarborPlugin, backfill_job + + +__all__ = ["HarborPlugin", "backfill_job"] diff --git a/py/src/braintrust/integrations/harbor/atif.py b/py/src/braintrust/integrations/harbor/atif.py new file mode 100644 index 000000000..6fb3f1d37 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/atif.py @@ -0,0 +1,412 @@ +"""Host-side ATIF to Braintrust span conversion.""" + +import json +import math +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from braintrust.logger import Attachment + +from .config import PluginConfig +from .identity import child_span_id, normalize_json + + +_INSTRUMENTATION = "braintrust.plugin.harbor" + + +@dataclass(frozen=True) +class ATIFImportResult: + final_message: Any = None + schema_version: str | None = None + root_extra: dict[str, Any] | None = None + warnings: tuple[str, ...] = () + repairs: tuple[str, ...] = () + imported_llm_spans: int = 0 + imported_tool_spans: int = 0 + + +def _timestamp(value: Any) -> float | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + except (ValueError, OverflowError): + return None + + +def _step_times(steps: list[dict[str, Any]], start: float, end: float) -> tuple[list[float], list[str]]: + if end < start: + end = start + repairs: list[str] = [] + parsed = [_timestamp(step.get("timestamp")) for step in steps] + count = max(len(steps), 1) + result: list[float] = [] + previous = start + for index, value in enumerate(parsed): + if value is None: + value = start + (end - start) * index / count + repairs.append(f"step {index + 1}: interpolated missing timestamp") + clamped = min(max(value, start), end) + if clamped != value: + repairs.append(f"step {index + 1}: clamped timestamp to agent phase") + if clamped < previous: + clamped = previous + repairs.append(f"step {index + 1}: repaired non-monotonic timestamp") + result.append(clamped) + previous = clamped + return result, repairs + + +def _provider(model: str | None) -> tuple[str | None, str | None]: + if not model: + return None, None + if "/" in model: + provider, model_name = model.split("/", 1) + return provider.lower(), model_name + return "unknown", model + + +def _valid_count(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def _valid_cost(value: Any) -> float | None: + return ( + float(value) + if isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and value >= 0 + else None + ) + + +def _usage_metrics(raw: Any) -> dict[str, int | float]: + if not isinstance(raw, dict): + return {} + prompt = _valid_count(raw.get("prompt_tokens")) + completion = _valid_count(raw.get("completion_tokens")) + cached = _valid_count(raw.get("cached_tokens")) + cost = _valid_cost(raw.get("cost_usd")) + metrics: dict[str, int | float] = {} + if prompt is not None: + metrics["prompt_tokens"] = prompt + if completion is not None: + metrics["completion_tokens"] = completion + if prompt is not None and completion is not None: + metrics["tokens"] = prompt + completion + if cached is not None: + metrics["prompt_cached_tokens"] = cached + if cost is not None: + metrics["estimated_cost"] = cost + extra = raw.get("extra") + if isinstance(extra, dict): + reasoning = _valid_count(extra.get("reasoning_tokens")) + first_token_ms = _valid_cost(extra.get("time_to_first_token_ms")) + cache_write = _valid_count(extra.get("cache_write_tokens")) + if reasoning is not None: + metrics["completion_reasoning_tokens"] = reasoning + if first_token_ms is not None: + metrics["time_to_first_token"] = first_token_ms / 1000 + if cache_write is not None: + metrics["prompt_cache_creation_tokens"] = cache_write + return metrics + + +def _bounded(value: Any, config: PluginConfig) -> Any: + return normalize_json( + value, + max_bytes=config.max_content_bytes, + redact_patterns=config.redact_patterns, + max_depth=10, + ).value + + +def _content(value: Any, trajectory_dir: Path, config: PluginConfig) -> tuple[Any, bool]: + if isinstance(value, str) or value is None: + return _bounded(value, config), True + if not isinstance(value, list): + return _bounded(value, config), False + result: list[Any] = [] + complete = True + for part in value: + if not isinstance(part, dict): + complete = False + result.append(_bounded(part, config)) + continue + if part.get("type") == "text" and isinstance(part.get("text"), str): + result.append({"type": "text", "text": _bounded(part["text"], config)}) + continue + source = part.get("source") + if part.get("type") == "image" and isinstance(source, dict) and isinstance(source.get("path"), str): + raw_path = Path(source["path"]) + if raw_path.is_absolute(): + complete = False + result.append({"type": "text", "text": "[image omitted: absolute path]"}) + continue + path = (trajectory_dir / raw_path).resolve() + try: + path.relative_to(trajectory_dir.resolve()) + data = path.read_bytes() + except (OSError, ValueError): + complete = False + result.append(_bounded(part, config)) + continue + if len(data) > config.max_attachment_bytes: + complete = False + result.append({"type": "text", "text": "[image omitted: size limit]"}) + continue + attachment = Attachment( + data=data, + filename=path.name, + content_type=source.get("media_type", "application/octet-stream"), + ) + result.append({"type": "image_url", "image_url": {"url": attachment}}) + continue + complete = False + result.append(_bounded(part, config)) + return result, complete + + +def _end_time(times: list[float], index: int, phase_end: float) -> float: + if index + 1 < len(times): + return max(times[index], times[index + 1]) + return max(times[index], phase_end) + + +def summarize_trajectory(trajectory_path: Path, config: PluginConfig) -> ATIFImportResult: + """Read bounded trajectory summary data without creating detailed leaves.""" + try: + if trajectory_path.stat().st_size > config.max_total_attachment_bytes: + return ATIFImportResult(warnings=("trajectory omitted: size limit",)) + trajectory = json.loads(trajectory_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + return ATIFImportResult(warnings=(f"trajectory unavailable or malformed: {exc}",)) + if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list): + return ATIFImportResult(warnings=("trajectory malformed: steps must be an array",)) + final_message = None + for step in trajectory["steps"]: + if isinstance(step, dict) and step.get("source") == "agent" and not step.get("is_copied_context"): + final_message = _bounded(step.get("message"), config) + extra = trajectory.get("extra") if isinstance(trajectory.get("extra"), dict) else None + final_metrics = trajectory.get("final_metrics") + root_extra = dict(extra or {}) + if isinstance(final_metrics, dict): + root_extra["final_metrics"] = _bounded(final_metrics, config) + return ATIFImportResult( + final_message=final_message, + schema_version=( + trajectory.get("schema_version") if isinstance(trajectory.get("schema_version"), str) else None + ), + root_extra=root_extra or None, + ) + + +def import_trajectory( + parent: Any, + trajectory_path: Path, + *, + trial_id: str, + semantic_prefix: str, + phase_start: float, + phase_end: float, + config: PluginConfig, + _trajectory_data: dict[str, Any] | None = None, +) -> ATIFImportResult: + warnings: list[str] = [] + if _trajectory_data is not None: + trajectory = _trajectory_data + else: + try: + if trajectory_path.stat().st_size > config.max_total_attachment_bytes: + return ATIFImportResult(warnings=("trajectory omitted: size limit",)) + trajectory = json.loads(trajectory_path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + return ATIFImportResult(warnings=(f"trajectory unavailable or malformed: {exc}",)) + if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list): + return ATIFImportResult(warnings=("trajectory malformed: steps must be an array",)) + + steps = [step for step in trajectory["steps"] if isinstance(step, dict)] + times, repairs = _step_times(steps, phase_start, phase_end) + agent = trajectory.get("agent") if isinstance(trajectory.get("agent"), dict) else {} + default_model = agent.get("model_name") + tools = agent.get("tool_definitions") if isinstance(agent.get("tool_definitions"), list) else None + messages: list[dict[str, Any]] = [] + final_message: Any = None + llm_count = 0 + tool_count = 0 + observations: dict[str, Any] = {} + for step in steps: + observation = step.get("observation") + if isinstance(observation, dict) and isinstance(observation.get("results"), list): + for result in observation["results"]: + if isinstance(result, dict) and isinstance(result.get("source_call_id"), str): + observations[result["source_call_id"]] = result + + for index, step in enumerate(steps): + source = step.get("source") + content, content_complete = _content(step.get("message"), trajectory_path.parent, config) + if source in {"system", "user"}: + if config.content_mode != "metadata": + messages.append({"role": source, "content": content}) + continue + if source != "agent": + warnings.append(f"step {index + 1}: unknown source") + continue + + tool_calls = step.get("tool_calls") if isinstance(step.get("tool_calls"), list) else [] + assistant_message: dict[str, Any] = {"role": "assistant", "content": content} + normalized_calls: list[dict[str, Any]] = [] + for call in tool_calls: + if not isinstance(call, dict): + continue + call_id, name, arguments = call.get("tool_call_id"), call.get("function_name"), call.get("arguments") + if isinstance(call_id, str) and isinstance(name, str) and isinstance(arguments, dict): + normalized_calls.append( + { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments, sort_keys=True)}, + } + ) + if normalized_calls: + assistant_message["tool_calls"] = normalized_calls + + llm_call_count = step.get("llm_call_count") + metrics = _usage_metrics(step.get("metrics")) + provider, model = _provider(step.get("model_name") or default_model) + can_be_llm = ( + config.content_mode != "metadata" + and llm_call_count == 1 + and content_complete + and provider is not None + and model is not None + and "tokens" in metrics + ) + path = f"{semantic_prefix}/turn/{step.get('step_id', index + 1)}" + if can_be_llm: + metadata: dict[str, Any] = {"provider": provider, "model": model} + if tools: + metadata["tools"] = _bounded(tools, config) + llm_span = parent.start_span( + name="chat.completions.create", + type="llm", + id=child_span_id(trial_id, f"{path}/llm"), + start_time=times[index], + set_current=False, + input=list(messages), + metadata=metadata, + internal={"instrumentation": _INSTRUMENTATION}, + ) + llm_span.log(output=assistant_message, metrics=metrics) + llm_span.end(end_time=_end_time(times, index, phase_end)) + llm_count += 1 + else: + reason = "not exactly one conforming model call" + if llm_call_count == 1 and "tokens" not in metrics: + reason = "model call missing token usage" + warnings.append(f"step {index + 1}: downgraded to task ({reason})") + summary_span = parent.start_span( + name=f"trajectory.step.{step.get('step_id', index + 1)}", + type="task", + id=child_span_id(trial_id, f"{path}/summary"), + start_time=times[index], + set_current=False, + input={"source": source}, + internal={"instrumentation": _INSTRUMENTATION}, + ) + summary_span.log(output={"message": content, "tool_call_count": len(normalized_calls)}) + summary_span.end(end_time=_end_time(times, index, phase_end)) + + if config.content_mode != "metadata": + messages.append(assistant_message) + if not step.get("is_copied_context"): + final_message = assistant_message + + for call_index, call in enumerate(tool_calls): + if not isinstance(call, dict): + continue + call_id, name, arguments = call.get("tool_call_id"), call.get("function_name"), call.get("arguments") + result = observations.get(call_id) if isinstance(call_id, str) else None + tool_path = f"{semantic_prefix}/tool/{call_id or call_index}" + if ( + isinstance(call_id, str) + and isinstance(name, str) + and isinstance(arguments, dict) + and isinstance(result, dict) + ): + tool_output, tool_complete = _content(result.get("content"), trajectory_path.parent, config) + result_extra = result.get("extra") if isinstance(result.get("extra"), dict) else {} + tool_error = result_extra.get("error") if isinstance(result_extra.get("error"), str) else None + has_result = result.get("content") is not None or tool_error is not None + if config.content_mode != "metadata" and tool_complete and has_result: + tool_span = parent.start_span( + name=name, + type="tool", + id=child_span_id(trial_id, tool_path), + start_time=times[index], + set_current=False, + input=_bounded(arguments, config), + metadata={"tool_call_id": call_id}, + internal={"instrumentation": _INSTRUMENTATION}, + ) + if tool_error is not None: + tool_span.log(error=tool_error) + else: + tool_span.log(output=tool_output) + tool_span.end(end_time=_end_time(times, index, phase_end)) + tool_count += 1 + else: + warnings.append(f"tool {call_id}: downgraded because payload is incomplete") + if config.content_mode != "metadata": + messages.append({"role": "tool", "tool_call_id": call_id, "content": tool_output}) + else: + warnings.append(f"tool {call_id or call_index}: missing correlated arguments or result") + + # Preserve subagents as explicit nested task trees. Their detailed leaves use + # the same conformance gate recursively. + for sub_index, subagent in enumerate(trajectory.get("subagent_trajectories") or []): + if not isinstance(subagent, dict): + continue + sub_parent = parent.start_span( + name=f"subagent:{(subagent.get('agent') or {}).get('name', sub_index)}", + type="task", + id=child_span_id(trial_id, f"{semantic_prefix}/subagent/{sub_index}"), + start_time=phase_start, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + ) + imported = import_trajectory( + sub_parent, + trajectory_path, + trial_id=trial_id, + semantic_prefix=f"{semantic_prefix}/subagent/{sub_index}", + phase_start=phase_start, + phase_end=phase_end, + config=config, + _trajectory_data=subagent, + ) + sub_parent.end(end_time=phase_end) + warnings.extend(imported.warnings) + repairs.extend(imported.repairs) + llm_count += imported.imported_llm_spans + tool_count += imported.imported_tool_spans + + extra = trajectory.get("extra") if isinstance(trajectory.get("extra"), dict) else None + root_extra = dict(extra or {}) + if isinstance(trajectory.get("final_metrics"), dict): + root_extra["final_metrics"] = _bounded(trajectory["final_metrics"], config) + return ATIFImportResult( + final_message=final_message, + schema_version=trajectory.get("schema_version") if isinstance(trajectory.get("schema_version"), str) else None, + root_extra=root_extra or None, + warnings=tuple(warnings), + repairs=tuple(repairs), + imported_llm_spans=llm_count, + imported_tool_spans=tool_count, + ) diff --git a/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml b/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml new file mode 100644 index 000000000..02e70b088 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/cassettes/latest/test_atif_import_round_trips_with_real_sdks.yaml @@ -0,0 +1,643 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MDU3NmNlM2YtZTMwNC00NDRkLThiMjktMzFkNTI3MzcxY2Ni'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 30 Jul 2026 23:21:19 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - MDU3NmNlM2YtZTMwNC00NDRkLThiMjktMzFkNTI3MzcxY2Ni + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::czd4t-1785453679008-8c7ac84e3979 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-harbor-tests", "project_id": null, "org_id": + "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": "harbor-atif-import", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '187' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-harbor-tests","description":null,"created":"2026-07-30T23:19:36.349Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"7aba8758-862e-41ef-9e77-6247c4b71287","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","name":"harbor-atif-import","description":null,"created":"2026-07-30T23:19:36.349Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '745' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-M2NmMjg2Y2EtMGRhYy00Y2I2LWEwZDUtMThlZjFiYTFhYzdh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 30 Jul 2026 23:21:19 GMT + Etag: + - '"29xhqzk7lxkp"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - M2NmMjg2Y2EtMGRhYy00Y2I2LWEwZDUtMThlZjFiYTFhYzdh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::9nsgt-1785453679247-50dbcbbf7967 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.9.0","date_version":"20260730","ff_version":42,"commit":"752127a4257d43c647e4f26fca64445645b9a982","deployment_mode":"lambda","deployment_type":"custom","brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 30 Jul 2026 23:21:19 GMT + Via: + - 1.1 24365d50ec90c9fb2b814e9d6c2f8b8c.cloudfront.net (CloudFront), 1.1 5e2f1ed3ba0ab1e08304bb3d134360de.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - ZBtiwqgmngo7Mzyi0v6hhgOM4yymO_VZ1yrn_TiBRCGR9yNenO8few== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a6bdc6f-4380093633a0d07c091aed99;Parent=061e32842fda1b33;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '557' + etag: + - W/"22d-ctakgszqYQZk3xpUQtT3XtS3QV4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - BV9hfGb_oAMEpJA= + x-amzn-Remapped-content-length: + - '557' + x-amzn-RequestId: + - 5ee5ce52-9412-4136-a4ca-66d01bb782f0 + x-bt-internal-trace-id: + - 6a6bdc6f000000000342bca29c861b13 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"_is_merge": false, "context": {"caller_filename": "[REDACTED_PATH]", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.31.0"}}, "created": "2026-07-30T23:21:18.800866+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "c7a87986-0192-5f40-9ac0-a535810f1fe7", + "metrics": {"start": 1767225600.0}, "root_span_id": "d43785839001054eaf1c4ac196c82112", + "span_attributes": {"exec_counter": 1, "name": "agent_execution", "type": "task"}, + "span_id": "2d261ed7ec06b709", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '831' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["c7a87986-0192-5f40-9ac0-a535810f1fe7"],"xact_id":"1000197603745303703"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 30 Jul 2026 23:21:19 GMT + Via: + - 1.1 24365d50ec90c9fb2b814e9d6c2f8b8c.cloudfront.net (CloudFront), 1.1 21c66eb5f493a6e3ddbaa803cebfe014.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - w1Bxd6CZxXMD_Nw5oNsqPzb9Vpc_O0-EZahRLhV_o5eClmDkar1rQQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a6bdc6f-7cae18ce350b8212466a6df7;Parent=47d54f416b4487bf;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '80' + etag: + - W/"50-Hr4297gud1rJiuyYUUm47RNn6Wc" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BV9hhFaOIAMEfBQ= + x-amzn-RequestId: + - dfd3e7b4-33ab-4eb8-a3f6-e79356557070 + x-bt-internal-trace-id: + - 6a6bdc6f000000006eef670d3893fe97 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.0"}}, "created": "2026-07-30T23:21:18.801549+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "7eb93a96-dfea-5318-bfff-63cf37ce89c6", + "input": [{"content": "What is 2+2?", "role": "user"}], "metadata": {"model": + "gpt-4o-mini", "provider": "openai", "tools": [{"function": {"name": "calculator", + "parameters": {"type": "object"}}, "type": "function"}]}, "metrics": {"completion_tokens": + 4, "end": 1767225602.0, "estimated_cost": 0.001, "prompt_tokens": 10, "start": + 1767225601.0, "tokens": 14}, "output": {"content": "I''ll calculate it.", "role": + "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\": \"2+2\"}", + "name": "calculator"}, "id": "call_1", "type": "function"}]}, "root_span_id": + "d43785839001054eaf1c4ac196c82112", "span_attributes": {"exec_counter": 2, "name": + "chat.completions.create", "type": "llm"}, "span_id": "624573d17b3f2bb9", "span_parents": + ["2d261ed7ec06b709"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.0"}}, "created": "2026-07-30T23:21:18.801909+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "843c7c39-2f3a-5813-aa42-56da9fe5709f", + "input": {"expression": "2+2"}, "metadata": {"tool_call_id": "call_1"}, "metrics": + {"end": 1767225602.0, "start": 1767225601.0}, "output": "4", "root_span_id": + "d43785839001054eaf1c4ac196c82112", "span_attributes": {"exec_counter": 3, "name": + "calculator", "type": "tool"}, "span_id": "161550719b4ad9d1", "span_parents": + ["2d261ed7ec06b709"]},{"context": {"caller_filename": "[REDACTED_PATH]", "caller_functionname": + "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": {"instrumentation": + {"name": "braintrust.plugin.harbor"}, "name": "braintrust.sdk.python", "version": + "0.31.0"}}, "created": "2026-07-30T23:21:18.802634+00:00", "experiment_id": + "7aba8758-862e-41ef-9e77-6247c4b71287", "id": "a2713e08-b2cb-5762-99d7-be29120f9880", + "input": [{"content": "What is 2+2?", "role": "user"}, {"content": "I''ll calculate + it.", "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"expression\": + \"2+2\"}", "name": "calculator"}, "id": "call_1", "type": "function"}]}, {"content": + "4", "role": "tool", "tool_call_id": "call_1"}], "metadata": {"model": "gpt-4o-mini", + "provider": "openai", "tools": [{"function": {"name": "calculator", "parameters": + {"type": "object"}}, "type": "function"}]}, "metrics": {"completion_tokens": + 5, "end": 1767225603.0, "prompt_tokens": 15, "start": 1767225602.0, "tokens": + 20}, "output": {"content": "The answer is 4.", "role": "assistant"}, "root_span_id": + "d43785839001054eaf1c4ac196c82112", "span_attributes": {"exec_counter": 4, "name": + "chat.completions.create", "type": "llm"}, "span_id": "5683a67b695888f5", "span_parents": + ["2d261ed7ec06b709"]},{"_is_merge": true, "experiment_id": "7aba8758-862e-41ef-9e77-6247c4b71287", + "id": "c7a87986-0192-5f40-9ac0-a535810f1fe7", "metrics": {"end": 1767225603.0}, + "root_span_id": "d43785839001054eaf1c4ac196c82112", "span_id": "2d261ed7ec06b709", + "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '3935' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["7eb93a96-dfea-5318-bfff-63cf37ce89c6","843c7c39-2f3a-5813-aa42-56da9fe5709f","a2713e08-b2cb-5762-99d7-be29120f9880","c7a87986-0192-5f40-9ac0-a535810f1fe7"],"xact_id":"1000197603745304547"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 30 Jul 2026 23:21:20 GMT + Via: + - 1.1 76283331d5eee8cee95c8c29a2095f48.cloudfront.net (CloudFront), 1.1 41c02c3f5acef4f58284b65a8f7a983a.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - gIDJ4LIhOrrdGo2SR8MUxAK_NAn4YRuv1trLRJZDN0j1MODdyagy_A== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a6bdc6f-6b1f5ed431cd8c9f3c429215;Parent=33ac9aeaee88bb24;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '197' + etag: + - W/"c5-j4z4aU7oH9appcHSn+idIbx80ks" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - BV9hjFO9IAMErJg= + x-amzn-RequestId: + - b25b10db-541b-4a2a-8ac4-64fac383dfb6 + x-bt-internal-trace-id: + - 6a6bdc6f0000000064194dc671a38a75 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "7aba8758-862e-41ef-9e77-6247c4b71287"}]}, "cursor": null, "limit": 1000}, "use_columnstore": + false, "brainstore_realtime": true, "query_source": "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '332' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[{"_pagination_key":"p07668465162241900546","_xact_id":"1000197603745304547","audit_data":[{"_xact_id":"1000197603745304547","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.0"}},"created":"2026-07-30T23:21:18.802Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"a2713e08-b2cb-5762-99d7-be29120f9880","input":[{"content":"What + is 2+2?","role":"user"},{"content":"I''ll calculate it.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"expression\": + \"2+2\"}","name":"calculator"},"id":"call_1","type":"function"}]},{"content":"4","role":"tool","tool_call_id":"call_1"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai","tools":[{"function":{"name":"calculator","parameters":{"type":"object"}},"type":"function"}]},"metrics":{"completion_tokens":5,"end":1767225603,"prompt_tokens":15,"start":1767225602,"tokens":20},"origin":null,"output":{"content":"The + answer is 4.","role":"assistant"},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"d43785839001054eaf1c4ac196c82112","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":4,"name":"chat.completions.create","type":"llm"},"span_id":"5683a67b695888f5","span_parents":["2d261ed7ec06b709"],"tags":null},{"_pagination_key":"p07668465162241900545","_xact_id":"1000197603745304547","audit_data":[{"_xact_id":"1000197603745304547","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.0"}},"created":"2026-07-30T23:21:18.801Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"843c7c39-2f3a-5813-aa42-56da9fe5709f","input":{"expression":"2+2"},"is_root":false,"metadata":{"tool_call_id":"call_1"},"metrics":{"end":1767225602,"start":1767225601},"origin":null,"output":"4","project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"d43785839001054eaf1c4ac196c82112","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":3,"name":"calculator","type":"tool"},"span_id":"161550719b4ad9d1","span_parents":["2d261ed7ec06b709"],"tags":null},{"_pagination_key":"p07668465162241900544","_xact_id":"1000197603745304547","audit_data":[{"_xact_id":"1000197603745304547","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust.plugin.harbor"},"name":"braintrust.sdk.python","version":"0.31.0"}},"created":"2026-07-30T23:21:18.801Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"7eb93a96-dfea-5318-bfff-63cf37ce89c6","input":[{"content":"What + is 2+2?","role":"user"}],"is_root":false,"metadata":{"model":"gpt-4o-mini","provider":"openai","tools":[{"function":{"name":"calculator","parameters":{"type":"object"}},"type":"function"}]},"metrics":{"completion_tokens":4,"end":1767225602,"estimated_cost":0.001,"prompt_tokens":10,"start":1767225601,"tokens":14},"origin":null,"output":{"content":"I''ll + calculate it.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"expression\": + \"2+2\"}","name":"calculator"},"id":"call_1","type":"function"}]},"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"d43785839001054eaf1c4ac196c82112","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":2,"name":"chat.completions.create","type":"llm"},"span_id":"624573d17b3f2bb9","span_parents":["2d261ed7ec06b709"],"tags":null},{"_pagination_key":"p07668465162186588160","_xact_id":"1000197603745304547","audit_data":[{"_xact_id":"1000197603745303703","audit_data":{"action":"upsert"},"metadata":{},"source":"api"},{"_xact_id":"1000197603745304547","audit_data":{"action":"merge","from":null,"path":["metrics"],"to":{"end":1767225603}},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"context":{"caller_filename":"[REDACTED_PATH]","caller_functionname":"pytest_pyfunc_call","caller_lineno":167,"span_origin":{"instrumentation":{"name":"braintrust-python-logger"},"name":"braintrust.sdk.python","version":"0.31.0"}},"created":"2026-07-30T23:21:18.800Z","error":null,"expected":null,"experiment_id":"7aba8758-862e-41ef-9e77-6247c4b71287","facets":null,"id":"c7a87986-0192-5f40-9ac0-a535810f1fe7","input":null,"is_root":true,"metadata":null,"metrics":{"end":1767225603,"start":1767225600},"origin":null,"output":null,"project_id":"d982a2fd-c0b1-4b55-a0ac-b5af3ba3e229","root_span_id":"d43785839001054eaf1c4ac196c82112","scores":null,"span_attributes":{"created_by_api_key_id":"607d8072-e7a7-48d9-a326-de588dc07bf0","created_by_user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","exec_counter":1,"name":"agent_execution","type":"task"},"span_id":"2d261ed7ec06b709","span_parents":null,"tags":null}],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"cursor":"amvcb4yXAAA","realtime_state":{"type":"on","minimum_xact_id":"1000197603742781733","read_bytes":5773,"actual_xact_id":"1000197603745304547"},"freshness_state":{"last_processed_xact_id":"1000197603742781733","last_considered_xact_id":"1000197603745304547"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 30 Jul 2026 23:21:20 GMT + Via: + - 1.1 76283331d5eee8cee95c8c29a2095f48.cloudfront.net (CloudFront), 1.1 67dd4d73b80aece69a8e725c6d612b6e.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - BeGdyrYRJvLjlEgZGKDvbKaYBoMqcKT0Jp7bg_ppmGGPJU4m6jnAHQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a6bdc70-71a2712e571c4d1212db7add;Parent=4b2e90332ad7da7f;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '14150' + vary: + - Origin + x-amz-apigw-id: + - BV9hmF_nIAMEAsA= + x-amzn-RequestId: + - e72c7a5d-ff66-495c-a892-e0a61d9be286 + x-bt-api-duration-ms: + - '377' + x-bt-brainstore-duration-ms: + - '272' + x-bt-cursor: + - amvcb4yXAAA + x-bt-internal-trace-id: + - 6a6bdc70000000003f25eed7e7d88720 + status: + code: 200 + message: OK +- request: + body: '{"query": {"select": [{"op": "star"}], "from": {"op": "function", "name": + {"op": "ident", "name": ["experiment"]}, "args": [{"op": "literal", "value": + "7aba8758-862e-41ef-9e77-6247c4b71287"}]}, "cursor": "amvcb4yXAAA", "limit": + 1000}, "use_columnstore": false, "brainstore_realtime": true, "query_source": + "py_sdk_object_fetcher_experiment"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip + Connection: + - keep-alive + Content-Length: + - '341' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/btql + response: + body: + string: '{"data":[],"schema":{"type":"array","items":{"type":"object","properties":{"_pagination_key":{"description":"A + stable, time-ordered key that can be used to paginate over experiment events. + This field is auto-generated by Braintrust and only exists in Brainstore.","type":["string","null"]},"_xact_id":{"description":"The + transaction id of an event is unique to the network operation that processed + the event insertion. Transaction ids are monotonically increasing over time + and can be used to retrieve a versioned snapshot of the experiment (see the + `version` parameter)","type":"string"},"audit_data":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"classifications":{"anyOf":[{"additionalProperties":{"items":{"additionalProperties":false,"properties":{"confidence":{"description":"Optional + confidence score for the classification","type":["number","null"]},"id":{"description":"Stable + classification identifier","type":"string"},"label":{"description":"Original + label of the classification item, which is useful for search and indexing + purposes","type":"string"},"metadata":{"anyOf":[{"additionalProperties":{},"type":"object"},{"type":"null"}],"description":"Optional + metadata associated with the classification"},"source":{"anyOf":[{"anyOf":[{"additionalProperties":false,"properties":{"id":{"type":"string"},"type":{"const":"function","type":"string"},"version":{"description":"The + version of the function","type":"string"}},"required":["type","id"],"type":"object"},{"additionalProperties":false,"properties":{"function_type":{"default":"scorer","description":"The + type of global function. Defaults to ''scorer''.","enum":["llm","scorer","task","tool","custom_view","preprocessor","facet","classifier","tag","parameters","sandbox"],"type":"string"},"name":{"type":"string"},"type":{"const":"global","type":"string"}},"required":["type","name"],"type":"object"}]},{"type":"null"}],"description":"Optional + function identifier that produced the classification"}},"required":["id"],"type":"object"},"type":"array"},"properties":{},"type":"object"},{"type":"null"}]},"comments":{"anyOf":[{"items":{},"type":"array"},{"type":"null"}]},"context":{"anyOf":[{"additionalProperties":{},"properties":{"caller_filename":{"description":"Name + of the file in code where the experiment event was created","type":["string","null"]},"caller_functionname":{"description":"The + function in code which created the experiment event","type":["string","null"]},"caller_lineno":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"created":{"description":"The + timestamp the experiment event was created","format":"date-time","type":"string"},"error":{"description":"The + error that occurred, if any."},"expected":{"description":"The ground truth + value (an arbitrary, JSON serializable object) that you''d compare to `output` + to determine if your `output` value is correct or not. Braintrust currently + does not compare `output` to `expected` for you, since there are so many different + ways to do that correctly. Instead, these values are just used to help you + navigate your experiments while digging into analyses. However, we may later + use these values to re-score outputs or fine-tune your models"},"experiment_id":{"description":"Unique + identifier for the experiment","format":"uuid","type":"string"},"facets":{"anyOf":[{"additionalProperties":{"type":["string","null"]},"properties":{},"type":"object"},{"type":"null"}]},"id":{"description":"A + unique identifier for the experiment event. If you don''t provide one, Braintrust + will generate one for you","type":"string"},"input":{"description":"The arguments + that uniquely define a test case (an arbitrary, JSON serializable object). + Later on, Braintrust will use the `input` to know whether two test cases are + the same between experiments, so they should not contain experiment-specific + state. A simple rule of thumb is that if you run the same experiment twice, + the `input` should be identical"},"is_root":{"description":"Whether this span + is a root span","type":["boolean","null"]},"metadata":{"anyOf":[{"additionalProperties":{},"properties":{"model":{"description":"The + model used for this example","type":["string","null"]}},"type":"object"},{"type":"null"}]},"metrics":{"anyOf":[{"additionalProperties":{"type":"number"},"properties":{"caller_filename":{"description":"This + metric is deprecated"},"caller_functionname":{"description":"This metric is + deprecated"},"caller_lineno":{"description":"This metric is deprecated"},"completion_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"end":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event finished","type":["number","null"]},"prompt_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]},"start":{"description":"A + unix timestamp recording when the section of code which produced the experiment + event started","type":["number","null"]},"tokens":{"anyOf":[{"type":"integer"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"origin":{"anyOf":[{"description":"Reference + to the original object and event this was copied from.","properties":{"_xact_id":{"description":"Transaction + ID of the original event.","type":["string","null"]},"created":{"description":"Created + timestamp of the original event. Used to help sort in the UI","type":["string","null"]},"id":{"description":"ID + of the original event.","type":"string"},"object_id":{"description":"ID of + the object the event is originating from.","format":"uuid","type":"string"},"object_type":{"description":"Type + of the object the event is originating from.","enum":["project_logs","experiment","dataset","prompt","function","prompt_session"],"type":"string"}},"required":["object_type","object_id","id"],"type":"object"},{"type":"null"}]},"output":{"description":"The + output of your application, including post-processing (an arbitrary, JSON + serializable object), that allows you to determine whether the result is correct + or not. For example, in an app that generates SQL queries, the `output` should + be the _result_ of the SQL query generated by the model, not the query itself, + because there may be multiple valid queries that answer a single question"},"project_id":{"description":"Unique + identifier for the project that the experiment belongs under","format":"uuid","type":"string"},"root_span_id":{"description":"A + unique identifier for the trace this experiment event belongs to","type":"string"},"scores":{"anyOf":[{"additionalProperties":{"anyOf":[{"maximum":1,"minimum":0,"type":"number"},{"type":"null"}]},"properties":{},"type":"object"},{"type":"null"}]},"span_attributes":{"anyOf":[{"additionalProperties":{},"description":"Human-identifying + attributes of the span, such as name, type, etc.","properties":{"name":{"description":"Name + of the span, for display purposes only","type":["string","null"]},"purpose":{"anyOf":[{"enum":["scorer"],"type":"string"},{"type":"null"}]},"type":{"anyOf":[{"enum":["llm","score","function","eval","task","tool","automation","facet","preprocessor","classifier","review"],"type":"string"},{"type":"null"}]}},"type":"object"},{"type":"null"}]},"span_id":{"description":"A + unique identifier used to link different experiment events together as part + of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) + for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"tags":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]}}}},"realtime_state":{"type":"on","minimum_xact_id":"1000197603742781733","read_bytes":5773,"actual_xact_id":"1000197603745304547"},"freshness_state":{"last_processed_xact_id":"1000197603742781733","last_considered_xact_id":"1000197603745304547"},"warnings":[]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 30 Jul 2026 23:21:21 GMT + Via: + - 1.1 24365d50ec90c9fb2b814e9d6c2f8b8c.cloudfront.net (CloudFront), 1.1 cfcfb1d8fbf5ce2b107182799687a614.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - ywEVZgsvxUYOaNWI-63HoH8ThETSuXL689ZLt7HhDrcn6aglhJX4DQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a6bdc70-302b8fb80d62693c45f4e00d;Parent=31f73070c62514c5;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - private, no-cache + content-length: + - '7788' + vary: + - Origin + x-amz-apigw-id: + - BV9hrEfoIAMElbw= + x-amzn-RequestId: + - e9717258-d80e-4a76-81bf-a799f1175df6 + x-bt-api-duration-ms: + - '341' + x-bt-brainstore-duration-ms: + - '230' + x-bt-internal-trace-id: + - 6a6bdc70000000000c17b6dd7b6b1173 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/harbor/compat.py b/py/src/braintrust/integrations/harbor/compat.py new file mode 100644 index 000000000..821e15b7e --- /dev/null +++ b/py/src/braintrust/integrations/harbor/compat.py @@ -0,0 +1,263 @@ +"""The isolated Harbor-version compatibility boundary.""" + +# Harbor is optional and only supports Python 3.12+, while pylint runs across +# Braintrust's full Python matrix without installing Harbor. +# pylint: disable=import-error + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .identity import logical_task_key + + +@dataclass(frozen=True) +class TaskData: + logical_key: str + source: str + name: str + input: dict[str, Any] + expected: Any + metadata: dict[str, Any] + digest: str | None + schema_version: str | None + task_dir: Path | None + + +@dataclass(frozen=True) +class TrialPlan: + trial_name: str + trial_config: Any + trial_lock: Any + task: TaskData + attempt_index: int + + +@dataclass(frozen=True) +class JobSnapshot: + job_id: str + job_name: str + job_dir: Path + job_config: Any + job_lock: Any + plans: tuple[TrialPlan, ...] + is_resuming: bool = False + + +def _task_download_path(job: Any, trial_config: Any) -> Path | None: + downloads = getattr(job, "_task_download_results", {}) + try: + result = downloads[trial_config.task.get_task_id()] + except (KeyError, AttributeError): + return None + path = getattr(result, "path", None) + return Path(path) if path is not None else None + + +def _task_data(trial_config: Any, trial_lock: Any, task_dir: Path | None) -> TaskData: + task_obj = None + if task_dir is not None: + try: + from harbor.models.task.task import Task + + # Dataset input is task-authored semantics only. Run-specific extra + # instructions are attached to agent_execution by the converter. + task_obj = Task( + task_dir, + disable_verification=bool(getattr(getattr(trial_config, "verifier", None), "disable", False)), + ) + except Exception: + task_obj = None + + task_lock = getattr(trial_lock, "task", None) + source = ( + getattr(getattr(trial_config, "task", None), "source", None) or getattr(task_lock, "source", None) or "adhoc" + ) + name = ( + getattr(task_obj, "name", None) + or getattr(task_lock, "name", None) + or trial_config.task.get_task_id().get_name() + ) + key = logical_task_key(trial_config, trial_lock) + steps = getattr(getattr(task_obj, "config", None), "steps", None) or [] + if task_obj is not None and steps: + canonical_input = { + "task": name, + "steps": [{"name": step.name, "instruction": task_obj.step_instruction(step.name)} for step in steps], + } + else: + canonical_input = { + "task": name, + "instruction": getattr(task_obj, "instruction", ""), + } + + task_config = getattr(task_obj, "config", None) + user_metadata = dict(getattr(task_config, "metadata", None) or {}) + resources: dict[str, Any] = {} + environment = getattr(task_config, "environment", None) + if environment is not None: + for field in ("cpus", "memory_mb", "storage_mb", "gpus", "tpu", "os"): + value = getattr(environment, field, None) + if value is not None: + resources[field] = getattr(value, "value", value) + metadata = { + "harbor": { + "source": source, + "logical_task_key": key, + "task_digest": getattr(task_lock, "digest", None), + "schema_version": getattr(task_config, "schema_version", None), + "resources": resources, + "custom": user_metadata, + } + } + return TaskData( + logical_key=key, + source=source, + name=name, + input=canonical_input, + expected=None, + metadata=metadata, + digest=getattr(task_lock, "digest", None), + schema_version=getattr(task_config, "schema_version", None), + task_dir=task_dir, + ) + + +def snapshot_job(job: Any) -> JobSnapshot: + """Feature-detect the read-only resolved-plan fields allowed by the design.""" + trial_configs = tuple(getattr(job, "_trial_configs")) + if not trial_configs: + raise ValueError("Harbor job has no resolved trial configurations") + + job_lock = getattr(job, "_job_lock", None) + if job_lock is None: + from harbor.models.job.lock import build_job_lock + + job_lock = build_job_lock( + config=job.config, + trial_configs=trial_configs, + task_download_results=getattr(job, "_task_download_results"), + ) + locks = tuple(job_lock.trials) + if len(locks) != len(trial_configs): + raise ValueError("Harbor trial plan and lock have different lengths") + + attempts: dict[tuple[str, str], int] = {} + plans: list[TrialPlan] = [] + for trial_config, trial_lock in zip(trial_configs, locks): + task_dir = _task_download_path(job, trial_config) + task = _task_data(trial_config, trial_lock, task_dir) + agent = trial_config.agent + attempt_key = (task.logical_key, json.dumps(agent.model_dump(mode="json", exclude_none=True), sort_keys=True)) + attempt_index = attempts.get(attempt_key, 0) + attempts[attempt_key] = attempt_index + 1 + plans.append(TrialPlan(trial_config.trial_name, trial_config, trial_lock, task, attempt_index)) + + return JobSnapshot( + job_id=str(job.id), + job_name=str(job.config.job_name), + job_dir=Path(job.job_dir), + job_config=job.config, + job_lock=job_lock, + plans=tuple(plans), + is_resuming=bool(getattr(job, "is_resuming", False)), + ) + + +def trial_directory(result: Any) -> Path: + config = result.config + return Path(config.trials_dir) / result.trial_name + + +def trajectory_paths(result: Any) -> list[tuple[str | None, Path]]: + base = trial_directory(result) + if result.step_results: + return [ + (step.step_name, base / "steps" / step.step_name / "agent" / "trajectory.json") + for step in result.step_results + ] + return [(None, base / "agent" / "trajectory.json")] + + +def reward_details_paths(result: Any) -> list[Path]: + base = trial_directory(result) + if result.step_results: + return [base / "steps" / step.step_name / "verifier" / "reward-details.json" for step in result.step_results] + return [base / "verifier" / "reward-details.json"] + + +def artifact_manifest_paths(result: Any) -> list[Path]: + base = trial_directory(result) + if result.step_results: + return [base / "steps" / step.step_name / "artifacts" / "manifest.json" for step in result.step_results] + return [base / "artifacts" / "manifest.json"] + + +def load_backfill_snapshot(job_dir: str | Path) -> tuple[JobSnapshot, Any]: + """Load persisted Harbor models for offline backfill.""" + from harbor.models.job.config import JobConfig + from harbor.models.job.lock import JobLock, TrialLock + from harbor.models.job.result import JobResult + from harbor.models.trial.result import TrialResult + + directory = Path(job_dir).expanduser().resolve() + config = JobConfig.model_validate_json((directory / "config.json").read_text()) + lock = JobLock.model_validate_json((directory / "lock.json").read_text()) + job_result = JobResult.model_validate_json((directory / "result.json").read_text()) + results: list[Any] = [] + result_paths = { + result_path.parent: result_path + for pattern in ("*/results.json", "*/result.json") + for result_path in sorted(directory.glob(pattern)) + } + for result_path in result_paths.values(): + try: + trial_result = TrialResult.model_validate_json(result_path.read_text()) + # A job directory may be moved before backfill. Resolve trial files + # relative to the directory being backfilled, not the old jobs_dir. + trial_result.config.trials_dir = directory + results.append(trial_result) + except Exception: + continue + if not results and job_result.trial_results: + results = list(job_result.trial_results) + job_result.trial_results = results + + lock_by_digest: dict[str, list[Any]] = {} + for item in lock.trials: + lock_by_digest.setdefault(item.task.digest, []).append(item) + attempts: dict[tuple[str, str], int] = {} + plans: list[TrialPlan] = [] + for result in results: + trial_lock_path = trial_directory(result) / "lock.json" + if trial_lock_path.exists(): + trial_lock = TrialLock.model_validate_json(trial_lock_path.read_text()) + else: + candidates = lock_by_digest.get(f"sha256:{result.task_checksum}", []) or lock_by_digest.get( + result.task_checksum, [] + ) + trial_lock = candidates[0] if candidates else lock.trials[0] + task_dir = None + try: + candidate = result.config.task.get_task_id().get_local_path() + task_dir = candidate if candidate.exists() else None + except Exception: + pass + task = _task_data(result.config, trial_lock, task_dir) + agent_key = json.dumps(result.config.agent.model_dump(mode="json", exclude_none=True), sort_keys=True) + attempt_key = (task.logical_key, agent_key) + attempt_index = attempts.get(attempt_key, 0) + attempts[attempt_key] = attempt_index + 1 + plans.append(TrialPlan(result.trial_name, result.config, trial_lock, task, attempt_index)) + + snapshot = JobSnapshot( + job_id=str(job_result.id), + job_name=config.job_name, + job_dir=directory, + job_config=config, + job_lock=lock, + plans=tuple(plans), + is_resuming=True, + ) + return snapshot, job_result diff --git a/py/src/braintrust/integrations/harbor/config.py b/py/src/braintrust/integrations/harbor/config.py new file mode 100644 index 000000000..53cb6df94 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/config.py @@ -0,0 +1,202 @@ +"""Configuration for the Harbor job plugin.""" + +import fnmatch +import json +import math +import os +from dataclasses import dataclass, field, fields +from typing import Any + + +_UNSET = object() +_PREFIX = "HARBOR_BRAINTRUST_" + + +def _environment_value(name: str, default: Any) -> Any: + names = [f"{_PREFIX}{name.upper()}"] + if name == "project_name": + names.append(f"{_PREFIX}PROJECT") + for environment_name in names: + value = os.environ.get(environment_name) + if value is not None: + return value + return default + + +def _resolve(value: Any, name: str, default: Any) -> Any: + return _environment_value(name, default) if value is _UNSET else value + + +def _parse_bool(value: Any, name: str) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be a boolean") + + +def _parse_int(value: Any, name: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{name} must be a non-negative integer") + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a non-negative integer") from exc + if parsed < 0: + raise ValueError(f"{name} must be a non-negative integer") + return parsed + + +def _parse_json(value: Any, name: str, expected_type: type) -> Any: + if value is None: + return None + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} must be valid JSON") from exc + if not isinstance(value, expected_type): + raise ValueError(f"{name} must be a JSON {expected_type.__name__}") + return value + + +def _parse_patterns(value: Any, name: str) -> tuple[str, ...]: + if value is None: + return () + value = _parse_json(value, name, list) if isinstance(value, str) else value + if not isinstance(value, (list, tuple)) or not all(isinstance(item, str) and item for item in value): + raise ValueError(f"{name} must be a JSON array of non-empty strings") + return tuple(value) + + +def _patterns_overlap(left: str, right: str) -> bool: + # Exact equality and either pattern matching the other catch all useful, + # deterministic overlap cases without pretending to solve glob intersection. + return left == right or fnmatch.fnmatchcase(left, right) or fnmatch.fnmatchcase(right, left) + + +@dataclass(frozen=True) +class PluginConfig: + project_name: str | None = None + project_id: str | None = None + experiment_prefix: str | None = None + base_experiment_name: str | None = None + base_experiment_id: str | None = None + dataset_mode: str = "sync" + dataset_name: str | None = None + trajectory_mode: str = "atif" + content_mode: str = "messages" + include_custom_metadata: bool = True + max_custom_metadata_bytes: int = 100_000 + score_keys: tuple[str, ...] = () + metric_keys: tuple[str, ...] = () + reward_rules: dict[str, dict[str, Any]] = field(default_factory=dict) + classifier_rules: dict[str, str] = field(default_factory=dict) + invalid_score_policy: str = "metric" + include_tracebacks: bool = False + attachments: str = "verifier-details" + artifact_include: tuple[str, ...] = () + max_attachment_bytes: int = 5_000_000 + max_total_attachment_bytes: int = 20_000_000 + max_content_bytes: int = 20_000 + log_job_summary: bool = True + log_retry_attempts: bool = False + strict: bool = False + redact_patterns: tuple[str, ...] = () + + @classmethod + def from_options(cls, **options: Any) -> "PluginConfig": + defaults = cls() + values: dict[str, Any] = {} + for config_field in fields(defaults): + name = config_field.name + values[name] = _resolve(options.get(name, _UNSET), name, getattr(defaults, name)) + + for name in ( + "include_custom_metadata", + "include_tracebacks", + "log_job_summary", + "log_retry_attempts", + "strict", + ): + values[name] = _parse_bool(values[name], name) + for name in ( + "max_custom_metadata_bytes", + "max_attachment_bytes", + "max_total_attachment_bytes", + "max_content_bytes", + ): + values[name] = _parse_int(values[name], name) + for name in ("score_keys", "metric_keys", "artifact_include", "redact_patterns"): + values[name] = _parse_patterns(values[name], name) + for name in ("reward_rules", "classifier_rules"): + values[name] = _parse_json(values[name], name, dict) or {} + + config = cls(**values) + config.validate() + return config + + def validate(self) -> None: + if self.project_name and self.project_id: + raise ValueError("project_name and project_id are mutually exclusive") + if self.base_experiment_name and self.base_experiment_id: + raise ValueError("base_experiment_name and base_experiment_id are mutually exclusive") + if self.dataset_mode not in {"sync", "none"}: + raise ValueError("dataset_mode must be 'sync' or 'none'; 'existing' is not supported yet") + if self.dataset_name and self.dataset_mode != "sync": + raise ValueError("dataset_name requires dataset_mode='sync'") + if self.trajectory_mode not in {"atif", "summary", "native"}: + raise ValueError("trajectory_mode must be 'atif', 'summary', or 'native'") + if self.content_mode not in {"metadata", "messages", "full"}: + raise ValueError("content_mode must be 'metadata', 'messages', or 'full'") + if self.invalid_score_policy not in {"metric", "drop", "error"}: + raise ValueError("invalid_score_policy must be 'metric', 'drop', or 'error'") + if self.attachments not in {"none", "verifier-details", "all"}: + raise ValueError("attachments must be 'none', 'verifier-details', or 'all'") + if self.artifact_include and self.attachments != "all": + raise ValueError("artifact_include requires attachments='all'") + if self.max_total_attachment_bytes < self.max_attachment_bytes: + raise ValueError("max_total_attachment_bytes must be at least max_attachment_bytes") + + for score_pattern in self.score_keys: + for metric_pattern in self.metric_keys: + if _patterns_overlap(score_pattern, metric_pattern): + raise ValueError(f"score_keys and metric_keys overlap: {score_pattern!r}, {metric_pattern!r}") + + for key, rule in self.reward_rules.items(): + if not isinstance(key, str) or not key or not isinstance(rule, dict): + raise ValueError("reward_rules must map non-empty strings to objects") + rule_type = rule.get("type") + if rule_type not in {"score", "metric"}: + raise ValueError(f"reward_rules[{key!r}].type must be 'score' or 'metric'") + if rule_type == "metric" and any(field in rule for field in ("direction", "min", "max", "score_name")): + raise ValueError(f"metric reward rule {key!r} cannot define score normalization") + if rule_type == "score": + direction = rule.get("direction", "maximize") + if direction not in {"maximize", "minimize"}: + raise ValueError(f"reward_rules[{key!r}].direction must be 'maximize' or 'minimize'") + has_min, has_max = "min" in rule, "max" in rule + if has_min != has_max: + raise ValueError(f"reward_rules[{key!r}] must define both min and max") + if has_min: + minimum, maximum = rule["min"], rule["max"] + if ( + isinstance(minimum, bool) + or isinstance(maximum, bool) + or not isinstance(minimum, (int, float)) + or not isinstance(maximum, (int, float)) + or not math.isfinite(float(minimum)) + or not math.isfinite(float(maximum)) + or minimum >= maximum + ): + raise ValueError(f"reward_rules[{key!r}] requires finite min < max") + + if not all( + isinstance(name, str) and name and isinstance(path, str) and path + for name, path in self.classifier_rules.items() + ): + raise ValueError("classifier_rules must map non-empty names to non-empty JSON paths") diff --git a/py/src/braintrust/integrations/harbor/identity.py b/py/src/braintrust/integrations/harbor/identity.py new file mode 100644 index 000000000..e0f3c0541 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/identity.py @@ -0,0 +1,214 @@ +"""Deterministic identity and privacy-safe normalization helpers.""" + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePath +from typing import Any +from urllib.parse import urlsplit, urlunsplit +from uuid import UUID, uuid5 + + +PLUGIN_NAMESPACE = UUID("67ea9f8a-e42a-5f31-96d8-85bcf27ca4c9") +_SECRET_KEY = re.compile(r"(?:api[_-]?key|token|secret|password|credential|authorization|cookie)", re.IGNORECASE) +_ABSOLUTE_WINDOWS_PATH = re.compile(r"^[a-zA-Z]:[\\/]") +_TEMPLATE = re.compile(r"^\$\{[A-Za-z_][A-Za-z0-9_]*(?::-[^}]*)?\}$") + + +@dataclass(frozen=True) +class NormalizedValue: + value: Any + warnings: tuple[str, ...] = () + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False) + + +def stable_hash(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def deterministic_id(scope: str, value: str) -> str: + return str(uuid5(PLUGIN_NAMESPACE, f"{scope}:{value}")) + + +def dataset_record_id(dataset_scope: str, logical_task_key: str) -> str: + return deterministic_id("dataset-record", f"{dataset_scope}:{logical_task_key}") + + +def child_span_id(trial_id: str, semantic_path: str) -> str: + try: + namespace = UUID(str(trial_id)) + except ValueError: + namespace = uuid5(PLUGIN_NAMESPACE, str(trial_id)) + return str(uuid5(namespace, semantic_path)) + + +def _is_absolute_path(value: str) -> bool: + return value.startswith(("/", "~/", "file://")) or bool(_ABSOLUTE_WINDOWS_PATH.match(value)) + + +def _json_size(value: Any) -> int: + try: + return len(canonical_json(value).encode("utf-8")) + except (TypeError, ValueError): + return 10**18 + + +def normalize_json( + value: Any, + *, + max_bytes: int, + redact_patterns: tuple[str, ...] = (), + max_depth: int = 8, +) -> NormalizedValue: + """Normalize untrusted metadata while preserving JSON types and nulls.""" + warnings: list[str] = [] + compiled_patterns = tuple(re.compile(pattern) for pattern in redact_patterns) + + def walk(item: Any, path: str, depth: int, key: str | None = None) -> Any: + if depth > max_depth: + warnings.append(f"dropped {path}: depth limit") + return "[DROPPED: depth limit]" + if key is not None and _SECRET_KEY.search(key): + if isinstance(item, str) and _TEMPLATE.match(item): + return item + return "[REDACTED]" + if item is None or isinstance(item, (bool, int, float)): + return item + if isinstance(item, str): + if _is_absolute_path(item): + warnings.append(f"dropped {path}: absolute path") + return "[REDACTED PATH]" + result = item + for pattern in compiled_patterns: + result = pattern.sub("[REDACTED]", result) + return result + if isinstance(item, PurePath): + raw = str(item) + if item.is_absolute() or _is_absolute_path(raw): + warnings.append(f"dropped {path}: absolute path") + return "[REDACTED PATH]" + return item.as_posix() + if isinstance(item, dict): + normalized: dict[str, Any] = {} + for raw_key, child in item.items(): + child_key = str(raw_key) + child_path = f"{path}.{child_key}" if path else child_key + normalized[child_key] = walk(child, child_path, depth + 1, child_key) + return normalized + if isinstance(item, (list, tuple)): + return [walk(child, f"{path}[{index}]", depth + 1) for index, child in enumerate(item)] + model_dump = getattr(item, "model_dump", None) + if callable(model_dump): + try: + return walk(model_dump(mode="json", exclude_none=False), path, depth) + except Exception: + pass + warnings.append(f"dropped {path}: unsupported type {type(item).__name__}") + return f"[DROPPED: {type(item).__name__}]" + + normalized = walk(value, "", 0) + if _json_size(normalized) <= max_bytes: + return NormalizedValue(normalized, tuple(warnings)) + + warnings.append(f"dropped metadata: exceeded {max_bytes} bytes") + if isinstance(normalized, dict): + bounded: dict[str, Any] = {} + for key in sorted(normalized): + candidate = {**bounded, key: normalized[key]} + if _json_size(candidate) > max_bytes: + warnings.append(f"dropped {key}: size limit") + continue + bounded[key] = normalized[key] + normalized = bounded + elif isinstance(normalized, str): + normalized = normalized.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore") + else: + normalized = "[DROPPED: size limit]" + return NormalizedValue(normalized, tuple(warnings)) + + +def safe_git_url(value: str) -> str: + parsed = urlsplit(value) + hostname = parsed.hostname or "" + if parsed.port: + hostname = f"{hostname}:{parsed.port}" + return urlunsplit((parsed.scheme, hostname, parsed.path.rstrip("/"), "", "")) + + +def logical_task_key(task_config: Any, task_lock: Any | None = None) -> str: + """Choose a stable logical task identity without exposing local paths.""" + task = getattr(task_config, "task", None) + if task is not None: + name = getattr(task, "name", None) + ref = getattr(task, "ref", None) + if name: + return f"package:{name}@{ref or 'default'}" + + git_url = getattr(task, "git_url", None) + path = getattr(task, "path", None) + if git_url: + relative = Path(path).as_posix().lstrip("/") if path is not None else "" + return f"git:{safe_git_url(git_url)}#{relative}" + + lock_name = getattr(getattr(task_lock, "task", task_lock), "name", None) + source = getattr(task, "source", None) or getattr(getattr(task_lock, "task", task_lock), "source", None) + if lock_name: + return f"harbor:{source or 'adhoc'}:{lock_name}" + name = Path(path).name if path is not None else "task" + return f"local:{source or 'adhoc'}:{name}" + + +def dataset_scope(source: str, logical_keys: list[str]) -> str: + key_hash = stable_hash(sorted(logical_keys))[:8] + return f"{source}:tasks-{key_hash}" + + +def dataset_display_name(source: str, logical_keys: list[str]) -> str: + scope = dataset_scope(source, logical_keys) + return f"harbor · {source} · {scope.rsplit(':', 1)[-1]}" + + +def semantic_agent_config(agent: Any, skills: list[Any]) -> dict[str, Any]: + def safe_env(raw: Any) -> dict[str, str]: + result: dict[str, str] = {} + for key, value in (raw or {}).items(): + value = str(value) + if _SECRET_KEY.search(str(key)): + result[str(key)] = value if _TEMPLATE.match(value) else f"${{{key}}}" + else: + result[str(key)] = value + return result + + raw = { + "name": getattr(agent, "name", None), + "import_path": getattr(agent, "import_path", None), + "model": getattr(agent, "model_name", None), + "kwargs": getattr(agent, "kwargs", None) or {}, + "env": safe_env(getattr(agent, "env", None)), + "mcp_servers": getattr(agent, "mcp_servers", None) or [], + "resume_trajectory": bool(getattr(agent, "resume_trajectory", False)), + "load_trajectory": getattr(agent, "load_trajectory", None), + "skills": sorted( + [ + { + "name": getattr(skill, "name", None), + "digest": getattr(skill, "digest", None), + "git_url": safe_git_url(str(getattr(skill, "git_url"))) + if getattr(skill, "git_url", None) + else None, + "git_commit_id": getattr(skill, "git_commit_id", None), + } + for skill in skills + ], + key=canonical_json, + ), + } + return normalize_json(raw, max_bytes=200_000).value + + +def partition_key(dataset_key: str, agent_config: dict[str, Any]) -> str: + return stable_hash({"dataset": dataset_key, "agent": agent_config}) diff --git a/py/src/braintrust/integrations/harbor/plugin.py b/py/src/braintrust/integrations/harbor/plugin.py new file mode 100644 index 000000000..7386ba386 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/plugin.py @@ -0,0 +1,938 @@ +"""Native Braintrust job plugin for Harbor.""" + +# Harbor is optional and only supports Python 3.12+, while pylint runs across +# Braintrust's full Python matrix without installing Harbor. +# pylint: disable=import-error + +import asyncio +import fnmatch +import json +import logging +import os +from dataclasses import dataclass, field, fields +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from braintrust.logger import Attachment, flush, init, init_dataset, init_logger +from exceptiongroup import ExceptionGroup + +from .atif import _INSTRUMENTATION, ATIFImportResult, import_trajectory, summarize_trajectory +from .compat import ( + JobSnapshot, + TrialPlan, + artifact_manifest_paths, + load_backfill_snapshot, + reward_details_paths, + snapshot_job, + trajectory_paths, +) +from .config import _UNSET, PluginConfig +from .identity import ( + canonical_json, + child_span_id, + dataset_display_name, + dataset_record_id, + dataset_scope, + normalize_json, + partition_key, + semantic_agent_config, +) +from .rewards import classify_rewards, extract_json_path, validate_classifications +from .state import ( + JobEvent, + JobMachine, + TrialEvent, + TrialEventKind, + TrialMachine, + TrialStatus, + reduce_job, + reduce_trial, +) + + +logger = logging.getLogger(__name__) +_PLUGIN_VERSION = "1" +_MANIFEST_VERSION = 1 + + +@dataclass +class DatasetBinding: + scope: str + dataset: Any = None + origins: dict[str, dict[str, Any]] = field(default_factory=dict) + error: str | None = None + + +@dataclass +class Partition: + key: str + name: str + dataset_scope: str + experiment: Any = None + experiment_id: str | None = None + + +@dataclass +class RuntimeState: + snapshot: JobSnapshot + plan_by_trial: dict[str, TrialPlan] + partition_by_trial: dict[str, Partition] + datasets: dict[str, DatasetBinding] + partitions: dict[str, Partition] + + +def _seconds(value: Any, fallback: float) -> float: + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.timestamp() + return fallback + + +def _timing(value: Any, default_start: float, default_end: float) -> tuple[float, float]: + start = _seconds(getattr(value, "started_at", None), default_start) + end = _seconds(getattr(value, "finished_at", None), default_end) + if end < start: + end = start + return start, end + + +def _exception(result: Any, include_traceback: bool) -> tuple[str | None, str | None]: + info = getattr(result, "exception_info", None) + if info is None: + return None, None + error = f"{info.exception_type}: {info.exception_message}" + traceback_value = info.exception_traceback if include_traceback else None + return error, traceback_value + + +def _answer_from_metadata(result: Any) -> Any: + contexts = [] + if getattr(result, "agent_result", None) is not None: + contexts.append(result.agent_result) + for step in getattr(result, "step_results", None) or []: + if getattr(step, "agent_result", None) is not None: + contexts.append(step.agent_result) + for context in reversed(contexts): + metadata = getattr(context, "metadata", None) + if not isinstance(metadata, dict): + continue + for key in ("standardized_answer", "final_answer", "answer", "output", "response"): + if key in metadata: + return metadata[key] + return None + + +def _rewards(result: Any) -> dict[str, Any]: + verifier = getattr(result, "verifier_result", None) + raw = getattr(verifier, "rewards", None) + return dict(raw or {}) + + +def _read_json_summary(paths: list[Path], max_bytes: int) -> tuple[Any, list[str]]: + summaries: list[Any] = [] + warnings: list[str] = [] + for path in paths: + try: + size = path.stat().st_size + with path.open("rb") as file_obj: + data = file_obj.read(min(size, max_bytes) + 1) + if len(data) > max_bytes: + warnings.append(f"{path.name} omitted: size limit") + continue + summaries.append(json.loads(data)) + except FileNotFoundError: + continue + except (OSError, json.JSONDecodeError) as exc: + warnings.append(f"could not read {path.name}: {exc}") + if not summaries: + return None, warnings + return summaries[0] if len(summaries) == 1 else summaries, warnings + + +def _artifact_attachments(result: Any, config: PluginConfig) -> tuple[dict[str, Attachment], list[str]]: + if config.attachments != "all" or not config.artifact_include: + return {}, [] + attachments: dict[str, Attachment] = {} + warnings: list[str] = [] + total = 0 + for manifest_path in artifact_manifest_paths(result): + root = manifest_path.parent.resolve() + if not root.exists(): + continue + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == "manifest.json" or path.is_symlink(): + continue + try: + resolved = path.resolve() + relative = resolved.relative_to(root).as_posix() + except (OSError, ValueError): + warnings.append(f"artifact {path.name} omitted: unsafe path") + continue + if not any(fnmatch.fnmatchcase(relative, pattern) for pattern in config.artifact_include): + continue + try: + size = resolved.stat().st_size + if size > config.max_attachment_bytes or total + size > config.max_total_attachment_bytes: + warnings.append(f"artifact {relative} omitted: attachment size limit") + continue + data = resolved.read_bytes() + except OSError as exc: + warnings.append(f"artifact {relative} omitted: {exc}") + continue + total += len(data) + attachments[relative] = Attachment( + data=data, + filename=resolved.name, + content_type="application/octet-stream", + ) + return attachments, warnings + + +def _attachment(paths: list[Path], config: PluginConfig) -> tuple[Attachment | None, Any, list[str]]: + if config.attachments == "none": + return None, None, [] + total = 0 + complete: list[Any] = [] + warnings: list[str] = [] + for path in paths: + try: + data = path.read_bytes() + except FileNotFoundError: + continue + except OSError as exc: + warnings.append(f"could not read {path.name}: {exc}") + continue + if len(data) > config.max_attachment_bytes or total + len(data) > config.max_total_attachment_bytes: + warnings.append(f"{path.name} omitted: attachment size limit") + continue + try: + parsed = json.loads(data) + except json.JSONDecodeError: + warnings.append(f"{path.name} is not valid JSON") + continue + normalized = normalize_json( + parsed, + max_bytes=config.max_attachment_bytes, + redact_patterns=config.redact_patterns, + max_depth=20, + ) + warnings.extend(normalized.warnings) + complete.append(normalized.value) + total += len(data) + if not complete: + return None, None, warnings + summary = complete[0] if len(complete) == 1 else complete + attachment_data = (canonical_json(summary) + "\n").encode() + if len(attachment_data) > config.max_total_attachment_bytes: + warnings.append("reward-details.json omitted after redaction: total size limit") + return None, summary, warnings + return ( + Attachment(data=attachment_data, filename="reward-details.json", content_type="application/json"), + summary, + warnings, + ) + + +class HarborPlugin: + """Harbor plugin that reconciles final trials into Braintrust experiments.""" + + def __init__( + self, + project_name: Any = _UNSET, + project_id: Any = _UNSET, + experiment_prefix: Any = _UNSET, + base_experiment_name: Any = _UNSET, + base_experiment_id: Any = _UNSET, + dataset_mode: Any = _UNSET, + dataset_name: Any = _UNSET, + trajectory_mode: Any = _UNSET, + content_mode: Any = _UNSET, + include_custom_metadata: Any = _UNSET, + max_custom_metadata_bytes: Any = _UNSET, + score_keys: Any = _UNSET, + metric_keys: Any = _UNSET, + reward_rules: Any = _UNSET, + classifier_rules: Any = _UNSET, + invalid_score_policy: Any = _UNSET, + include_tracebacks: Any = _UNSET, + attachments: Any = _UNSET, + artifact_include: Any = _UNSET, + max_attachment_bytes: Any = _UNSET, + max_total_attachment_bytes: Any = _UNSET, + max_content_bytes: Any = _UNSET, + log_job_summary: Any = _UNSET, + log_retry_attempts: Any = _UNSET, + strict: Any = _UNSET, + **kwargs: Any, + ) -> None: + options = { + "project_name": project_name, + "project_id": project_id, + "experiment_prefix": experiment_prefix, + "base_experiment_name": base_experiment_name, + "base_experiment_id": base_experiment_id, + "dataset_mode": dataset_mode, + "dataset_name": dataset_name, + "trajectory_mode": trajectory_mode, + "content_mode": content_mode, + "include_custom_metadata": include_custom_metadata, + "max_custom_metadata_bytes": max_custom_metadata_bytes, + "score_keys": score_keys, + "metric_keys": metric_keys, + "reward_rules": reward_rules, + "classifier_rules": classifier_rules, + "invalid_score_policy": invalid_score_policy, + "include_tracebacks": include_tracebacks, + "attachments": attachments, + "artifact_include": artifact_include, + "max_attachment_bytes": max_attachment_bytes, + "max_total_attachment_bytes": max_total_attachment_bytes, + "max_content_bytes": max_content_bytes, + "log_job_summary": log_job_summary, + "log_retry_attempts": log_retry_attempts, + "strict": strict, + **kwargs, + } + unknown = set(options) - {config_field.name for config_field in fields(PluginConfig)} + if unknown: + raise TypeError(f"Unexpected HarborPlugin options: {', '.join(sorted(unknown))}") + self.config = PluginConfig.from_options(**options) + self._job_machine = JobMachine() + self._trial_machines: dict[str, TrialMachine] = {} + self._trial_locks: dict[str, asyncio.Lock] = {} + self._runtime: RuntimeState | None = None + self._snapshot: JobSnapshot | None = None + self._errors: list[str] = [] + self._warnings: list[str] = [] + self._manifest: dict[str, Any] = {} + self._disabled_reason: str | None = None + + async def on_job_start(self, job: Any) -> None: + self._job_machine = reduce_job(self._job_machine, JobEvent.INITIALIZE, strict=self.config.strict) + try: + snapshot = await asyncio.to_thread(snapshot_job, job) + self._snapshot = snapshot + self._runtime = await asyncio.to_thread(self._initialize, snapshot) + for plan in snapshot.plans: + self._trial_machines[plan.trial_name] = TrialMachine(plan.trial_name) + self._trial_locks[plan.trial_name] = asyncio.Lock() + self._register_hooks(job) + self._job_machine = reduce_job(self._job_machine, JobEvent.READY, strict=self.config.strict) + await asyncio.to_thread(self._persist_manifest, False) + except Exception as exc: + self._disable(f"Braintrust initialization failed: {exc}") + if self._snapshot is not None: + try: + await asyncio.to_thread(self._persist_disabled_manifest) + except OSError as persist_exc: + self._errors.append(f"could not persist disabled manifest: {persist_exc}") + if self.config.strict: + raise + + async def on_job_end(self, job_result: Any) -> None: + if self._runtime is None: + return + self._job_machine = reduce_job(self._job_machine, JobEvent.RECONCILE, strict=self.config.strict) + final_names = {result.trial_name for result in job_result.trial_results} + failures: list[BaseException] = [] + + async def reconcile(result: Any) -> None: + try: + await self._dispatch(result.trial_name, TrialEvent(TrialEventKind.FINAL_RESULT, payload=result)) + await asyncio.to_thread(self._sync_final_result, result) + await self._dispatch(result.trial_name, TrialEvent(TrialEventKind.SYNCED)) + except Exception as exc: + failures.append(exc) + self._errors.append(f"trial {result.trial_name}: {exc}") + try: + await self._dispatch(result.trial_name, TrialEvent(TrialEventKind.SYNC_FAILED, payload=str(exc))) + except Exception: + pass + + await asyncio.gather(*(reconcile(result) for result in job_result.trial_results)) + for name in set(self._trial_machines) - final_names: + await self._dispatch(name, TrialEvent(TrialEventKind.OMIT)) + try: + await asyncio.to_thread(self._finalize, job_result) + except Exception as exc: + failures.append(exc) + self._errors.append(f"final flush: {exc}") + self._job_machine = reduce_job(self._job_machine, JobEvent.CLOSE, strict=self.config.strict) + try: + await asyncio.to_thread(self._persist_manifest, not failures) + except Exception as exc: + failures.append(exc) + self._errors.append(f"manifest persistence: {exc}") + logger.warning("Could not persist Harbor Braintrust sync manifest", exc_info=True) + if failures and self.config.strict: + raise ExceptionGroup("Braintrust Harbor synchronization failed", failures) + + def _disable(self, message: str) -> None: + self._disabled_reason = message + self._errors.append(message) + self._job_machine = reduce_job(self._job_machine, JobEvent.DISABLE) + logger.warning(message, exc_info=True) + + def _register_hooks(self, job: Any) -> None: + from harbor.trial.hooks import TrialEvent as HarborTrialEvent + + mapping = { + HarborTrialEvent.START: TrialEventKind.START, + HarborTrialEvent.ENVIRONMENT_START: TrialEventKind.ENVIRONMENT_START, + HarborTrialEvent.AGENT_START: TrialEventKind.AGENT_START, + HarborTrialEvent.AGENT_END: TrialEventKind.AGENT_END, + HarborTrialEvent.VERIFICATION_START: TrialEventKind.VERIFICATION_START, + HarborTrialEvent.END: TrialEventKind.END, + HarborTrialEvent.CANCEL: TrialEventKind.CANCEL, + } + max_retries = int(getattr(getattr(job.config, "retry", None), "max_retries", 0) or 0) + for harbor_event, internal_kind in mapping.items(): + + async def callback(event: Any, kind: TrialEventKind = internal_kind) -> None: + try: + machine = self._trial_machines.get(event.trial_name) + retry_predicted = bool( + kind == TrialEventKind.END + and machine is not None + and machine.retry_index < max_retries + and getattr(event.result, "exception_info", None) is not None + ) + await self._dispatch( + event.trial_name, + TrialEvent( + kind, + timestamp=event.timestamp.timestamp(), + payload=event.result if kind == TrialEventKind.END else None, + retry_predicted=retry_predicted, + ), + ) + except Exception as exc: + self._errors.append(f"hook {kind.value} for {event.trial_name}: {exc}") + if self.config.strict: + raise + + job.add_hook(harbor_event, callback) + + async def _dispatch(self, identity: str, event: TrialEvent) -> None: + if self._job_machine.status.value not in {"active", "reconciling"} and event.kind not in { + TrialEventKind.FINAL_RESULT, + TrialEventKind.SYNCED, + TrialEventKind.SYNC_FAILED, + TrialEventKind.OMIT, + }: + return + if identity not in self._trial_machines: + self._trial_machines[identity] = TrialMachine(identity) + self._trial_locks[identity] = asyncio.Lock() + async with self._trial_locks[identity]: + new_state, _effects = reduce_trial( + self._trial_machines[identity], + event, + strict=self.config.strict, + ) + self._trial_machines[identity] = new_state + + def _initialize(self, snapshot: JobSnapshot) -> RuntimeState: + previous = self._load_manifest(snapshot.job_dir) + self._manifest = previous + plan_by_trial = {plan.trial_name: plan for plan in snapshot.plans} + datasets: dict[str, DatasetBinding] = {} + source_tasks: dict[str, dict[str, Any]] = {} + for plan in snapshot.plans: + source_tasks.setdefault(plan.task.source, {})[plan.task.logical_key] = plan.task + + for source, task_map in source_tasks.items(): + scope = dataset_scope(source, sorted(task_map)) + binding = DatasetBinding(scope) + datasets[scope] = binding + if self.config.dataset_mode != "sync": + continue + try: + if self.config.dataset_name and len(source_tasks) == 1: + name = self.config.dataset_name + elif self.config.dataset_name: + name = f"{self.config.dataset_name} · {source} · {scope.rsplit(':', 1)[-1]}" + else: + name = dataset_display_name(source, sorted(task_map)) + dataset = init_dataset( + project=self.config.project_name, + project_id=self.config.project_id, + name=name, + use_output=False, + metadata={"harbor": {"source": source, "scope": scope, "schema_version": _PLUGIN_VERSION}}, + ) + for task in task_map.values(): + normalized = normalize_json( + task.metadata, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + self._warnings.extend(normalized.warnings) + dataset.insert( + id=dataset_record_id(scope, task.logical_key), + input=task.input, + expected=task.expected, + metadata=normalized.value, + ) + dataset.flush() + rows = list(dataset) + for row in rows: + if row.get("id") and row.get("_xact_id"): + binding.origins[row["id"]] = { + "object_type": "dataset", + "object_id": dataset.id, + "id": row["id"], + "created": row.get("created"), + "_xact_id": row["_xact_id"], + } + binding.dataset = dataset + except Exception as exc: + binding.error = str(exc) + self._warnings.append(f"dataset {scope} sync failed; continuing without association: {exc}") + + partitions: dict[str, Partition] = {} + partition_by_trial: dict[str, Partition] = {} + for plan in snapshot.plans: + scope = dataset_scope(plan.task.source, sorted(source_tasks[plan.task.source])) + semantic = semantic_agent_config( + plan.trial_config.agent, list(getattr(plan.trial_lock, "skills", []) or []) + ) + key = partition_key(scope, semantic) + partition = partitions.get(key) + if partition is None: + agent_name = ( + getattr(plan.trial_config.agent, "name", None) + or getattr(plan.trial_config.agent, "import_path", None) + or "agent" + ) + model = getattr(plan.trial_config.agent, "model_name", None) or "default" + prefix = self.config.experiment_prefix or snapshot.job_name + name = f"{prefix}-{snapshot.job_id[:8]} · {agent_name}@{model} · {plan.task.source} · {key[:8]}" + metadata = { + "harbor": { + "job_id": snapshot.job_id, + "job_name": snapshot.job_name, + "partition_key": key, + "semantic_agent_config": semantic, + } + } + dataset = datasets[scope].dataset + experiment = init( + project=self.config.project_name, + project_id=self.config.project_id, + experiment=name, + update=True, + dataset=dataset, + metadata=metadata, + base_experiment=self.config.base_experiment_name, + base_experiment_id=self.config.base_experiment_id, + ) + partition = Partition(key=key, name=name, dataset_scope=scope, experiment=experiment) + # Resolve lazy metadata now so initialization/auth failures are isolated. + partition.experiment_id = experiment.id + partitions[key] = partition + partition_by_trial[plan.trial_name] = partition + + return RuntimeState(snapshot, plan_by_trial, partition_by_trial, datasets, partitions) + + def _root_metadata(self, result: Any, plan: TrialPlan, machine: TrialMachine) -> dict[str, Any]: + raw_rewards = _rewards(result) + trial_custom = getattr(getattr(result, "agent_result", None), "metadata", None) or {} + normalized = normalize_json( + trial_custom if self.config.include_custom_metadata else {}, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + task_custom = normalize_json( + plan.task.metadata.get("harbor", {}).get("custom", {}) if self.config.include_custom_metadata else {}, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + self._warnings.extend((*normalized.warnings, *task_custom.warnings)) + error, traceback_value = _exception(result, self.config.include_tracebacks) + metadata: dict[str, Any] = { + "harbor": { + "job_id": self._runtime.snapshot.job_id if self._runtime else None, + "trial_id": str(result.id), + "task_name": result.task_name, + "agent": result.agent_info.name, + "model": result.agent_info.model_info.name if result.agent_info.model_info else None, + "attempt_index": plan.attempt_index, + "retry_index": machine.retry_index, + "raw_rewards": raw_rewards, + "custom": {"task": task_custom.value, "trial": normalized.value}, + "warnings": list(machine.warnings), + } + } + if error and traceback_value: + metadata["harbor"]["exception_traceback"] = traceback_value + return metadata + + def _start_phase( + self, + task_span: Any, + result: Any, + name: str, + timing_name: str, + trial_id: str, + root_start: float, + root_end: float, + **event: Any, + ) -> Any: + start, end = _timing(getattr(result, timing_name, None), root_start, root_end) + span = task_span.start_span( + name=name, + type="task", + id=child_span_id(trial_id, f"task/{name}"), + start_time=start, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + **event, + ) + span.end(end_time=end) + return span + + def _sync_final_result(self, result: Any) -> None: + if self._runtime is None: + raise RuntimeError("plugin is not initialized") + plan = self._runtime.plan_by_trial.get(result.trial_name) + partition = self._runtime.partition_by_trial.get(result.trial_name) + if plan is None or partition is None: + raise ValueError(f"final result {result.trial_name!r} is absent from the resolved plan") + machine = self._trial_machines[result.trial_name] + trial_id = str(result.id) + now = datetime.now().timestamp() + root_start = _seconds(getattr(result, "started_at", None), now) + root_end = _seconds(getattr(result, "finished_at", None), root_start) + if root_end < root_start: + root_end = root_start + error, _ = _exception(result, self.config.include_tracebacks) + metadata = self._root_metadata(result, plan, machine) + rewards = _rewards(result) + conversion = classify_rewards(rewards, self.config) + metadata["harbor"]["warnings"].extend(conversion.warnings) + if not rewards and error is None: + metadata["harbor"]["warnings"].append("trial has no reward and is unevaluated") + + binding = self._runtime.datasets[partition.dataset_scope] + record_id = dataset_record_id(partition.dataset_scope, plan.task.logical_key) + origin = binding.origins.get(record_id) + root_metrics = dict(conversion.metrics) + if machine.completed_attempts: + root_metrics["retries"] = max(machine.completed_attempts - 1, machine.retry_index) + root_event: dict[str, Any] = { + "id": trial_id, + "name": "eval", + "type": "eval", + "start_time": root_start, + "set_current": False, + "input": plan.task.input, + "expected": plan.task.expected, + "metadata": metadata, + "metrics": root_metrics, + } + if origin: + root_event["origin"] = origin + if error: + root_event["error"] = error + root = partition.experiment.start_span( + internal={"instrumentation": _INSTRUMENTATION}, + **root_event, + ) + task = root.start_span( + name="task", + type="task", + id=child_span_id(trial_id, "task"), + start_time=root_start, + set_current=False, + input=plan.task.input, + expected=plan.task.expected, + error=error, + internal={"instrumentation": _INSTRUMENTATION}, + ) + + self._start_phase(task, result, "environment_setup", "environment_setup", trial_id, root_start, root_end) + self._start_phase(task, result, "agent_setup", "agent_setup", trial_id, root_start, root_end) + agent_start, agent_end = _timing(getattr(result, "agent_execution", None), root_start, root_end) + execution_input: dict[str, Any] = {"task": plan.task.input} + extra_instructions: list[str] = [] + for path in getattr(result.config, "extra_instruction_paths", []) or []: + try: + extra_instructions.append(Path(path).read_text()) + except OSError: + continue + if extra_instructions: + execution_input["extra_instructions"] = extra_instructions + selected_artifacts, artifact_attachment_warnings = _artifact_attachments(result, self.config) + metadata["harbor"]["warnings"].extend(artifact_attachment_warnings) + agent_span = task.start_span( + name="agent_execution", + type="task", + id=child_span_id(trial_id, "task/agent_execution"), + start_time=agent_start, + set_current=False, + input=normalize_json( + execution_input, max_bytes=self.config.max_content_bytes, redact_patterns=self.config.redact_patterns + ).value, + internal={"instrumentation": _INSTRUMENTATION}, + ) + + atif_results: list[tuple[str | None, ATIFImportResult]] = [] + if self.config.trajectory_mode in {"atif", "summary"}: + for step_name, path in trajectory_paths(result): + if self.config.trajectory_mode == "summary": + imported = summarize_trajectory(path, self.config) + else: + prefix = "task/agent_execution" if step_name is None else f"task/step:{step_name}/agent_execution" + imported = import_trajectory( + agent_span, + path, + trial_id=trial_id, + semantic_prefix=prefix, + phase_start=agent_start, + phase_end=agent_end, + config=self.config, + ) + atif_results.append((step_name, imported)) + if selected_artifacts: + agent_span.log(output={"artifacts": selected_artifacts}) + agent_span.end(end_time=agent_end) + self._start_phase(task, result, "verification", "verifier", trial_id, root_start, root_end) + + for step in getattr(result, "step_results", None) or []: + step_start, step_end = _timing(getattr(step, "agent_execution", None), root_start, root_end) + step_span = task.start_span( + name=f"step:{step.step_name}", + type="task", + id=child_span_id(trial_id, f"task/step:{step.step_name}"), + start_time=step_start, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + ) + step_error, _ = _exception(step, self.config.include_tracebacks) + if step_error: + step_span.log(error=step_error) + step_span.end(end_time=step_end) + + trajectory_warnings = [warning for _, imported in atif_results for warning in imported.warnings] + repairs = [repair for _, imported in atif_results for repair in imported.repairs] + metadata["harbor"]["warnings"].extend(trajectory_warnings) + metadata["harbor"]["trajectory"] = { + "present": bool(atif_results), + "schema_version": next( + (imported.schema_version for _, imported in atif_results if imported.schema_version), None + ), + "repairs": repairs, + } + if atif_results and atif_results[0][1].root_extra and self.config.include_custom_metadata: + normalized_extra = normalize_json( + atif_results[0][1].root_extra, + max_bytes=self.config.max_custom_metadata_bytes, + redact_patterns=self.config.redact_patterns, + ) + metadata["harbor"]["trajectory"]["custom"] = normalized_extra.value + metadata["harbor"]["warnings"].extend(normalized_extra.warnings) + + output = _answer_from_metadata(result) + if output is None: + final_messages = [ + (name, imported.final_message) for name, imported in atif_results if imported.final_message is not None + ] + if len(final_messages) == 1: + output = final_messages[0][1] + elif final_messages: + output = {name or "final": message for name, message in final_messages} + else: + output = {"status": "completed" if error is None else "error"} + output = normalize_json( + output, max_bytes=self.config.max_content_bytes, redact_patterns=self.config.redact_patterns + ).value + if error is None: + task.log(output=output) + root.log(output=output) + task.log(metadata={"harbor": {"warnings": trajectory_warnings}}) + task.end(end_time=root_end) + + details_attachment, details_summary, detail_warnings = _attachment(reward_details_paths(result), self.config) + metadata["harbor"]["warnings"].extend(detail_warnings) + for score in conversion.scores: + scorer = root.start_span( + name=score.name, + type="score", + span_attributes={"purpose": "scorer"}, + id=child_span_id(trial_id, f"scorer/{score.source_key}"), + start_time=root_end, + set_current=False, + input={"reward": score.raw_value}, + internal={"instrumentation": _INSTRUMENTATION}, + ) + scorer_output: dict[str, Any] = {"score": score.value, "raw_reward": score.raw_value} + if details_summary is not None: + scorer_output["reward_details_summary"] = normalize_json( + details_summary, max_bytes=self.config.max_content_bytes + ).value + if details_attachment is not None: + scorer_output["reward_details"] = details_attachment + scorer.log(output=scorer_output, scores={score.name: score.value}) + scorer.end(end_time=root_end) + + classifications: dict[str, list[dict[str, Any]]] = {} + for source_name, path in self.config.classifier_rules.items(): + classifier = root.start_span( + name=source_name, + type="classifier", + span_attributes={"purpose": "scorer"}, + id=child_span_id(trial_id, f"classifier/{source_name}"), + start_time=root_end, + set_current=False, + internal={"instrumentation": _INSTRUMENTATION}, + ) + try: + items = validate_classifications(extract_json_path(result, path)) + if items: + classifications[source_name] = items + classifier.log(output=items[0] if len(items) == 1 else items) + except Exception as exc: + classifier.log(error=f"invalid classifier {source_name}: {exc}") + metadata["harbor"]["warnings"].append(f"classifier {source_name!r} was malformed: {exc}") + classifier.end(end_time=root_end) + if classifications: + root.log(classifications=classifications) + + manifests, manifest_warnings = _read_json_summary( + artifact_manifest_paths(result), self.config.max_content_bytes + ) + metadata["harbor"]["warnings"].extend(manifest_warnings) + if manifests is not None: + metadata["harbor"]["artifact_manifest"] = normalize_json( + manifests, max_bytes=self.config.max_content_bytes + ).value + root.log(metadata=metadata) + root.end(end_time=root_end) + + def _finalize(self, job_result: Any) -> None: + if self._runtime is None: + return + if self.config.log_job_summary: + project_logger = init_logger( + project=self.config.project_name, + project_id=self.config.project_id, + set_current=False, + ) + summary = project_logger.start_span( + name="harbor.job.summary", + type="task", + id=f"harbor-job-summary-{self._runtime.snapshot.job_id}", + set_current=False, + input={"job_id": self._runtime.snapshot.job_id}, + metadata={ + "harbor": { + "job_id": self._runtime.snapshot.job_id, + "experiments": [ + {"id": partition.experiment_id, "name": partition.name} + for partition in self._runtime.partitions.values() + ], + } + }, + internal={"instrumentation": _INSTRUMENTATION}, + ) + summary.log(output=job_result.stats.model_dump(mode="json", exclude_none=False)) + summary.end(end_time=_seconds(getattr(job_result, "finished_at", None), datetime.now().timestamp())) + flush() + + def _persist_disabled_manifest(self) -> None: + if self._snapshot is None: + return + manifest = { + "manifest_version": _MANIFEST_VERSION, + "plugin_version": _PLUGIN_VERSION, + "job_id": self._snapshot.job_id, + "project": {"name": self.config.project_name, "id": self.config.project_id}, + "datasets": {}, + "experiments": {}, + "trials": {}, + "synced_trial_ids": [], + "warnings": self._warnings, + "errors": self._errors, + "disabled_reason": self._disabled_reason, + "completed": False, + } + path = self._snapshot.job_dir / "braintrust-sync.json" + temp_path = path.with_suffix(".json.tmp") + temp_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + os.replace(temp_path, path) + self._manifest = manifest + + @staticmethod + def _load_manifest(job_dir: Path) -> dict[str, Any]: + path = job_dir / "braintrust-sync.json" + try: + data = json.loads(path.read_text()) + return data if isinstance(data, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + + def _persist_manifest(self, completed: bool) -> None: + if self._runtime is None: + return + snapshot = self._runtime.snapshot + manifest = { + "manifest_version": _MANIFEST_VERSION, + "plugin_version": _PLUGIN_VERSION, + "job_id": snapshot.job_id, + "project": {"name": self.config.project_name, "id": self.config.project_id}, + "datasets": { + scope: { + "id": binding.dataset.id if binding.dataset is not None else None, + "version": binding.dataset.version if binding.dataset is not None else None, + "error": binding.error, + } + for scope, binding in self._runtime.datasets.items() + }, + "experiments": { + key: {"id": partition.experiment_id, "name": partition.name} + for key, partition in self._runtime.partitions.items() + }, + "trials": { + name: { + "status": machine.status.value, + "retry_count": machine.retry_index, + "completed_attempts": machine.completed_attempts, + "warnings": list(machine.warnings), + } + for name, machine in self._trial_machines.items() + }, + "synced_trial_ids": sorted( + str(machine.final_result.id) + for machine in self._trial_machines.values() + if machine.status == TrialStatus.SYNCED and machine.final_result is not None + ), + "warnings": self._warnings, + "errors": self._errors, + "disabled_reason": self._disabled_reason, + "completed": completed, + } + path = snapshot.job_dir / "braintrust-sync.json" + temp_path = path.with_suffix(".json.tmp") + temp_path.write_text(json.dumps(manifest, indent=2, sort_keys=True, default=str) + "\n") + os.replace(temp_path, path) + self._manifest = manifest + + async def sync_job_directory(self, job_dir: str | Path) -> None: + """Backfill a persisted Harbor job directory with the online conversion core.""" + self._job_machine = reduce_job(self._job_machine, JobEvent.INITIALIZE, strict=self.config.strict) + snapshot, result = await asyncio.to_thread(load_backfill_snapshot, job_dir) + self._snapshot = snapshot + self._runtime = await asyncio.to_thread(self._initialize, snapshot) + for plan in snapshot.plans: + self._trial_machines[plan.trial_name] = TrialMachine(plan.trial_name) + self._trial_locks[plan.trial_name] = asyncio.Lock() + self._job_machine = reduce_job(self._job_machine, JobEvent.READY, strict=self.config.strict) + await self.on_job_end(result) + + +async def backfill_job(job_dir: str | Path, **plugin_options: Any) -> None: + """Backfill a Harbor job directory into Braintrust.""" + await HarborPlugin(**plugin_options).sync_job_directory(job_dir) diff --git a/py/src/braintrust/integrations/harbor/rewards.py b/py/src/braintrust/integrations/harbor/rewards.py new file mode 100644 index 000000000..d1895ee5c --- /dev/null +++ b/py/src/braintrust/integrations/harbor/rewards.py @@ -0,0 +1,152 @@ +"""Harbor reward and classifier conversion.""" + +import fnmatch +import math +from dataclasses import dataclass, field +from numbers import Real +from typing import Any + +from .config import PluginConfig + + +@dataclass(frozen=True) +class ScoreValue: + name: str + value: float + source_key: str + raw_value: int | float + transformed: bool = False + + +@dataclass(frozen=True) +class RewardConversion: + scores: tuple[ScoreValue, ...] = () + metrics: dict[str, int | float] = field(default_factory=dict) + warnings: tuple[str, ...] = () + + +def _numeric(value: Any) -> bool: + return isinstance(value, Real) and not isinstance(value, bool) and math.isfinite(float(value)) + + +def _matches(key: str, patterns: tuple[str, ...]) -> bool: + return any(fnmatch.fnmatchcase(key, pattern) for pattern in patterns) + + +def _metric_name(key: str) -> str: + # These keys have standardized Braintrust meanings. Harbor rewards are not + # presumed to share them, so retain an explicit integration namespace. + standard = { + "start", + "end", + "duration", + "tokens", + "prompt_tokens", + "completion_tokens", + "estimated_cost", + "time_to_first_token", + } + return f"harbor_reward.{key}" if key in standard else key + + +def classify_rewards(rewards: dict[str, Any] | None, config: PluginConfig) -> RewardConversion: + scores: list[ScoreValue] = [] + metrics: dict[str, int | float] = {} + warnings: list[str] = [] + if not rewards: + return RewardConversion() + + for key, raw in rewards.items(): + if not _numeric(raw): + warnings.append(f"reward {key!r} is not a finite number and was omitted") + continue + value = float(raw) + rule = config.reward_rules.get(key) + requested_type: str | None = rule.get("type") if rule else None + if requested_type is None: + if _matches(key, config.score_keys): + requested_type = "score" + elif _matches(key, config.metric_keys): + requested_type = "metric" + elif key == "reward" and 0 <= value <= 1: + requested_type = "score" + else: + requested_type = "metric" + + if requested_type == "metric": + metrics[_metric_name(key)] = raw + continue + + score_name = str(rule.get("score_name", key)) if rule else key + transformed = False + normalized = value + if rule and "min" in rule and "max" in rule: + minimum, maximum = float(rule["min"]), float(rule["max"]) + if minimum <= value <= maximum: + direction = rule.get("direction", "maximize") + normalized = (value - minimum) / (maximum - minimum) + if direction == "minimize": + normalized = (maximum - value) / (maximum - minimum) + transformed = True + else: + normalized = float("nan") + + if not math.isfinite(normalized) or not 0 <= normalized <= 1: + warning = f"configured score {key!r} has invalid value {raw!r}" + if config.invalid_score_policy == "error": + raise ValueError(warning) + warnings.append(warning) + if config.invalid_score_policy == "metric": + metrics[_metric_name(key)] = raw + continue + + scores.append( + ScoreValue( + name=score_name, + value=normalized, + source_key=key, + raw_value=raw, + transformed=transformed, + ) + ) + if transformed: + metrics[f"harbor_reward.raw.{key}"] = raw + + return RewardConversion(tuple(scores), metrics, tuple(warnings)) + + +def extract_json_path(value: Any, path: str) -> Any: + """Extract a documented dotted/JSON-pointer-like path from a model or mapping.""" + parts = [part for part in path.replace("/", ".").split(".") if part] + current = value + for part in parts: + if isinstance(current, dict): + if part not in current: + raise KeyError(path) + current = current[part] + elif isinstance(current, (list, tuple)): + current = current[int(part)] + else: + model_dump = getattr(current, "model_dump", None) + if callable(model_dump): + current = model_dump(mode="python", exclude_none=False) + if part not in current: + raise KeyError(path) + current = current[part] + else: + raise KeyError(path) + return current + + +def validate_classifications(value: Any) -> list[dict[str, Any]]: + items = value if isinstance(value, list) else [value] + validated: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict) or not isinstance(item.get("id"), str) or not item["id"]: + raise ValueError("classification items require a non-empty string id") + if "label" in item and item["label"] is not None and not isinstance(item["label"], str): + raise ValueError("classification item label must be a string or null") + if "metadata" in item and item["metadata"] is not None and not isinstance(item["metadata"], dict): + raise ValueError("classification item metadata must be a JSON object or null") + validated.append({key: item[key] for key in ("id", "label", "metadata") if key in item}) + return validated diff --git a/py/src/braintrust/integrations/harbor/state.py b/py/src/braintrust/integrations/harbor/state.py new file mode 100644 index 000000000..99a2d5394 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/state.py @@ -0,0 +1,220 @@ +"""Pure reducer-based Harbor lifecycle state machines.""" + +from dataclasses import dataclass, replace +from enum import Enum +from typing import Any + + +class JobStatus(str, Enum): + NEW = "new" + INITIALIZING = "initializing" + ACTIVE = "active" + RECONCILING = "reconciling" + CLOSED = "closed" + DISABLED = "disabled" + FAILED = "failed" + + +class JobEvent(str, Enum): + INITIALIZE = "initialize" + READY = "ready" + RECONCILE = "reconcile" + CLOSE = "close" + DISABLE = "disable" + FAIL = "fail" + + +@dataclass(frozen=True) +class JobMachine: + status: JobStatus = JobStatus.NEW + warnings: tuple[str, ...] = () + + +def reduce_job(state: JobMachine, event: JobEvent, *, strict: bool = False) -> JobMachine: + transitions = { + (JobStatus.NEW, JobEvent.INITIALIZE): JobStatus.INITIALIZING, + (JobStatus.INITIALIZING, JobEvent.READY): JobStatus.ACTIVE, + (JobStatus.ACTIVE, JobEvent.RECONCILE): JobStatus.RECONCILING, + (JobStatus.RECONCILING, JobEvent.CLOSE): JobStatus.CLOSED, + } + if event == JobEvent.DISABLE and state.status not in {JobStatus.CLOSED, JobStatus.FAILED}: + return replace(state, status=JobStatus.DISABLED) + if event == JobEvent.FAIL and state.status not in {JobStatus.CLOSED, JobStatus.DISABLED}: + return replace(state, status=JobStatus.FAILED) + target = transitions.get((state.status, event)) + if target is not None: + return replace(state, status=target) + message = f"illegal job transition {state.status.value} + {event.value}" + if strict: + raise ValueError(message) + return replace(state, warnings=(*state.warnings, message)) + + +class TrialStatus(str, Enum): + PENDING = "pending" + ACTIVE = "active" + WAITING_RETRY = "waiting_retry" + FINAL_CANDIDATE = "final_candidate" + CANCELLED = "cancelled" + FINALIZING = "finalizing" + SYNCED = "synced" + OMITTED = "omitted" + + +class TrialPhase(str, Enum): + STARTED = "started" + ENVIRONMENT = "environment" + AGENT = "agent" + AGENT_DONE = "agent_done" + VERIFICATION = "verification" + + +_PHASE_ORDER = { + TrialPhase.STARTED: 0, + TrialPhase.ENVIRONMENT: 1, + TrialPhase.AGENT: 2, + TrialPhase.AGENT_DONE: 3, + TrialPhase.VERIFICATION: 4, +} + + +class TrialEventKind(str, Enum): + START = "start" + ENVIRONMENT_START = "environment-start" + AGENT_START = "agent-start" + AGENT_END = "agent-end" + VERIFICATION_START = "verification-start" + END = "end" + CANCEL = "cancel" + FINAL_RESULT = "final_result" + OMIT = "omit" + SYNCED = "synced" + SYNC_FAILED = "sync_failed" + + +@dataclass(frozen=True) +class TrialEvent: + kind: TrialEventKind + timestamp: float | None = None + payload: Any = None + retry_predicted: bool = False + + +class EffectKind(str, Enum): + STAGE_RESULT = "stage_result" + RECORD_RETRY = "record_retry" + RECORD_CANCELLATION = "record_cancellation" + SYNC_FINAL = "sync_final" + CLOSE_OMITTED = "close_omitted" + + +@dataclass(frozen=True) +class Effect: + kind: EffectKind + payload: Any = None + + +@dataclass(frozen=True) +class TrialMachine: + identity: str + status: TrialStatus = TrialStatus.PENDING + phase: TrialPhase | None = None + retry_index: int = 0 + completed_attempts: int = 0 + warnings: tuple[str, ...] = () + final_result: Any = None + + +def _warn(state: TrialMachine, message: str, strict: bool) -> tuple[TrialMachine, tuple[Effect, ...]]: + if strict: + raise ValueError(message) + return replace(state, warnings=(*state.warnings, message)), () + + +def reduce_trial( + state: TrialMachine, + event: TrialEvent, + *, + strict: bool = False, +) -> tuple[TrialMachine, tuple[Effect, ...]]: + """Reduce one lifecycle event without performing I/O.""" + kind = event.kind + if kind == TrialEventKind.START: + if state.status == TrialStatus.ACTIVE: + return state, () + if state.status in { + TrialStatus.PENDING, + TrialStatus.WAITING_RETRY, + TrialStatus.FINAL_CANDIDATE, + TrialStatus.CANCELLED, + }: + retry_index = state.retry_index + effects: tuple[Effect, ...] = () + if state.status != TrialStatus.PENDING: + retry_index += 1 + effects = (Effect(EffectKind.RECORD_RETRY, retry_index),) + return replace( + state, status=TrialStatus.ACTIVE, phase=TrialPhase.STARTED, retry_index=retry_index + ), effects + return _warn(state, f"START after terminal state {state.status.value}", strict) + + phase_for_event = { + TrialEventKind.ENVIRONMENT_START: TrialPhase.ENVIRONMENT, + TrialEventKind.AGENT_START: TrialPhase.AGENT, + TrialEventKind.AGENT_END: TrialPhase.AGENT_DONE, + TrialEventKind.VERIFICATION_START: TrialPhase.VERIFICATION, + }.get(kind) + if phase_for_event is not None: + if state.status != TrialStatus.ACTIVE or state.phase is None: + return _warn(state, f"{kind.value} while {state.status.value}", strict) + current_order = _PHASE_ORDER[state.phase] + next_order = _PHASE_ORDER[phase_for_event] + if next_order == current_order: + return state, () + if next_order < current_order: + return _warn(state, f"backward phase {state.phase.value} -> {phase_for_event.value}", strict) + return replace(state, phase=phase_for_event), () + + if kind == TrialEventKind.END: + if state.status in {TrialStatus.WAITING_RETRY, TrialStatus.FINAL_CANDIDATE, TrialStatus.CANCELLED}: + return state, () + if state.status != TrialStatus.ACTIVE: + return _warn(state, f"END while {state.status.value}", strict) + status = TrialStatus.WAITING_RETRY if event.retry_predicted else TrialStatus.FINAL_CANDIDATE + return ( + replace(state, status=status, completed_attempts=state.completed_attempts + 1), + (Effect(EffectKind.STAGE_RESULT, event.payload),), + ) + + if kind == TrialEventKind.CANCEL: + if state.status == TrialStatus.CANCELLED: + return state, () + if state.status in {TrialStatus.SYNCED, TrialStatus.OMITTED}: + return _warn(state, f"CANCEL after terminal state {state.status.value}", strict) + return replace(state, status=TrialStatus.CANCELLED), (Effect(EffectKind.RECORD_CANCELLATION),) + + if kind == TrialEventKind.FINAL_RESULT: + if state.status == TrialStatus.SYNCED: + return state, () + if state.status == TrialStatus.OMITTED: + return _warn(state, "FINAL_RESULT after OMIT", strict) + return replace(state, status=TrialStatus.FINALIZING, final_result=event.payload), ( + Effect(EffectKind.SYNC_FINAL, event.payload), + ) + + if kind == TrialEventKind.SYNCED: + if state.status != TrialStatus.FINALIZING: + return _warn(state, f"SYNCED while {state.status.value}", strict) + return replace(state, status=TrialStatus.SYNCED), () + + if kind == TrialEventKind.SYNC_FAILED: + if state.status != TrialStatus.FINALIZING: + return _warn(state, f"SYNC_FAILED while {state.status.value}", strict) + return _warn(replace(state, status=TrialStatus.FINAL_CANDIDATE), str(event.payload), False) + + if kind == TrialEventKind.OMIT: + if state.status in {TrialStatus.SYNCED, TrialStatus.OMITTED}: + return state, () + return replace(state, status=TrialStatus.OMITTED), (Effect(EffectKind.CLOSE_OMITTED),) + + return _warn(state, f"unknown trial event {kind}", strict) diff --git a/py/src/braintrust/integrations/harbor/test_harbor.py b/py/src/braintrust/integrations/harbor/test_harbor.py new file mode 100644 index 000000000..eef20adc2 --- /dev/null +++ b/py/src/braintrust/integrations/harbor/test_harbor.py @@ -0,0 +1,363 @@ +# Harbor is installed by the dedicated Python 3.12+ nox session, not by the +# cross-version pylint environment. +# pylint: disable=import-error + +import json +import os +import re +from datetime import datetime +from pathlib import Path + +import pytest +from braintrust import flush, init +from braintrust.conftest import get_vcr_config +from braintrust.git_fields import GitMetadataSettings +from braintrust.integrations.harbor.atif import _usage_metrics, import_trajectory, summarize_trajectory +from braintrust.integrations.harbor.config import PluginConfig +from braintrust.integrations.harbor.identity import ( + child_span_id, + dataset_record_id, + logical_task_key, + normalize_json, + partition_key, + semantic_agent_config, +) +from braintrust.integrations.harbor.rewards import classify_rewards, validate_classifications +from braintrust.integrations.harbor.state import ( + TrialEvent, + TrialEventKind, + TrialMachine, + TrialPhase, + TrialStatus, + reduce_trial, +) +from harbor.models.job.lock import AgentSkillLock, TaskLock, TrialLock +from harbor.models.trajectories.trajectory import Trajectory +from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig, VerifierConfig + + +_ABSOLUTE_PATH_RE = re.compile(r"(?:/(?:Users|private|home)/[^\"\\\\\s]+|[A-Za-z]:\\\\[^\"\\\\\s]+)") + + +def _redact_cassette_body(body): + if not isinstance(body, (str, bytes)): + return body + is_bytes = isinstance(body, bytes) + text = body.decode("utf-8", errors="replace") if is_bytes else body + redacted = _ABSOLUTE_PATH_RE.sub("[REDACTED_PATH]", text) + return redacted.encode() if is_bytes else redacted + + +@pytest.fixture(scope="module") +def vcr_config(): + config = get_vcr_config() + scrub_response = config["before_record_response"] + + def before_record_request(request): + request.body = _redact_cassette_body(request.body) + return request + + def before_record_response(response): + response = scrub_response(response) + body = response.get("body", {}) + if "string" in body: + body["string"] = _redact_cassette_body(body["string"]) + return response + + return { + **config, + "before_record_request": before_record_request, + "before_record_response": before_record_response, + } + + +def test_config_environment_fallback_and_explicit_precedence(): + names = { + "HARBOR_BRAINTRUST_PROJECT": "harbor-project", + "HARBOR_BRAINTRUST_DATASET_MODE": "none", + "HARBOR_BRAINTRUST_SCORE_KEYS": '["reward", "correct*"]', + "HARBOR_BRAINTRUST_STRICT": "true", + } + original = {name: os.environ.get(name) for name in names} + try: + os.environ.update(names) + from braintrust.integrations.harbor import HarborPlugin + + config = HarborPlugin(strict=False).config + assert config.project_name == "harbor-project" + assert config.dataset_mode == "none" + assert config.score_keys == ("reward", "correct*") + assert config.strict is False + finally: + for name, value in original.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def test_config_rejects_overlapping_reward_patterns_and_invalid_bounds(): + with pytest.raises(ValueError, match="overlap"): + PluginConfig.from_options(score_keys=["correct*"], metric_keys=["correctness"]) + with pytest.raises(ValueError, match="min < max"): + PluginConfig.from_options(reward_rules={"latency": {"type": "score", "min": 1, "max": 1}}) + + +def test_reward_classification_is_semantic_not_range_based(): + config = PluginConfig.from_options( + reward_rules={ + "error_rate": { + "type": "score", + "direction": "minimize", + "min": 0, + "max": 10, + "score_name": "reliability", + } + }, + score_keys=["correctness"], + ) + result = classify_rewards( + {"reward": 0.8, "quality": 0.7, "correctness": 1, "error_rate": 2, "tokens": 40}, + config, + ) + + assert {score.name: score.value for score in result.scores} == { + "reward": 0.8, + "correctness": 1, + "reliability": 0.8, + } + assert result.metrics == { + "quality": 0.7, + "harbor_reward.raw.error_rate": 2, + "harbor_reward.tokens": 40, + } + + +def test_invalid_configured_score_defaults_to_metric(): + config = PluginConfig.from_options(score_keys=["raw"]) + result = classify_rewards({"raw": 5}, config) + assert result.scores == () + assert result.metrics == {"raw": 5} + assert result.warnings + + +def test_classification_validation_is_atomic_and_preserves_duplicates(): + items = validate_classifications( + [ + {"id": "cat", "label": "Cat", "metadata": {"confidence": "high"}}, + {"id": "cat", "label": None}, + ] + ) + assert items[0]["id"] == items[1]["id"] == "cat" + with pytest.raises(ValueError): + validate_classifications([{"id": "ok"}, {"label": "missing id"}]) + + +def test_metadata_normalization_redacts_secrets_paths_and_bounds_size(tmp_path): + normalized = normalize_json( + { + "api_key": "secret", + "nested": {"keep": None, "path": str(tmp_path / "private")}, + "message": "token=abc", + }, + max_bytes=1_000, + redact_patterns=(r"token=[a-z]+",), + ) + assert normalized.value["api_key"] == "[REDACTED]" + assert normalized.value["nested"]["keep"] is None + assert normalized.value["nested"]["path"] == "[REDACTED PATH]" + assert normalized.value["message"] == "[REDACTED]" + assert any("absolute path" in warning for warning in normalized.warnings) + + +def test_ids_and_partition_are_deterministic_and_do_not_include_concurrency(tmp_path): + task_config = TrialConfig( + task=TaskConfig(path=Path("relative/task"), source="suite"), + agent=AgentConfig( + name="agent", + model_name="provider/model", + n_concurrent=1, + env={"API_KEY": "actual-secret", "MODE": "careful"}, + ), + ) + task_lock = TrialLock( + task=TaskLock(name="task", type="local", digest="sha256:" + "a" * 64, source="suite"), + agent=task_config.agent, + skills=[ + AgentSkillLock( + name="skill", + source=tmp_path / "skill", + digest="sha256:" + "b" * 64, + ) + ], + environment=EnvironmentConfig(), + verifier=VerifierConfig(), + ) + semantic = semantic_agent_config(task_config.agent, task_lock.skills) + key = logical_task_key(task_config, task_lock) + + assert "actual-secret" not in json.dumps(semantic) + assert semantic["env"]["API_KEY"] == "${API_KEY}" + assert partition_key(key, semantic) == partition_key(key, semantic) + assert dataset_record_id("scope", key) == dataset_record_id("scope", key) + assert child_span_id("trial", "task/verification") == child_span_id("trial", "task/verification") + + changed_concurrency = AgentConfig( + name="agent", + model_name="provider/model", + n_concurrent=99, + env={"API_KEY": "actual-secret", "MODE": "careful"}, + ) + assert semantic_agent_config(changed_concurrency, task_lock.skills) == semantic + + +def test_trial_reducer_retry_duplicate_backward_and_reconcile(): + state = TrialMachine("trial") + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.START)) + assert state.status == TrialStatus.ACTIVE + assert state.phase == TrialPhase.STARTED + + duplicate, effects = reduce_trial(state, TrialEvent(TrialEventKind.START)) + assert duplicate == state + assert effects == () + + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.AGENT_START)) + backward, _ = reduce_trial(state, TrialEvent(TrialEventKind.ENVIRONMENT_START)) + assert backward.phase == TrialPhase.AGENT + assert backward.warnings + + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.END, retry_predicted=True)) + assert state.status == TrialStatus.WAITING_RETRY + assert state.completed_attempts == 1 + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.START)) + assert state.retry_index == 1 + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.END)) + assert state.status == TrialStatus.FINAL_CANDIDATE + + final = "authoritative-final-result" + state, effects = reduce_trial(state, TrialEvent(TrialEventKind.FINAL_RESULT, payload=final)) + assert state.status == TrialStatus.FINALIZING + assert effects[0].payload is final + state, _ = reduce_trial(state, TrialEvent(TrialEventKind.SYNCED)) + assert state.status == TrialStatus.SYNCED + + +@pytest.mark.vcr +def test_atif_import_round_trips_with_real_sdks(tmp_path): + trajectory_path = tmp_path / "trajectory.json" + trajectory = Trajectory.model_validate( + { + "schema_version": "ATIF-v1.7", + "agent": { + "name": "test-agent", + "version": "1", + "model_name": "openai/gpt-4o-mini", + "tool_definitions": [ + { + "type": "function", + "function": {"name": "calculator", "parameters": {"type": "object"}}, + } + ], + }, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-01-01T00:00:00Z", + "source": "user", + "message": "What is 2+2?", + }, + { + "step_id": 2, + "timestamp": "2026-01-01T00:00:01Z", + "source": "agent", + "message": "I'll calculate it.", + "llm_call_count": 1, + "metrics": {"prompt_tokens": 10, "completion_tokens": 4, "cost_usd": 0.001}, + "tool_calls": [ + { + "tool_call_id": "call_1", + "function_name": "calculator", + "arguments": {"expression": "2+2"}, + } + ], + "observation": {"results": [{"source_call_id": "call_1", "content": "4"}]}, + }, + { + "step_id": 3, + "timestamp": "2026-01-01T00:00:02Z", + "source": "agent", + "message": "The answer is 4.", + "llm_call_count": 1, + "metrics": {"prompt_tokens": 15, "completion_tokens": 5}, + }, + ], + } + ) + trajectory_path.write_text(trajectory.model_dump_json()) + summary = summarize_trajectory(trajectory_path, PluginConfig.from_options()) + + assert summary.schema_version == "ATIF-v1.7" + assert summary.final_message == "The answer is 4." + assert summary.warnings == () + assert _usage_metrics(trajectory.steps[1].metrics.model_dump(mode="python")) == { + "prompt_tokens": 10, + "completion_tokens": 4, + "tokens": 14, + "estimated_cost": 0.001, + } + + phase_start = datetime.fromisoformat("2026-01-01T00:00:00+00:00").timestamp() + phase_end = datetime.fromisoformat("2026-01-01T00:00:03+00:00").timestamp() + experiment = init( + project="python-sdk-harbor-tests", + experiment="harbor-atif-import", + update=True, + set_current=False, + git_metadata_settings=GitMetadataSettings(collect="none"), + api_key=os.environ.get("BRAINTRUST_API_KEY", "test-api-key-for-vcr-playback"), + ) + parent_id = "c7a87986-0192-5f40-9ac0-a535810f1fe7" + parent = experiment.start_span( + name="agent_execution", + type="task", + id=parent_id, + start_time=phase_start, + set_current=False, + ) + imported = import_trajectory( + parent, + trajectory_path, + trial_id="trial-1", + semantic_prefix="task/agent_execution", + phase_start=phase_start, + phase_end=phase_end, + config=PluginConfig.from_options(), + ) + parent.end(end_time=phase_end) + flush() + + expected_ids = { + parent_id, + child_span_id("trial-1", "task/agent_execution/turn/2/llm"), + child_span_id("trial-1", "task/agent_execution/tool/call_1"), + child_span_id("trial-1", "task/agent_execution/turn/3/llm"), + } + spans = [span for span in experiment if span["id"] in expected_ids] + leaves = sorted( + (span for span in spans if span["span_attributes"]["type"] in {"llm", "tool"}), + key=lambda span: (span["metrics"]["start"], span["span_attributes"]["exec_counter"]), + ) + + assert imported.imported_llm_spans == 2 + assert imported.imported_tool_spans == 1 + assert [span["span_attributes"]["type"] for span in leaves] == ["llm", "tool", "llm"] + assert leaves[0]["metadata"] == { + "provider": "openai", + "model": "gpt-4o-mini", + "tools": [{"type": "function", "function": {"name": "calculator", "parameters": {"type": "object"}}}], + } + assert leaves[0]["metrics"]["tokens"] == 14 + assert leaves[1]["input"] == {"expression": "2+2"} + assert all( + span["context"]["span_origin"]["instrumentation"]["name"] == "braintrust.plugin.harbor" for span in leaves + ) From fd1e54aa9e2c77821e5d16e6b3deaf32771deba4 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 31 Jul 2026 09:58:43 -0400 Subject: [PATCH 3/5] docs(examples): add Harbor plugin walkthrough --- examples/README.md | 1 + examples/harbor/.env.example | 3 ++ examples/harbor/.gitignore | 1 + examples/harbor/README.md | 60 +++++++++++++++++++++ examples/harbor/backfill.py | 22 ++++++++ examples/harbor/pyproject.toml | 12 +++++ examples/harbor/task/environment/Dockerfile | 3 ++ examples/harbor/task/instruction.md | 1 + examples/harbor/task/task.toml | 24 +++++++++ examples/harbor/task/tests/test.sh | 10 ++++ 10 files changed, 137 insertions(+) create mode 100644 examples/harbor/.env.example create mode 100644 examples/harbor/.gitignore create mode 100644 examples/harbor/README.md create mode 100755 examples/harbor/backfill.py create mode 100644 examples/harbor/pyproject.toml create mode 100644 examples/harbor/task/environment/Dockerfile create mode 100644 examples/harbor/task/instruction.md create mode 100644 examples/harbor/task/task.toml create mode 100755 examples/harbor/task/tests/test.sh diff --git a/examples/README.md b/examples/README.md index b7ec21e8d..c9bb52e28 100644 --- a/examples/README.md +++ b/examples/README.md @@ -69,6 +69,7 @@ Unless noted otherwise, every example below uses `braintrust.auto_instrument()`. | `dspy/` | DSPy `ReAct` agent with two tools (LiteLLM token metrics propagate) | | `evals/` | The `Eval` framework — does **not** use `auto_instrument()` | | `google_genai/` | Google GenAI `generate_content` against Gemini | +| `harbor/` | Native Harbor evaluation plugin — does **not** use `auto_instrument()` | | `langchain/` | LangChain `prompt | model` chain — global handler installed by `auto_instrument()` | | `langsmith/` | Migration helper for projects coming from LangSmith — uses `setup_langsmith()` | | `litellm/` | LiteLLM `completion` | diff --git a/examples/harbor/.env.example b/examples/harbor/.env.example new file mode 100644 index 000000000..b55d5af38 --- /dev/null +++ b/examples/harbor/.env.example @@ -0,0 +1,3 @@ +BRAINTRUST_API_KEY= +OPENAI_API_KEY= +HARBOR_BRAINTRUST_PROJECT=example-harbor diff --git a/examples/harbor/.gitignore b/examples/harbor/.gitignore new file mode 100644 index 000000000..1c18760e8 --- /dev/null +++ b/examples/harbor/.gitignore @@ -0,0 +1 @@ +jobs/ diff --git a/examples/harbor/README.md b/examples/harbor/README.md new file mode 100644 index 000000000..a7813a982 --- /dev/null +++ b/examples/harbor/README.md @@ -0,0 +1,60 @@ +# Harbor + Braintrust + +Runs a small, self-contained [Harbor](https://harborframework.com/) evaluation and uses Harbor's native Braintrust job plugin to sync the result. Braintrust receives a managed dataset, an experiment row for the final trial, verifier rewards, and the Harbor lifecycle and ATIF trace. + +The plugin is discovered automatically through Harbor's `braintrust` entry point. The Braintrust API key remains in the host process; it is not passed into the task container. + +## Setup + +Install the example's dependencies: + +```bash +uv sync +``` + +The command below reads credentials from the repository's root `.env`. It requires: + +```dotenv +BRAINTRUST_API_KEY=... +OPENAI_API_KEY=... +``` + +Alternatively, copy `.env.example` to `.env` in this directory and change `--env-file ../../.env` below to `--env-file .env`. + +## Run + +Docker must be running. From this directory, run: + +```bash +uv run harbor run \ + --path task \ + --agent terminus-2 \ + --model openai/gpt-4.1-mini \ + --job-name braintrust-harbor-example \ + --jobs-dir jobs \ + --env-file ../../.env \ + --plugin braintrust \ + --plugin-kwarg project_name=example-harbor \ + --yes +``` + +The agent solves the task in `task/`, and Harbor's verifier emits a normalized `reward` plus an `answer_length` metric. The plugin creates `jobs/braintrust-harbor-example/braintrust-sync.json` after synchronization. + +Harbor also accepts plugin options through `HARBOR_BRAINTRUST_*` variables. For example, setting this in `.env` removes the need for the `project_name` plugin argument: + +```dotenv +HARBOR_BRAINTRUST_PROJECT=example-harbor +``` + +Then omit `--plugin-kwarg project_name=example-harbor` from the command. + +## Backfill an existing job + +To synchronize the persisted job again without rerunning the agent or verifier: + +```bash +uv run --env-file ../../.env python backfill.py jobs/braintrust-harbor-example \ + --project example-harbor +``` + +Backfill uses the same deterministic dataset, experiment, and span identities, so it reconciles the existing Braintrust data instead of creating duplicate rows. diff --git a/examples/harbor/backfill.py b/examples/harbor/backfill.py new file mode 100755 index 000000000..69baa0cca --- /dev/null +++ b/examples/harbor/backfill.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Backfill a persisted Harbor job into Braintrust.""" + +import argparse +import asyncio +from pathlib import Path + +from braintrust.integrations.harbor import backfill_job + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_dir", type=Path, help="Persisted Harbor job directory") + parser.add_argument("--project", help="Braintrust project name (otherwise read from the environment)") + args = parser.parse_args() + + options = {"project_name": args.project} if args.project else {} + asyncio.run(backfill_job(args.job_dir, **options)) + + +if __name__ == "__main__": + main() diff --git a/examples/harbor/pyproject.toml b/examples/harbor/pyproject.toml new file mode 100644 index 000000000..c0a26474b --- /dev/null +++ b/examples/harbor/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "braintrust-harbor-example" +version = "0.1.0" +description = "Run a Harbor evaluation and sync it to Braintrust" +requires-python = ">=3.12" +dependencies = [ + "braintrust", + "harbor==0.20.0", +] + +[tool.uv.sources] +braintrust = { path = "../../py", editable = true } diff --git a/examples/harbor/task/environment/Dockerfile b/examples/harbor/task/environment/Dockerfile new file mode 100644 index 000000000..ddf0680c4 --- /dev/null +++ b/examples/harbor/task/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM ubuntu:24.04 + +WORKDIR /app diff --git a/examples/harbor/task/instruction.md b/examples/harbor/task/instruction.md new file mode 100644 index 000000000..ad4d0dd7b --- /dev/null +++ b/examples/harbor/task/instruction.md @@ -0,0 +1 @@ +Calculate 17 × 6. Create `/app/answer.txt` containing only the decimal result and a trailing newline. diff --git a/examples/harbor/task/task.toml b/examples/harbor/task/task.toml new file mode 100644 index 000000000..146d31f8e --- /dev/null +++ b/examples/harbor/task/task.toml @@ -0,0 +1,24 @@ +schema_version = "1.3" +artifacts = [] + +[metadata] +category = "arithmetic" + +[verifier] +timeout_sec = 60.0 +collect = [] + +[verifier.env] + +[agent] +timeout_sec = 300.0 + +[environment] +network_mode = "public" +build_timeout_sec = 300.0 +os = "linux" +mcp_servers = [] + +[environment.env] + +[solution.env] diff --git a/examples/harbor/task/tests/test.sh b/examples/harbor/task/tests/test.sh new file mode 100755 index 000000000..1901adbf1 --- /dev/null +++ b/examples/harbor/task/tests/test.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +answer="$(tr -d '[:space:]' < /app/answer.txt 2>/dev/null || true)" +if [ "$answer" = "102" ]; then + reward=1 +else + reward=0 +fi + +printf '{"reward":%s,"answer_length":%s}\n' "$reward" "${#answer}" > /logs/verifier/reward.json From f82dded37341db9c9c6137dab6b399e951bc2957 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 31 Jul 2026 10:15:20 -0400 Subject: [PATCH 4/5] fix(harbor): preserve timing and Terminus LLM spans --- docs/harbor-braintrust-plugin-design.md | 1 + py/src/braintrust/integrations/harbor/atif.py | 16 ++++++++++ .../braintrust/integrations/harbor/plugin.py | 12 +++++--- .../integrations/harbor/test_harbor.py | 30 ++++++++++++++++--- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/docs/harbor-braintrust-plugin-design.md b/docs/harbor-braintrust-plugin-design.md index 0d2b878ce..040929c17 100644 --- a/docs/harbor-braintrust-plugin-design.md +++ b/docs/harbor-braintrust-plugin-design.md @@ -331,6 +331,7 @@ An ATIF step may be an `llm` span only when it represents exactly one model call - `llm_call_count == 0` is deterministic work; - unknown or multiple calls are not one LLM call unless ATIF exposes each call; +- a pinned, producer-specific compatibility adapter may repair an omitted count only when the tested producer guarantees one LLM call per agent step; record the repair on the eval root; - a streaming call without measurable time-to-first-token is downgraded; - redaction that removes required payload content also downgrades the leaf. diff --git a/py/src/braintrust/integrations/harbor/atif.py b/py/src/braintrust/integrations/harbor/atif.py index 6fb3f1d37..43ff15f47 100644 --- a/py/src/braintrust/integrations/harbor/atif.py +++ b/py/src/braintrust/integrations/harbor/atif.py @@ -71,6 +71,19 @@ def _provider(model: str | None) -> tuple[str | None, str | None]: return "unknown", model +def _known_single_llm_step(agent: dict[str, Any], step: dict[str, Any], metrics: dict[str, int | float]) -> bool: + # Harbor 0.20's Terminus 2 producer creates one agent step immediately + # after each LLM interaction but omits ATIF-v1.7's llm_call_count field. + # Keep this exception producer/version-specific rather than inferring from + # token usage for arbitrary ATIF producers. + return ( + agent.get("name") == "terminus-2" + and agent.get("version") == "2.0.0" + and step.get("llm_call_count") is None + and "tokens" in metrics + ) + + def _valid_count(value: Any) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None @@ -279,6 +292,9 @@ def import_trajectory( llm_call_count = step.get("llm_call_count") metrics = _usage_metrics(step.get("metrics")) + if _known_single_llm_step(agent, step, metrics): + llm_call_count = 1 + repairs.append(f"step {index + 1}: inferred one model call from terminus-2 2.0.0 trajectory") provider, model = _provider(step.get("model_name") or default_model) can_be_llm = ( config.content_mode != "metadata" diff --git a/py/src/braintrust/integrations/harbor/plugin.py b/py/src/braintrust/integrations/harbor/plugin.py index 7386ba386..c3653312a 100644 --- a/py/src/braintrust/integrations/harbor/plugin.py +++ b/py/src/braintrust/integrations/harbor/plugin.py @@ -10,7 +10,7 @@ import logging import os from dataclasses import dataclass, field, fields -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Any @@ -84,8 +84,9 @@ class RuntimeState: def _seconds(value: Any, fallback: float) -> float: if isinstance(value, datetime): - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) + # Harbor trial timestamps are timezone-aware, but job timestamps are + # currently naive local datetimes. datetime.timestamp() preserves both + # conventions; assigning UTC to a naive value shifts non-UTC jobs. return value.timestamp() return fallback @@ -821,10 +822,13 @@ def _finalize(self, job_result: Any) -> None: project_id=self.config.project_id, set_current=False, ) + now = datetime.now().timestamp() + summary_start, summary_end = _timing(job_result, now, now) summary = project_logger.start_span( name="harbor.job.summary", type="task", id=f"harbor-job-summary-{self._runtime.snapshot.job_id}", + start_time=summary_start, set_current=False, input={"job_id": self._runtime.snapshot.job_id}, metadata={ @@ -839,7 +843,7 @@ def _finalize(self, job_result: Any) -> None: internal={"instrumentation": _INSTRUMENTATION}, ) summary.log(output=job_result.stats.model_dump(mode="json", exclude_none=False)) - summary.end(end_time=_seconds(getattr(job_result, "finished_at", None), datetime.now().timestamp())) + summary.end(end_time=summary_end) flush() def _persist_disabled_manifest(self) -> None: diff --git a/py/src/braintrust/integrations/harbor/test_harbor.py b/py/src/braintrust/integrations/harbor/test_harbor.py index eef20adc2..19ac3be56 100644 --- a/py/src/braintrust/integrations/harbor/test_harbor.py +++ b/py/src/braintrust/integrations/harbor/test_harbor.py @@ -22,6 +22,7 @@ partition_key, semantic_agent_config, ) +from braintrust.integrations.harbor.plugin import _seconds, _timing from braintrust.integrations.harbor.rewards import classify_rewards, validate_classifications from braintrust.integrations.harbor.state import ( TrialEvent, @@ -34,6 +35,7 @@ from harbor.models.job.lock import AgentSkillLock, TaskLock, TrialLock from harbor.models.trajectories.trajectory import Trajectory from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig, VerifierConfig +from harbor.models.trial.result import TimingInfo _ABSOLUTE_PATH_RE = re.compile(r"(?:/(?:Users|private|home)/[^\"\\\\\s]+|[A-Za-z]:\\\\[^\"\\\\\s]+)") @@ -211,6 +213,23 @@ def test_ids_and_partition_are_deterministic_and_do_not_include_concurrency(tmp_ assert semantic_agent_config(changed_concurrency, task_lock.skills) == semantic +def test_harbor_naive_datetimes_use_the_host_timezone_and_timings_never_run_backward(): + started_at = datetime(2026, 7, 31, 9, 57, 7) + finished_at = datetime(2026, 7, 31, 9, 57, 41) + timing = TimingInfo(started_at=started_at, finished_at=finished_at) + + start, end = _timing(timing, 0, 0) + + assert start == started_at.timestamp() + assert end == finished_at.timestamp() + assert end >= start + assert _seconds(started_at, 0) == started_at.timestamp() + + backwards = TimingInfo(started_at=finished_at, finished_at=started_at) + backwards_start, backwards_end = _timing(backwards, 0, 0) + assert backwards_end == backwards_start + + def test_trial_reducer_retry_duplicate_backward_and_reconcile(): state = TrialMachine("trial") state, _ = reduce_trial(state, TrialEvent(TrialEventKind.START)) @@ -249,8 +268,8 @@ def test_atif_import_round_trips_with_real_sdks(tmp_path): { "schema_version": "ATIF-v1.7", "agent": { - "name": "test-agent", - "version": "1", + "name": "terminus-2", + "version": "2.0.0", "model_name": "openai/gpt-4o-mini", "tool_definitions": [ { @@ -271,7 +290,6 @@ def test_atif_import_round_trips_with_real_sdks(tmp_path): "timestamp": "2026-01-01T00:00:01Z", "source": "agent", "message": "I'll calculate it.", - "llm_call_count": 1, "metrics": {"prompt_tokens": 10, "completion_tokens": 4, "cost_usd": 0.001}, "tool_calls": [ { @@ -287,7 +305,6 @@ def test_atif_import_round_trips_with_real_sdks(tmp_path): "timestamp": "2026-01-01T00:00:02Z", "source": "agent", "message": "The answer is 4.", - "llm_call_count": 1, "metrics": {"prompt_tokens": 15, "completion_tokens": 5}, }, ], @@ -299,6 +316,7 @@ def test_atif_import_round_trips_with_real_sdks(tmp_path): assert summary.schema_version == "ATIF-v1.7" assert summary.final_message == "The answer is 4." assert summary.warnings == () + assert trajectory.steps[1].llm_call_count is None assert _usage_metrics(trajectory.steps[1].metrics.model_dump(mode="python")) == { "prompt_tokens": 10, "completion_tokens": 4, @@ -350,6 +368,10 @@ def test_atif_import_round_trips_with_real_sdks(tmp_path): assert imported.imported_llm_spans == 2 assert imported.imported_tool_spans == 1 + assert imported.repairs == ( + "step 2: inferred one model call from terminus-2 2.0.0 trajectory", + "step 3: inferred one model call from terminus-2 2.0.0 trajectory", + ) assert [span["span_attributes"]["type"] for span in leaves] == ["llm", "tool", "llm"] assert leaves[0]["metadata"] == { "provider": "openai", From 6bd9cfa195107d8b3d68d254b606a8bfb7f45c67 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 31 Jul 2026 10:35:48 -0400 Subject: [PATCH 5/5] refactor(harbor): remove project log summary --- docs/harbor-braintrust-plugin-design.md | 9 ++--- .../braintrust/integrations/harbor/config.py | 2 - .../braintrust/integrations/harbor/plugin.py | 39 +------------------ 3 files changed, 6 insertions(+), 44 deletions(-) diff --git a/docs/harbor-braintrust-plugin-design.md b/docs/harbor-braintrust-plugin-design.md index 040929c17..f0989d3b9 100644 --- a/docs/harbor-braintrust-plugin-design.md +++ b/docs/harbor-braintrust-plugin-design.md @@ -16,7 +16,7 @@ Harbor final trial → Braintrust root eval span Harbor verifier rewards → Braintrust scores or metrics Harbor verifier labels → Braintrust classifications Harbor lifecycle + ATIF → child spans -Harbor JobResult → final reconciliation + job summary +Harbor JobResult → final reconciliation ``` ### Key decisions @@ -57,7 +57,7 @@ Version 1 does not need to: | Harbor | Braintrust | Notes | |---|---|---| -| Job | Shared metadata + optional project-log summary | A job may create several experiments. | +| Job | Shared metadata on experiments and eval roots | A job may create several experiments. | | Resolved dataset/task set | Dataset | Scope is the exact logical task selection. | | Task | Dataset record | Deterministic ID and canonical task input. | | Eval group/variant | Experiment | Partition by dataset and semantic agent config. | @@ -307,7 +307,7 @@ Version 1 should only read labels from known Harbor adapters or explicit `classi If `reward-details.json` exists, put a bounded summary in the scorer output and optionally attach the complete JSON. Do not create criterion-level or judge-LLM spans until Rewardkit has a stable, tested mapping. -Harbor's job metrics, custom `metric.py`, and pass@k remain authoritative aggregate results. Do not create a synthetic eval row for aggregates. Optionally write one project-log trace, `harbor.job.summary`, containing exact Harbor aggregates and links to partition experiments. +Harbor's job metrics, custom `metric.py`, and pass@k remain authoritative aggregate results. Do not create a synthetic eval row or project-log trace for aggregates. ## ATIF import @@ -495,7 +495,6 @@ HarborPlugin( max_attachment_bytes=5_000_000, max_total_attachment_bytes=20_000_000, max_content_bytes=20_000, - log_job_summary=True, log_retry_attempts=False, strict=False, ) @@ -527,7 +526,7 @@ Do not mutate private fields or depend on queue, metrics, progress, existing-tri 2. Validate config/auth and read resolved tasks, lock, and custom metadata. 3. Normalize metadata, build partitions, sync datasets, and initialize experiments. 4. Register all trial hooks with one thin dispatcher. -5. Dispatch job `READY`; optionally start a project-log job trace. +5. Dispatch job `READY`. Keep blocking SDK and filesystem work off Harbor's event loop. Preserve per-trial effect ordering while allowing independent trials to make progress concurrently. diff --git a/py/src/braintrust/integrations/harbor/config.py b/py/src/braintrust/integrations/harbor/config.py index 53cb6df94..6f4640d08 100644 --- a/py/src/braintrust/integrations/harbor/config.py +++ b/py/src/braintrust/integrations/harbor/config.py @@ -103,7 +103,6 @@ class PluginConfig: max_attachment_bytes: int = 5_000_000 max_total_attachment_bytes: int = 20_000_000 max_content_bytes: int = 20_000 - log_job_summary: bool = True log_retry_attempts: bool = False strict: bool = False redact_patterns: tuple[str, ...] = () @@ -119,7 +118,6 @@ def from_options(cls, **options: Any) -> "PluginConfig": for name in ( "include_custom_metadata", "include_tracebacks", - "log_job_summary", "log_retry_attempts", "strict", ): diff --git a/py/src/braintrust/integrations/harbor/plugin.py b/py/src/braintrust/integrations/harbor/plugin.py index c3653312a..9899b0209 100644 --- a/py/src/braintrust/integrations/harbor/plugin.py +++ b/py/src/braintrust/integrations/harbor/plugin.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -from braintrust.logger import Attachment, flush, init, init_dataset, init_logger +from braintrust.logger import Attachment, flush, init, init_dataset from exceptiongroup import ExceptionGroup from .atif import _INSTRUMENTATION, ATIFImportResult, import_trajectory, summarize_trajectory @@ -263,7 +263,6 @@ def __init__( max_attachment_bytes: Any = _UNSET, max_total_attachment_bytes: Any = _UNSET, max_content_bytes: Any = _UNSET, - log_job_summary: Any = _UNSET, log_retry_attempts: Any = _UNSET, strict: Any = _UNSET, **kwargs: Any, @@ -291,7 +290,6 @@ def __init__( "max_attachment_bytes": max_attachment_bytes, "max_total_attachment_bytes": max_total_attachment_bytes, "max_content_bytes": max_content_bytes, - "log_job_summary": log_job_summary, "log_retry_attempts": log_retry_attempts, "strict": strict, **kwargs, @@ -356,7 +354,7 @@ async def reconcile(result: Any) -> None: for name in set(self._trial_machines) - final_names: await self._dispatch(name, TrialEvent(TrialEventKind.OMIT)) try: - await asyncio.to_thread(self._finalize, job_result) + await asyncio.to_thread(flush) except Exception as exc: failures.append(exc) self._errors.append(f"final flush: {exc}") @@ -813,39 +811,6 @@ def _sync_final_result(self, result: Any) -> None: root.log(metadata=metadata) root.end(end_time=root_end) - def _finalize(self, job_result: Any) -> None: - if self._runtime is None: - return - if self.config.log_job_summary: - project_logger = init_logger( - project=self.config.project_name, - project_id=self.config.project_id, - set_current=False, - ) - now = datetime.now().timestamp() - summary_start, summary_end = _timing(job_result, now, now) - summary = project_logger.start_span( - name="harbor.job.summary", - type="task", - id=f"harbor-job-summary-{self._runtime.snapshot.job_id}", - start_time=summary_start, - set_current=False, - input={"job_id": self._runtime.snapshot.job_id}, - metadata={ - "harbor": { - "job_id": self._runtime.snapshot.job_id, - "experiments": [ - {"id": partition.experiment_id, "name": partition.name} - for partition in self._runtime.partitions.values() - ], - } - }, - internal={"instrumentation": _INSTRUMENTATION}, - ) - summary.log(output=job_result.stats.model_dump(mode="json", exclude_none=False)) - summary.end(end_time=summary_end) - flush() - def _persist_disabled_manifest(self) -> None: if self._snapshot is None: return