diff --git a/README.md b/README.md
index 7b2a179..c5ee391 100644
--- a/README.md
+++ b/README.md
@@ -99,6 +99,91 @@ import { Factuality } from "autoevals";
+## Evaluating Agent Behavior
+
+The `Behavior` LLM judge evaluates an agent against an [Agent Behavior](https://github.com/braintrustdata/agentbehavior) spec. It returns `1` for compliance, `0` for non-compliance, and `null`/`None` when the behavior is not applicable or cannot be judged.
+
+In a Braintrust eval:
+
+- `input` is the dataset case passed to your task—for example, the user's request and any agent context.
+- `output` is the value returned by your task—for example, the agent's final answer or a structured trajectory.
+- `expected` is optional reference data. The behavior spec is supplied separately through `behavior`.
+- When the agent is instrumented with Braintrust, the scorer also receives its trace thread automatically.
+
+
+
+### TypeScript
+
+```typescript
+import { Behavior } from "autoevals";
+import { Eval } from "braintrust";
+
+Eval("Support agent", {
+ data: () => [
+ {
+ input: {
+ message: "Our API is returning 401s and production is blocked.",
+ },
+ },
+ ],
+ task: async (input) => runSupportAgent(input.message),
+ scores: [Behavior],
+});
+```
+
+### Python
+
+```python
+from autoevals import Behavior
+from braintrust import Eval
+
+Eval(
+ "Support agent",
+ data=[
+ {
+ "input": {
+ "message": "Our API is returning 401s and production is blocked.",
+ },
+ },
+ ],
+ task=lambda input: run_support_agent(input["message"]),
+ scores=[Behavior],
+)
+```
+
+
+
+This usage discovers the behavior automatically when the project contains exactly one valid `.agents/behaviors//BEHAVIOR.md`.
+
+If the project contains multiple behaviors, select one when configuring the scorer:
+
+```typescript
+scores: [Behavior.partial({ behavior: "support-ticket-triage" })];
+```
+
+```python
+scores=[Behavior(behavior="support-ticket-triage")]
+```
+
+You can also select a behavior with a `BEHAVIOR.md` path, complete file content, or loaded behavior object.
+
+The scorer can also be called directly:
+
+```typescript
+const result = await Behavior({
+ behavior: "support-ticket-triage",
+ input: userRequest,
+ output: agentResult,
+});
+```
+
+```python
+result = Behavior(behavior="support-ticket-triage").eval(
+ input=user_request,
+ output=agent_result,
+)
+```
+
## Using other AI providers
When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will look for a `BRAINTRUST_AI_GATEWAY_URL` environment variable and then default to the [Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway).
diff --git a/SCORERS.md b/SCORERS.md
index 5689324..51383d4 100644
--- a/SCORERS.md
+++ b/SCORERS.md
@@ -16,6 +16,28 @@ Complete reference for all scorers available in Autoevals, including parameters,
These scorers use language models to evaluate outputs based on semantic understanding.
+### Behavior
+
+Evaluates observable agent conduct against an [Agent Behavior](https://github.com/braintrustdata/agentbehavior) spec. It accepts text outputs, structured trajectories, or a trace thread.
+
+**Parameters:**
+
+- `output` (required): Task return value—the agent's final answer or a structured trajectory
+- `behavior` (optional): Loaded behavior, behavior name, `BEHAVIOR.md` path/directory, or complete spec content
+- `behaviorRoot` / `behavior_root` (optional): Project root for discovery and relative paths (default: current directory)
+- `input` (optional): Dataset case passed to the task, such as the user request and agent context
+- `expected` (optional): Reference data that may help judge the output; this is not the behavior spec
+- `trace` (optional): Trace whose thread should be judged; Braintrust Eval supplies this automatically
+- `model` (optional): Model to use
+
+When used in `Braintrust Eval`, `input`, `output`, `expected`, `metadata`, and the trace are passed to the scorer automatically. If `behavior` is omitted, exactly one valid spec must be discoverable under `.agents/behaviors/`.
+
+**Score Range:**
+
+- `1.0` = Applicable behavior is satisfied
+- `0.0` = Applicable behavior is violated
+- `null` / `None` = Behavior is not applicable or cannot be judged
+
### Factuality
Evaluates whether the output is factually consistent with the expected answer.
diff --git a/js/behavior.test.ts b/js/behavior.test.ts
new file mode 100644
index 0000000..86105ae
--- /dev/null
+++ b/js/behavior.test.ts
@@ -0,0 +1,197 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { http, HttpResponse } from "msw";
+import { setupServer } from "msw/node";
+import { OpenAI } from "openai";
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";
+
+import { Behavior, discoverAgentBehaviors } from "./llm";
+
+const server = setupServer();
+const behaviorProjects = new Set();
+
+beforeAll(() => server.listen({ onUnhandledRequest: "error" }));
+afterEach(async () => {
+ server.resetHandlers();
+ await Promise.all(
+ [...behaviorProjects].map((root) =>
+ fs.rm(root, { recursive: true, force: true }),
+ ),
+ );
+ behaviorProjects.clear();
+});
+afterAll(() => server.close());
+
+async function createBehaviorProject(name = "verify-work") {
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "autoevals-behavior-"));
+ behaviorProjects.add(root);
+ const directory = path.join(root, ".agents", "behaviors", name);
+ await fs.mkdir(directory, { recursive: true });
+ await fs.writeFile(
+ path.join(directory, "BEHAVIOR.md"),
+ `---\nname: ${name}\ndescription: Verify work before answering.\n---\n# Verify work\n\nThe agent MUST show its calculation.\n`,
+ );
+ return root;
+}
+
+describe("Behavior", () => {
+ test("discovers a valid BEHAVIOR.md from a project root", async () => {
+ const root = await createBehaviorProject();
+ const behaviors = await discoverAgentBehaviors(root);
+
+ expect(behaviors).toHaveLength(1);
+ expect(behaviors[0]).toMatchObject({
+ name: "verify-work",
+ description: "Verify work before answering.",
+ });
+ expect(behaviors[0]?.location).toBe(
+ path.join(root, ".agents", "behaviors", "verify-work", "BEHAVIOR.md"),
+ );
+ });
+
+ test("judges structured output against an explicitly provided behavior", async () => {
+ let prompt = "";
+ let choice: "true" | "na" = "true";
+ server.use(
+ http.post(
+ "https://api.openai.com/v1/chat/completions",
+ async ({ request }) => {
+ const body = (await request.json()) as {
+ messages: Array<{ content: string }>;
+ };
+ prompt = body.messages[0]!.content;
+ return HttpResponse.json({
+ id: "chatcmpl-behavior",
+ object: "chat.completion",
+ created: 0,
+ model: "gpt-4o-mini",
+ choices: [
+ {
+ index: 0,
+ finish_reason: "tool_calls",
+ message: {
+ role: "assistant",
+ content: null,
+ tool_calls: [
+ {
+ id: "call-behavior",
+ type: "function",
+ function: {
+ name: "select_choice",
+ arguments: JSON.stringify({
+ choice,
+ reasons: "The calculation is visible.",
+ }),
+ },
+ },
+ ],
+ },
+ },
+ ],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ });
+ },
+ ),
+ );
+
+ const result = await Behavior({
+ behavior: {
+ name: "verify-work",
+ description: "Verify work before answering.",
+ body: "# Verify work\n\nThe agent MUST show its calculation.",
+ },
+ input: { question: "What is 2 + 2?" },
+ output: { events: [{ type: "answer", content: "2 + 2 = 4" }] },
+ trace: {
+ getThread: async () => [
+ { role: "system", content: "Use the calculator." },
+ { role: "user", content: "What is 2 + 2?" },
+ { role: "assistant", content: "2 + 2 = 4" },
+ ],
+ },
+ model: "gpt-4o-mini",
+ client: new OpenAI({
+ apiKey: "test",
+ baseURL: "https://api.openai.com/v1",
+ }),
+ });
+
+ expect(result.score).toBe(1);
+ expect(result.metadata?.choice).toBe("true");
+ expect(result.metadata?.behavior).toMatchObject({ name: "verify-work" });
+ expect(prompt).toContain("The agent MUST show its calculation.");
+ expect(prompt).toContain('"question":"What is 2 + 2?"');
+ expect(prompt).toContain("System:\n Use the calculator.");
+
+ choice = "na";
+ const notApplicable = await Behavior({
+ behavior: {
+ name: "verify-work",
+ description: "Verify work before answering.",
+ body: "# Verify work\n\nThe agent MUST show its calculation.",
+ },
+ output: "No calculation was requested.",
+ model: "gpt-4o-mini",
+ client: new OpenAI({
+ apiKey: "test",
+ baseURL: "https://api.openai.com/v1",
+ }),
+ });
+ expect(notApplicable.score).toBeNull();
+
+ const root = await createBehaviorProject();
+ await fs.writeFile(
+ path.join(root, "verify-work"),
+ "unrelated project file",
+ );
+ choice = "true";
+ const namedBehavior = await Behavior({
+ behavior: "verify-work",
+ behaviorRoot: root,
+ output: "2 + 2 = 4",
+ model: "gpt-4o-mini",
+ client: new OpenAI({
+ apiKey: "test",
+ baseURL: "https://api.openai.com/v1",
+ }),
+ });
+ expect(namedBehavior.metadata?.behavior).toMatchObject({
+ name: "verify-work",
+ });
+ });
+
+ test("includes invalid spec diagnostics when discovery finds no valid behavior", async () => {
+ const root = await createBehaviorProject();
+ const behaviorFile = path.join(
+ root,
+ ".agents",
+ "behaviors",
+ "verify-work",
+ "BEHAVIOR.md",
+ );
+ await fs.writeFile(
+ behaviorFile,
+ "---\nname: INVALID\ndescription: Invalid behavior.\n---\n# Invalid\n",
+ );
+
+ await expect(
+ Behavior({ output: "done", behaviorRoot: root, model: "gpt-4o-mini" }),
+ ).rejects.toThrow("Diagnostics: Agent Behavior name");
+ });
+
+ test("requires an explicit selection when discovery finds multiple behaviors", async () => {
+ const root = await createBehaviorProject("first-behavior");
+ const second = path.join(root, ".agents", "behaviors", "second-behavior");
+ await fs.mkdir(second, { recursive: true });
+ await fs.writeFile(
+ path.join(second, "BEHAVIOR.md"),
+ "---\nname: second-behavior\ndescription: A second behavior.\n---\n# Second\n",
+ );
+
+ await expect(
+ Behavior({ output: "done", behaviorRoot: root, model: "gpt-4o-mini" }),
+ ).rejects.toThrow("Multiple Agent Behavior specs were discovered");
+ });
+});
diff --git a/js/llm.ts b/js/llm.ts
index 1b7f318..f561396 100644
--- a/js/llm.ts
+++ b/js/llm.ts
@@ -14,6 +14,7 @@ import {
import type { ReasoningEffort } from "openai/resources/shared";
import { makePartial, ScorerWithPartial } from "./partial";
import { renderMessages } from "./render-messages";
+import * as yaml from "js-yaml";
import {
computeThreadTemplateVars,
type ThreadTemplateVars,
@@ -140,7 +141,7 @@ export type OpenAIClassifierArgs = {
name: string;
model: string;
messages: ChatCompletionMessageParam[];
- choiceScores: Record;
+ choiceScores: Record;
classificationTools: ChatCompletionTool[];
cache?: ChatCache;
} & LLMArgs &
@@ -251,9 +252,9 @@ export async function OpenAIClassifier(
function parseResponse(
resp: ChatCompletionMessage,
- choiceScores: Record,
+ choiceScores: Record,
): Omit {
- let score = 0;
+ let score: number | null = 0;
const metadata: Record = {};
if (!resp.tool_calls || resp.tool_calls.length === 0) {
@@ -294,7 +295,7 @@ export type LLMClassifierArgs = {
} & LLMArgs &
RenderArgs;
-export function LLMClassifierFromTemplate({
+export function LLMClassifierFromTemplate({
name,
promptTemplate,
choiceScores,
@@ -309,7 +310,7 @@ export function LLMClassifierFromTemplate({
}: {
name: string;
promptTemplate: string;
- choiceScores: Record;
+ choiceScores: Record;
model?: string;
useCoT?: boolean;
temperature?: number;
@@ -318,10 +319,10 @@ export function LLMClassifierFromTemplate({
reasoningEnabled?: boolean;
reasoningBudget?: number;
useResponsesApi?: boolean;
-}): Scorer> {
+}): Scorer