feat: Add Agent Behavior scorer for Python and TypeScript

README.md

@@ -99,6 +99,91 @@ import { Factuality } from "autoevals";
 
 </div>
 
+## Evaluating Agent Behavior
+
+The `Behavior` LLM judge evaluates an agent against an [Agent Behavior](https://github.com/braintrustdata/agentbehavior) spec. It returns `1` for compliance, `0` for non-compliance, and `null`/`None` when the behavior is not applicable or cannot be judged.
+
+In a Braintrust eval:
+
+- `input` is the dataset case passed to your task—for example, the user's request and any agent context.
+- `output` is the value returned by your task—for example, the agent's final answer or a structured trajectory.
+- `expected` is optional reference data. The behavior spec is supplied separately through `behavior`.
+- When the agent is instrumented with Braintrust, the scorer also receives its trace thread automatically.
+
+<div className="tabs">
+
+### TypeScript
+
+```typescript
+import { Behavior } from "autoevals";
+import { Eval } from "braintrust";
+
+Eval("Support agent", {
+  data: () => [
+    {
+      input: {
+        message: "Our API is returning 401s and production is blocked.",
+      },
+    },
+  ],
+  task: async (input) => runSupportAgent(input.message),
+  scores: [Behavior],
+});
+```
+
+### Python
+
+```python
+from autoevals import Behavior
+from braintrust import Eval
+
+Eval(
+    "Support agent",
+    data=[
+        {
+            "input": {
+                "message": "Our API is returning 401s and production is blocked.",
+            },
+        },
+    ],
+    task=lambda input: run_support_agent(input["message"]),
+    scores=[Behavior],
+)
+```
+
+</div>
+
+This usage discovers the behavior automatically when the project contains exactly one valid `.agents/behaviors/<name>/BEHAVIOR.md`.
+
+If the project contains multiple behaviors, select one when configuring the scorer:
+
+```typescript
+scores: [Behavior.partial({ behavior: "support-ticket-triage" })];
+```
+
+```python
+scores=[Behavior(behavior="support-ticket-triage")]
+```
+
+You can also select a behavior with a `BEHAVIOR.md` path, complete file content, or loaded behavior object.
+
+The scorer can also be called directly:
+
+```typescript
+const result = await Behavior({
+  behavior: "support-ticket-triage",
+  input: userRequest,
+  output: agentResult,
+});
+```
+
+```python
+result = Behavior(behavior="support-ticket-triage").eval(
+    input=user_request,
+    output=agent_result,
+)
+```
+
 ## Using other AI providers
 
 When you use Autoevals, it will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI-compatible API. If `OPENAI_BASE_URL` is not set, it will look for a `BRAINTRUST_AI_GATEWAY_URL` environment variable and then default to the [Braintrust Gateway](https://www.braintrust.dev/docs/deploy/gateway).

SCORERS.md

@@ -16,6 +16,28 @@ Complete reference for all scorers available in Autoevals, including parameters,
 
 These scorers use language models to evaluate outputs based on semantic understanding.
 
+### Behavior
+
+Evaluates observable agent conduct against an [Agent Behavior](https://github.com/braintrustdata/agentbehavior) spec. It accepts text outputs, structured trajectories, or a trace thread.
+
+**Parameters:**
+
+- `output` (required): Task return value—the agent's final answer or a structured trajectory
+- `behavior` (optional): Loaded behavior, behavior name, `BEHAVIOR.md` path/directory, or complete spec content
+- `behaviorRoot` / `behavior_root` (optional): Project root for discovery and relative paths (default: current directory)
+- `input` (optional): Dataset case passed to the task, such as the user request and agent context
+- `expected` (optional): Reference data that may help judge the output; this is not the behavior spec
+- `trace` (optional): Trace whose thread should be judged; Braintrust Eval supplies this automatically
+- `model` (optional): Model to use
+
+When used in `Braintrust Eval`, `input`, `output`, `expected`, `metadata`, and the trace are passed to the scorer automatically. If `behavior` is omitted, exactly one valid spec must be discoverable under `.agents/behaviors/`.
+
+**Score Range:**
+
+- `1.0` = Applicable behavior is satisfied
+- `0.0` = Applicable behavior is violated
+- `null` / `None` = Behavior is not applicable or cannot be judged
+
 ### Factuality
 
 Evaluates whether the output is factually consistent with the expected answer.

js/behavior.test.ts

@@ -0,0 +1,197 @@
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { http, HttpResponse } from "msw";
+import { setupServer } from "msw/node";
+import { OpenAI } from "openai";
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";
+
+import { Behavior, discoverAgentBehaviors } from "./llm";
+
+const server = setupServer();
+const behaviorProjects = new Set<string>();
+
+beforeAll(() => server.listen({ onUnhandledRequest: "error" }));
+afterEach(async () => {
+  server.resetHandlers();
+  await Promise.all(
+    [...behaviorProjects].map((root) =>
+      fs.rm(root, { recursive: true, force: true }),
+    ),
+  );
+  behaviorProjects.clear();
+});
+afterAll(() => server.close());
+
+async function createBehaviorProject(name = "verify-work") {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), "autoevals-behavior-"));
+  behaviorProjects.add(root);
+  const directory = path.join(root, ".agents", "behaviors", name);
+  await fs.mkdir(directory, { recursive: true });
+  await fs.writeFile(
+    path.join(directory, "BEHAVIOR.md"),
+    `---\nname: ${name}\ndescription: Verify work before answering.\n---\n# Verify work\n\nThe agent MUST show its calculation.\n`,
+  );
+  return root;
+}
+
+describe("Behavior", () => {
+  test("discovers a valid BEHAVIOR.md from a project root", async () => {
+    const root = await createBehaviorProject();
+    const behaviors = await discoverAgentBehaviors(root);
+
+    expect(behaviors).toHaveLength(1);
+    expect(behaviors[0]).toMatchObject({
+      name: "verify-work",
+      description: "Verify work before answering.",
+    });
+    expect(behaviors[0]?.location).toBe(
+      path.join(root, ".agents", "behaviors", "verify-work", "BEHAVIOR.md"),
+    );
+  });
+
+  test("judges structured output against an explicitly provided behavior", async () => {
+    let prompt = "";
+    let choice: "true" | "na" = "true";
+    server.use(
+      http.post(
+        "https://api.openai.com/v1/chat/completions",
+        async ({ request }) => {
+          const body = (await request.json()) as {
+            messages: Array<{ content: string }>;
+          };
+          prompt = body.messages[0]!.content;
+          return HttpResponse.json({
+            id: "chatcmpl-behavior",
+            object: "chat.completion",
+            created: 0,
+            model: "gpt-4o-mini",
+            choices: [
+              {
+                index: 0,
+                finish_reason: "tool_calls",
+                message: {
+                  role: "assistant",
+                  content: null,
+                  tool_calls: [
+                    {
+                      id: "call-behavior",
+                      type: "function",
+                      function: {
+                        name: "select_choice",
+                        arguments: JSON.stringify({
+                          choice,
+                          reasons: "The calculation is visible.",
+                        }),
+                      },
+                    },
+                  ],
+                },
+              },
+            ],
+            usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+          });
+        },
+      ),
+    );
+
+    const result = await Behavior({
+      behavior: {
+        name: "verify-work",
+        description: "Verify work before answering.",
+        body: "# Verify work\n\nThe agent MUST show its calculation.",
+      },
+      input: { question: "What is 2 + 2?" },
+      output: { events: [{ type: "answer", content: "2 + 2 = 4" }] },
+      trace: {
+        getThread: async () => [
+          { role: "system", content: "Use the calculator." },
+          { role: "user", content: "What is 2 + 2?" },
+          { role: "assistant", content: "2 + 2 = 4" },
+        ],
+      },
+      model: "gpt-4o-mini",
+      client: new OpenAI({
+        apiKey: "test",
+        baseURL: "https://api.openai.com/v1",
+      }),
+    });
+
+    expect(result.score).toBe(1);
+    expect(result.metadata?.choice).toBe("true");
+    expect(result.metadata?.behavior).toMatchObject({ name: "verify-work" });
+    expect(prompt).toContain("The agent MUST show its calculation.");
+    expect(prompt).toContain('"question":"What is 2 + 2?"');
+    expect(prompt).toContain("System:\n  Use the calculator.");
+
+    choice = "na";
+    const notApplicable = await Behavior({
+      behavior: {
+        name: "verify-work",
+        description: "Verify work before answering.",
+        body: "# Verify work\n\nThe agent MUST show its calculation.",
+      },
+      output: "No calculation was requested.",
+      model: "gpt-4o-mini",
+      client: new OpenAI({
+        apiKey: "test",
+        baseURL: "https://api.openai.com/v1",
+      }),
+    });
+    expect(notApplicable.score).toBeNull();
+
+    const root = await createBehaviorProject();
+    await fs.writeFile(
+      path.join(root, "verify-work"),
+      "unrelated project file",
+    );
+    choice = "true";
+    const namedBehavior = await Behavior({
+      behavior: "verify-work",
+      behaviorRoot: root,
+      output: "2 + 2 = 4",
+      model: "gpt-4o-mini",
+      client: new OpenAI({
+        apiKey: "test",
+        baseURL: "https://api.openai.com/v1",
+      }),
+    });
+    expect(namedBehavior.metadata?.behavior).toMatchObject({
+      name: "verify-work",
+    });
+  });
+
+  test("includes invalid spec diagnostics when discovery finds no valid behavior", async () => {
+    const root = await createBehaviorProject();
+    const behaviorFile = path.join(
+      root,
+      ".agents",
+      "behaviors",
+      "verify-work",
+      "BEHAVIOR.md",
+    );
+    await fs.writeFile(
+      behaviorFile,
+      "---\nname: INVALID\ndescription: Invalid behavior.\n---\n# Invalid\n",
+    );
+
+    await expect(
+      Behavior({ output: "done", behaviorRoot: root, model: "gpt-4o-mini" }),
+    ).rejects.toThrow("Diagnostics: Agent Behavior name");
+  });
+
+  test("requires an explicit selection when discovery finds multiple behaviors", async () => {
+    const root = await createBehaviorProject("first-behavior");
+    const second = path.join(root, ".agents", "behaviors", "second-behavior");
+    await fs.mkdir(second, { recursive: true });
+    await fs.writeFile(
+      path.join(second, "BEHAVIOR.md"),
+      "---\nname: second-behavior\ndescription: A second behavior.\n---\n# Second\n",
+    );
+
+    await expect(
+      Behavior({ output: "done", behaviorRoot: root, model: "gpt-4o-mini" }),
+    ).rejects.toThrow("Multiple Agent Behavior specs were discovered");
+  });
+});