From 7e0497d42a6af258ade96d2d43d89bbb7415d4e7 Mon Sep 17 00:00:00 2001 From: Lee Yamin Date: Wed, 26 Aug 2026 12:50:00 +0300 Subject: [PATCH] Add AgentTrust evaluation demo. Runs an AgentTrust scope-compliance evaluation as a Kubernetes job against a live A2A agent. Closes #492. Signed-off-by: Lee Yamin --- demos/agenttrust-evaluation/README.md | 89 +++++++++++++++++++ demos/agenttrust-evaluation/configmap.yaml | 10 +++ demos/agenttrust-evaluation/job.yaml | 46 ++++++++++ .../agenttrust-evaluation/kustomization.yaml | 5 ++ .../agenttrust-evaluation/secret.example.yaml | 7 ++ 5 files changed, 157 insertions(+) create mode 100644 demos/agenttrust-evaluation/README.md create mode 100644 demos/agenttrust-evaluation/configmap.yaml create mode 100644 demos/agenttrust-evaluation/job.yaml create mode 100644 demos/agenttrust-evaluation/kustomization.yaml create mode 100644 demos/agenttrust-evaluation/secret.example.yaml diff --git a/demos/agenttrust-evaluation/README.md b/demos/agenttrust-evaluation/README.md new file mode 100644 index 00000000..572fb68c --- /dev/null +++ b/demos/agenttrust-evaluation/README.md @@ -0,0 +1,89 @@ +# AgentTrust Evaluation Demo + +Run an [AgentTrust](https://github.com/leeyamin/agent-trust) evaluation as a Kubernetes Job. AgentTrust evaluates whether an AI agent's behavior aligns with its declared capabilities by probing it across in-scope, out-of-scope, and near-miss requests. + +This demo is report-only — results are logged but no action is automatically enforced. + +## Prerequisites + +- A Kubernetes cluster with kubectl access +- A running A2A agent with an Agent Card endpoint +- An Anthropic API key for the LLM judge (AgentTrust uses the Claude Agent SDK, which also supports Vertex AI and Bedrock via Claude Code CLI configuration) + +## Setup + +### 1. Create the credentials secret + +```bash +kubectl create secret generic agenttrust-judge-credentials \ + --from-literal=ANTHROPIC_API_KEY= +``` + +See `secret.example.yaml` for the expected format. + +### 2. Configure the evaluation + +Edit `configmap.yaml` to set the target agent and evaluation parameters: + +| Variable | Default | Description | +|----------|---------|-------------| +| `AGENT_URL` | `http://weather-agent.agents.svc:8000` | A2A endpoint of the target agent | +| `AGENTTRUST_NUM_PROBES` | `5` | Probes per scope | +| `AGENTTRUST_TRACE_SOURCE` | `none` | `mlflow` or `none` | +| `MLFLOW_EXPERIMENT_NAME` | `agenttrust-evaluations` | MLflow experiment name | +| `CLAUDE_CONFIG_DIR` | `/work` | Writable directory for CLI session data | + +For the full list of configuration options, see the [AgentTrust CLI Reference](https://github.com/leeyamin/agent-trust#cli-reference). + +### 3. Deploy + +```bash +kubectl apply -k demos/agenttrust-evaluation/ +``` + +### 4. View results + +```bash +kubectl logs job/agenttrust-evaluation +``` + +The Job outputs a compact JSON result to stdout: + +```json +{ + "schemaVersion": "v1", + "evaluationId": "abc-123", + "agent": "weather_agent", + "cardHash": "sha256:...", + "outcome": "completed", + "alignmentPassed": true, + "score": 85, + "evidenceMode": "text", + "completedAt": "2026-01-01T00:00:00+00:00", + "reportURI": null +} +``` + +| Field | Description | +|-------|-------------| +| `outcome` | `completed` or `error` | +| `alignmentPassed` | `true` if score >= alignment threshold | +| `score` | Trust score (0-100) | +| `evidenceMode` | `text` or `text+trace` | +| `reportURI` | MLflow report URI when `MLFLOW_TRACKING_URI` is set | + +## Cleanup and rerun + +```bash +kubectl delete job agenttrust-evaluation +kubectl apply -k demos/agenttrust-evaluation/ +``` + +K8s Jobs are immutable — delete before re-applying. + +## Security + +- Only evaluate sandboxed agents with read-only tools +- The container runs as non-root with a read-only filesystem +- No Kubernetes API permissions are granted to the pod +- No evaluation result is automatically enforced — results are informational only diff --git a/demos/agenttrust-evaluation/configmap.yaml b/demos/agenttrust-evaluation/configmap.yaml new file mode 100644 index 00000000..2dfcd2d7 --- /dev/null +++ b/demos/agenttrust-evaluation/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agenttrust-evaluation-config +data: + AGENT_URL: "http://weather-agent.agents.svc:8000" + AGENTTRUST_NUM_PROBES: "5" + AGENTTRUST_TRACE_SOURCE: "none" + MLFLOW_EXPERIMENT_NAME: "agenttrust-evaluations" + CLAUDE_CONFIG_DIR: "/work" diff --git a/demos/agenttrust-evaluation/job.yaml b/demos/agenttrust-evaluation/job.yaml new file mode 100644 index 00000000..042956f3 --- /dev/null +++ b/demos/agenttrust-evaluation/job.yaml @@ -0,0 +1,46 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: agenttrust-evaluation +spec: + backoffLimit: 0 + activeDeadlineSeconds: 900 + template: + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + containers: + - name: agenttrust + image: ghcr.io/leeyamin/agenttrust:latest + envFrom: + - configMapRef: + name: agenttrust-evaluation-config + - secretRef: + name: agenttrust-judge-credentials + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: work + mountPath: /work + - name: tmp + mountPath: /tmp + volumes: + - name: work + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/demos/agenttrust-evaluation/kustomization.yaml b/demos/agenttrust-evaluation/kustomization.yaml new file mode 100644 index 00000000..ad4c98ee --- /dev/null +++ b/demos/agenttrust-evaluation/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - configmap.yaml + - job.yaml diff --git a/demos/agenttrust-evaluation/secret.example.yaml b/demos/agenttrust-evaluation/secret.example.yaml new file mode 100644 index 00000000..e51a9ba0 --- /dev/null +++ b/demos/agenttrust-evaluation/secret.example.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: agenttrust-judge-credentials +type: Opaque +stringData: + ANTHROPIC_API_KEY: "REPLACE_ME"