Add PR review performance telemetry and dashboards

.github/actions/install-agent-harnesses/action.yml

@@ -9,5 +9,5 @@ runs:
       shell: pwsh
 
     - name: Install GitHub Copilot CLI
-      run: npm install -g @github/copilot@1.0.80
+      run: npm install -g @github/copilot@1.0.79
       shell: pwsh

.github/workflows/pr-review-evaluation.yml

@@ -115,7 +115,7 @@ jobs:
         uses: actions/checkout@v5
         with:
           repository: microsoft/BC-ALAgents
-          ref: f2ac8704bf8d39000f8002bcf2d287f1f5b5e9ba
+          ref: 533dd39dfe29218c09e5e31c39c78bb72fa20aa2
           path: bc-alagents-engine
           token: ${{ github.token }}
 

docs/code-review.md

@@ -39,7 +39,9 @@ bcbench evaluate claude <entry> --category code-review
 bcbench evaluate pr-review <entry>
 ```
 
-The evaluation workflow pins BC-ALAgents to a commit SHA, and each result records the exact engine revision and filtered BCQuality content. Engine updates require a new BC-Bench version and must record that SHA in the release notes.
+The evaluation workflow pins BC-ALAgents to a commit SHA. Engine updates require a new BC-Bench version and must record that SHA in the release notes. GitHub Copilot CLI stays pinned to `1.0.79` because `1.0.80` is not published to npm and `1.0.79` is the version whose structured telemetry contract was validated.
+
+BC PR Review records wall-clock duration, prompt/completion/total tokens, actual model API calls, exact AI credits, and two structural BCQuality counts: Markdown knowledge files available after filtering and knowledge files removed by the filter. Usage values come from the engine's strictly validated schema-v1 `_run-metrics.json`, never from console transcripts. Additional producer diagnostics remain in that raw artifact rather than being promoted into BC-Bench result and leaderboard schemas.
 
 ## Baseline Leaderboard
 
@@ -77,6 +79,48 @@ The evaluation workflow pins BC-ALAgents to a commit SHA, and each result record
 <p><em>No results available yet. Check back soon!</em></p>
 {% endif %}
 
+## Performance Leaderboard
+
+{% if site.data.code-review.aggregate and site.data.code-review.aggregate.size > 0 %}
+<table>
+  <thead>
+    <tr>
+      <th>Agent</th>
+      <th>Model</th>
+      <th>Avg Time</th>
+      <th>Avg Prompt Tokens</th>
+      <th>Avg Completion Tokens</th>
+      <th>Avg Total Tokens</th>
+      <th>Avg API Calls</th>
+      <th>Avg AI Credits</th>
+      <th>Avg Knowledge Files</th>
+      <th>Avg Knowledge Pruned</th>
+      <th>Ver</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% assign performance_results = site.data.code-review.aggregate | sort: "average_duration" %}
+    {% for agg in performance_results %}
+    <tr>
+      <td>{{ agg.agent_name }}</td>
+      <td>{{ agg.model }}</td>
+      <td>{{ agg.average_duration | round: 1 }}s</td>
+      <td>{% if agg.average_prompt_tokens != null %}{{ agg.average_prompt_tokens | round: 0 }}{% else %}—{% endif %}</td>
+      <td>{% if agg.average_completion_tokens != null %}{{ agg.average_completion_tokens | round: 0 }}{% else %}—{% endif %}</td>
+      <td>{% if agg.average_total_tokens != null %}{{ agg.average_total_tokens | round: 0 }}{% else %}—{% endif %}</td>
+      <td>{% if agg.average_api_calls != null %}{{ agg.average_api_calls | round: 1 }}{% else %}—{% endif %}</td>
+      <td>{% if agg.average_ai_credits != null %}{{ agg.average_ai_credits | round: 4 }}{% else %}—{% endif %}</td>
+      <td>{% if agg.average_knowledge_files != null %}{{ agg.average_knowledge_files | round: 1 }}{% else %}—{% endif %}</td>
+      <td>{% if agg.average_knowledge_pruned != null %}{{ agg.average_knowledge_pruned | round: 1 }}{% else %}—{% endif %}</td>
+      <td><a href="https://github.com/microsoft/BC-Bench/releases/tag/v{{ agg.benchmark_version }}" target="_blank">{{ agg.benchmark_version }}</a></td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{% else %}
+<p><em>No performance results available yet. Check back soon!</em></p>
+{% endif %}
+
 ## Experiment Leaderboard
 
 Compares review-knowledge configurations for the same model (see the Baseline Leaderboard above for the plain agent):

src/bcbench/agent/pr_review/agent.py

@@ -20,6 +20,7 @@
 
 import yaml
 
+from bcbench.agent.pr_review.metrics import build_pr_review_metrics
 from bcbench.agent.pr_review.review_output import engine_report_to_review_comments, load_engine_report
 from bcbench.config import get_config
 from bcbench.dataset import BaseDatasetEntry
@@ -229,4 +230,4 @@ def run_pr_review_agent(
         logger.exception("Unexpected error running engine review")
         raise
     else:
-        return AgentMetrics(execution_time=time.monotonic() - start), config
+        return build_pr_review_metrics(output_dir, bcquality_root, time.monotonic() - start), config

src/bcbench/agent/pr_review/metrics.py

@@ -0,0 +1,127 @@
+import json
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
+
+from bcbench.exceptions import AgentError
+from bcbench.types import AgentMetrics
+
+FILTER_REPORT_FILE_NAME = "_filter-report.json"
+RUN_METRICS_FILE_NAME = "_run-metrics.json"
+_KNOWLEDGE_LAYERS = {"microsoft", "community", "custom"}
+_NonNegativeInt = Annotated[int, Field(ge=0)]
+_NonNegativeFloat = Annotated[float, Field(ge=0)]
+
+
+class _FilterRemoval(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    kind: Literal["knowledge", "skill"]
+
+
+class _FilterReport(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    removed: list[_FilterRemoval]
+
+
+class _RunMetrics(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+    schema_version: Literal[1]
+    metrics_source: Literal["copilot-cli-otel", "not-applicable"]
+    cli_version: str | None
+    wall_time_seconds: _NonNegativeFloat | None
+    prompt_tokens: _NonNegativeInt | None
+    cached_tokens: _NonNegativeInt | None
+    cache_creation_tokens: _NonNegativeInt | None
+    completion_tokens: _NonNegativeInt | None
+    reasoning_tokens: _NonNegativeInt | None
+    total_tokens: _NonNegativeInt | None
+    api_calls: _NonNegativeInt | None
+    failed_api_calls: _NonNegativeInt | None
+    usage_api_calls: _NonNegativeInt | None
+    ai_credits: _NonNegativeFloat | None
+    premium_requests: _NonNegativeFloat | None
+    models: list[str]
+    usage_complete: bool
+    malformed_records: _NonNegativeInt
+
+    @model_validator(mode="after")
+    def validate_not_applicable_shape(self) -> "_RunMetrics":
+        if self.metrics_source != "not-applicable":
+            return self
+        expected = {
+            "cli_version": None,
+            "wall_time_seconds": 0,
+            "prompt_tokens": 0,
+            "cached_tokens": 0,
+            "cache_creation_tokens": 0,
+            "completion_tokens": 0,
+            "reasoning_tokens": None,
+            "total_tokens": 0,
+            "api_calls": 0,
+            "failed_api_calls": 0,
+            "usage_api_calls": 0,
+            "ai_credits": 0.0,
+            "premium_requests": None,
+            "models": [],
+            "usage_complete": True,
+            "malformed_records": 0,
+        }
+        invalid = [name for name, value in expected.items() if getattr(self, name) != value]
+        if invalid:
+            raise ValueError(f"not-applicable metrics have invalid fields: {', '.join(invalid)}")
+        return self
+
+
+def _load_run_metrics(path: Path) -> _RunMetrics:
+    if not path.exists():
+        raise AgentError(f"Engine run metrics artifact not found at {path}.")
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    except (json.JSONDecodeError, OSError) as exc:
+        raise AgentError(f"Could not read engine run metrics artifact {path}: {exc}") from exc
+    try:
+        return _RunMetrics.model_validate(payload)
+    except ValidationError as exc:
+        raise AgentError(f"Engine run metrics artifact {path} does not satisfy schema version 1: {exc}") from exc
+
+
+def _load_filter_report(path: Path) -> _FilterReport:
+    if not path.exists():
+        raise AgentError(f"BCQuality filter report not found at {path}.")
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    except (json.JSONDecodeError, OSError) as exc:
+        raise AgentError(f"Could not read BCQuality filter report {path}: {exc}") from exc
+    try:
+        return _FilterReport.model_validate(payload)
+    except ValidationError as exc:
+        raise AgentError(f"BCQuality filter report {path} has an invalid shape: {exc}") from exc
+
+
+def _count_available_knowledge(bcquality_root: Path) -> int:
+    def is_knowledge_file(path: Path) -> bool:
+        parts = path.relative_to(bcquality_root).parts
+        return len(parts) >= 3 and parts[0].lower() in _KNOWLEDGE_LAYERS and parts[1].lower() == "knowledge"
+
+    return sum(1 for path in bcquality_root.rglob("*.md") if path.is_file() and is_knowledge_file(path))
+
+
+def build_pr_review_metrics(output_dir: Path, bcquality_root: Path, execution_time: float) -> AgentMetrics:
+    run = _load_run_metrics(output_dir / RUN_METRICS_FILE_NAME)
+    report = _load_filter_report(bcquality_root / FILTER_REPORT_FILE_NAME)
+    usage_values_available = run.malformed_records == 0
+    token_values_available = usage_values_available and run.usage_complete
+    return AgentMetrics(
+        execution_time=execution_time,
+        prompt_tokens=run.prompt_tokens if token_values_available else None,
+        completion_tokens=run.completion_tokens if token_values_available else None,
+        total_tokens=run.total_tokens if token_values_available else None,
+        api_calls=run.api_calls if usage_values_available else None,
+        ai_credits=run.ai_credits if usage_values_available else None,
+        knowledge_files=_count_available_knowledge(bcquality_root),
+        knowledge_pruned=sum(1 for item in report.removed if item.kind == "knowledge"),
+    )

src/bcbench/results/codereview.py

@@ -288,6 +288,9 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary):
     Macro metrics average per-task scores (each task weighted equally).
     """
 
+    average_prompt_tokens: float | None = None
+    average_completion_tokens: float | None = None
+
     generated_comment_count: int = Field(default=0, ge=0)
     expected_comment_count: int = Field(default=0, ge=0)
     matched_comment_count: int = Field(default=0, ge=0)
@@ -310,10 +313,30 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary):
     severity_mae: float = 0.0
     valid_review_output_rate: float = Field(default=0.0, ge=0.0, le=1.0)
 
+    average_total_tokens: float | None = None
+    average_api_calls: float | None = None
+    average_knowledge_files: float | None = None
+    average_knowledge_pruned: float | None = None
+
     # Per-task F1 keyed by instance_id, retained so the leaderboard can bootstrap a confidence
     # interval over tasks (meaningful even for a single run) instead of only over runs.
     instance_results: dict[str, float] = Field(default_factory=dict)
 
+    def _performance_markdown(self) -> str:
+        def metric(value: float | None, digits: int = 1) -> str:
+            return f"{value:.{digits}f}" if value is not None else "n/a"
+
+        return (
+            "## Performance\n"
+            "\n"
+            "| Avg duration (s) | Avg prompt tokens | Avg completion tokens | Avg total tokens | Avg API calls | Avg AI credits | Avg knowledge files | Avg knowledge pruned |\n"
+            "|-----------------:|------------------:|----------------------:|-----------------:|--------------:|---------------:|--------------------:|---------------------:|\n"
+            f"| {self.average_duration:.1f} | {metric(self.average_prompt_tokens)} | {metric(self.average_completion_tokens)} | "
+            f"{metric(self.average_total_tokens)} | {metric(self.average_api_calls)} | {metric(self.average_ai_credits, 4)} | "
+            f"{metric(self.average_knowledge_files)} | {metric(self.average_knowledge_pruned)} |\n"
+            "\n"
+        )
+
     def render_github_metrics_markdown(self) -> str:
         micro_p = self.precision * 100
         micro_r = self.recall * 100
@@ -351,6 +374,7 @@ def render_github_metrics_markdown(self) -> str:
             "|-------------:|-------------------------:|\n"
             f"| {self.severity_mae:.3f} | {valid_rate:.1f}% |\n"
             "\n"
+            f"{self._performance_markdown()}"
             f"{_METRIC_EXPLANATIONS}"
         )
 
@@ -397,6 +421,20 @@ def render_console_metrics(self) -> RenderableType:
                 ["Severity MAE", "Valid review output rate"],
                 [f"{self.severity_mae:.3f}", f"{self.valid_review_output_rate * 100:.1f}%"],
             ),
+            _build_console_table(
+                "Performance",
+                ["Avg duration (s)", "Prompt", "Completion", "Total", "API calls", "AI credits", "Knowledge files", "Knowledge pruned"],
+                [
+                    f"{self.average_duration:.1f}",
+                    f"{self.average_prompt_tokens:.1f}" if self.average_prompt_tokens is not None else "n/a",
+                    f"{self.average_completion_tokens:.1f}" if self.average_completion_tokens is not None else "n/a",
+                    f"{self.average_total_tokens:.1f}" if self.average_total_tokens is not None else "n/a",
+                    f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a",
+                    f"{self.average_ai_credits:.4f}" if self.average_ai_credits is not None else "n/a",
+                    f"{self.average_knowledge_files:.1f}" if self.average_knowledge_files is not None else "n/a",
+                    f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a",
+                ],
+            ),
             Panel(
                 _CONSOLE_METRIC_EXPLANATIONS,
                 title="📖 How to read these metrics",
@@ -453,6 +491,10 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> "
         valid_output_count: int = sum(1 for r in code_review_results if r.valid_review_output)
         valid_output_rate: float = valid_output_count / total_results
 
+        def average_metric(name: str) -> float | None:
+            values = [value for result in code_review_results if result.metrics and (value := getattr(result.metrics, name)) is not None]
+            return sum(values) / len(values) if values else None
+
         return summary.model_copy(
             update={
                 "generated_comment_count": generated_total,
@@ -474,5 +516,12 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> "
                 "severity_mae": round(severity_mae, 3),
                 "valid_review_output_rate": round(valid_output_rate, 3),
                 "instance_results": {r.instance_id: round(r.f1, 6) for r in code_review_results},
+                "average_prompt_tokens": average_metric("prompt_tokens"),
+                "average_completion_tokens": average_metric("completion_tokens"),
+                "average_total_tokens": average_metric("total_tokens"),
+                "average_api_calls": average_metric("api_calls"),
+                "average_ai_credits": average_metric("ai_credits"),
+                "average_knowledge_files": average_metric("knowledge_files"),
+                "average_knowledge_pruned": average_metric("knowledge_pruned"),
             }
         )

src/bcbench/results/leaderboard.py

@@ -147,6 +147,14 @@ class CodeReviewLeaderboardAggregate(JudgeBasedLeaderboardAggregate):
     macro_precision: float = 0.0
     macro_recall: float = 0.0
 
+    average_prompt_tokens: float | None = None
+    average_completion_tokens: float | None = None
+    average_total_tokens: float | None = None
+    average_api_calls: float | None = None
+    average_ai_credits: float | None = None
+    average_knowledge_files: float | None = None
+    average_knowledge_pruned: float | None = None
+
     @classmethod
     def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeaderboardAggregate":
         from bcbench.results.codereview import CodeReviewResultSummary
@@ -157,6 +165,10 @@ def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeader
         cr_runs: list[CodeReviewResultSummary] = [run for run in runs if isinstance(run, CodeReviewResultSummary)]
         n = len(cr_runs)
 
+        def mean_metric(name: str) -> float | None:
+            values = [value for run in cr_runs if (value := getattr(run, name)) is not None]
+            return sum(values) / len(values) if values else None
+
         # The micro headline pools every comment across the dataset, so there is no per-task
         # decomposition to resample; its CI is intentionally over run-level means and captures
         # run-to-run reproducibility (None unless >=2 runs with variance).
@@ -184,6 +196,13 @@ def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeader
                 "macro_f_beta_2": sum(r.macro_f_beta_2 for r in cr_runs) / n,
                 "macro_precision": sum(r.macro_precision for r in cr_runs) / n,
                 "macro_recall": sum(r.macro_recall for r in cr_runs) / n,
+                "average_prompt_tokens": mean_metric("average_prompt_tokens"),
+                "average_completion_tokens": mean_metric("average_completion_tokens"),
+                "average_total_tokens": mean_metric("average_total_tokens"),
+                "average_api_calls": mean_metric("average_api_calls"),
+                "average_ai_credits": mean_metric("average_ai_credits"),
+                "average_knowledge_files": mean_metric("average_knowledge_files"),
+                "average_knowledge_pruned": mean_metric("average_knowledge_pruned"),
             }
         )
 

src/bcbench/results/summary.py

@@ -123,10 +123,9 @@ def from_json(cls, payload: dict[str, Any]) -> "EvaluationResultSummary":
     def to_dict(self) -> dict[str, Any]:
         data = self.model_dump(mode="json")
         data["average_duration"] = round(data["average_duration"], 1)
-        data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1)
-        data["average_completion_tokens"] = round(data["average_completion_tokens"], 1)
+        data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1) if data["average_prompt_tokens"] is not None else None
+        data["average_completion_tokens"] = round(data["average_completion_tokens"], 1) if data["average_completion_tokens"] is not None else None
         data["average_llm_duration"] = round(data["average_llm_duration"], 1) if data["average_llm_duration"] is not None else None
-        data["average_ai_credits"] = round(data["average_ai_credits"], 2) if data["average_ai_credits"] is not None else None
         return data
 
     def save(self, output_dir: Path, summary_file: str) -> None: