diff --git a/.github/actions/install-agent-harnesses/action.yml b/.github/actions/install-agent-harnesses/action.yml index 1549e0fdf..8233ea820 100644 --- a/.github/actions/install-agent-harnesses/action.yml +++ b/.github/actions/install-agent-harnesses/action.yml @@ -9,5 +9,5 @@ runs: shell: pwsh - name: Install GitHub Copilot CLI - run: npm install -g @github/copilot@1.0.80 + run: npm install -g @github/copilot@1.0.79 shell: pwsh diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml index 7b019c468..ceded2fb8 100644 --- a/.github/workflows/pr-review-evaluation.yml +++ b/.github/workflows/pr-review-evaluation.yml @@ -115,7 +115,7 @@ jobs: uses: actions/checkout@v5 with: repository: microsoft/BC-ALAgents - ref: f2ac8704bf8d39000f8002bcf2d287f1f5b5e9ba + ref: 533dd39dfe29218c09e5e31c39c78bb72fa20aa2 path: bc-alagents-engine token: ${{ github.token }} diff --git a/docs/code-review.md b/docs/code-review.md index 85f6b0f93..450d82d01 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -39,7 +39,9 @@ bcbench evaluate claude --category code-review bcbench evaluate pr-review ``` -The evaluation workflow pins BC-ALAgents to a commit SHA, and each result records the exact engine revision and filtered BCQuality content. Engine updates require a new BC-Bench version and must record that SHA in the release notes. +The evaluation workflow pins BC-ALAgents to a commit SHA. Engine updates require a new BC-Bench version and must record that SHA in the release notes. GitHub Copilot CLI stays pinned to `1.0.79` because `1.0.80` is not published to npm and `1.0.79` is the version whose structured telemetry contract was validated. + +BC PR Review records wall-clock duration, prompt/completion/total tokens, actual model API calls, exact AI credits, and two structural BCQuality counts: Markdown knowledge files available after filtering and knowledge files removed by the filter. Usage values come from the engine's strictly validated schema-v1 `_run-metrics.json`, never from console transcripts. Additional producer diagnostics remain in that raw artifact rather than being promoted into BC-Bench result and leaderboard schemas. ## Baseline Leaderboard @@ -77,6 +79,48 @@ The evaluation workflow pins BC-ALAgents to a commit SHA, and each result record

No results available yet. Check back soon!

{% endif %} +## Performance Leaderboard + +{% if site.data.code-review.aggregate and site.data.code-review.aggregate.size > 0 %} + + + + + + + + + + + + + + + + + + {% assign performance_results = site.data.code-review.aggregate | sort: "average_duration" %} + {% for agg in performance_results %} + + + + + + + + + + + + + + {% endfor %} + +
AgentModelAvg TimeAvg Prompt TokensAvg Completion TokensAvg Total TokensAvg API CallsAvg AI CreditsAvg Knowledge FilesAvg Knowledge PrunedVer
{{ agg.agent_name }}{{ agg.model }}{{ agg.average_duration | round: 1 }}s{% if agg.average_prompt_tokens != null %}{{ agg.average_prompt_tokens | round: 0 }}{% else %}—{% endif %}{% if agg.average_completion_tokens != null %}{{ agg.average_completion_tokens | round: 0 }}{% else %}—{% endif %}{% if agg.average_total_tokens != null %}{{ agg.average_total_tokens | round: 0 }}{% else %}—{% endif %}{% if agg.average_api_calls != null %}{{ agg.average_api_calls | round: 1 }}{% else %}—{% endif %}{% if agg.average_ai_credits != null %}{{ agg.average_ai_credits | round: 4 }}{% else %}—{% endif %}{% if agg.average_knowledge_files != null %}{{ agg.average_knowledge_files | round: 1 }}{% else %}—{% endif %}{% if agg.average_knowledge_pruned != null %}{{ agg.average_knowledge_pruned | round: 1 }}{% else %}—{% endif %}{{ agg.benchmark_version }}
+{% else %} +

No performance results available yet. Check back soon!

+{% endif %} + ## Experiment Leaderboard Compares review-knowledge configurations for the same model (see the Baseline Leaderboard above for the plain agent): diff --git a/src/bcbench/agent/pr_review/agent.py b/src/bcbench/agent/pr_review/agent.py index 4c7f4eb59..a4f90f194 100644 --- a/src/bcbench/agent/pr_review/agent.py +++ b/src/bcbench/agent/pr_review/agent.py @@ -20,6 +20,7 @@ import yaml +from bcbench.agent.pr_review.metrics import build_pr_review_metrics from bcbench.agent.pr_review.review_output import engine_report_to_review_comments, load_engine_report from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry @@ -229,4 +230,4 @@ def run_pr_review_agent( logger.exception("Unexpected error running engine review") raise else: - return AgentMetrics(execution_time=time.monotonic() - start), config + return build_pr_review_metrics(output_dir, bcquality_root, time.monotonic() - start), config diff --git a/src/bcbench/agent/pr_review/metrics.py b/src/bcbench/agent/pr_review/metrics.py new file mode 100644 index 000000000..73eb7462f --- /dev/null +++ b/src/bcbench/agent/pr_review/metrics.py @@ -0,0 +1,127 @@ +import json +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator + +from bcbench.exceptions import AgentError +from bcbench.types import AgentMetrics + +FILTER_REPORT_FILE_NAME = "_filter-report.json" +RUN_METRICS_FILE_NAME = "_run-metrics.json" +_KNOWLEDGE_LAYERS = {"microsoft", "community", "custom"} +_NonNegativeInt = Annotated[int, Field(ge=0)] +_NonNegativeFloat = Annotated[float, Field(ge=0)] + + +class _FilterRemoval(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + kind: Literal["knowledge", "skill"] + + +class _FilterReport(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + removed: list[_FilterRemoval] + + +class _RunMetrics(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + schema_version: Literal[1] + metrics_source: Literal["copilot-cli-otel", "not-applicable"] + cli_version: str | None + wall_time_seconds: _NonNegativeFloat | None + prompt_tokens: _NonNegativeInt | None + cached_tokens: _NonNegativeInt | None + cache_creation_tokens: _NonNegativeInt | None + completion_tokens: _NonNegativeInt | None + reasoning_tokens: _NonNegativeInt | None + total_tokens: _NonNegativeInt | None + api_calls: _NonNegativeInt | None + failed_api_calls: _NonNegativeInt | None + usage_api_calls: _NonNegativeInt | None + ai_credits: _NonNegativeFloat | None + premium_requests: _NonNegativeFloat | None + models: list[str] + usage_complete: bool + malformed_records: _NonNegativeInt + + @model_validator(mode="after") + def validate_not_applicable_shape(self) -> "_RunMetrics": + if self.metrics_source != "not-applicable": + return self + expected = { + "cli_version": None, + "wall_time_seconds": 0, + "prompt_tokens": 0, + "cached_tokens": 0, + "cache_creation_tokens": 0, + "completion_tokens": 0, + "reasoning_tokens": None, + "total_tokens": 0, + "api_calls": 0, + "failed_api_calls": 0, + "usage_api_calls": 0, + "ai_credits": 0.0, + "premium_requests": None, + "models": [], + "usage_complete": True, + "malformed_records": 0, + } + invalid = [name for name, value in expected.items() if getattr(self, name) != value] + if invalid: + raise ValueError(f"not-applicable metrics have invalid fields: {', '.join(invalid)}") + return self + + +def _load_run_metrics(path: Path) -> _RunMetrics: + if not path.exists(): + raise AgentError(f"Engine run metrics artifact not found at {path}.") + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except (json.JSONDecodeError, OSError) as exc: + raise AgentError(f"Could not read engine run metrics artifact {path}: {exc}") from exc + try: + return _RunMetrics.model_validate(payload) + except ValidationError as exc: + raise AgentError(f"Engine run metrics artifact {path} does not satisfy schema version 1: {exc}") from exc + + +def _load_filter_report(path: Path) -> _FilterReport: + if not path.exists(): + raise AgentError(f"BCQuality filter report not found at {path}.") + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except (json.JSONDecodeError, OSError) as exc: + raise AgentError(f"Could not read BCQuality filter report {path}: {exc}") from exc + try: + return _FilterReport.model_validate(payload) + except ValidationError as exc: + raise AgentError(f"BCQuality filter report {path} has an invalid shape: {exc}") from exc + + +def _count_available_knowledge(bcquality_root: Path) -> int: + def is_knowledge_file(path: Path) -> bool: + parts = path.relative_to(bcquality_root).parts + return len(parts) >= 3 and parts[0].lower() in _KNOWLEDGE_LAYERS and parts[1].lower() == "knowledge" + + return sum(1 for path in bcquality_root.rglob("*.md") if path.is_file() and is_knowledge_file(path)) + + +def build_pr_review_metrics(output_dir: Path, bcquality_root: Path, execution_time: float) -> AgentMetrics: + run = _load_run_metrics(output_dir / RUN_METRICS_FILE_NAME) + report = _load_filter_report(bcquality_root / FILTER_REPORT_FILE_NAME) + usage_values_available = run.malformed_records == 0 + token_values_available = usage_values_available and run.usage_complete + return AgentMetrics( + execution_time=execution_time, + prompt_tokens=run.prompt_tokens if token_values_available else None, + completion_tokens=run.completion_tokens if token_values_available else None, + total_tokens=run.total_tokens if token_values_available else None, + api_calls=run.api_calls if usage_values_available else None, + ai_credits=run.ai_credits if usage_values_available else None, + knowledge_files=_count_available_knowledge(bcquality_root), + knowledge_pruned=sum(1 for item in report.removed if item.kind == "knowledge"), + ) diff --git a/src/bcbench/results/codereview.py b/src/bcbench/results/codereview.py index fbf21debf..5d61f6b69 100644 --- a/src/bcbench/results/codereview.py +++ b/src/bcbench/results/codereview.py @@ -288,6 +288,9 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary): Macro metrics average per-task scores (each task weighted equally). """ + average_prompt_tokens: float | None = None + average_completion_tokens: float | None = None + generated_comment_count: int = Field(default=0, ge=0) expected_comment_count: int = Field(default=0, ge=0) matched_comment_count: int = Field(default=0, ge=0) @@ -310,10 +313,30 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary): severity_mae: float = 0.0 valid_review_output_rate: float = Field(default=0.0, ge=0.0, le=1.0) + average_total_tokens: float | None = None + average_api_calls: float | None = None + average_knowledge_files: float | None = None + average_knowledge_pruned: float | None = None + # Per-task F1 keyed by instance_id, retained so the leaderboard can bootstrap a confidence # interval over tasks (meaningful even for a single run) instead of only over runs. instance_results: dict[str, float] = Field(default_factory=dict) + def _performance_markdown(self) -> str: + def metric(value: float | None, digits: int = 1) -> str: + return f"{value:.{digits}f}" if value is not None else "n/a" + + return ( + "## Performance\n" + "\n" + "| Avg duration (s) | Avg prompt tokens | Avg completion tokens | Avg total tokens | Avg API calls | Avg AI credits | Avg knowledge files | Avg knowledge pruned |\n" + "|-----------------:|------------------:|----------------------:|-----------------:|--------------:|---------------:|--------------------:|---------------------:|\n" + f"| {self.average_duration:.1f} | {metric(self.average_prompt_tokens)} | {metric(self.average_completion_tokens)} | " + f"{metric(self.average_total_tokens)} | {metric(self.average_api_calls)} | {metric(self.average_ai_credits, 4)} | " + f"{metric(self.average_knowledge_files)} | {metric(self.average_knowledge_pruned)} |\n" + "\n" + ) + def render_github_metrics_markdown(self) -> str: micro_p = self.precision * 100 micro_r = self.recall * 100 @@ -351,6 +374,7 @@ def render_github_metrics_markdown(self) -> str: "|-------------:|-------------------------:|\n" f"| {self.severity_mae:.3f} | {valid_rate:.1f}% |\n" "\n" + f"{self._performance_markdown()}" f"{_METRIC_EXPLANATIONS}" ) @@ -397,6 +421,20 @@ def render_console_metrics(self) -> RenderableType: ["Severity MAE", "Valid review output rate"], [f"{self.severity_mae:.3f}", f"{self.valid_review_output_rate * 100:.1f}%"], ), + _build_console_table( + "Performance", + ["Avg duration (s)", "Prompt", "Completion", "Total", "API calls", "AI credits", "Knowledge files", "Knowledge pruned"], + [ + f"{self.average_duration:.1f}", + f"{self.average_prompt_tokens:.1f}" if self.average_prompt_tokens is not None else "n/a", + f"{self.average_completion_tokens:.1f}" if self.average_completion_tokens is not None else "n/a", + f"{self.average_total_tokens:.1f}" if self.average_total_tokens is not None else "n/a", + f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a", + f"{self.average_ai_credits:.4f}" if self.average_ai_credits is not None else "n/a", + f"{self.average_knowledge_files:.1f}" if self.average_knowledge_files is not None else "n/a", + f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a", + ], + ), Panel( _CONSOLE_METRIC_EXPLANATIONS, title="📖 How to read these metrics", @@ -453,6 +491,10 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " valid_output_count: int = sum(1 for r in code_review_results if r.valid_review_output) valid_output_rate: float = valid_output_count / total_results + def average_metric(name: str) -> float | None: + values = [value for result in code_review_results if result.metrics and (value := getattr(result.metrics, name)) is not None] + return sum(values) / len(values) if values else None + return summary.model_copy( update={ "generated_comment_count": generated_total, @@ -474,5 +516,12 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " "severity_mae": round(severity_mae, 3), "valid_review_output_rate": round(valid_output_rate, 3), "instance_results": {r.instance_id: round(r.f1, 6) for r in code_review_results}, + "average_prompt_tokens": average_metric("prompt_tokens"), + "average_completion_tokens": average_metric("completion_tokens"), + "average_total_tokens": average_metric("total_tokens"), + "average_api_calls": average_metric("api_calls"), + "average_ai_credits": average_metric("ai_credits"), + "average_knowledge_files": average_metric("knowledge_files"), + "average_knowledge_pruned": average_metric("knowledge_pruned"), } ) diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index 146476b0f..7d0303761 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -147,6 +147,14 @@ class CodeReviewLeaderboardAggregate(JudgeBasedLeaderboardAggregate): macro_precision: float = 0.0 macro_recall: float = 0.0 + average_prompt_tokens: float | None = None + average_completion_tokens: float | None = None + average_total_tokens: float | None = None + average_api_calls: float | None = None + average_ai_credits: float | None = None + average_knowledge_files: float | None = None + average_knowledge_pruned: float | None = None + @classmethod def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeaderboardAggregate": from bcbench.results.codereview import CodeReviewResultSummary @@ -157,6 +165,10 @@ def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeader cr_runs: list[CodeReviewResultSummary] = [run for run in runs if isinstance(run, CodeReviewResultSummary)] n = len(cr_runs) + def mean_metric(name: str) -> float | None: + values = [value for run in cr_runs if (value := getattr(run, name)) is not None] + return sum(values) / len(values) if values else None + # The micro headline pools every comment across the dataset, so there is no per-task # decomposition to resample; its CI is intentionally over run-level means and captures # run-to-run reproducibility (None unless >=2 runs with variance). @@ -184,6 +196,13 @@ def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeader "macro_f_beta_2": sum(r.macro_f_beta_2 for r in cr_runs) / n, "macro_precision": sum(r.macro_precision for r in cr_runs) / n, "macro_recall": sum(r.macro_recall for r in cr_runs) / n, + "average_prompt_tokens": mean_metric("average_prompt_tokens"), + "average_completion_tokens": mean_metric("average_completion_tokens"), + "average_total_tokens": mean_metric("average_total_tokens"), + "average_api_calls": mean_metric("average_api_calls"), + "average_ai_credits": mean_metric("average_ai_credits"), + "average_knowledge_files": mean_metric("average_knowledge_files"), + "average_knowledge_pruned": mean_metric("average_knowledge_pruned"), } ) diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index 826318dc9..0c4c3812c 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -123,10 +123,9 @@ def from_json(cls, payload: dict[str, Any]) -> "EvaluationResultSummary": def to_dict(self) -> dict[str, Any]: data = self.model_dump(mode="json") data["average_duration"] = round(data["average_duration"], 1) - data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1) - data["average_completion_tokens"] = round(data["average_completion_tokens"], 1) + data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1) if data["average_prompt_tokens"] is not None else None + data["average_completion_tokens"] = round(data["average_completion_tokens"], 1) if data["average_completion_tokens"] is not None else None data["average_llm_duration"] = round(data["average_llm_duration"], 1) if data["average_llm_duration"] is not None else None - data["average_ai_credits"] = round(data["average_ai_credits"], 2) if data["average_ai_credits"] is not None else None return data def save(self, output_dir: Path, summary_file: str) -> None: diff --git a/src/bcbench/types.py b/src/bcbench/types.py index ec75a2896..9b664b6f0 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -79,9 +79,16 @@ class AgentMetrics(BaseModel): prompt_tokens: int | None = None completion_tokens: int | None = None + total_tokens: int | None = None + # Actual model requests, including nested calls and retries; distinct from conversational turns. + api_calls: int | None = None + # Tool usage statistics from agent logs tool_usage: dict[str, int] | None = None + knowledge_files: int | None = None + knowledge_pruned: int | None = None + class ExperimentConfiguration(BaseModel): """Configuration for agent experiment execution. @@ -197,8 +204,19 @@ def expected_metrics(self) -> frozenset[str]: completion_tokens=None, tool_usage=None, ) - case AgentHarness.BCAL | AgentHarness.PR_REVIEW: + case AgentHarness.BCAL: expected = AgentMetrics(execution_time=None) + case AgentHarness.PR_REVIEW: + expected = AgentMetrics( + execution_time=None, + prompt_tokens=None, + completion_tokens=None, + total_tokens=None, + api_calls=None, + ai_credits=None, + knowledge_files=None, + knowledge_pruned=None, + ) case _: raise ValueError(f"Unknown AgentHarness: {self}") diff --git a/tests/test_evaluation_summary.py b/tests/test_evaluation_summary.py index eeeca5ade..6dd1ba64a 100644 --- a/tests/test_evaluation_summary.py +++ b/tests/test_evaluation_summary.py @@ -226,6 +226,18 @@ def test_from_results_leaves_ai_credits_none_when_harness_reports_none(self): assert summary.average_ai_credits is None assert summary.to_dict()["average_ai_credits"] is None + def test_to_dict_preserves_exact_ai_credit_precision(self): + result = create_bugfix_result( + instance_id="test__1", + project="app", + resolved=True, + metrics=AgentMetrics(execution_time=100.0, ai_credits=0.123456), + ) + + summary = ExecutionBasedEvaluationResultSummary.from_results([result], run_id="test_run_123") + + assert summary.to_dict()["average_ai_credits"] == pytest.approx(0.123456) + def test_from_results_calculates_average_tool_usage(self): results = [ create_bugfix_result( diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index c780ea6e9..4a881222f 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -90,6 +90,38 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: } completed = subprocess.CompletedProcess(args=["pwsh"], returncode=0, stdout="✓", stderr="") entry = create_codereview_entry(repo="microsoft/BCApps") + bcquality_root = tmp_path / "bcquality" + knowledge_root = bcquality_root / "microsoft" / "knowledge" / "performance" + knowledge_root.mkdir(parents=True) + (knowledge_root / "one.md").write_text("# One", encoding="utf-8") + (bcquality_root / "_filter-report.json").write_text('{"removed": []}', encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "_run-metrics.json").write_text( + json.dumps( + { + "schema_version": 1, + "metrics_source": "copilot-cli-otel", + "cli_version": "1.0.81-0", + "wall_time_seconds": 2.4, + "prompt_tokens": 100, + "cached_tokens": 20, + "cache_creation_tokens": 5, + "completion_tokens": 10, + "reasoning_tokens": 4, + "total_tokens": 110, + "api_calls": 2, + "failed_api_calls": 0, + "usage_api_calls": 2, + "ai_credits": 0.25, + "premium_requests": 0.5, + "models": ["gpt-5.6-luna"], + "usage_complete": True, + "malformed_records": 0, + } + ), + encoding="utf-8", + ) with ( patch("bcbench.agent.pr_review.agent._load_pr_review_settings", return_value=settings), @@ -97,7 +129,7 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: patch("bcbench.agent.pr_review.agent._resolve_pwsh", return_value="pwsh"), patch("bcbench.agent.pr_review.agent._commit_patch_as_head"), patch("bcbench.agent.pr_review.agent._init_trusted_workspace", return_value=tmp_path / "trusted"), - patch("bcbench.agent.pr_review.agent._prepare_bcquality_root", return_value=tmp_path / "bcquality"), + patch("bcbench.agent.pr_review.agent._prepare_bcquality_root", return_value=bcquality_root), patch("bcbench.agent.pr_review.agent._write_review_json", return_value=0), patch("bcbench.agent.pr_review.agent.time.monotonic", side_effect=[10.0, 12.5]), patch("bcbench.agent.pr_review.agent.subprocess.run", return_value=completed) as run_process, @@ -113,6 +145,13 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: assert metrics is not None assert metrics.execution_time == 2.5 + assert metrics.prompt_tokens == 100 + assert metrics.completion_tokens == 10 + assert metrics.total_tokens == 110 + assert metrics.api_calls == 2 + assert metrics.ai_credits == 0.25 + assert metrics.knowledge_files == 1 + assert metrics.knowledge_pruned == 0 assert config.is_empty() assert run_process.call_args.kwargs["encoding"] == "utf-8" assert run_process.call_args.kwargs["cwd"] == str((tmp_path / "repo").resolve()) diff --git a/tests/test_pr_review_metrics.py b/tests/test_pr_review_metrics.py new file mode 100644 index 000000000..98ca53430 --- /dev/null +++ b/tests/test_pr_review_metrics.py @@ -0,0 +1,292 @@ +import json +from pathlib import Path + +import pytest + +from bcbench.agent.pr_review.metrics import FILTER_REPORT_FILE_NAME, RUN_METRICS_FILE_NAME, build_pr_review_metrics +from bcbench.exceptions import AgentError + + +def _write_filter_report(root: Path, removed: object) -> None: + (root / FILTER_REPORT_FILE_NAME).write_text(json.dumps({"removed": removed}), encoding="utf-8") + + +def _run_metrics(**overrides: object) -> dict[str, object]: + payload: dict[str, object] = { + "schema_version": 1, + "metrics_source": "copilot-cli-otel", + "cli_version": "1.0.81-0", + "wall_time_seconds": 12.346, + "prompt_tokens": 150, + "cached_tokens": 60, + "cache_creation_tokens": 10, + "completion_tokens": 28, + "reasoning_tokens": 7, + "total_tokens": 178, + "api_calls": 2, + "failed_api_calls": 1, + "usage_api_calls": 2, + "ai_credits": 1.75, + "premium_requests": 1.75, + "models": ["gpt-5.4-mini", "gpt-5.6-sol"], + "usage_complete": True, + "malformed_records": 0, + } + return {**payload, **overrides} + + +def _write_run_metrics(root: Path, **overrides: object) -> None: + (root / RUN_METRICS_FILE_NAME).write_text(json.dumps(_run_metrics(**overrides)), encoding="utf-8") + + +def test_build_metrics_promotes_public_usage_and_filtered_knowledge(tmp_path: Path) -> None: + knowledge = tmp_path / "microsoft" / "knowledge" / "performance" + knowledge.mkdir(parents=True) + (knowledge / "one.md").write_text("# One", encoding="utf-8") + (knowledge / "two.md").write_text("# Two", encoding="utf-8") + (knowledge / "two.good.al").write_text("", encoding="utf-8") + (tmp_path / "skills").mkdir() + (tmp_path / "skills" / "entry.md").write_text("# Entry", encoding="utf-8") + _write_filter_report( + tmp_path, + [ + {"path": "community/knowledge/old.md", "kind": "knowledge", "reason": "layer-disabled"}, + {"path": "community/skills/old.md", "kind": "skill", "reason": "layer-disabled"}, + ], + ) + _write_run_metrics(tmp_path) + + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=12.5) + + assert metrics.execution_time == 12.5 + assert metrics.prompt_tokens == 150 + assert metrics.completion_tokens == 28 + assert metrics.total_tokens == 178 + assert metrics.api_calls == 2 + assert metrics.ai_credits == 1.75 + assert metrics.knowledge_files == 2 + assert metrics.knowledge_pruned == 1 + + +def test_legal_null_optional_fields_and_multiple_models_are_accepted(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + cli_version=None, + wall_time_seconds=None, + cached_tokens=None, + cache_creation_tokens=None, + reasoning_tokens=None, + ai_credits=None, + premium_requests=None, + models=["gpt-5.4-mini", "gpt-5.6-sol"], + ) + + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) + + assert metrics.ai_credits is None + assert metrics.total_tokens == 178 + + +def test_malformed_records_suppress_all_usage_metrics(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + prompt_tokens=25, + cached_tokens=None, + cache_creation_tokens=None, + completion_tokens=5, + total_tokens=30, + api_calls=2, + failed_api_calls=1, + usage_api_calls=1, + ai_credits=0.1, + reasoning_tokens=None, + premium_requests=None, + usage_complete=False, + malformed_records=3, + ) + + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) + + assert metrics.prompt_tokens is None + assert metrics.completion_tokens is None + assert metrics.total_tokens is None + assert metrics.api_calls is None + assert metrics.ai_credits is None + + +def test_incomplete_usage_suppresses_tokens_but_preserves_exact_calls_and_credits(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + prompt_tokens=25, + completion_tokens=5, + total_tokens=30, + api_calls=2, + ai_credits=0.1, + usage_complete=False, + malformed_records=0, + ) + + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) + + assert metrics.prompt_tokens is None + assert metrics.completion_tokens is None + assert metrics.total_tokens is None + assert metrics.api_calls == 2 + assert metrics.ai_credits == 0.1 + + +def test_missing_run_metrics_raises(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + + with pytest.raises(AgentError, match="run metrics artifact not found"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_invalid_run_metrics_json_raises(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + (tmp_path / RUN_METRICS_FILE_NAME).write_text("not json", encoding="utf-8") + + with pytest.raises(AgentError, match="Could not read engine run metrics artifact"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +@pytest.mark.parametrize( + "overrides", + [ + {"schema_version": 2}, + {"metrics_source": "console-transcript"}, + {"api_calls": "2"}, + {"usage_complete": 1}, + {"reasoning_tokens": "5"}, + {"premium_requests": "1.0"}, + {"cli_version": 79}, + {"models": ["gpt-5.6-sol", 5]}, + {"unexpected": "field"}, + ], +) +def test_invalid_run_metrics_contract_raises(tmp_path: Path, overrides: dict[str, object]) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics(tmp_path, **overrides) + + with pytest.raises(AgentError, match="does not satisfy schema version 1"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_missing_filter_report_raises(tmp_path: Path) -> None: + _write_run_metrics(tmp_path) + + with pytest.raises(AgentError, match="not found"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_missing_run_metrics_key_raises(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + payload = _run_metrics() + del payload["models"] + (tmp_path / RUN_METRICS_FILE_NAME).write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(AgentError, match="does not satisfy schema version 1"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_not_applicable_zero_shape_is_accepted(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + metrics_source="not-applicable", + cli_version=None, + wall_time_seconds=0, + prompt_tokens=0, + cached_tokens=0, + cache_creation_tokens=0, + completion_tokens=0, + reasoning_tokens=None, + total_tokens=0, + api_calls=0, + failed_api_calls=0, + usage_api_calls=0, + ai_credits=0.0, + premium_requests=None, + models=[], + usage_complete=True, + malformed_records=0, + ) + + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=0.25) + + assert metrics.execution_time == 0.25 + assert metrics.prompt_tokens == 0 + assert metrics.completion_tokens == 0 + assert metrics.total_tokens == 0 + assert metrics.api_calls == 0 + assert metrics.ai_credits == 0.0 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("cli_version", "1.0.79"), + ("wall_time_seconds", 1.0), + ("prompt_tokens", None), + ("reasoning_tokens", 0), + ("premium_requests", 0.0), + ("models", ["gpt-5.6-sol"]), + ("usage_complete", False), + ("malformed_records", 1), + ], +) +def test_not_applicable_rejects_noncanonical_shape(tmp_path: Path, field: str, value: object) -> None: + _write_filter_report(tmp_path, []) + not_applicable = { + "metrics_source": "not-applicable", + "cli_version": None, + "wall_time_seconds": 0, + "prompt_tokens": 0, + "cached_tokens": 0, + "cache_creation_tokens": 0, + "completion_tokens": 0, + "reasoning_tokens": None, + "total_tokens": 0, + "api_calls": 0, + "failed_api_calls": 0, + "usage_api_calls": 0, + "ai_credits": 0.0, + "premium_requests": None, + "models": [], + "usage_complete": True, + "malformed_records": 0, + field: value, + } + _write_run_metrics(tmp_path, **not_applicable) + + with pytest.raises(AgentError, match="not-applicable metrics have invalid fields"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +@pytest.mark.parametrize( + "payload", + [ + [], + {}, + {"removed": "invalid"}, + {"removed": [{"kind": "unknown"}]}, + {"removed": ["invalid"]}, + ], +) +def test_malformed_filter_report_raises(tmp_path: Path, payload: object) -> None: + (tmp_path / FILTER_REPORT_FILE_NAME).write_text(json.dumps(payload), encoding="utf-8") + _write_run_metrics(tmp_path) + + with pytest.raises(AgentError, match="filter report"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_invalid_filter_report_json_raises(tmp_path: Path) -> None: + (tmp_path / FILTER_REPORT_FILE_NAME).write_text("not json", encoding="utf-8") + _write_run_metrics(tmp_path) + + with pytest.raises(AgentError, match="Could not read"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) diff --git a/tests/test_pr_review_metrics_reporting.py b/tests/test_pr_review_metrics_reporting.py new file mode 100644 index 000000000..bd1f3080f --- /dev/null +++ b/tests/test_pr_review_metrics_reporting.py @@ -0,0 +1,141 @@ +import json + +from bcbench.results.codereview import CodeReviewResultSummary +from bcbench.results.leaderboard import CodeReviewLeaderboardAggregate +from bcbench.types import AgentMetrics +from tests.conftest import create_codereview_result + + +def _metrics(*, duration: float, scale: int) -> AgentMetrics: + return AgentMetrics( + execution_time=duration, + prompt_tokens=900 * scale, + completion_tokens=100 * scale, + total_tokens=1000 * scale, + api_calls=10 * scale, + ai_credits=0.5 * scale, + knowledge_files=20 * scale, + knowledge_pruned=4 * scale, + ) + + +def test_summary_aggregates_public_pr_review_metrics() -> None: + summary = CodeReviewResultSummary.from_results( + [ + create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1)), + create_codereview_result(instance_id="proj__review-2", metrics=_metrics(duration=6.0, scale=2)), + ], + run_id="run", + ) + + assert summary.average_duration == 5 + assert summary.average_prompt_tokens == 1350 + assert summary.average_completion_tokens == 150 + assert summary.average_total_tokens == 1500 + assert summary.average_api_calls == 15 + assert summary.average_ai_credits == 0.75 + assert summary.average_knowledge_files == 30 + assert summary.average_knowledge_pruned == 6 + + +def test_summary_preserves_unavailable_usage_as_none() -> None: + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(metrics=AgentMetrics(execution_time=4.0, knowledge_files=20, knowledge_pruned=4))], + run_id="run", + ) + + serialized = summary.to_dict() + + assert serialized["average_prompt_tokens"] is None + assert serialized["average_completion_tokens"] is None + assert serialized["average_total_tokens"] is None + assert serialized["average_api_calls"] is None + assert serialized["average_ai_credits"] is None + + +def test_leaderboard_propagates_public_pr_review_metrics() -> None: + first = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1))], + run_id="one", + ) + second = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=6.0, scale=2))], + run_id="two", + ) + + aggregate = CodeReviewLeaderboardAggregate.from_runs([first, second]) + + assert aggregate.average_duration == 5 + assert aggregate.average_prompt_tokens == 1350 + assert aggregate.average_completion_tokens == 150 + assert aggregate.average_total_tokens == 1500 + assert aggregate.average_api_calls == 15 + assert aggregate.average_ai_credits == 0.75 + assert aggregate.average_knowledge_files == 30 + assert aggregate.average_knowledge_pruned == 6 + + +def test_github_summary_renders_only_public_performance_metrics() -> None: + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1))], + run_id="run", + ) + + markdown = summary.render_github_metrics_markdown() + + assert "## Performance" in markdown + assert "Avg prompt tokens" in markdown + assert "Avg completion tokens" in markdown + assert "Avg total tokens" in markdown + assert "Avg API calls" in markdown + assert "Avg AI credits" in markdown + assert "Avg knowledge files" in markdown + for diagnostic in ("cached", "reasoning", "failed API", "usage", "premium", "malformed"): + assert diagnostic not in markdown + + +def test_result_json_excludes_raw_only_diagnostics(tmp_path) -> None: + result = create_codereview_result(metrics=_metrics(duration=4.0, scale=1)) + result.save(tmp_path, "results.jsonl") + + saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] + + assert saved_metrics["prompt_tokens"] == 900 + assert saved_metrics["completion_tokens"] == 100 + assert saved_metrics["total_tokens"] == 1000 + assert saved_metrics["api_calls"] == 10 + assert saved_metrics["ai_credits"] == 0.5 + assert saved_metrics["knowledge_files"] == 20 + assert saved_metrics["knowledge_pruned"] == 4 + for diagnostic in ( + "cached_tokens", + "cache_creation_tokens", + "reasoning_tokens", + "failed_api_calls", + "usage_api_calls", + "premium_requests", + "usage_complete", + "malformed_records", + ): + assert diagnostic not in saved_metrics + + +def test_summary_and_leaderboard_schemas_exclude_raw_only_diagnostics() -> None: + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(metrics=_metrics(duration=4.0, scale=1))], + run_id="run", + ) + aggregate = CodeReviewLeaderboardAggregate.from_runs([summary]) + + for payload in (summary.model_dump(), aggregate.model_dump()): + for diagnostic in ( + "average_cached_tokens", + "average_cache_creation_tokens", + "average_reasoning_tokens", + "average_failed_api_calls", + "average_usage_api_calls", + "average_premium_requests", + "structured_usage_complete_rate", + "average_malformed_records", + ): + assert diagnostic not in payload diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index 8548b33e5..8e9c8f4b3 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -3,6 +3,7 @@ import yaml WORKFLOWS = Path(__file__).parents[1] / ".github" / "workflows" +ACTIONS = Path(__file__).parents[1] / ".github" / "actions" def _workflow(name: str) -> str: @@ -35,7 +36,7 @@ def test_pr_review_workflow_is_fixed_to_code_review() -> None: assert "category: code-review" in workflow assert "bcbench evaluate pr-review" in workflow assert "repository: microsoft/BC-ALAgents" in workflow - assert "f2ac8704bf8d39000f8002bcf2d287f1f5b5e9ba" in workflow + assert "533dd39dfe29218c09e5e31c39c78bb72fa20aa2" in workflow assert "ref: main" not in workflow assert '--engine-path "${{ github.workspace }}/bc-alagents-engine"' in workflow assert "BC_PR_REVIEW_ROOT:" not in workflow @@ -47,3 +48,10 @@ def test_pr_review_workflow_is_fixed_to_code_review() -> None: assert "mai-code-1-flash-picker" not in workflow for input_name in ("model:", "test-run:", "repeat:", "git-ref:"): assert input_name in workflow + + +def test_agent_harness_action_pins_published_copilot_version() -> None: + action = (ACTIONS / "install-agent-harnesses" / "action.yml").read_text(encoding="utf-8") + + assert "@github/copilot@1.0.79" in action + assert "@github/copilot@1.0.80" not in action