From fd7f34767de7171ddcd7ecd614c9faad16dd9317 Mon Sep 17 00:00:00 2001 From: dayland Date: Thu, 13 Aug 2026 16:50:51 +0100 Subject: [PATCH 1/3] Add PR review performance telemetry Collect BC-ALAgents usage and knowledge metrics, expose Code Review performance dashboards, and make engine and BCQuality experiment sources independently configurable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/copilot-evaluation.yml | 13 -- docs/code-review.md | 178 ++++++++++++--- src/bcbench/agent/copilot/pr_review/agent.py | 207 ++++++++++++------ .../agent/copilot/pr_review/metrics.py | 164 ++++++++++++++ .../scripts/Prepare-BCQualityRoot.ps1 | 21 +- src/bcbench/agent/shared/config.yaml | 14 +- src/bcbench/commands/evaluate.py | 18 +- src/bcbench/commands/run.py | 18 +- src/bcbench/results/base.py | 11 + src/bcbench/results/codereview.py | 83 ++++++- src/bcbench/results/leaderboard.py | 15 ++ src/bcbench/types.py | 19 +- tests/test_pr_review_agent.py | 115 +++++++++- tests/test_pr_review_metrics.py | 110 ++++++++++ tests/test_pr_review_metrics_reporting.py | 95 ++++++++ 15 files changed, 941 insertions(+), 140 deletions(-) create mode 100644 src/bcbench/agent/copilot/pr_review/metrics.py create mode 100644 tests/test_pr_review_metrics.py create mode 100644 tests/test_pr_review_metrics_reporting.py diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index 307ca7b9f..b4e21c48f 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -139,18 +139,6 @@ jobs: - name: Install evaluation CLIs uses: $/.github/actions/install-agent-harnesses - # code-review is evaluated through the BC-ALAgents review engine (the real PROD - # generate half + BCQuality). Check it out only for that category. To evaluate the - # engine at a different ref, change `ref` below on your private branch. - - name: Checkout BC-ALAgents review engine - if: ${{ inputs.category == 'code-review' }} - uses: actions/checkout@v5 - with: - repository: microsoft/BC-ALAgents - ref: main - path: bc-alagents-engine - token: ${{ github.token }} - - name: Run code-review engine for entry ${{ matrix.entry }} if: ${{ inputs.category == 'code-review' }} timeout-minutes: 120 @@ -158,7 +146,6 @@ jobs: env: COPILOT_GITHUB_TOKEN: ${{ github.token }} GH_TOKEN: ${{ github.token }} - BC_PR_REVIEW_ROOT: ${{ github.workspace }}/bc-alagents-engine run: | Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN" diff --git a/docs/code-review.md b/docs/code-review.md index 29092d777..e3c1513f3 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -6,7 +6,7 @@ title: Code Review - BC-Bench # Code Review @@ -27,6 +45,24 @@ Unlike the pass/fail categories, code review is scored with **Precision / Recall A gold entry may also declare **`ignored_comments`** — legitimate-but-optional observations (out-of-scope nitpicks, maintainer-judgment calls) that should be neither required nor penalized. Ignored comments are structurally paired against the generated comments and validated by the same LLM judge, in a single judge pass alongside the expected comments. Any generated comment the judge confirms as an ignored match is dropped from scoring entirely: it earns no recall and does not count against precision. Expected always takes precedence, so a comment that could match both is credited as a real find; a comment that does not hold up as an expected match can still be neutralized as ignored rather than counting as a false positive. Most entries leave `ignored_comments` empty, which scores identically to before. +## Configuring Engine Experiments + +Code Review runs the production BC-ALAgents generate path. A BC-Bench experiment branch can independently select BC-ALAgents and BCQuality sources in `src/bcbench/agent/shared/config.yaml`: + +```yaml +pr_review: + engine: + repo: microsoft/BC-ALAgents + ref: main + local_path: null + bcquality: + repo: microsoft/BCQuality + ref: main + local_path: null +``` + +`ref` accepts a branch, tag, or commit. Set either `local_path` for an unpushed local checkout; `BC_PR_REVIEW_ROOT` remains the highest-priority BC-ALAgents local override. The `run code-review` and `evaluate code-review` commands also expose `--engine-repo`, `--engine-ref`, `--engine-local-path`, and matching `--bcquality-*` options. Results record the resolved commits for both sources. + ## Baseline Leaderboard {% if site.data.code-review.aggregate and site.data.code-review.aggregate.size > 0 %} @@ -68,43 +104,112 @@ A gold entry may also declare **`ignored_comments`** — legitimate-but-optional Compares review-knowledge configurations for the same model (see the Baseline Leaderboard above for the plain agent): - **Inline knowledge (pre-#8700)** — the review checklists BCApps shipped inline before adopting BCQuality, injected as custom instructions. +- **PR-review engine** — BC-ALAgents runs against a configured BCQuality revision, with performance and context-filtering metrics captured alongside review quality. {% assign experiment_rows = site.data.code-review.aggregate | where_exp: "agg", "agg.experiment" %} {% if experiment_rows and experiment_rows.size > 0 %} - - - - - - - - - - - - - - - - {% assign experiment_results = experiment_rows | sort: "f1" | reverse %} - {% for agg in experiment_results %} - - - - - - - - - - - - {% endfor %} - -
VariantAgentModelMicro F1 (95% CI)Macro F1 (95% CI)PrecisionRecallAvg TimeVer
- {%- if agg.experiment.custom_instructions -%}Inline knowledge (pre-#8700) - {%- else -%}Other{%- endif -%} - {{ agg.agent_name }}{{ agg.model }}{{ agg.f1 | times: 100.0 | round: 1 }}%{% if agg.f1_ci_low %} ({{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.macro_f1 | times: 100.0 | round: 1 }}%{% if agg.macro_f1_ci_low %} ({{ agg.macro_f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.macro_f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.precision | times: 100.0 | round: 1 }}%{{ agg.recall | times: 100.0 | round: 1 }}%{{ agg.average_duration | round: 1 }}s{{ agg.benchmark_version }}
+{% assign experiment_results = experiment_rows | sort: "f1" | reverse %} +
+ + +
+ +
+ + + + + + + + + + + + + + + + {% for agg in experiment_results %} + + + + + + + + + + + + {% endfor %} + +
VariantEngine / BCQualityAgentModelMicro F1 (95% CI)Macro F1 (95% CI)PrecisionRecallVer
{% if agg.experiment.custom_agent == "bc-review-engine" %}PR-review engine{% elsif agg.experiment.custom_instructions %}Inline knowledge (pre-#8700){% else %}Other{% endif %} + {% if agg.experiment.custom_agent == "bc-review-engine" and agg.experiment.plugins %} + {% for plugin in agg.experiment.plugins %} + {% assign plugin_parts = plugin | split: "@" %} + {% if plugin contains "bc-review-engine@" or plugin contains "BCQuality@" %}{{ plugin_parts[0] }}@{{ plugin_parts[1] | slice: 0, 7 }}{% unless forloop.last %}
{% endunless %}{% endif %} + {% endfor %} + {% elsif agg.experiment.custom_agent == "bc-review-engine" or agg.experiment.custom_instructions %}self-contained + {% else %}—{% endif %} + 		{{ agg.agent_name }}{{ agg.model }}{{ agg.f1 | times: 100.0 | round: 1 }}%{% if agg.f1_ci_low %} ({{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.macro_f1 | times: 100.0 | round: 1 }}%{% if agg.macro_f1_ci_low %} ({{ agg.macro_f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.macro_f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.precision | times: 100.0 | round: 1 }}%{{ agg.recall | times: 100.0 | round: 1 }}%{{ agg.benchmark_version }}
+
+ + + + {% else %}

No experiment results available yet. Check back soon!

{% endif %} @@ -121,5 +226,8 @@ Compares review-knowledge configurations for the same model (see the Baseline Le - **Valid output rate** — fraction of tasks whose output parsed into a structured review. Failures score zero on every other metric. (Reported per run.) - **Micro vs. Macro** — *Micro* sums matched, scorable generated (generated minus ignored), and expected across all tasks (tasks with many comments dominate); *Macro* averages per-task scores (every task counts equally). - **95% CI** — confidence interval bootstrapped over the per-task F1 scores, so the leaderboard reports sampling uncertainty even for a single run. The micro `F1` CI resamples runs; the `Macro F1` CI resamples tasks. +- **Avg Tokens / API Calls / Estimated Credits** — mean PR-review engine usage per evaluated entry. Estimated credits use the engine's configured token prices and are not a currency value. +- **Knowledge Used** — mean number of BCQuality knowledge articles remaining after filtering and available to the reviewer. +- **Knowledge Pruned** — mean number of BCQuality knowledge articles removed by the engine's filtering step before review. [← Back to Home](index.md) diff --git a/src/bcbench/agent/copilot/pr_review/agent.py b/src/bcbench/agent/copilot/pr_review/agent.py index db1ea696c..30fa12851 100644 --- a/src/bcbench/agent/copilot/pr_review/agent.py +++ b/src/bcbench/agent/copilot/pr_review/agent.py @@ -15,17 +15,21 @@ import shutil import subprocess import time +from collections.abc import Generator +from contextlib import contextmanager from pathlib import Path from typing import Any import yaml +from bcbench.agent.copilot.pr_review.metrics import build_pr_review_metrics from bcbench.agent.copilot.pr_review.review_output import engine_report_to_review_comments, load_engine_report from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry from bcbench.dataset.codereview import CodeReviewEntry from bcbench.exceptions import AgentError, AgentTimeoutError from bcbench.logger import get_logger +from bcbench.operations.git_operations import clone_repo_at_revision, remove_tree from bcbench.types import AgentMetrics, EvaluationCategory, ExperimentConfiguration logger = get_logger(__name__) @@ -42,17 +46,50 @@ def _load_pr_review_settings() -> dict[str, Any]: return data.get("pr_review") or {} -def _resolve_pr_review_root(settings: dict[str, Any]) -> Path: - raw = os.environ.get("BC_PR_REVIEW_ROOT") or settings.get("path") - if not raw: - raise AgentError("Engine root not configured. Set 'pr_review.path' in the shared agent config.yaml or the BC_PR_REVIEW_ROOT environment variable.") +def _validate_engine_root(raw: str | Path) -> Path: root = Path(raw).expanduser() shell = root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" if not shell.exists(): - raise AgentError(f"Engine review shell not found at {shell}. Check 'pr_review.path' points at a BC-ALAgents checkout.") + raise AgentError(f"Engine review shell not found at {shell}. Check the configured BC-ALAgents source.") return root +@contextmanager +def _prepare_engine_root( + settings: dict[str, Any], + destination: Path, + engine_ref: str | None = None, + engine_repo: str | None = None, + engine_local_path: str | None = None, +) -> Generator[Path]: + engine_cfg = settings.get("engine") or {} + environment_root = os.environ.get("BC_PR_REVIEW_ROOT") + if environment_root: + yield _validate_engine_root(environment_root) + return + + if engine_local_path and (engine_repo or engine_ref): + raise AgentError("--engine-local-path cannot be combined with --engine-repo or --engine-ref.") + + cli_remote_source = engine_repo is not None or engine_ref is not None + local_path = engine_local_path if engine_local_path is not None else None if cli_remote_source else engine_cfg.get("local_path") + if local_path: + yield _validate_engine_root(local_path) + return + + repo = engine_repo or engine_cfg.get("repo") + ref = engine_ref or engine_cfg.get("ref") + if not repo or not ref: + raise AgentError("Engine source not configured. Set pr_review.engine.repo/ref, use --engine-repo/--engine-ref, or provide BC_PR_REVIEW_ROOT.") + + try: + clone_repo_at_revision(str(repo), str(ref), destination) + yield _validate_engine_root(destination) + finally: + if destination.exists(): + remove_tree(destination) + + def _resolve_engine_revision(engine_root: Path) -> str: """Resolve the engine checkout's git revision (with a dirty marker) for provenance.""" head = subprocess.run(["git", "-C", str(engine_root), "rev-parse", "HEAD"], capture_output=True, text=True, check=False) @@ -151,6 +188,25 @@ def _prepare_bcquality_root( return root, sha +def _resolve_bcquality_source( + settings: dict[str, Any], + bcquality_ref: str | None, + bcquality_repo: str | None, + bcquality_local_path: str | None, +) -> tuple[str | None, str | None, str | None]: + bcquality_cfg = settings.get("bcquality") or {} + if bcquality_local_path and (bcquality_repo or bcquality_ref): + raise AgentError("--bcquality-local-path cannot be combined with --bcquality-repo or --bcquality-ref.") + + remote_override = bcquality_repo is not None or bcquality_ref is not None + resolved_repo = bcquality_repo or bcquality_cfg.get("repo") + resolved_ref = bcquality_ref or bcquality_cfg.get("ref") + resolved_local_path = bcquality_local_path + if resolved_local_path is None and not remote_override: + resolved_local_path = bcquality_cfg.get("local_path") + return resolved_ref, resolved_repo, resolved_local_path + + def _write_review_json(output_dir: Path, repo_path: Path) -> int: agent_output = output_dir / _AGENT_OUTPUT_FILE if not agent_output.exists(): @@ -174,6 +230,9 @@ def run_pr_review_agent( bcquality_ref: str | None = None, bcquality_repo: str | None = None, bcquality_local_path: str | None = None, + engine_ref: str | None = None, + engine_repo: str | None = None, + engine_local_path: str | None = None, min_severity: str | None = None, ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run the engine's generate half on a code-review entry and write review.json. @@ -190,80 +249,88 @@ def run_pr_review_agent( raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.") if not isinstance(entry, CodeReviewEntry): raise AgentError(f"The engine agent requires a CodeReviewEntry, got {type(entry).__name__}.") - settings = _load_pr_review_settings() - engine_root = _resolve_pr_review_root(settings) pwsh = _resolve_pwsh() gh_token = _resolve_gh_token() agent_version = str(settings.get("agent_version", "0.0.0")) severity = min_severity or settings.get("min_severity") or "Low" - bcquality_cfg = settings.get("bcquality") or {} - bcquality_repo = bcquality_repo or bcquality_cfg.get("repo") - bcquality_ref = bcquality_ref or bcquality_cfg.get("ref") - bcquality_local_path = bcquality_local_path or bcquality_cfg.get("local_path") - - output_dir.mkdir(parents=True, exist_ok=True) - logger.info(f"Running BC-ALAgents review engine on: {entry.instance_id}") - - _commit_patch_as_head(repo_path) - trusted_workspace = _init_trusted_workspace(output_dir / "trusted") - bcquality_root, bcquality_sha = _prepare_bcquality_root( - engine_root, - pwsh, - output_dir / "bcquality", + bcquality_ref, bcquality_repo, bcquality_local_path = _resolve_bcquality_source( + settings, bcquality_ref, bcquality_repo, bcquality_local_path, ) - shell = engine_root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" - env = { - **os.environ, - "REVIEW_SOURCE": "local", - "BASE_REF": entry.base_commit, - "REVIEW_TARGET_WORKSPACE": str(repo_path), - "REVIEW_WORKSPACE": str(trusted_workspace), - "REVIEW_OUTPUT_DIR": str(output_dir), - "BCQUALITY_ROOT": str(bcquality_root), - "COPILOT_MODEL": model, - "COPILOT_REVIEW_AGENT_VERSION": agent_version, - "AGENT_MINIMUM_SEVERITY": severity, - "GH_TOKEN": gh_token, - } - - plugins = [f"bc-review-engine@{_resolve_engine_revision(engine_root)}"] - if bcquality_sha: - plugins.append(f"BCQuality@{bcquality_sha}") - config = ExperimentConfiguration( - custom_agent="bc-review-engine", - plugins=plugins, - ) + output_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Running BC-ALAgents review engine on: {entry.instance_id}") - start = time.monotonic() - try: - result = subprocess.run( - [pwsh, "-NoProfile", "-File", str(shell), "-GenerateOnly", "-OutputDir", str(output_dir)], - cwd=str(repo_path), - env=env, - capture_output=True, - text=True, - timeout=_config.timeout.agent_execution, - check=True, + with _prepare_engine_root( + settings, + output_dir / "engine", + engine_ref=engine_ref, + engine_repo=engine_repo, + engine_local_path=engine_local_path, + ) as engine_root: + _commit_patch_as_head(repo_path) + trusted_workspace = _init_trusted_workspace(output_dir / "trusted") + bcquality_root, bcquality_sha = _prepare_bcquality_root( + engine_root, + pwsh, + output_dir / "bcquality", + bcquality_ref, + bcquality_repo, + bcquality_local_path, ) - logger.debug(f"Engine stdout:\n{result.stdout}") - if result.stderr: - logger.debug(f"Engine stderr:\n{result.stderr}") - count = _write_review_json(output_dir, repo_path) - logger.info(f"Engine review complete for {entry.instance_id}: wrote {count} comment(s) to {_REVIEW_OUTPUT_FILE}") - except subprocess.TimeoutExpired: - logger.exception(f"Engine review timed out after {_config.timeout.agent_execution} seconds") - metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) - raise AgentTimeoutError("Engine review timed out", metrics=metrics, config=config) from None - except subprocess.CalledProcessError as e: - logger.exception(f"Engine review failed (exit {e.returncode}):\n{e.stdout}\n{e.stderr}") - raise AgentError(f"Engine review execution failed: {e}") from None - except Exception: - logger.exception("Unexpected error running engine review") - raise - else: - return AgentMetrics(execution_time=time.monotonic() - start), config + + shell = engine_root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" + env = { + **os.environ, + "REVIEW_SOURCE": "local", + "BASE_REF": entry.base_commit, + "REVIEW_TARGET_WORKSPACE": str(repo_path), + "REVIEW_WORKSPACE": str(trusted_workspace), + "REVIEW_OUTPUT_DIR": str(output_dir), + "BCQUALITY_ROOT": str(bcquality_root), + "COPILOT_MODEL": model, + "COPILOT_REVIEW_AGENT_VERSION": agent_version, + "COPILOT_REVIEW_LOG_LEVEL": "debug", + "AGENT_MINIMUM_SEVERITY": severity, + "GH_TOKEN": gh_token, + } + + plugins = [f"bc-review-engine@{_resolve_engine_revision(engine_root)}"] + if bcquality_sha: + plugins.append(f"BCQuality@{bcquality_sha}") + config = ExperimentConfiguration( + custom_agent="bc-review-engine", + plugins=plugins, + ) + + start = time.monotonic() + try: + result = subprocess.run( + [pwsh, "-NoProfile", "-File", str(shell), "-GenerateOnly", "-OutputDir", str(output_dir)], + cwd=str(repo_path), + env=env, + capture_output=True, + text=True, + timeout=_config.timeout.agent_execution, + check=True, + ) + logger.debug(f"Engine stdout:\n{result.stdout}") + if result.stderr: + logger.debug(f"Engine stderr:\n{result.stderr}") + count = _write_review_json(output_dir, repo_path) + logger.info(f"Engine review complete for {entry.instance_id}: wrote {count} comment(s) to {_REVIEW_OUTPUT_FILE}") + except subprocess.TimeoutExpired: + logger.exception(f"Engine review timed out after {_config.timeout.agent_execution} seconds") + metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) + raise AgentTimeoutError("Engine review timed out", metrics=metrics, config=config) from None + except subprocess.CalledProcessError as e: + logger.exception(f"Engine review failed (exit {e.returncode}):\n{e.stdout}\n{e.stderr}") + raise AgentError(f"Engine review execution failed: {e}") from None + except Exception: + logger.exception("Unexpected error running engine review") + raise + else: + return build_pr_review_metrics(output_dir, bcquality_root, time.monotonic() - start), config diff --git a/src/bcbench/agent/copilot/pr_review/metrics.py b/src/bcbench/agent/copilot/pr_review/metrics.py new file mode 100644 index 000000000..9286d0509 --- /dev/null +++ b/src/bcbench/agent/copilot/pr_review/metrics.py @@ -0,0 +1,164 @@ +import json +import re +from pathlib import Path +from typing import Any + +from bcbench.agent.copilot.metrics import parse_metrics +from bcbench.logger import get_logger +from bcbench.types import AgentMetrics + +logger = get_logger(__name__) + +RUN_METRICS_FILE_NAME = "_run-metrics.json" +FILTER_REPORT_FILE_NAME = "_filter-report.json" +TRANSCRIPT_FILE_NAME = "agent-transcript.log" +METRIC_NUMBER_PATTERN = r"[0-9][0-9,]*(?:\.[0-9]+)?[kKmM]?" +AI_CREDITS_PATTERN = re.compile(rf"(?m)^(?:err:\s*)?AI Credits\s+({METRIC_NUMBER_PATTERN})") +PREMIUM_REQUESTS_PATTERN = re.compile(rf"(?:Requests\s+|Total usage est:\s*)({METRIC_NUMBER_PATTERN})\s+Premium", re.IGNORECASE) +TOKENS_PATTERN = re.compile( + rf"(?m)^(?:err:\s*)?Tokens\s+↑\s*({METRIC_NUMBER_PATTERN})" + rf"(?:\s+\(({METRIC_NUMBER_PATTERN})\s+cached(?:,\s*{METRIC_NUMBER_PATTERN}\s+written)?\))?" + rf"\s+•\s+↓\s*({METRIC_NUMBER_PATTERN})" + rf"(?:\s+\(({METRIC_NUMBER_PATTERN})\s+reasoning\))?" +) + + +def _load_json(path: Path) -> dict[str, Any] | None: + if not path.exists(): + logger.debug(f"Engine perf file not found: {path}") + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + logger.warning(f"Could not read engine perf file {path}: {exc}") + return None + if not isinstance(payload, dict): + logger.warning(f"Engine perf file {path} is not a JSON object; ignoring") + return None + return payload + + +def _as_int(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _as_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _parse_compact_number(value: str) -> float: + normalized = value.replace(",", "").lower() + multiplier = 1.0 + if normalized.endswith("k"): + normalized = normalized[:-1] + multiplier = 1_000.0 + elif normalized.endswith("m"): + normalized = normalized[:-1] + multiplier = 1_000_000.0 + return float(normalized) * multiplier + + +def parse_run_metrics(path: Path) -> dict[str, Any]: + payload = _load_json(path) + if payload is None: + return {} + + result: dict[str, Any] = {} + for source_key, target_key, coerce in ( + ("prompt_tokens", "prompt_tokens", _as_int), + ("completion_tokens", "completion_tokens", _as_int), + ("total_tokens", "total_tokens", _as_int), + ("api_calls", "api_calls", _as_int), + ("estimated_credits", "estimated_credits", _as_float), + ("wall_time_seconds", "wall_time_seconds", _as_float), + ): + value = coerce(payload.get(source_key)) + if value is not None: + result[target_key] = value + + if "total_tokens" not in result and "prompt_tokens" in result and "completion_tokens" in result: + result["total_tokens"] = int(result["prompt_tokens"]) + int(result["completion_tokens"]) + return result + + +def parse_transcript_metrics(path: Path) -> dict[str, int | float]: + if not path.exists(): + logger.debug(f"Engine transcript not found: {path}") + return {} + try: + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + except OSError as exc: + logger.warning(f"Could not read engine transcript {path}: {exc}") + return {} + + parsed = parse_metrics(lines, session_log_path=path) + result: dict[str, int | float] = {} + if parsed: + if parsed.prompt_tokens is not None: + result["prompt_tokens"] = parsed.prompt_tokens + if parsed.completion_tokens is not None: + result["completion_tokens"] = parsed.completion_tokens + if parsed.turn_count is not None: + result["api_calls"] = parsed.turn_count + + transcript = "".join(lines) + token_matches = list(TOKENS_PATTERN.finditer(transcript)) + if token_matches: + token_match = token_matches[-1] + result["prompt_tokens"] = int(_parse_compact_number(token_match.group(1))) + result["completion_tokens"] = int(_parse_compact_number(token_match.group(3))) + + credit_matches = list(AI_CREDITS_PATTERN.finditer(transcript)) + if not credit_matches: + credit_matches = list(PREMIUM_REQUESTS_PATTERN.finditer(transcript)) + if credit_matches: + result["estimated_credits"] = _parse_compact_number(credit_matches[-1].group(1)) + if "prompt_tokens" in result and "completion_tokens" in result: + result["total_tokens"] = int(result["prompt_tokens"]) + int(result["completion_tokens"]) + return result + + +def _count_filtered_knowledge(bcquality_root: Path) -> int: + return sum(1 for path in bcquality_root.rglob("*.md") if path.is_file() and "knowledge" in {part.lower() for part in path.relative_to(bcquality_root).parts[:-1]}) + + +def parse_filter_report(path: Path, bcquality_root: Path) -> dict[str, int]: + payload = _load_json(path) + if payload is None: + return {} + removed = payload.get("removed") + if not isinstance(removed, list): + return {} + return { + "knowledge_pruned": sum(1 for item in removed if isinstance(item, dict) and item.get("kind") == "knowledge"), + "knowledge_used": _count_filtered_knowledge(bcquality_root), + } + + +def build_pr_review_metrics(output_dir: Path, bcquality_root: Path, execution_time: float) -> AgentMetrics: + transcript = parse_transcript_metrics(output_dir / TRANSCRIPT_FILE_NAME) + run = {**transcript, **parse_run_metrics(output_dir / RUN_METRICS_FILE_NAME)} + filter_report = output_dir / FILTER_REPORT_FILE_NAME + if not filter_report.exists(): + filter_report = bcquality_root / FILTER_REPORT_FILE_NAME + knowledge = parse_filter_report(filter_report, bcquality_root) + return AgentMetrics( + execution_time=execution_time, + prompt_tokens=_as_int(run.get("prompt_tokens")), + completion_tokens=_as_int(run.get("completion_tokens")), + total_tokens=_as_int(run.get("total_tokens")), + api_calls=_as_int(run.get("api_calls")), + estimated_credits=_as_float(run.get("estimated_credits")), + knowledge_used=knowledge.get("knowledge_used"), + knowledge_pruned=knowledge.get("knowledge_pruned"), + ) diff --git a/src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 b/src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 index 7b224453f..5c3711169 100644 --- a/src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 +++ b/src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 @@ -67,13 +67,20 @@ else { Write-Host "Fetching BCQuality from $repo@$ref into $Root" if (Test-Path -LiteralPath $Root) { Remove-Item -LiteralPath $Root -Recurse -Force } - New-Item -ItemType Directory -Force -Path $Root | Out-Null - git -C $Root init -q - git -C $Root remote add origin $repo - git -C $Root fetch --depth=1 origin "$ref" - if ($LASTEXITCODE -ne 0) { throw "git fetch of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } - git -C $Root checkout -q FETCH_HEAD - if ($LASTEXITCODE -ne 0) { throw "git checkout of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Root) | Out-Null + if ($repo -match '^[^/\\]+/[^/\\]+$') { + gh repo clone $repo $Root -- --depth=1 "--revision=$ref" --quiet + if ($LASTEXITCODE -ne 0) { throw "gh clone of BCQuality ref '$repo@$ref' failed (exit $LASTEXITCODE)" } + } + else { + New-Item -ItemType Directory -Force -Path $Root | Out-Null + git -C $Root init -q + git -C $Root remote add origin $repo + git -C $Root fetch --depth=1 origin "$ref" + if ($LASTEXITCODE -ne 0) { throw "git fetch of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } + git -C $Root checkout -q FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw "git checkout of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } + } $resolvedSha = (& git -C $Root rev-parse HEAD).Trim() Write-Host "BCQuality resolved SHA: $resolvedSha" diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index b6ebb72c9..8b362841e 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -170,16 +170,18 @@ mcp: # generate half (Invoke-PRReviewShell.ps1 -GenerateOnly) in local mode instead of a # bespoke review prompt, so it measures the real PROD engine + BCQuality. These knobs # live here in the shared config so they are easy to tweak on a private branch. -# path: local microsoft/BC-ALAgents checkout, e.g. "C:/depot/BC-ALAgents". Left -# null in the repo; CI sets BC_PR_REVIEW_ROOT (which takes precedence). -# For a local run, set that env var or put your checkout path here on your -# own branch (do not commit a machine-specific path). +# engine: BC-ALAgents source. repo/ref are fetched for reproducible CI runs. +# Set local_path for local development without pushing. BC_PR_REVIEW_ROOT +# remains an environment override and takes precedence over these values. # bcquality: content source for the engine run; all optional (null = engine's pinned # repo/ref). Set local_path to iterate on a local BCQuality checkout without # pushing (it is copied and filtered; the original is never modified). -# CLI flags (--bcquality-repo/-ref/-local-path) override these. +# CLI flags for either source override these values. pr_review: - path: null + engine: + repo: microsoft/BC-ALAgents + ref: main + local_path: null bcquality: repo: null ref: null diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 4af96b754..9f6a51fab 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -48,6 +48,9 @@ def _run_pr_review_evaluation( repo_path: Path, output_dir: Path, run_id: str, + engine_ref: str | None = None, + engine_repo: str | None = None, + engine_local_path: str | None = None, bcquality_ref: str | None = None, bcquality_repo: str | None = None, bcquality_local_path: str | None = None, @@ -84,6 +87,9 @@ def _run_pr_review_evaluation( category=category, model=ctx.model, output_dir=ctx.result_dir, + engine_ref=engine_ref, + engine_repo=engine_repo, + engine_local_path=engine_local_path, bcquality_ref=bcquality_ref, bcquality_repo=bcquality_repo, bcquality_local_path=bcquality_local_path, @@ -214,8 +220,11 @@ def evaluate_code_review( repo_path: RepoPath = _config.paths.testbed_path, output_dir: OutputDir = _config.paths.evaluation_results_path, run_id: RunId = "pr_review_test_run", + engine_ref: Annotated[str | None, typer.Option(help="Override the BC-ALAgents ref (defaults to pr_review.engine.ref)")] = None, + engine_repo: Annotated[str | None, typer.Option(help="Override the BC-ALAgents repo (defaults to pr_review.engine.repo)")] = None, + engine_local_path: Annotated[str | None, typer.Option(help="Use a local BC-ALAgents checkout instead of fetching")] = None, bcquality_ref: Annotated[str | None, typer.Option(help="Override the BCQuality ref (defaults to the engine's pinned ref)")] = None, - bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to config/engine)")] = None, + bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to pr_review.bcquality.repo or the engine pin)")] = None, bcquality_local_path: Annotated[str | None, typer.Option(help="Use a local BCQuality checkout (copied + filtered, never modified) instead of fetching")] = None, min_severity: Annotated[str | None, typer.Option(help="AGENT_MINIMUM_SEVERITY floor (defaults to config)")] = None, ) -> None: @@ -224,8 +233,8 @@ def evaluate_code_review( code-review is not a general harness choice: it always runs the engine's own generate shell in local mode - the real PROD generate path - then scores the resulting - review.json with the standard code-review judge. Requires a local BC-ALAgents checkout - (pr_review.path in config.yaml or BC_PR_REVIEW_ROOT), PowerShell 7+, and GH_TOKEN. + review.json with the standard code-review judge. BC-ALAgents and BCQuality sources can + be configured in config.yaml or overridden with command options. To only generate review.json without scoring, use 'bcbench run code-review' instead. """ @@ -235,6 +244,9 @@ def evaluate_code_review( repo_path=repo_path, output_dir=output_dir, run_id=run_id, + engine_ref=engine_ref, + engine_repo=engine_repo, + engine_local_path=engine_local_path, bcquality_ref=bcquality_ref, bcquality_repo=bcquality_repo, bcquality_local_path=bcquality_local_path, diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index eaf3b8a92..959c477cf 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -34,6 +34,9 @@ def _run_pr_review( model: str, repo_path: Path, output_dir: Path, + engine_ref: str | None = None, + engine_repo: str | None = None, + engine_local_path: str | None = None, bcquality_ref: str | None = None, bcquality_repo: str | None = None, bcquality_local_path: str | None = None, @@ -55,6 +58,9 @@ def _run_pr_review( model=model, category=category, output_dir=output_dir, + engine_ref=engine_ref, + engine_repo=engine_repo, + engine_local_path=engine_local_path, bcquality_ref=bcquality_ref, bcquality_repo=bcquality_repo, bcquality_local_path=bcquality_local_path, @@ -138,8 +144,11 @@ def run_code_review( model: CopilotModel = "claude-sonnet-5", repo_path: RepoPath = _config.paths.testbed_path, output_dir: OutputDir = _config.paths.evaluation_results_path, + engine_ref: Annotated[str | None, typer.Option(help="Override the BC-ALAgents ref (defaults to pr_review.engine.ref)")] = None, + engine_repo: Annotated[str | None, typer.Option(help="Override the BC-ALAgents repo (defaults to pr_review.engine.repo)")] = None, + engine_local_path: Annotated[str | None, typer.Option(help="Use a local BC-ALAgents checkout instead of fetching")] = None, bcquality_ref: Annotated[str | None, typer.Option(help="Override the BCQuality ref (defaults to the engine's pinned ref)")] = None, - bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to config/engine)")] = None, + bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to pr_review.bcquality.repo or the engine pin)")] = None, bcquality_local_path: Annotated[str | None, typer.Option(help="Use a local BCQuality checkout (copied + filtered, never modified) instead of fetching")] = None, min_severity: Annotated[str | None, typer.Option(help="AGENT_MINIMUM_SEVERITY floor (defaults to config)")] = None, ) -> None: @@ -149,8 +158,8 @@ def run_code_review( code-review is not a general harness choice: it always runs the engine's real generate half (never a bespoke prompt), so it has its own command instead of a copilot/claude sub-command. Writes review.json in the repo root without scoring; for full evaluation - use 'bcbench evaluate code-review'. Requires a local BC-ALAgents checkout - (pr_review.path in config.yaml or BC_PR_REVIEW_ROOT), PowerShell 7+, and GH_TOKEN. + use 'bcbench evaluate code-review'. BC-ALAgents and BCQuality sources can be + configured in config.yaml or overridden with command options. Example: uv run bcbench run code-review synthetic__style-018 --repo-path /path/to/testbed @@ -160,6 +169,9 @@ def run_code_review( model=model, repo_path=repo_path, output_dir=output_dir, + engine_ref=engine_ref, + engine_repo=engine_repo, + engine_local_path=engine_local_path, bcquality_ref=bcquality_ref, bcquality_repo=bcquality_repo, bcquality_local_path=bcquality_local_path, diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 15eda4c3f..0204f8924 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -11,6 +11,14 @@ logger = get_logger(__name__) +_CODE_REVIEW_METRIC_FIELDS = ( + "total_tokens", + "api_calls", + "estimated_credits", + "knowledge_used", + "knowledge_pruned", +) + class BaseEvaluationResult(BaseModel): """Base class for all evaluation results with shared metrics across categories.""" @@ -55,6 +63,9 @@ def save(self, output_dir: Path, result_file: str) -> None: output_dir.mkdir(parents=True, exist_ok=True) with output_file.open("a", encoding="utf-8") as f: result_dict = self.model_dump(mode="json") + if self.category is not EvaluationCategory.CODE_REVIEW and result_dict["metrics"]: + for field in _CODE_REVIEW_METRIC_FIELDS: + result_dict["metrics"].pop(field, None) # Per-instance JSONL result files are uploaded as workflow artifacts and are the only inputs required by the summarize-results workflow. f.write(json.dumps(result_dict) + "\n") diff --git a/src/bcbench/results/codereview.py b/src/bcbench/results/codereview.py index fbf21debf..0ebf372d5 100644 --- a/src/bcbench/results/codereview.py +++ b/src/bcbench/results/codereview.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import NamedTuple, Self +from typing import Any, NamedTuple, Self import numpy as np from pydantic import Field @@ -310,10 +310,42 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary): severity_mae: float = 0.0 valid_review_output_rate: float = Field(default=0.0, ge=0.0, le=1.0) + average_total_tokens: float | None = None + average_api_calls: float | None = None + average_estimated_credits: float | None = None + average_knowledge_used: float | None = None + average_knowledge_pruned: float | None = None + # Per-task F1 keyed by instance_id, retained so the leaderboard can bootstrap a confidence # interval over tasks (meaningful even for a single run) instead of only over runs. instance_results: dict[str, float] = Field(default_factory=dict) + def _perf_markdown(self) -> str: + if all( + value is None + for value in ( + self.average_total_tokens, + self.average_api_calls, + self.average_estimated_credits, + self.average_knowledge_used, + self.average_knowledge_pruned, + ) + ): + return "" + tokens = f"{self.average_total_tokens:.0f}" if self.average_total_tokens is not None else "n/a" + api_calls = f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a" + credits = f"{self.average_estimated_credits:.4f}" if self.average_estimated_credits is not None else "n/a" + used = f"{self.average_knowledge_used:.1f}" if self.average_knowledge_used is not None else "n/a" + pruned = f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a" + return ( + "## Performance\n" + "\n" + "| Avg duration (s) | Avg total tokens | Avg API calls | Avg est. credits | Avg knowledge used | Avg knowledge pruned |\n" + "|-----------------:|-----------------:|--------------:|-----------------:|-------------------:|---------------------:|\n" + f"| {self.average_duration:.1f} | {tokens} | {api_calls} | {credits} | {used} | {pruned} |\n" + "\n" + ) + def render_github_metrics_markdown(self) -> str: micro_p = self.precision * 100 micro_r = self.recall * 100 @@ -351,9 +383,35 @@ def render_github_metrics_markdown(self) -> str: "|-------------:|-------------------------:|\n" f"| {self.severity_mae:.3f} | {valid_rate:.1f}% |\n" "\n" + f"{self._perf_markdown()}" f"{_METRIC_EXPLANATIONS}" ) + def _perf_console_tables(self) -> list[RenderableType]: + if all( + value is None + for value in ( + self.average_total_tokens, + self.average_api_calls, + self.average_estimated_credits, + self.average_knowledge_used, + self.average_knowledge_pruned, + ) + ): + return [] + tokens = f"{self.average_total_tokens:.0f}" if self.average_total_tokens is not None else "n/a" + api_calls = f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a" + credits = f"{self.average_estimated_credits:.4f}" if self.average_estimated_credits is not None else "n/a" + used = f"{self.average_knowledge_used:.1f}" if self.average_knowledge_used is not None else "n/a" + pruned = f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a" + return [ + _build_console_table( + "Performance", + ["Avg duration (s)", "Avg total tokens", "Avg API calls", "Avg est. credits", "Avg knowledge used", "Avg knowledge pruned"], + [f"{self.average_duration:.1f}", tokens, api_calls, credits, used, pruned], + ) + ] + def render_console_metrics(self) -> RenderableType: metric_columns = ["Precision", "Recall", "F1", "Fβ (β=0.5)", "Fβ (β=2)"] @@ -397,6 +455,7 @@ def render_console_metrics(self) -> RenderableType: ["Severity MAE", "Valid review output rate"], [f"{self.severity_mae:.3f}", f"{self.valid_review_output_rate * 100:.1f}%"], ), + *self._perf_console_tables(), Panel( _CONSOLE_METRIC_EXPLANATIONS, title="📖 How to read these metrics", @@ -453,6 +512,10 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " valid_output_count: int = sum(1 for r in code_review_results if r.valid_review_output) valid_output_rate: float = valid_output_count / total_results + def average_metric(name: str) -> float | None: + values = [value for result in code_review_results if result.metrics and (value := getattr(result.metrics, name)) is not None] + return sum(values) / len(values) if values else None + return summary.model_copy( update={ "generated_comment_count": generated_total, @@ -474,5 +537,23 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " "severity_mae": round(severity_mae, 3), "valid_review_output_rate": round(valid_output_rate, 3), "instance_results": {r.instance_id: round(r.f1, 6) for r in code_review_results}, + "average_total_tokens": average_metric("total_tokens"), + "average_api_calls": average_metric("api_calls"), + "average_estimated_credits": average_metric("estimated_credits"), + "average_knowledge_used": average_metric("knowledge_used"), + "average_knowledge_pruned": average_metric("knowledge_pruned"), } ) + + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + for key, digits in ( + ("average_total_tokens", 1), + ("average_api_calls", 2), + ("average_estimated_credits", 4), + ("average_knowledge_used", 2), + ("average_knowledge_pruned", 2), + ): + if data[key] is not None: + data[key] = round(float(data[key]), digits) + return data diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index 146476b0f..8a59e3437 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -147,6 +147,12 @@ class CodeReviewLeaderboardAggregate(JudgeBasedLeaderboardAggregate): macro_precision: float = 0.0 macro_recall: float = 0.0 + average_total_tokens: float | None = None + average_api_calls: float | None = None + average_estimated_credits: float | None = None + average_knowledge_used: float | None = None + average_knowledge_pruned: float | None = None + @classmethod def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeaderboardAggregate": from bcbench.results.codereview import CodeReviewResultSummary @@ -157,6 +163,10 @@ def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeader cr_runs: list[CodeReviewResultSummary] = [run for run in runs if isinstance(run, CodeReviewResultSummary)] n = len(cr_runs) + def mean_metric(name: str) -> float | None: + values = [value for run in cr_runs if (value := getattr(run, name)) is not None] + return sum(values) / len(values) if values else None + # The micro headline pools every comment across the dataset, so there is no per-task # decomposition to resample; its CI is intentionally over run-level means and captures # run-to-run reproducibility (None unless >=2 runs with variance). @@ -184,6 +194,11 @@ def from_runs(cls, runs: Sequence[EvaluationResultSummary]) -> "CodeReviewLeader "macro_f_beta_2": sum(r.macro_f_beta_2 for r in cr_runs) / n, "macro_precision": sum(r.macro_precision for r in cr_runs) / n, "macro_recall": sum(r.macro_recall for r in cr_runs) / n, + "average_total_tokens": mean_metric("average_total_tokens"), + "average_api_calls": mean_metric("average_api_calls"), + "average_estimated_credits": mean_metric("average_estimated_credits"), + "average_knowledge_used": mean_metric("average_knowledge_used"), + "average_knowledge_pruned": mean_metric("average_knowledge_pruned"), } ) diff --git a/src/bcbench/types.py b/src/bcbench/types.py index add22630d..654472f81 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -76,6 +76,12 @@ class AgentMetrics(BaseModel): prompt_tokens: int | None = None completion_tokens: int | None = None + total_tokens: int | None = None + api_calls: int | None = None + estimated_credits: float | None = None + knowledge_used: int | None = None + knowledge_pruned: int | None = None + # Tool usage statistics from agent logs tool_usage: dict[str, int] | None = None @@ -194,8 +200,19 @@ def expected_metrics(self) -> frozenset[str]: completion_tokens=None, tool_usage=None, ) - case AgentHarness.BCAL | AgentHarness.PR_REVIEW: + case AgentHarness.BCAL: expected = AgentMetrics(execution_time=None) + case AgentHarness.PR_REVIEW: + expected = AgentMetrics( + execution_time=None, + prompt_tokens=None, + completion_tokens=None, + total_tokens=None, + api_calls=None, + estimated_credits=None, + knowledge_used=None, + knowledge_pruned=None, + ) case _: raise ValueError(f"Unknown AgentHarness: {self}") diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index 33e0f30e2..d1e193a92 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -3,7 +3,7 @@ import pytest -from bcbench.agent.copilot.pr_review.agent import _write_review_json +from bcbench.agent.copilot.pr_review.agent import _prepare_engine_root, _resolve_bcquality_source, _write_review_json from bcbench.exceptions import AgentError @@ -19,6 +19,119 @@ def _write_output(output_dir: Path, text: str) -> None: (output_dir / "agent-output.txt").write_text(text, encoding="utf-8") +def _write_engine_shell(root: Path) -> None: + shell = root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" + shell.parent.mkdir(parents=True) + shell.write_text("", encoding="utf-8") + + +def test_prepare_engine_root_uses_configured_local_path(tmp_path: Path) -> None: + engine = tmp_path / "local-engine" + _write_engine_shell(engine) + + with _prepare_engine_root( + {"engine": {"repo": "microsoft/BC-ALAgents", "ref": "main", "local_path": str(engine)}}, + tmp_path / "clone", + ) as resolved: + assert resolved == engine + + +def test_prepare_engine_root_environment_override_takes_precedence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + configured = tmp_path / "configured" + override = tmp_path / "override" + _write_engine_shell(configured) + _write_engine_shell(override) + monkeypatch.setenv("BC_PR_REVIEW_ROOT", str(override)) + + with _prepare_engine_root( + {"engine": {"local_path": str(configured)}}, + tmp_path / "clone", + engine_local_path=str(configured), + ) as resolved: + assert resolved == override + + +def test_prepare_engine_root_clones_configured_ref_and_cleans_up(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + destination = tmp_path / "engine" + configured_local = tmp_path / "configured-local" + clone_args: list[object] = [] + _write_engine_shell(configured_local) + + def fake_clone(repo: str, revision: str, target: Path) -> None: + clone_args.extend([repo, revision, target]) + _write_engine_shell(target) + + monkeypatch.delenv("BC_PR_REVIEW_ROOT", raising=False) + monkeypatch.setattr("bcbench.agent.copilot.pr_review.agent.clone_repo_at_revision", fake_clone) + + with _prepare_engine_root( + {"engine": {"repo": "contoso/BC-ALAgents", "ref": "feature/review", "local_path": str(configured_local)}}, + destination, + engine_repo="fabrikam/BC-ALAgents", + engine_ref="experiment/engine", + ) as resolved: + assert resolved == destination + assert destination.exists() + + assert clone_args == ["fabrikam/BC-ALAgents", "experiment/engine", destination] + assert not destination.exists() + + +def test_prepare_engine_root_cleans_up_failed_clone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + destination = tmp_path / "engine" + + def failed_clone(repo: str, revision: str, target: Path) -> None: + target.mkdir(parents=True) + (target / "partial").write_text("", encoding="utf-8") + raise RuntimeError(f"Could not clone {repo}@{revision}") + + monkeypatch.delenv("BC_PR_REVIEW_ROOT", raising=False) + monkeypatch.setattr("bcbench.agent.copilot.pr_review.agent.clone_repo_at_revision", failed_clone) + + with ( + pytest.raises(RuntimeError, match="Could not clone"), + _prepare_engine_root( + {"engine": {"repo": "contoso/BC-ALAgents", "ref": "feature/review"}}, + destination, + ), + ): + pass + + assert not destination.exists() + + +def test_remote_bcquality_override_ignores_configured_local_path() -> None: + resolved = _resolve_bcquality_source( + {"bcquality": {"repo": "microsoft/BCQuality", "ref": "main", "local_path": "C:/local/BCQuality"}}, + bcquality_ref="feature/knowledge", + bcquality_repo=None, + bcquality_local_path=None, + ) + + assert resolved == ("feature/knowledge", "microsoft/BCQuality", None) + + +def test_local_bcquality_override_uses_configured_remote_defaults() -> None: + resolved = _resolve_bcquality_source( + {"bcquality": {"repo": "microsoft/BCQuality", "ref": "main", "local_path": None}}, + bcquality_ref=None, + bcquality_repo=None, + bcquality_local_path="C:/local/BCQuality", + ) + + assert resolved == ("main", "microsoft/BCQuality", "C:/local/BCQuality") + + +def test_conflicting_bcquality_cli_sources_raise() -> None: + with pytest.raises(AgentError, match="cannot be combined"): + _resolve_bcquality_source( + {"bcquality": {}}, + bcquality_ref="feature/knowledge", + bcquality_repo=None, + bcquality_local_path="C:/local/BCQuality", + ) + + def test_valid_empty_findings_is_a_clean_review(tmp_path: Path) -> None: out, repo = _dirs(tmp_path) _write_output(out, json.dumps({"outcome": "completed", "outcome-reason": "", "findings": []})) diff --git a/tests/test_pr_review_metrics.py b/tests/test_pr_review_metrics.py new file mode 100644 index 000000000..6e4c9e82f --- /dev/null +++ b/tests/test_pr_review_metrics.py @@ -0,0 +1,110 @@ +import json +from pathlib import Path + +from bcbench.agent.copilot.pr_review.metrics import ( + FILTER_REPORT_FILE_NAME, + RUN_METRICS_FILE_NAME, + TRANSCRIPT_FILE_NAME, + build_pr_review_metrics, + parse_filter_report, + parse_run_metrics, + parse_transcript_metrics, +) + + +def _write(path: Path, payload: object) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def test_parse_run_metrics(tmp_path: Path) -> None: + path = tmp_path / RUN_METRICS_FILE_NAME + _write( + path, + { + "prompt_tokens": 1200, + "completion_tokens": 300, + "api_calls": 7, + "estimated_credits": 0.33, + }, + ) + + metrics = parse_run_metrics(path) + + assert metrics == { + "prompt_tokens": 1200, + "completion_tokens": 300, + "total_tokens": 1500, + "api_calls": 7, + "estimated_credits": 0.33, + } + + +def test_parse_transcript_metrics_from_engine_artifact(tmp_path: Path) -> None: + path = tmp_path / TRANSCRIPT_FILE_NAME + path.write_text( + """err: --- Start of group: Sending request to the AI model --- +err: --- Start of group: Sending request to the AI model --- +err: AI Credits 138 (1m 20s) +err: Tokens ↑ 1,234,567 (1,000,000 cached) • ↓ 86,543 (500 reasoning)""", + encoding="utf-8", + ) + + metrics = parse_transcript_metrics(path) + + assert metrics == { + "prompt_tokens": 1234567, + "completion_tokens": 86543, + "total_tokens": 1321110, + "api_calls": 2, + "estimated_credits": 138, + } + + +def test_parse_filter_report_counts_used_and_pruned_knowledge(tmp_path: Path) -> None: + knowledge = tmp_path / "content" / "knowledge" + knowledge.mkdir(parents=True) + (knowledge / "one.md").write_text("# One", encoding="utf-8") + (knowledge / "two.md").write_text("# Two", encoding="utf-8") + report = tmp_path / FILTER_REPORT_FILE_NAME + _write(report, {"removed": [{"kind": "knowledge"}, {"kind": "skill"}, {"kind": "knowledge"}]}) + + metrics = parse_filter_report(report, tmp_path) + + assert metrics == {"knowledge_pruned": 2, "knowledge_used": 2} + + +def test_build_metrics_degrades_when_side_files_are_missing(tmp_path: Path) -> None: + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=12.5) + + assert metrics.execution_time == 12.5 + assert metrics.total_tokens is None + assert metrics.knowledge_used is None + assert metrics.knowledge_pruned is None + + +def test_build_metrics_combines_engine_and_knowledge_signals(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + bcquality = tmp_path / "bcquality" + knowledge = bcquality / "knowledge" + knowledge.mkdir(parents=True) + (knowledge / "used.md").write_text("# Used", encoding="utf-8") + _write( + output / RUN_METRICS_FILE_NAME, + { + "prompt_tokens": 1000, + "completion_tokens": 200, + "total_tokens": 1200, + "api_calls": 5, + "estimated_credits": 0.25, + }, + ) + _write(bcquality / FILTER_REPORT_FILE_NAME, {"removed": [{"kind": "knowledge"}]}) + + metrics = build_pr_review_metrics(output, bcquality, execution_time=8.0) + + assert metrics.total_tokens == 1200 + assert metrics.api_calls == 5 + assert metrics.estimated_credits == 0.25 + assert metrics.knowledge_used == 1 + assert metrics.knowledge_pruned == 1 diff --git a/tests/test_pr_review_metrics_reporting.py b/tests/test_pr_review_metrics_reporting.py new file mode 100644 index 000000000..aa155effb --- /dev/null +++ b/tests/test_pr_review_metrics_reporting.py @@ -0,0 +1,95 @@ +import json +from pathlib import Path + +from bcbench.results.codereview import CodeReviewResultSummary +from bcbench.results.leaderboard import CodeReviewLeaderboardAggregate, ExecutionBasedLeaderboardAggregate +from bcbench.results.summary import ExecutionBasedEvaluationResultSummary +from bcbench.types import AgentMetrics +from tests.conftest import create_bugfix_result, create_codereview_result + + +def _metrics(tokens: int, calls: int, credits: float, used: int, pruned: int) -> AgentMetrics: + return AgentMetrics( + execution_time=4.0, + prompt_tokens=tokens - 100, + completion_tokens=100, + total_tokens=tokens, + api_calls=calls, + estimated_credits=credits, + knowledge_used=used, + knowledge_pruned=pruned, + ) + + +def test_summary_aggregates_pr_review_metrics() -> None: + summary = CodeReviewResultSummary.from_results( + [ + create_codereview_result(instance_id="proj__review-1", metrics=_metrics(1000, 10, 0.5, 20, 4)), + create_codereview_result(instance_id="proj__review-2", metrics=_metrics(2000, 20, 1.5, 30, 8)), + ], + run_id="run", + ) + + assert summary.average_total_tokens == 1500 + assert summary.average_api_calls == 15 + assert summary.average_estimated_credits == 1 + assert summary.average_knowledge_used == 25 + assert summary.average_knowledge_pruned == 6 + + +def test_leaderboard_propagates_pr_review_metrics() -> None: + first = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(1000, 10, 0.5, 20, 4))], + run_id="one", + ) + second = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(2000, 20, 1.5, 30, 8))], + run_id="two", + ) + + aggregate = CodeReviewLeaderboardAggregate.from_runs([first, second]) + + assert aggregate.average_total_tokens == 1500 + assert aggregate.average_knowledge_used == 25 + assert aggregate.average_knowledge_pruned == 6 + + +def test_github_summary_renders_performance_metrics() -> None: + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(1500, 12, 0.75, 24, 5))], + run_id="run", + ) + + markdown = summary.render_github_metrics_markdown() + + assert "## Performance" in markdown + assert "Avg knowledge used" in markdown + assert "0.7500" in markdown + + +def test_github_summary_renders_knowledge_only_metrics() -> None: + metrics = AgentMetrics(execution_time=4.0, knowledge_used=24, knowledge_pruned=5) + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=metrics)], + run_id="run", + ) + + markdown = summary.render_github_metrics_markdown() + + assert "## Performance" in markdown + assert "| 4.0 | n/a | n/a | n/a | 24.0 | 5.0 |" in markdown + + +def test_execution_based_models_do_not_serialize_code_review_metrics(tmp_path: Path) -> None: + result = create_bugfix_result(metrics=AgentMetrics(execution_time=4.0)) + summary = ExecutionBasedEvaluationResultSummary.from_results([result], run_id="run") + aggregate = ExecutionBasedLeaderboardAggregate.from_runs([summary]) + result.save(tmp_path, "results.jsonl") + saved_result = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8")) + + assert "total_tokens" not in saved_result["metrics"] + assert "knowledge_used" not in saved_result["metrics"] + assert "average_total_tokens" not in summary.to_dict() + assert "average_knowledge_used" not in summary.to_dict() + assert "average_total_tokens" not in aggregate.model_dump(mode="json") + assert "average_knowledge_used" not in aggregate.model_dump(mode="json") From c0eabd1ce904a73d6ab5a755ed9735edcf3b2d75 Mon Sep 17 00:00:00 2001 From: Wenjie Fan <31087545+gggdttt@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:29:52 +0200 Subject: [PATCH 2/3] Propose PR #807 conflict resolution on the dedicated runner (#808) Co-authored-by: wenjiefan Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: dayland Copilot-Session: 045bef88-d7d4-4553-a681-d7d73cdb2285 Copilot-Session: 003aa3eb-6da6-49f0-bb28-49c8bb0e0d3f Copilot-Session: 15d832b0-a404-440c-932e-186ceebbb9a7 --- .../install-agent-harnesses/action.yml | 2 +- .github/workflows/copilot-evaluation.yml | 18 +- .github/workflows/pr-review-evaluation.yml | 171 +++++++++ CONTRIBUTING.md | 7 + README.md | 4 + dataset/codereview.jsonl | 26 +- docs/code-review.md | 226 +++++------- notebooks/code-review-coverage.ipynb | 22 +- pyproject.toml | 6 +- src/bcbench/agent/__init__.py | 7 +- .../agent/copilot/pr_review/__init__.py | 3 - src/bcbench/agent/copilot/pr_review/agent.py | 336 ----------------- .../agent/copilot/pr_review/metrics.py | 164 --------- src/bcbench/agent/pr_review/__init__.py | 3 + src/bcbench/agent/pr_review/agent.py | 233 ++++++++++++ src/bcbench/agent/pr_review/metrics.py | 133 +++++++ .../{copilot => }/pr_review/review_output.py | 34 +- .../scripts/Prepare-BCQualityRoot.ps1 | 32 +- src/bcbench/agent/shared/config.yaml | 44 ++- src/bcbench/cli_options.py | 36 +- src/bcbench/commands/evaluate.py | 139 +++---- src/bcbench/commands/run.py | 96 ++--- src/bcbench/dataset/codereview.py | 22 +- src/bcbench/evaluate/review_parsing.py | 11 +- src/bcbench/operations/__init__.py | 4 + src/bcbench/operations/git_operations.py | 20 +- src/bcbench/results/base.py | 11 - src/bcbench/results/codereview.py | 175 +++++---- src/bcbench/results/leaderboard.py | 30 +- src/bcbench/results/summary.py | 4 +- src/bcbench/types.py | 34 +- tests/test_bcquality_article_coverage.py | 40 +- tests/test_copilot_prompt.py | 21 ++ tests/test_git_operations.py | 21 +- tests/test_pr_review_agent.py | 229 ++++++------ tests/test_pr_review_metrics.py | 341 +++++++++++++----- tests/test_pr_review_metrics_reporting.py | 192 +++++++--- tests/test_pr_review_output.py | 37 +- tests/test_review_runners.py | 137 +++++++ tests/test_review_workflows.py | 57 +++ uv.lock | 4 +- 41 files changed, 1834 insertions(+), 1298 deletions(-) create mode 100644 .github/workflows/pr-review-evaluation.yml delete mode 100644 src/bcbench/agent/copilot/pr_review/__init__.py delete mode 100644 src/bcbench/agent/copilot/pr_review/agent.py delete mode 100644 src/bcbench/agent/copilot/pr_review/metrics.py create mode 100644 src/bcbench/agent/pr_review/__init__.py create mode 100644 src/bcbench/agent/pr_review/agent.py create mode 100644 src/bcbench/agent/pr_review/metrics.py rename src/bcbench/agent/{copilot => }/pr_review/review_output.py (72%) rename src/bcbench/agent/{copilot => }/pr_review/scripts/Prepare-BCQualityRoot.ps1 (66%) create mode 100644 tests/test_review_runners.py create mode 100644 tests/test_review_workflows.py diff --git a/.github/actions/install-agent-harnesses/action.yml b/.github/actions/install-agent-harnesses/action.yml index 1549e0fdf..8233ea820 100644 --- a/.github/actions/install-agent-harnesses/action.yml +++ b/.github/actions/install-agent-harnesses/action.yml @@ -9,5 +9,5 @@ runs: shell: pwsh - name: Install GitHub Copilot CLI - run: npm install -g @github/copilot@1.0.80 + run: npm install -g @github/copilot@1.0.79 shell: pwsh diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index b4e21c48f..5eeeeda84 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -139,23 +139,7 @@ jobs: - name: Install evaluation CLIs uses: $/.github/actions/install-agent-harnesses - - name: Run code-review engine for entry ${{ matrix.entry }} - if: ${{ inputs.category == 'code-review' }} - timeout-minutes: 120 - shell: pwsh - env: - COPILOT_GITHUB_TOKEN: ${{ github.token }} - GH_TOKEN: ${{ github.token }} - run: | - Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN" - - uv run bcbench evaluate code-review "${{ matrix.entry }}" ` - --model "${{ inputs.model }}" ` - --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` - --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" - - name: Run GitHub Copilot CLI for entry ${{ matrix.entry }} - if: ${{ inputs.category != 'code-review' }} timeout-minutes: 120 shell: pwsh env: @@ -189,7 +173,7 @@ jobs: with: results-dir: ${{ needs.evaluate-with-copilot-cli.outputs.results-dir }} model: ${{ inputs.model }} - agent: ${{ inputs.category == 'code-review' && 'BC PR Review' || 'GitHub Copilot CLI' }} + agent: "GitHub Copilot CLI" mock: ${{ inputs.test-run }} category: ${{ inputs.category }} git-ref: ${{ inputs.git-ref || github.ref_name }} diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml new file mode 100644 index 000000000..ceded2fb8 --- /dev/null +++ b/.github/workflows/pr-review-evaluation.yml @@ -0,0 +1,171 @@ +name: Evaluation with BC PR Review +permissions: + contents: read + actions: write + +on: + workflow_dispatch: + inputs: + model: + description: "Copilot model used internally by BC PR Review" + required: false + default: "gpt-5.6-luna" + type: choice + options: + - "claude-sonnet-5" + - "claude-opus-5" + - "gpt-5.6-sol" + - "gpt-5.6-terra" + - "gpt-5.6-luna" + - "gpt-5.3-codex" + - "mai-code-1.1-flash" + - "gemini-3.6-flash" + test-run: + description: "Indicate this is a test run (with few entries)" + required: false + default: true + type: boolean + repeat: + description: "Number of times to run sequentially (ignored for test runs)" + required: false + default: "1" + type: choice + options: + - "1" + - "2" + - "3" + - "4" + - "5" + git-ref: + description: "Branch to record for experiment tracking (auto-filled on requeue; leave blank for manual runs)" + required: false + default: "" + type: string + +concurrency: + group: pr-review-evaluation-${{ inputs.test-run && 'test' || 'full' }} + cancel-in-progress: false + +env: + EVALUATION_RESULTS_DIR: evaluation_results + +jobs: + pin-commit: + uses: $/.github/workflows/pin-evaluation-commit.yml + permissions: + contents: write + with: + agent: pr-review + test-run: ${{ inputs.test-run }} + repeat: ${{ inputs.repeat }} + + get-entries: + uses: $/.github/workflows/get-entries.yml + with: + test-run: ${{ inputs.test-run }} + category: code-review + + evaluate-with-pr-review: + runs-on: ${{ needs.get-entries.outputs.runner }} + needs: get-entries + outputs: + results-dir: ${{ env.EVALUATION_RESULTS_DIR }} + if: needs.get-entries.outputs.entries != '[]' + environment: + name: ado-read + deployment: false + permissions: + contents: read + id-token: write + copilot-requests: write + name: ${{ matrix.entry }} + strategy: + fail-fast: false + max-parallel: 64 + matrix: + entry: ${{ fromJson(needs.get-entries.outputs.entries) }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Setup Repository + id: setup-env + timeout-minutes: 40 + uses: $/.github/actions/setup-bc-container-repo + with: + instance-id: ${{ matrix.entry }} + category: code-review + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-container: true + skip-repo: false + + - name: Setup Python with UV + uses: $/.github/actions/setup-python-uv + + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Install evaluation CLIs + uses: $/.github/actions/install-agent-harnesses + + - name: Checkout BC-ALAgents review engine + uses: actions/checkout@v5 + with: + repository: microsoft/BC-ALAgents + ref: 533dd39dfe29218c09e5e31c39c78bb72fa20aa2 + path: bc-alagents-engine + token: ${{ github.token }} + + - name: Run BC PR Review for entry ${{ matrix.entry }} + timeout-minutes: 120 + shell: pwsh + env: + COPILOT_GITHUB_TOKEN: ${{ github.token }} + run: | + Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN" + + uv run bcbench evaluate pr-review "${{ matrix.entry }}" ` + --model "${{ inputs.model }}" ` + --engine-path "${{ github.workspace }}/bc-alagents-engine" ` + --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` + --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" + + - name: Upload evaluation results + uses: actions/upload-artifact@v6 + if: always() + with: + name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} + path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + retention-days: ${{ inputs.test-run && 1 || 30 }} + + summarize-results: + needs: evaluate-with-pr-review + uses: $/.github/workflows/summarize-results.yml + permissions: + contents: write + id-token: write + with: + results-dir: ${{ needs.evaluate-with-pr-review.outputs.results-dir }} + model: ${{ inputs.model }} + agent: "BC PR Review" + mock: ${{ inputs.test-run }} + category: code-review + git-ref: ${{ inputs.git-ref || github.ref_name }} + secrets: inherit + + requeue: + needs: [summarize-results, pin-commit] + if: ${{ !cancelled() && !failure() && !inputs.test-run }} + uses: $/.github/workflows/requeue-evaluation.yml + permissions: + contents: write + actions: write + with: + workflow-file: pr-review-evaluation.yml + repeat: ${{ inputs.repeat }} + existing-tag: ${{ needs.pin-commit.outputs.tag-name }} + workflow-inputs: | + {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c76d7680..395f53077 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,6 +133,13 @@ Keep evaluation tools pinned so benchmark runs remain reproducible. For example, 4. Run focused tests for the integration, then perform a test evaluation for tools exposed to the agent. 5. Bump the benchmark version according to the Versioning Policy. Tool changes that may affect evaluation results normally require a minor bump. +### Bump the BC PR Review engine + +1. Update the pinned `microsoft/BC-ALAgents` commit in `.github/workflows/pr-review-evaluation.yml` +2. Run a test evaluation through the `pr-review` workflow +3. Bump the BC-Bench version following the Versioning Policy +4. Include the exact BC-ALAgents commit SHA in the BC-Bench release notes + ### Create a new release After you bump the version in [pyproject.toml](https://github.com/microsoft/BC-Bench/blob/main/pyproject.toml#L7) following the Versioning Policy, use the repository's [`create-release` skill](.github/skills/create-release/SKILL.md) to prepare release notes after pushing your changes. The skill screens merged PRs since the previous version tag and returns Markdown covering only changes that may affect evaluation results, without creating the tag or release. diff --git a/README.md b/README.md index 0ac5de7f7..a0a02d039 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ The [GitHub Copilot CLI](https://github.com/github/copilot-cli) supports MCP ser [Claude Code](https://docs.anthropic.com/en/docs/claude-code) is Anthropic's agentic coding tool. It supports MCP servers, custom system prompts, and agent mode. BC-Bench integrates with Claude Code using the same shared configuration as Copilot. +### BC PR Review + +BC PR Review is the production-fidelity BC-ALAgents + BCQuality runner for the `code-review` category. It remains separate from the category contract so its results can be compared with GitHub Copilot CLI and Claude Code on the same dataset and scorer. + ## Getting Started BC-Bench is open source, and you're welcome to fork and adapt it for your own use. We are not accepting external contributions in this repository at this time. You can run evaluations locally and replace the dataset under `dataset/` with tasks from your own codebase. diff --git a/dataset/codereview.jsonl b/dataset/codereview.jsonl index fd5f5795f..97e6e3491 100644 --- a/dataset/codereview.jsonl +++ b/dataset/codereview.jsonl @@ -1,18 +1,18 @@ -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-001", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/SecureAPIManager.Codeunit.al b/src/SecureAPIManager.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SecureAPIManager.Codeunit.al\n@@ -0,0 +1,46 @@\n+codeunit 50100 \"Secure API Manager\"\n+{\n+ Access = Internal;\n+\n+ var\n+ KeyNotConfiguredErr: Label 'API key is not configured for %1.', Comment = '%1 = configuration code';\n+ RequestFailedErr: Label 'The API request failed. Check the configuration.', Comment = 'Shown when an outbound API call fails.';\n+ EndpointTok: Label 'https://api.businesscentral.dynamics.com/v2.0/data', Locked = true;\n+ BearerTok: Label 'Bearer %1', Locked = true;\n+ StorageKeyTok: Label 'ApiKey_%1', Locked = true;\n+\n+ [NonDebuggable]\n+ procedure StoreKey(ConfigCode: Code[20]; KeyValue: SecretText)\n+ begin\n+ IsolatedStorage.SetEncrypted(GetStorageKey(ConfigCode), KeyValue, DataScope::Module);\n+ end;\n+\n+ [NonDebuggable]\n+ procedure GetKey(ConfigCode: Code[20]) Result: SecretText\n+ begin\n+ if not IsolatedStorage.Contains(GetStorageKey(ConfigCode), DataScope::Module) then\n+ Error(KeyNotConfiguredErr, ConfigCode);\n+ IsolatedStorage.Get(GetStorageKey(ConfigCode), DataScope::Module, Result);\n+ end;\n+\n+ procedure CallEndpoint(ConfigCode: Code[20])\n+ var\n+ Client: HttpClient;\n+ Headers: HttpHeaders;\n+ Response: HttpResponseMessage;\n+ AuthHeader: SecretText;\n+ begin\n+ AuthHeader := SecretStrSubstNo(BearerTok, GetKey(ConfigCode));\n+ Headers := Client.DefaultRequestHeaders();\n+ Headers.Add('Authorization', AuthHeader);\n+ if not Client.Get(EndpointTok, Response) then\n+ Error(RequestFailedErr);\n+ if not Response.IsSuccessStatusCode() then\n+ Error(RequestFailedErr);\n+ end;\n+\n+ local procedure GetStorageKey(ConfigCode: Code[20]): Text[50]\n+ begin\n+ exit(CopyStr(StrSubstNo(StorageKeyTok, ConfigCode), 1, 50));\n+ end;\n+}\ndiff --git a/src/HardcodedSecretClient.Codeunit.al b/src/HardcodedSecretClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/HardcodedSecretClient.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50301 \"Hardcoded Secret Client\"\n+{\n+ procedure CallApi()\n+ var\n+ HttpClient: HttpClient;\n+ HttpHeaders: HttpHeaders;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpHeaders := HttpClient.DefaultRequestHeaders();\n+ HttpHeaders.Add('X-Api-Key', this.GetApiKey());\n+ HttpClient.Get('https://api.contoso.com/data', HttpResponseMessage);\n+ end;\n+\n+ local procedure GetApiKey(): Text\n+ begin\n+ exit('sk-1234567890abcdef');\n+ end;\n+}\n", "expected_comments": [{"file": "src/HardcodedSecretClient.Codeunit.al", "line_start": 16, "line_end": 16, "severity": "critical", "domain": "security", "body": "Hardcoded API key in source code. Retrieve secrets from encrypted isolated storage or another secure store instead.", "article": "security/secrettext-for-credentials"}], "category": "code-review", "description": "Clean codeunit using SecretText, NonDebuggable, IsolatedStorage.SetEncrypted, and HTTPS enforcement with no security issues", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-002", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/SecureKeyManager.Codeunit.al b/src/SecureKeyManager.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SecureKeyManager.Codeunit.al\n@@ -0,0 +1,26 @@\n+codeunit 50101 \"Secure Key Manager\"\n+{\n+ Access = Internal;\n+\n+ var\n+ KeyNotFoundErr: Label 'The requested key was not found. Configure it before use.';\n+\n+ [NonDebuggable]\n+ procedure StoreEncryptedKey(KeyName: Code[20]; KeyValue: SecretText)\n+ begin\n+ IsolatedStorage.SetEncrypted(KeyName, KeyValue, DataScope::Module);\n+ end;\n+\n+ [NonDebuggable]\n+ procedure RetrieveKey(KeyName: Code[20]) Result: SecretText\n+ begin\n+ if not IsolatedStorage.Contains(KeyName, DataScope::Module) then\n+ Error(KeyNotFoundErr);\n+ IsolatedStorage.Get(KeyName, DataScope::Module, Result);\n+ end;\n+\n+ procedure HasKey(KeyName: Code[20]): Boolean\n+ begin\n+ exit(IsolatedStorage.Contains(KeyName, DataScope::Module));\n+ end;\n+}\ndiff --git a/src/PlainTokenHeaderClient.Codeunit.al b/src/PlainTokenHeaderClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/PlainTokenHeaderClient.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50302 \"Plain Token Header Client\"\n+{\n+ procedure SendRequest(AccessToken: Text)\n+ var\n+ HttpClient: HttpClient;\n+ HttpHeaders: HttpHeaders;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpHeaders := HttpClient.DefaultRequestHeaders();\n+ HttpHeaders.Add('Authorization', 'Bearer ' + AccessToken);\n+ HttpClient.Get(this.GetOrdersEndpoint(), HttpResponseMessage);\n+ end;\n+\n+ local procedure GetOrdersEndpoint(): Text\n+ begin\n+ exit('https://api.contoso.com/orders');\n+ end;\n+}\n", "expected_comments": [{"file": "src/PlainTokenHeaderClient.Codeunit.al", "line_start": 10, "line_end": 10, "severity": "high", "domain": "security", "body": "Bearer token is concatenated into a plain Text authorization header. Build the header with SecretStrSubstNo() and add it as SecretText.", "article": "security/secretstrsubstno-for-composing-secrets"}], "category": "code-review", "description": "Clean codeunit correctly storing and retrieving API keys using IsolatedStorage.SetEncrypted, SecretText, and NonDebuggable", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-003", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/SafeErrorHandler.Codeunit.al b/src/SafeErrorHandler.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SafeErrorHandler.Codeunit.al\n@@ -0,0 +1,41 @@\n+codeunit 50102 \"Safe Error Handler\"\n+{\n+ Access = Internal;\n+\n+ var\n+ InvalidRequestTxt: Label 'Invalid request. Please check your input.';\n+ AuthFailedTxt: Label 'Authentication failed. Please verify your credentials.';\n+ ForbiddenTxt: Label 'You do not have permission for this operation.';\n+ NotFoundTxt: Label 'The requested resource was not found.';\n+ UnexpectedTxt: Label 'An unexpected error occurred. Contact your administrator.';\n+ PostFailedErr: Label 'Could not post document %1.', Comment = '%1 = document number';\n+\n+ procedure GetApiResponseMessage(StatusCode: Integer): Text\n+ begin\n+ case StatusCode of\n+ 200, 201:\n+ exit('');\n+ 400:\n+ exit(InvalidRequestTxt);\n+ 401:\n+ exit(AuthFailedTxt);\n+ 403:\n+ exit(ForbiddenTxt);\n+ 404:\n+ exit(NotFoundTxt);\n+ else\n+ exit(UnexpectedTxt);\n+ end;\n+ end;\n+\n+ procedure PostDocument(DocNo: Code[20])\n+ begin\n+ if not TryPost(DocNo) then\n+ Error(PostFailedErr, DocNo);\n+ end;\n+\n+ [TryFunction]\n+ local procedure TryPost(DocNo: Code[20])\n+ begin\n+ end;\n+}\ndiff --git a/src/UnwrappedSecretClient.Codeunit.al b/src/UnwrappedSecretClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/UnwrappedSecretClient.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50303 \"Unwrapped Secret Client\"\n+{\n+ procedure SendRequest(SessionToken: SecretText)\n+ var\n+ HttpClient: HttpClient;\n+ HttpHeaders: HttpHeaders;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpHeaders := HttpClient.DefaultRequestHeaders();\n+ HttpHeaders.Add('Authorization', this.BuildHeader(SessionToken));\n+ HttpClient.Get('https://api.contoso.com/data', HttpResponseMessage);\n+ end;\n+\n+ local procedure BuildHeader(SessionToken: SecretText): Text\n+ begin\n+ exit('Bearer ' + SessionToken.Unwrap());\n+ end;\n+}\n", "expected_comments": [{"file": "src/UnwrappedSecretClient.Codeunit.al", "line_start": 16, "line_end": 16, "severity": "high", "domain": "security", "body": "SecretText.Unwrap() exposes the secret as plain Text without a [NonDebuggable] procedure. Add [NonDebuggable] or avoid unwrapping by using SecretStrSubstNo().", "article": "security/nondebuggable-required-when-unwrapping-secrettext"}], "category": "code-review", "description": "Clean codeunit with proper error handling: generic user-facing messages, no system details exposed, no GetLastErrorText shown to user", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-004", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/AppConstants.Codeunit.al b/src/AppConstants.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/AppConstants.Codeunit.al\n@@ -0,0 +1,30 @@\n+codeunit 50103 \"App Constants\"\n+{\n+ Access = Internal;\n+\n+ var\n+ ApiVersionTok: Label 'v2.0', Locked = true;\n+ DefaultCurrencyTok: Label 'USD', Locked = true;\n+ DateFormatTok: Label 'yyyy-MM-dd', Locked = true;\n+ AppIdTok: Label 'BC-INVENTORY-APP', Locked = true;\n+\n+ procedure GetApiVersion(): Text\n+ begin\n+ exit(ApiVersionTok);\n+ end;\n+\n+ procedure GetDefaultCurrency(): Code[10]\n+ begin\n+ exit(DefaultCurrencyTok);\n+ end;\n+\n+ procedure GetDateFormat(): Text\n+ begin\n+ exit(DateFormatTok);\n+ end;\n+\n+ procedure GetAppId(): Text\n+ begin\n+ exit(AppIdTok);\n+ end;\n+}\ndiff --git a/src/QueryStringSecretClient.Codeunit.al b/src/QueryStringSecretClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/QueryStringSecretClient.Codeunit.al\n@@ -0,0 +1,15 @@\n+codeunit 50304 \"Query String Secret Client\"\n+{\n+ procedure FetchAccount(ApiKey: Text)\n+ var\n+ HttpClient: HttpClient;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpClient.Get(this.BuildAccountUrl(ApiKey), HttpResponseMessage);\n+ end;\n+\n+ local procedure BuildAccountUrl(ApiKey: Text): Text\n+ begin\n+ exit('https://api.contoso.com/accounts?api_key=' + ApiKey);\n+ end;\n+}\n", "expected_comments": [{"file": "src/QueryStringSecretClient.Codeunit.al", "line_start": 13, "line_end": 13, "severity": "high", "domain": "security", "body": "API key is placed in the URL query string. Use an Authorization header, or SetSecretRequestUri() if a secret URI is unavoidable.", "article": "security/secrettext-with-httpclient"}], "category": "code-review", "description": "Clean codeunit with configuration constants that are not secrets: API version, currency codes, labels, and format strings", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-005", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/ValidatedImportConfig.Table.al b/src/ValidatedImportConfig.Table.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ValidatedImportConfig.Table.al\n@@ -0,0 +1,34 @@\n+table 50104 \"Validated Import Config\"\n+{\n+ Caption = 'Validated Import Configuration';\n+ DataClassification = CustomerContent;\n+\n+ fields\n+ {\n+ field(1; \"Code\"; Code[20])\n+ {\n+ Caption = 'Code';\n+ NotBlank = true;\n+ }\n+ field(2; \"Source Table ID\"; Integer)\n+ {\n+ Caption = 'Source Table';\n+ TableRelation = AllObjWithCaption.\"Object ID\" where(\"Object Type\" = const(Table));\n+ ValidateTableRelation = true;\n+ }\n+ field(3; \"Max Records\"; Integer)\n+ {\n+ Caption = 'Maximum Records';\n+ MinValue = 1;\n+ MaxValue = 10000;\n+ }\n+ }\n+\n+ keys\n+ {\n+ key(PK; \"Code\")\n+ {\n+ Clustered = true;\n+ }\n+ }\n+}\ndiff --git a/src/BroadFinanceAccess.PermissionSet.al b/src/BroadFinanceAccess.PermissionSet.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/BroadFinanceAccess.PermissionSet.al\n@@ -0,0 +1,15 @@\n+permissionset 50305 \"Broad Finance Access\"\n+{\n+ Assignable = true;\n+ Caption = 'Broad Finance Access', Locked = true;\n+ Permissions =\n+ tabledata * = RIMD,\n+ table * = X,\n+ tabledata Customer = R,\n+ tabledata Vendor = R,\n+ tabledata Item = R,\n+ tabledata \"Sales Header\" = R,\n+ tabledata \"Sales Line\" = R,\n+ codeunit \"Release Sales Document\" = X,\n+ codeunit \"Sales-Post\" = X;\n+}\n", "expected_comments": [{"file": "src/BroadFinanceAccess.PermissionSet.al", "line_start": 6, "line_end": 6, "severity": "critical", "domain": "security", "body": "Permission set grants RIMD on all table data. Replace the wildcard with the minimum specific tabledata permissions required.", "article": "security/permission-set-avoid-wildcard-grants"}, {"file": "src/BroadFinanceAccess.PermissionSet.al", "line_start": 7, "line_end": 7, "severity": "high", "domain": "security", "body": "Permission set grants execute permission on all tables. Grant execute only on the specific objects this role requires.", "article": "security/permission-set-avoid-wildcard-grants"}], "category": "code-review", "description": "Clean table with proper input validation: ValidateTableRelation, OnValidate triggers, MinValue/MaxValue, Editable=false on system fields", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-006", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/InventoryReader.PermissionSet.al b/src/InventoryReader.PermissionSet.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/InventoryReader.PermissionSet.al\n@@ -0,0 +1,11 @@\n+permissionset 50106 \"Inventory Reader\"\n+{\n+ Caption = 'Inventory Reader';\n+ Assignable = true;\n+\n+ Permissions =\n+ tabledata Item = r,\n+ tabledata \"Item Ledger Entry\" = r,\n+ tabledata \"Item Category\" = r,\n+ codeunit \"Inventory Lookup\" = X;\n+}\ndiff --git a/src/InventoryLookup.Codeunit.al b/src/InventoryLookup.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/InventoryLookup.Codeunit.al\n@@ -0,0 +1,15 @@\n+codeunit 50108 \"Inventory Lookup\"\n+{\n+ Access = Internal;\n+ Permissions = tabledata Item = r;\n+\n+ procedure GetItemDescription(ItemNo: Code[20]): Text[100]\n+ var\n+ Item: Record Item;\n+ begin\n+ Item.SetLoadFields(Description);\n+ if Item.Get(ItemNo) then\n+ exit(Item.Description);\n+ exit('');\n+ end;\n+}\ndiff --git a/src/ExcessiveInherentAccess.Codeunit.al b/src/ExcessiveInherentAccess.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExcessiveInherentAccess.Codeunit.al\n@@ -0,0 +1,17 @@\n+codeunit 50306 \"Excessive Inherent Access\"\n+{\n+ procedure LookupCustomerName(CustomerNo: Code[20]): Text\n+ begin\n+ exit(this.GetCustomerName(CustomerNo));\n+ end;\n+\n+ [InherentPermissions(PermissionObjectType::TableData, Database::Customer, 'RIMD')]\n+ local procedure GetCustomerName(CustomerNo: Code[20]): Text\n+ var\n+ Customer: Record Customer;\n+ begin\n+ if Customer.Get(CustomerNo) then\n+ exit(Customer.Name);\n+ exit('');\n+ end;\n+}\n", "expected_comments": [{"file": "src/ExcessiveInherentAccess.Codeunit.al", "line_start": 8, "line_end": 8, "severity": "high", "domain": "security", "body": "InherentPermissions grants RIMD tabledata access even though this procedure only reads Customer. Reduce the permission to the minimal read access required.", "article": "security/inherent-permissions-minimal-grant"}], "category": "code-review", "description": "Clean permission sets with least-privilege access: read-only for readers, read-insert for editors, no RIMD grants", "expect_findings": true, "source": "vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-001","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/SecureAPIManager.Codeunit.al b/src/SecureAPIManager.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SecureAPIManager.Codeunit.al\n@@ -0,0 +1,46 @@\n+codeunit 50100 \"Secure API Manager\"\n+{\n+ Access = Internal;\n+\n+ var\n+ KeyNotConfiguredErr: Label 'API key is not configured for %1.', Comment = '%1 = configuration code';\n+ RequestFailedErr: Label 'The API request failed. Check the configuration.', Comment = 'Shown when an outbound API call fails.';\n+ EndpointTok: Label 'https://api.businesscentral.dynamics.com/v2.0/data', Locked = true;\n+ BearerTok: Label 'Bearer %1', Locked = true;\n+ StorageKeyTok: Label 'ApiKey_%1', Locked = true;\n+\n+ [NonDebuggable]\n+ procedure StoreKey(ConfigCode: Code[20]; KeyValue: SecretText)\n+ begin\n+ IsolatedStorage.SetEncrypted(GetStorageKey(ConfigCode), KeyValue, DataScope::Module);\n+ end;\n+\n+ [NonDebuggable]\n+ procedure GetKey(ConfigCode: Code[20]) Result: SecretText\n+ begin\n+ if not IsolatedStorage.Contains(GetStorageKey(ConfigCode), DataScope::Module) then\n+ Error(KeyNotConfiguredErr, ConfigCode);\n+ IsolatedStorage.Get(GetStorageKey(ConfigCode), DataScope::Module, Result);\n+ end;\n+\n+ procedure CallEndpoint(ConfigCode: Code[20])\n+ var\n+ Client: HttpClient;\n+ Headers: HttpHeaders;\n+ Response: HttpResponseMessage;\n+ AuthHeader: SecretText;\n+ begin\n+ AuthHeader := SecretStrSubstNo(BearerTok, GetKey(ConfigCode));\n+ Headers := Client.DefaultRequestHeaders();\n+ Headers.Add('Authorization', AuthHeader);\n+ if not Client.Get(EndpointTok, Response) then\n+ Error(RequestFailedErr);\n+ if not Response.IsSuccessStatusCode() then\n+ Error(RequestFailedErr);\n+ end;\n+\n+ local procedure GetStorageKey(ConfigCode: Code[20]): Text[50]\n+ begin\n+ exit(CopyStr(StrSubstNo(StorageKeyTok, ConfigCode), 1, 50));\n+ end;\n+}\ndiff --git a/src/HardcodedSecretClient.Codeunit.al b/src/HardcodedSecretClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/HardcodedSecretClient.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50301 \"Hardcoded Secret Client\"\n+{\n+ procedure CallApi()\n+ var\n+ HttpClient: HttpClient;\n+ HttpHeaders: HttpHeaders;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpHeaders := HttpClient.DefaultRequestHeaders();\n+ HttpHeaders.Add('X-Api-Key', this.GetApiKey());\n+ HttpClient.Get('https://api.contoso.com/data', HttpResponseMessage);\n+ end;\n+\n+ local procedure GetApiKey(): Text\n+ begin\n+ exit('sk-1234567890abcdef');\n+ end;\n+}\n","expected_comments":[{"file":"src/HardcodedSecretClient.Codeunit.al","line_start":16,"line_end":16,"severity":"critical","domain":"security","body":"Hardcoded API key in source code. Retrieve secrets from encrypted isolated storage or another secure store instead.","articles":["security/secrettext-for-credentials"]}],"category":"code-review","description":"Clean codeunit using SecretText, NonDebuggable, IsolatedStorage.SetEncrypted, and HTTPS enforcement with no security issues","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-002","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/SecureKeyManager.Codeunit.al b/src/SecureKeyManager.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SecureKeyManager.Codeunit.al\n@@ -0,0 +1,26 @@\n+codeunit 50101 \"Secure Key Manager\"\n+{\n+ Access = Internal;\n+\n+ var\n+ KeyNotFoundErr: Label 'The requested key was not found. Configure it before use.';\n+\n+ [NonDebuggable]\n+ procedure StoreEncryptedKey(KeyName: Code[20]; KeyValue: SecretText)\n+ begin\n+ IsolatedStorage.SetEncrypted(KeyName, KeyValue, DataScope::Module);\n+ end;\n+\n+ [NonDebuggable]\n+ procedure RetrieveKey(KeyName: Code[20]) Result: SecretText\n+ begin\n+ if not IsolatedStorage.Contains(KeyName, DataScope::Module) then\n+ Error(KeyNotFoundErr);\n+ IsolatedStorage.Get(KeyName, DataScope::Module, Result);\n+ end;\n+\n+ procedure HasKey(KeyName: Code[20]): Boolean\n+ begin\n+ exit(IsolatedStorage.Contains(KeyName, DataScope::Module));\n+ end;\n+}\ndiff --git a/src/PlainTokenHeaderClient.Codeunit.al b/src/PlainTokenHeaderClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/PlainTokenHeaderClient.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50302 \"Plain Token Header Client\"\n+{\n+ procedure SendRequest(AccessToken: Text)\n+ var\n+ HttpClient: HttpClient;\n+ HttpHeaders: HttpHeaders;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpHeaders := HttpClient.DefaultRequestHeaders();\n+ HttpHeaders.Add('Authorization', 'Bearer ' + AccessToken);\n+ HttpClient.Get(this.GetOrdersEndpoint(), HttpResponseMessage);\n+ end;\n+\n+ local procedure GetOrdersEndpoint(): Text\n+ begin\n+ exit('https://api.contoso.com/orders');\n+ end;\n+}\n","expected_comments":[{"file":"src/PlainTokenHeaderClient.Codeunit.al","line_start":10,"line_end":10,"severity":"high","domain":"security","body":"Bearer token is concatenated into a plain Text authorization header. Build the header with SecretStrSubstNo() and add it as SecretText.","articles":["security/secretstrsubstno-for-composing-secrets"]}],"category":"code-review","description":"Clean codeunit correctly storing and retrieving API keys using IsolatedStorage.SetEncrypted, SecretText, and NonDebuggable","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-003","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/SafeErrorHandler.Codeunit.al b/src/SafeErrorHandler.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SafeErrorHandler.Codeunit.al\n@@ -0,0 +1,41 @@\n+codeunit 50102 \"Safe Error Handler\"\n+{\n+ Access = Internal;\n+\n+ var\n+ InvalidRequestTxt: Label 'Invalid request. Please check your input.';\n+ AuthFailedTxt: Label 'Authentication failed. Please verify your credentials.';\n+ ForbiddenTxt: Label 'You do not have permission for this operation.';\n+ NotFoundTxt: Label 'The requested resource was not found.';\n+ UnexpectedTxt: Label 'An unexpected error occurred. Contact your administrator.';\n+ PostFailedErr: Label 'Could not post document %1.', Comment = '%1 = document number';\n+\n+ procedure GetApiResponseMessage(StatusCode: Integer): Text\n+ begin\n+ case StatusCode of\n+ 200, 201:\n+ exit('');\n+ 400:\n+ exit(InvalidRequestTxt);\n+ 401:\n+ exit(AuthFailedTxt);\n+ 403:\n+ exit(ForbiddenTxt);\n+ 404:\n+ exit(NotFoundTxt);\n+ else\n+ exit(UnexpectedTxt);\n+ end;\n+ end;\n+\n+ procedure PostDocument(DocNo: Code[20])\n+ begin\n+ if not TryPost(DocNo) then\n+ Error(PostFailedErr, DocNo);\n+ end;\n+\n+ [TryFunction]\n+ local procedure TryPost(DocNo: Code[20])\n+ begin\n+ end;\n+}\ndiff --git a/src/UnwrappedSecretClient.Codeunit.al b/src/UnwrappedSecretClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/UnwrappedSecretClient.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50303 \"Unwrapped Secret Client\"\n+{\n+ procedure SendRequest(SessionToken: SecretText)\n+ var\n+ HttpClient: HttpClient;\n+ HttpHeaders: HttpHeaders;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpHeaders := HttpClient.DefaultRequestHeaders();\n+ HttpHeaders.Add('Authorization', this.BuildHeader(SessionToken));\n+ HttpClient.Get('https://api.contoso.com/data', HttpResponseMessage);\n+ end;\n+\n+ local procedure BuildHeader(SessionToken: SecretText): Text\n+ begin\n+ exit('Bearer ' + SessionToken.Unwrap());\n+ end;\n+}\n","expected_comments":[{"file":"src/UnwrappedSecretClient.Codeunit.al","line_start":16,"line_end":16,"severity":"high","domain":"security","body":"SecretText.Unwrap() exposes the secret as plain Text without a [NonDebuggable] procedure. Add [NonDebuggable] or avoid unwrapping by using SecretStrSubstNo().","articles":["security/nondebuggable-required-when-unwrapping-secrettext"]}],"category":"code-review","description":"Clean codeunit with proper error handling: generic user-facing messages, no system details exposed, no GetLastErrorText shown to user","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-004","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/AppConstants.Codeunit.al b/src/AppConstants.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/AppConstants.Codeunit.al\n@@ -0,0 +1,30 @@\n+codeunit 50103 \"App Constants\"\n+{\n+ Access = Internal;\n+\n+ var\n+ ApiVersionTok: Label 'v2.0', Locked = true;\n+ DefaultCurrencyTok: Label 'USD', Locked = true;\n+ DateFormatTok: Label 'yyyy-MM-dd', Locked = true;\n+ AppIdTok: Label 'BC-INVENTORY-APP', Locked = true;\n+\n+ procedure GetApiVersion(): Text\n+ begin\n+ exit(ApiVersionTok);\n+ end;\n+\n+ procedure GetDefaultCurrency(): Code[10]\n+ begin\n+ exit(DefaultCurrencyTok);\n+ end;\n+\n+ procedure GetDateFormat(): Text\n+ begin\n+ exit(DateFormatTok);\n+ end;\n+\n+ procedure GetAppId(): Text\n+ begin\n+ exit(AppIdTok);\n+ end;\n+}\ndiff --git a/src/QueryStringSecretClient.Codeunit.al b/src/QueryStringSecretClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/QueryStringSecretClient.Codeunit.al\n@@ -0,0 +1,15 @@\n+codeunit 50304 \"Query String Secret Client\"\n+{\n+ procedure FetchAccount(ApiKey: Text)\n+ var\n+ HttpClient: HttpClient;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpClient.Get(this.BuildAccountUrl(ApiKey), HttpResponseMessage);\n+ end;\n+\n+ local procedure BuildAccountUrl(ApiKey: Text): Text\n+ begin\n+ exit('https://api.contoso.com/accounts?api_key=' + ApiKey);\n+ end;\n+}\n","expected_comments":[{"file":"src/QueryStringSecretClient.Codeunit.al","line_start":13,"line_end":13,"severity":"high","domain":"security","body":"API key is placed in the URL query string. Use an Authorization header, or SetSecretRequestUri() if a secret URI is unavoidable.","articles":["security/secrettext-with-httpclient"]}],"category":"code-review","description":"Clean codeunit with configuration constants that are not secrets: API version, currency codes, labels, and format strings","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-005","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/ValidatedImportConfig.Table.al b/src/ValidatedImportConfig.Table.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ValidatedImportConfig.Table.al\n@@ -0,0 +1,34 @@\n+table 50104 \"Validated Import Config\"\n+{\n+ Caption = 'Validated Import Configuration';\n+ DataClassification = CustomerContent;\n+\n+ fields\n+ {\n+ field(1; \"Code\"; Code[20])\n+ {\n+ Caption = 'Code';\n+ NotBlank = true;\n+ }\n+ field(2; \"Source Table ID\"; Integer)\n+ {\n+ Caption = 'Source Table';\n+ TableRelation = AllObjWithCaption.\"Object ID\" where(\"Object Type\" = const(Table));\n+ ValidateTableRelation = true;\n+ }\n+ field(3; \"Max Records\"; Integer)\n+ {\n+ Caption = 'Maximum Records';\n+ MinValue = 1;\n+ MaxValue = 10000;\n+ }\n+ }\n+\n+ keys\n+ {\n+ key(PK; \"Code\")\n+ {\n+ Clustered = true;\n+ }\n+ }\n+}\ndiff --git a/src/BroadFinanceAccess.PermissionSet.al b/src/BroadFinanceAccess.PermissionSet.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/BroadFinanceAccess.PermissionSet.al\n@@ -0,0 +1,15 @@\n+permissionset 50305 \"Broad Finance Access\"\n+{\n+ Assignable = true;\n+ Caption = 'Broad Finance Access', Locked = true;\n+ Permissions =\n+ tabledata * = RIMD,\n+ table * = X,\n+ tabledata Customer = R,\n+ tabledata Vendor = R,\n+ tabledata Item = R,\n+ tabledata \"Sales Header\" = R,\n+ tabledata \"Sales Line\" = R,\n+ codeunit \"Release Sales Document\" = X,\n+ codeunit \"Sales-Post\" = X;\n+}\n","expected_comments":[{"file":"src/BroadFinanceAccess.PermissionSet.al","line_start":6,"line_end":6,"severity":"critical","domain":"security","body":"Permission set grants RIMD on all table data. Replace the wildcard with the minimum specific tabledata permissions required.","articles":["security/permission-set-avoid-wildcard-grants"]},{"file":"src/BroadFinanceAccess.PermissionSet.al","line_start":7,"line_end":7,"severity":"high","domain":"security","body":"Permission set grants execute permission on all tables. Grant execute only on the specific objects this role requires.","articles":["security/permission-set-avoid-wildcard-grants"]}],"category":"code-review","description":"Clean table with proper input validation: ValidateTableRelation, OnValidate triggers, MinValue/MaxValue, Editable=false on system fields","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-006","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/InventoryReader.PermissionSet.al b/src/InventoryReader.PermissionSet.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/InventoryReader.PermissionSet.al\n@@ -0,0 +1,11 @@\n+permissionset 50106 \"Inventory Reader\"\n+{\n+ Caption = 'Inventory Reader';\n+ Assignable = true;\n+\n+ Permissions =\n+ tabledata Item = r,\n+ tabledata \"Item Ledger Entry\" = r,\n+ tabledata \"Item Category\" = r,\n+ codeunit \"Inventory Lookup\" = X;\n+}\ndiff --git a/src/InventoryLookup.Codeunit.al b/src/InventoryLookup.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/InventoryLookup.Codeunit.al\n@@ -0,0 +1,15 @@\n+codeunit 50108 \"Inventory Lookup\"\n+{\n+ Access = Internal;\n+ Permissions = tabledata Item = r;\n+\n+ procedure GetItemDescription(ItemNo: Code[20]): Text[100]\n+ var\n+ Item: Record Item;\n+ begin\n+ Item.SetLoadFields(Description);\n+ if Item.Get(ItemNo) then\n+ exit(Item.Description);\n+ exit('');\n+ end;\n+}\ndiff --git a/src/ExcessiveInherentAccess.Codeunit.al b/src/ExcessiveInherentAccess.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExcessiveInherentAccess.Codeunit.al\n@@ -0,0 +1,17 @@\n+codeunit 50306 \"Excessive Inherent Access\"\n+{\n+ procedure LookupCustomerName(CustomerNo: Code[20]): Text\n+ begin\n+ exit(this.GetCustomerName(CustomerNo));\n+ end;\n+\n+ [InherentPermissions(PermissionObjectType::TableData, Database::Customer, 'RIMD')]\n+ local procedure GetCustomerName(CustomerNo: Code[20]): Text\n+ var\n+ Customer: Record Customer;\n+ begin\n+ if Customer.Get(CustomerNo) then\n+ exit(Customer.Name);\n+ exit('');\n+ end;\n+}\n","expected_comments":[{"file":"src/ExcessiveInherentAccess.Codeunit.al","line_start":8,"line_end":8,"severity":"high","domain":"security","body":"InherentPermissions grants RIMD tabledata access even though this procedure only reads Customer. Reduce the permission to the minimal read access required.","articles":["security/inherent-permissions-minimal-grant"]}],"category":"code-review","description":"Clean permission sets with least-privilege access: read-only for readers, read-insert for editors, no RIMD grants","expect_findings":true,"source":"vsoadmin"} {"repo": "microsoft/BCApps", "instance_id": "synthetic__security-007", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/SafeRecordQuery.Codeunit.al b/src/SafeRecordQuery.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SafeRecordQuery.Codeunit.al\n@@ -0,0 +1,31 @@\n+codeunit 50109 \"Safe Record Query\"\n+{\n+ Access = Internal;\n+\n+ procedure CustomerExists(CustomerNo: Code[20]): Boolean\n+ var\n+ Customer: Record Customer;\n+ begin\n+ Customer.SetLoadFields(\"No.\");\n+ exit(Customer.Get(CustomerNo));\n+ end;\n+\n+ procedure HasOpenSalesOrders(CustomerNo: Code[20]): Boolean\n+ var\n+ SalesHeader: Record \"Sales Header\";\n+ begin\n+ SalesHeader.SetRange(\"Document Type\", SalesHeader.\"Document Type\"::Order);\n+ SalesHeader.SetRange(\"Sell-to Customer No.\", CustomerNo);\n+ exit(not SalesHeader.IsEmpty());\n+ end;\n+\n+ procedure GetItemDescription(ItemNo: Code[20]): Text[100]\n+ var\n+ Item: Record Item;\n+ begin\n+ Item.SetLoadFields(Description);\n+ if Item.Get(ItemNo) then\n+ exit(Item.Description);\n+ exit('');\n+ end;\n+}\ndiff --git a/src/InsecureEndpointClient.Codeunit.al b/src/InsecureEndpointClient.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/InsecureEndpointClient.Codeunit.al\n@@ -0,0 +1,17 @@\n+codeunit 50307 \"Insecure Endpoint Client\"\n+{\n+ procedure SendSession(SessionToken: SecretText)\n+ var\n+ HttpClient: HttpClient;\n+ HttpContent: HttpContent;\n+ HttpResponseMessage: HttpResponseMessage;\n+ begin\n+ HttpContent.WriteFrom(SessionToken);\n+ HttpClient.Post(this.GetEndpoint(), HttpContent, HttpResponseMessage);\n+ end;\n+\n+ local procedure GetEndpoint(): Text\n+ begin\n+ exit('http://api.contoso.com/session');\n+ end;\n+}\n", "expected_comments": [{"file": "src/InsecureEndpointClient.Codeunit.al", "line_start": 15, "line_end": 15, "severity": "high", "domain": "security", "body": "External service endpoint uses HTTP instead of HTTPS. Use HTTPS for all external HTTP calls, especially when sending session tokens."}], "category": "code-review", "description": "Clean codeunit using proper BC record operations: SetRange, SetFilter, FindSet, Count — no string concatenation or dynamic SQL", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-008", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/PartnerConfigKeyManager.Codeunit.al b/src/PartnerConfigKeyManager.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/PartnerConfigKeyManager.Codeunit.al\n@@ -0,0 +1,9 @@\n+codeunit 50100 \"Partner Config Key Manager\"\n+{\n+ Access = Internal;\n+\n+ internal procedure StoreApiKey(KeyName: Text; ApiKey: Text)\n+ begin\n+ IsolatedStorage.SetEncrypted(KeyName, ApiKey, DataScope::Module);\n+ end;\n+}\n", "expected_comments": [{"file": "src/PartnerConfigKeyManager.Codeunit.al", "line_start": 5, "line_end": 5, "domain": "security", "severity": "high", "body": "The API key is accepted as a plain Text parameter instead of SecretText, so the secret is exposed in memory and to anyone inspecting the call stack or debugger.", "article": "security/secrettext-for-credentials"}], "category": "code-review", "description": "True positive security findings: encryption (trimmed to 5 representative findings)", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-009", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/OutlookAddinDeployer.Codeunit.al b/src/OutlookAddinDeployer.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/OutlookAddinDeployer.Codeunit.al\n@@ -0,0 +1,37 @@\n+namespace Microsoft.Integration.Outlook;\n+\n+codeunit 50104 \"Outlook Addin Deployer\"\n+{\n+ Access = Internal;\n+\n+ var\n+ EndpointTok: Label 'https://outlook.office365.com/api/v2.0/addins/deploy', Locked = true;\n+ StatusErr: Label 'Deployment failed (HTTP %1): %2', Comment = '%1 is the HTTP status code, %2 is the raw response body.';\n+ ConnectErr: Label 'Failed to connect to the deployment service: %1', Comment = '%1 is the underlying error text.';\n+\n+ procedure DeployAddin(ManifestPath: Text)\n+ var\n+ Client: HttpClient;\n+ Content: HttpContent;\n+ Response: HttpResponseMessage;\n+ Headers: HttpHeaders;\n+ Payload: Text;\n+ ResponseText: Text;\n+ begin\n+ Payload := '{\"manifest\":\"' + ManifestPath + '\"}';\n+ Content.WriteFrom(Payload);\n+ Content.GetHeaders(Headers);\n+ Headers.Add('Authorization', 'Bearer ' + GetAccessToken());\n+ if Client.Post(EndpointTok, Content, Response) then begin\n+ Response.Content.ReadAs(ResponseText);\n+ if not Response.IsSuccessStatusCode() then\n+ Error(StatusErr, Response.HttpStatusCode(), ResponseText);\n+ end else\n+ Error(ConnectErr, GetLastErrorText());\n+ end;\n+\n+ local procedure GetAccessToken(): Text\n+ begin\n+ exit('dummy_access_token_for_testing');\n+ end;\n+}\n", "expected_comments": [{"file": "src/OutlookAddinDeployer.Codeunit.al", "line_start": 21, "line_end": 21, "domain": "security", "severity": "medium", "body": "The manifest path is concatenated directly into a JSON payload, allowing JSON injection. Build the payload with a JsonObject so values are escaped."}, {"file": "src/OutlookAddinDeployer.Codeunit.al", "line_start": 28, "line_end": 28, "domain": "security", "severity": "medium", "body": "The error surfaces the raw HTTP status code and full response body to the user, leaking internal service details. Log the details and show a generic message."}, {"file": "src/OutlookAddinDeployer.Codeunit.al", "line_start": 30, "line_end": 30, "domain": "security", "severity": "medium", "body": "GetLastErrorText() is shown to the user, exposing internal system details. Log the raw error and present a sanitized message."}, {"file": "src/OutlookAddinDeployer.Codeunit.al", "line_start": 33, "line_end": 33, "domain": "security", "severity": "high", "body": "The access token is returned as plain Text instead of SecretText, exposing it in memory and to the debugger. Return and handle it as SecretText.", "article": "security/secrettext-for-credentials"}, {"file": "src/OutlookAddinDeployer.Codeunit.al", "line_start": 35, "line_end": 35, "domain": "security", "severity": "high", "body": "A hardcoded access token is embedded in source code. Retrieve the token from a secure store or OAuth flow instead of hardcoding it.", "article": "security/secrettext-for-credentials"}], "category": "code-review", "description": "True positive security findings: error_exposure (verified line numbers)", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-011", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/ElecVATSubmission.Codeunit.al b/src/ElecVATSubmission.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ElecVATSubmission.Codeunit.al\n@@ -0,0 +1,24 @@\n+namespace Microsoft.Finance.VAT;\n+\n+codeunit 13610 \"Elec VAT Submission\"\n+{\n+ Access = Internal;\n+\n+ procedure SubmitReturn(AuthorityUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Content: HttpContent;\n+ Response: HttpResponseMessage;\n+ begin\n+ Content.WriteFrom('{}');\n+ exit(Client.Post(AuthorityUrl, Content, Response));\n+ end;\n+\n+ procedure CheckHealth(ServiceUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Response: HttpResponseMessage;\n+ begin\n+ exit(Client.Get(ServiceUrl, Response));\n+ end;\n+}\n", "expected_comments": [{"file": "src/ElecVATSubmission.Codeunit.al", "line_start": 14, "line_end": 14, "domain": "security", "severity": "high", "body": "A caller-supplied URL is passed to HttpClient.Post without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.", "article": "security/validate-user-configurable-urls"}, {"file": "src/ElecVATSubmission.Codeunit.al", "line_start": 22, "line_end": 22, "domain": "security", "severity": "high", "body": "A caller-supplied URL is passed to HttpClient.Get without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.", "article": "security/validate-user-configurable-urls"}], "category": "code-review", "description": "True positive security findings: input_validation (trimmed to core input validation cases)", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-012", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/HttpAuthenticationBasic.Codeunit.al b/src/HttpAuthenticationBasic.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/HttpAuthenticationBasic.Codeunit.al\n@@ -0,0 +1,37 @@\n+codeunit 2359 \"Http Authentication Basic\"\n+{\n+ Access = Public;\n+ InherentEntitlements = X;\n+ InherentPermissions = X;\n+\n+ var\n+ Credential: SecretText;\n+ UsernameDomainTok: Label '%1\\%2', Comment = '%1 = domain, %2 = user name', Locked = true;\n+\n+ [NonDebuggable]\n+ procedure Initialize(Username: SecretText; Domain: Text; Password: SecretText)\n+ begin\n+ Credential := SecretStrSubstNo('%1:%2', QualifyUser(Username, Domain), Password);\n+ end;\n+\n+ procedure GetAuthorizationHeader() Header: SecretText\n+ begin\n+ Header := ToBase64(Credential);\n+ end;\n+\n+ [NonDebuggable]\n+ local procedure QualifyUser(Username: SecretText; Domain: Text): SecretText\n+ begin\n+ if Domain = '' then\n+ exit(Username);\n+ exit(SecretStrSubstNo(UsernameDomainTok, Domain, Username));\n+ end;\n+\n+ local procedure ToBase64(Value: SecretText) Base64Value: SecretText\n+ var\n+ Convert: DotNet Convert;\n+ Encoding: DotNet Encoding;\n+ begin\n+ Base64Value := Convert.ToBase64String(Encoding.UTF8().GetBytes(Value.Unwrap()));\n+ end;\n+}\n", "expected_comments": [{"file": "src/HttpAuthenticationBasic.Codeunit.al", "line_start": 30, "line_end": 30, "domain": "security", "severity": "medium", "body": "ToBase64 transforms SecretText credential material and calls Unwrap() without [NonDebuggable], so the plaintext credential is visible in the debugger.", "article": "security/nondebuggable-required-when-unwrapping-secrettext"}], "category": "code-review", "description": "True positive security findings: procedures handling passwords or SecretText values without [NonDebuggable]", "expect_findings": true, "source": "vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-008","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/PartnerConfigKeyManager.Codeunit.al b/src/PartnerConfigKeyManager.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/PartnerConfigKeyManager.Codeunit.al\n@@ -0,0 +1,9 @@\n+codeunit 50100 \"Partner Config Key Manager\"\n+{\n+ Access = Internal;\n+\n+ internal procedure StoreApiKey(KeyName: Text; ApiKey: Text)\n+ begin\n+ IsolatedStorage.SetEncrypted(KeyName, ApiKey, DataScope::Module);\n+ end;\n+}\n","expected_comments":[{"file":"src/PartnerConfigKeyManager.Codeunit.al","line_start":5,"line_end":5,"domain":"security","severity":"high","body":"The API key is accepted as a plain Text parameter instead of SecretText, so the secret is exposed in memory and to anyone inspecting the call stack or debugger.","articles":["security/secrettext-for-credentials"]}],"category":"code-review","description":"True positive security findings: encryption (trimmed to 5 representative findings)","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-009","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/OutlookAddinDeployer.Codeunit.al b/src/OutlookAddinDeployer.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/OutlookAddinDeployer.Codeunit.al\n@@ -0,0 +1,37 @@\n+namespace Microsoft.Integration.Outlook;\n+\n+codeunit 50104 \"Outlook Addin Deployer\"\n+{\n+ Access = Internal;\n+\n+ var\n+ EndpointTok: Label 'https://outlook.office365.com/api/v2.0/addins/deploy', Locked = true;\n+ StatusErr: Label 'Deployment failed (HTTP %1): %2', Comment = '%1 is the HTTP status code, %2 is the raw response body.';\n+ ConnectErr: Label 'Failed to connect to the deployment service: %1', Comment = '%1 is the underlying error text.';\n+\n+ procedure DeployAddin(ManifestPath: Text)\n+ var\n+ Client: HttpClient;\n+ Content: HttpContent;\n+ Response: HttpResponseMessage;\n+ Headers: HttpHeaders;\n+ Payload: Text;\n+ ResponseText: Text;\n+ begin\n+ Payload := '{\"manifest\":\"' + ManifestPath + '\"}';\n+ Content.WriteFrom(Payload);\n+ Content.GetHeaders(Headers);\n+ Headers.Add('Authorization', 'Bearer ' + GetAccessToken());\n+ if Client.Post(EndpointTok, Content, Response) then begin\n+ Response.Content.ReadAs(ResponseText);\n+ if not Response.IsSuccessStatusCode() then\n+ Error(StatusErr, Response.HttpStatusCode(), ResponseText);\n+ end else\n+ Error(ConnectErr, GetLastErrorText());\n+ end;\n+\n+ local procedure GetAccessToken(): Text\n+ begin\n+ exit('dummy_access_token_for_testing');\n+ end;\n+}\n","expected_comments":[{"file":"src/OutlookAddinDeployer.Codeunit.al","line_start":21,"line_end":21,"domain":"security","severity":"medium","body":"The manifest path is concatenated directly into a JSON payload, allowing JSON injection. Build the payload with a JsonObject so values are escaped."},{"file":"src/OutlookAddinDeployer.Codeunit.al","line_start":28,"line_end":28,"domain":"security","severity":"medium","body":"The error surfaces the raw HTTP status code and full response body to the user, leaking internal service details. Log the details and show a generic message."},{"file":"src/OutlookAddinDeployer.Codeunit.al","line_start":30,"line_end":30,"domain":"security","severity":"medium","body":"GetLastErrorText() is shown to the user, exposing internal system details. Log the raw error and present a sanitized message."},{"file":"src/OutlookAddinDeployer.Codeunit.al","line_start":33,"line_end":33,"domain":"security","severity":"high","body":"The access token is returned as plain Text instead of SecretText, exposing it in memory and to the debugger. Return and handle it as SecretText.","articles":["security/secrettext-for-credentials"]},{"file":"src/OutlookAddinDeployer.Codeunit.al","line_start":35,"line_end":35,"domain":"security","severity":"high","body":"A hardcoded access token is embedded in source code. Retrieve the token from a secure store or OAuth flow instead of hardcoding it.","articles":["security/secrettext-for-credentials"]}],"category":"code-review","description":"True positive security findings: error_exposure (verified line numbers)","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-011","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/ElecVATSubmission.Codeunit.al b/src/ElecVATSubmission.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ElecVATSubmission.Codeunit.al\n@@ -0,0 +1,24 @@\n+namespace Microsoft.Finance.VAT;\n+\n+codeunit 13610 \"Elec VAT Submission\"\n+{\n+ Access = Internal;\n+\n+ procedure SubmitReturn(AuthorityUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Content: HttpContent;\n+ Response: HttpResponseMessage;\n+ begin\n+ Content.WriteFrom('{}');\n+ exit(Client.Post(AuthorityUrl, Content, Response));\n+ end;\n+\n+ procedure CheckHealth(ServiceUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Response: HttpResponseMessage;\n+ begin\n+ exit(Client.Get(ServiceUrl, Response));\n+ end;\n+}\n","expected_comments":[{"file":"src/ElecVATSubmission.Codeunit.al","line_start":14,"line_end":14,"domain":"security","severity":"high","body":"A caller-supplied URL is passed to HttpClient.Post without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.","articles":["security/validate-user-configurable-urls"]},{"file":"src/ElecVATSubmission.Codeunit.al","line_start":22,"line_end":22,"domain":"security","severity":"high","body":"A caller-supplied URL is passed to HttpClient.Get without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.","articles":["security/validate-user-configurable-urls"]}],"category":"code-review","description":"True positive security findings: input_validation (trimmed to core input validation cases)","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-012","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/HttpAuthenticationBasic.Codeunit.al b/src/HttpAuthenticationBasic.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/HttpAuthenticationBasic.Codeunit.al\n@@ -0,0 +1,37 @@\n+codeunit 2359 \"Http Authentication Basic\"\n+{\n+ Access = Public;\n+ InherentEntitlements = X;\n+ InherentPermissions = X;\n+\n+ var\n+ Credential: SecretText;\n+ UsernameDomainTok: Label '%1\\%2', Comment = '%1 = domain, %2 = user name', Locked = true;\n+\n+ [NonDebuggable]\n+ procedure Initialize(Username: SecretText; Domain: Text; Password: SecretText)\n+ begin\n+ Credential := SecretStrSubstNo('%1:%2', QualifyUser(Username, Domain), Password);\n+ end;\n+\n+ procedure GetAuthorizationHeader() Header: SecretText\n+ begin\n+ Header := ToBase64(Credential);\n+ end;\n+\n+ [NonDebuggable]\n+ local procedure QualifyUser(Username: SecretText; Domain: Text): SecretText\n+ begin\n+ if Domain = '' then\n+ exit(Username);\n+ exit(SecretStrSubstNo(UsernameDomainTok, Domain, Username));\n+ end;\n+\n+ local procedure ToBase64(Value: SecretText) Base64Value: SecretText\n+ var\n+ Convert: DotNet Convert;\n+ Encoding: DotNet Encoding;\n+ begin\n+ Base64Value := Convert.ToBase64String(Encoding.UTF8().GetBytes(Value.Unwrap()));\n+ end;\n+}\n","expected_comments":[{"file":"src/HttpAuthenticationBasic.Codeunit.al","line_start":30,"line_end":30,"domain":"security","severity":"medium","body":"ToBase64 transforms SecretText credential material and calls Unwrap() without [NonDebuggable], so the plaintext credential is visible in the debugger.","articles":["security/nondebuggable-required-when-unwrapping-secrettext"]}],"category":"code-review","description":"True positive security findings: procedures handling passwords or SecretText values without [NonDebuggable]","expect_findings":true,"source":"vsoadmin"} {"repo": "microsoft/BCApps", "instance_id": "synthetic__security-013", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/ExpenseAgentAdmin.PermissionSet.al b/src/ExpenseAgentAdmin.PermissionSet.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExpenseAgentAdmin.PermissionSet.al\n@@ -0,0 +1,15 @@\n+// ------------------------------------------------------------------------------------------------\n+// Copyright (c) Microsoft Corporation. All rights reserved.\n+// Licensed under the MIT License. See License.txt in the project root for license information.\n+// ------------------------------------------------------------------------------------------------\n+namespace Microsoft.Agents.Expense;\n+\n+permissionset 50700 \"Expense Agent Admin\"\n+{\n+ Assignable = true;\n+ Caption = 'Expense Agent Administration';\n+\n+ Permissions =\n+ tabledata \"Agent Creation Control\" = RIMD,\n+ tabledata \"Expense Report Rule Violation\" = IMD;\n+}\ndiff --git a/src/ExpenseAgentConsumption.Table.al b/src/ExpenseAgentConsumption.Table.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExpenseAgentConsumption.Table.al\n@@ -0,0 +1,51 @@\n+// ------------------------------------------------------------------------------------------------\n+// Copyright (c) Microsoft Corporation. All rights reserved.\n+// Licensed under the MIT License. See License.txt in the project root for license information.\n+// ------------------------------------------------------------------------------------------------\n+namespace Microsoft.Agents.Expense;\n+\n+table 50600 \"Expense Agent Consumption\"\n+{\n+ Caption = 'Expense Agent Consumption';\n+ DataClassification = CustomerContent;\n+ InherentEntitlements = RIX;\n+ InherentPermissions = RIX;\n+\n+ fields\n+ {\n+ field(1; \"Entry No.\"; Integer)\n+ {\n+ Caption = 'Entry No.';\n+ DataClassification = SystemMetadata;\n+ AutoIncrement = true;\n+ }\n+ field(10; Amount; Decimal)\n+ {\n+ Caption = 'Amount';\n+ DataClassification = CustomerContent;\n+ }\n+ field(20; \"User Security ID\"; Guid)\n+ {\n+ Caption = 'User Security ID';\n+ DataClassification = EndUserPseudonymousIdentifiers;\n+ }\n+ }\n+\n+ keys\n+ {\n+ key(PK; \"Entry No.\")\n+ {\n+ Clustered = true;\n+ }\n+ }\n+\n+ procedure LogConsumption(CallerSecurityId: Guid; ConsumptionAmount: Decimal)\n+ var\n+ ConsumptionEntry: Record \"Expense Agent Consumption\";\n+ begin\n+ ConsumptionEntry.Init();\n+ ConsumptionEntry.\"User Security ID\" := CallerSecurityId;\n+ ConsumptionEntry.Amount := ConsumptionAmount;\n+ ConsumptionEntry.Insert();\n+ end;\n+}\n", "expected_comments": [{"file": "src/ExpenseAgentConsumption.Table.al", "line_start": 11, "line_end": 11, "domain": "security", "severity": "medium", "body": "InherentEntitlements and InherentPermissions of RIX grant read/insert/execute to every user regardless of assigned permission sets. Remove the inherent grants and control access explicitly."}, {"file": "src/ExpenseAgentConsumption.Table.al", "line_start": 42, "line_end": 42, "domain": "security", "severity": "medium", "body": "The procedure accepts an arbitrary UserSecurityId, letting a caller log consumption against any user's identity. Derive the user from UserSecurityId() instead of trusting the parameter."}, {"file": "src/ExpenseAgentAdmin.PermissionSet.al", "line_start": 13, "line_end": 13, "domain": "security", "severity": "medium", "body": "RIMD on Agent Creation Control lets assigned users delete creation-control records, removing a security guardrail. Grant only the permissions actually required."}, {"file": "src/ExpenseAgentAdmin.PermissionSet.al", "line_start": 14, "line_end": 14, "domain": "security", "severity": "medium", "body": "IMD on Expense Report Rule Violation lets users delete recorded policy violations, enabling them to hide their own violations. Remove delete and modify access."}], "category": "code-review", "description": "True positive security findings: permission (trimmed to 5 representative findings)", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-014", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/SecureOperationHelper.Codeunit.al b/src/SecureOperationHelper.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SecureOperationHelper.Codeunit.al\n@@ -0,0 +1,13 @@\n+codeunit 50105 \"Secure Operation Helper\"\n+{\n+ Access = Internal;\n+\n+ internal procedure DeleteAllRecords(TableNo: Integer)\n+ var\n+ RecRef: RecordRef;\n+ begin\n+ RecRef.Open(TableNo);\n+ RecRef.DeleteAll();\n+ RecRef.Close();\n+ end;\n+}\n", "expected_comments": [{"file": "src/SecureOperationHelper.Codeunit.al", "line_start": 9, "line_end": 10, "domain": "security", "severity": "high", "body": "A caller-provided table number is opened with RecordRef.Open and then DeleteAll is called, letting any caller delete every record in an arbitrary table. Restrict the allowed tables and enforce permission checks.", "article": "security/recordref-open-with-caller-table-must-not-be-public"}], "category": "code-review", "description": "True positive: public procedure uses RecordRef.Open with caller-provided table number, allowing any extension to delete all records from any table through this codeunit's permissions", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-015", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/ExternalIntegrationMgt.Codeunit.al b/src/ExternalIntegrationMgt.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExternalIntegrationMgt.Codeunit.al\n@@ -0,0 +1,24 @@\n+namespace Microsoft.Integration.Partner;\n+\n+codeunit 50205 \"External Integration Mgt.\"\n+{\n+ Access = Internal;\n+\n+ procedure PostToPartner(EndpointUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Content: HttpContent;\n+ Response: HttpResponseMessage;\n+ begin\n+ Content.WriteFrom('{}');\n+ exit(Client.Post(EndpointUrl, Content, Response));\n+ end;\n+\n+ procedure GetFromProvider(EndpointUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Response: HttpResponseMessage;\n+ begin\n+ exit(Client.Get(EndpointUrl, Response));\n+ end;\n+}\n", "expected_comments": [{"file": "src/ExternalIntegrationMgt.Codeunit.al", "line_start": 14, "line_end": 14, "domain": "security", "severity": "high", "body": "A caller-supplied URL is passed to HttpClient.Post without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.", "article": "security/validate-user-configurable-urls"}, {"file": "src/ExternalIntegrationMgt.Codeunit.al", "line_start": 22, "line_end": 22, "domain": "security", "severity": "high", "body": "A caller-supplied URL is passed to HttpClient.Get without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.", "article": "security/validate-user-configurable-urls"}], "category": "code-review", "description": "True positive security findings: URLs from table fields used in HTTP requests without validation (SSRF risk). Three procedures use user-configurable URLs directly, while two procedures correctly validate using Uri.AreURIsHaveSameHost and Uri.IsValidURIPattern.", "expect_findings": true, "source": "vsoadmin"} -{"repo": "microsoft/BCApps", "instance_id": "synthetic__security-016", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "security"}, "patch": "diff --git a/src/ExpenseHtmlNotifier.Codeunit.al b/src/ExpenseHtmlNotifier.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExpenseHtmlNotifier.Codeunit.al\n@@ -0,0 +1,12 @@\n+codeunit 50900 \"Expense Html Notifier\"\n+{\n+ Access = Internal;\n+\n+ var\n+ BodyTemplateTok: Label '

Dear %1,

%2

', Locked = true;\n+\n+ internal procedure BuildNotificationBody(EmployeeName: Text; Description: Text): Text\n+ begin\n+ exit(StrSubstNo(BodyTemplateTok, EmployeeName, Description));\n+ end;\n+}\n", "expected_comments": [{"file": "src/ExpenseHtmlNotifier.Codeunit.al", "line_start": 10, "line_end": 10, "domain": "security", "severity": "high", "body": "User-supplied EmployeeName and Description are substituted into the HTML body without encoding, enabling stored or reflected XSS. HTML-encode the values before embedding them.", "article": "security/al-has-no-built-in-htmlencode"}], "category": "code-review", "description": "True positive security findings: xss (user-supplied data embedded in HTML without encoding)", "expect_findings": true, "source": "vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-014","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/SecureOperationHelper.Codeunit.al b/src/SecureOperationHelper.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SecureOperationHelper.Codeunit.al\n@@ -0,0 +1,13 @@\n+codeunit 50105 \"Secure Operation Helper\"\n+{\n+ Access = Internal;\n+\n+ internal procedure DeleteAllRecords(TableNo: Integer)\n+ var\n+ RecRef: RecordRef;\n+ begin\n+ RecRef.Open(TableNo);\n+ RecRef.DeleteAll();\n+ RecRef.Close();\n+ end;\n+}\n","expected_comments":[{"file":"src/SecureOperationHelper.Codeunit.al","line_start":9,"line_end":10,"domain":"security","severity":"high","body":"A caller-provided table number is opened with RecordRef.Open and then DeleteAll is called, letting any caller delete every record in an arbitrary table. Restrict the allowed tables and enforce permission checks.","articles":["security/recordref-open-with-caller-table-must-not-be-public"]}],"category":"code-review","description":"True positive: public procedure uses RecordRef.Open with caller-provided table number, allowing any extension to delete all records from any table through this codeunit's permissions","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-015","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/ExternalIntegrationMgt.Codeunit.al b/src/ExternalIntegrationMgt.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExternalIntegrationMgt.Codeunit.al\n@@ -0,0 +1,24 @@\n+namespace Microsoft.Integration.Partner;\n+\n+codeunit 50205 \"External Integration Mgt.\"\n+{\n+ Access = Internal;\n+\n+ procedure PostToPartner(EndpointUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Content: HttpContent;\n+ Response: HttpResponseMessage;\n+ begin\n+ Content.WriteFrom('{}');\n+ exit(Client.Post(EndpointUrl, Content, Response));\n+ end;\n+\n+ procedure GetFromProvider(EndpointUrl: Text): Boolean\n+ var\n+ Client: HttpClient;\n+ Response: HttpResponseMessage;\n+ begin\n+ exit(Client.Get(EndpointUrl, Response));\n+ end;\n+}\n","expected_comments":[{"file":"src/ExternalIntegrationMgt.Codeunit.al","line_start":14,"line_end":14,"domain":"security","severity":"high","body":"A caller-supplied URL is passed to HttpClient.Post without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.","articles":["security/validate-user-configurable-urls"]},{"file":"src/ExternalIntegrationMgt.Codeunit.al","line_start":22,"line_end":22,"domain":"security","severity":"high","body":"A caller-supplied URL is passed to HttpClient.Get without host or scheme validation, allowing SSRF requests to internal or attacker-chosen endpoints.","articles":["security/validate-user-configurable-urls"]}],"category":"code-review","description":"True positive security findings: URLs from table fields used in HTTP requests without validation (SSRF risk). Three procedures use user-configurable URLs directly, while two procedures correctly validate using Uri.AreURIsHaveSameHost and Uri.IsValidURIPattern.","expect_findings":true,"source":"vsoadmin"} +{"repo":"microsoft/BCApps","instance_id":"synthetic__security-016","base_commit":"70fd0246a0a4dbc72cb183ca719106722c03be4d","created_at":"2026-05-15T00:00:00Z","environment_setup_version":"27.0","project_paths":[],"metadata":{"area":"security"},"patch":"diff --git a/src/ExpenseHtmlNotifier.Codeunit.al b/src/ExpenseHtmlNotifier.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/ExpenseHtmlNotifier.Codeunit.al\n@@ -0,0 +1,12 @@\n+codeunit 50900 \"Expense Html Notifier\"\n+{\n+ Access = Internal;\n+\n+ var\n+ BodyTemplateTok: Label '

Dear %1,

%2

', Locked = true;\n+\n+ internal procedure BuildNotificationBody(EmployeeName: Text; Description: Text): Text\n+ begin\n+ exit(StrSubstNo(BodyTemplateTok, EmployeeName, Description));\n+ end;\n+}\n","expected_comments":[{"file":"src/ExpenseHtmlNotifier.Codeunit.al","line_start":10,"line_end":10,"domain":"security","severity":"high","body":"User-supplied EmployeeName and Description are substituted into the HTML body without encoding, enabling stored or reflected XSS. HTML-encode the values before embedding them.","articles":["security/al-has-no-built-in-htmlencode"]}],"category":"code-review","description":"True positive security findings: xss (user-supplied data embedded in HTML without encoding)","expect_findings":true,"source":"vsoadmin"} {"repo": "microsoft/BCApps", "instance_id": "synthetic__performance-001", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "performance"}, "patch": "diff --git a/src/FADepreciationBook.Table.al b/src/FADepreciationBook.Table.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/FADepreciationBook.Table.al\n@@ -0,0 +1,78 @@\n+table 50200 \"FA Depreciation Book FP\"\n+{\n+ DataClassification = CustomerContent;\n+\n+ fields\n+ {\n+ field(1; \"FA No.\"; Code[20])\n+ {\n+ Caption = 'FA No.';\n+ TableRelation = \"Fixed Asset\";\n+ }\n+\n+ field(2; \"Depreciation Book Code\"; Code[10])\n+ {\n+ Caption = 'Depreciation Book Code';\n+ TableRelation = \"Depreciation Book\";\n+ }\n+\n+ field(3; Depreciation; Decimal)\n+ {\n+ FieldClass = FlowField;\n+ CalcFormula = sum(\"FA Ledger Entry\".Amount where(\"FA No.\" = field(\"FA No.\"),\n+ \"Depreciation Book Code\" = field(\"Depreciation Book Code\"),\n+ \"FA Posting Category\" = const(Depreciation)));\n+ Caption = 'Depreciation';\n+ }\n+\n+ field(4; \"Bonus Depr. Applied Amount\"; Decimal)\n+ {\n+ FieldClass = FlowField;\n+ CalcFormula = sum(\"FA Ledger Entry\".Amount where(\"FA No.\" = field(\"FA No.\"),\n+ \"Depreciation Book Code\" = field(\"Depreciation Book Code\"),\n+ \"FA Posting Type\" = const(\"Bonus Depreciation\")));\n+ Caption = 'Bonus Depr. Applied Amount';\n+ }\n+\n+ field(5; \"Use Half-Year Convention\"; Boolean)\n+ {\n+ Caption = 'Use Half-Year Convention';\n+\n+ trigger OnValidate()\n+ var\n+ CannotChangeHalfYearErr: Label 'Cannot change half-year convention after depreciation has been posted.';\n+ CannotChangeBonusErr: Label 'Cannot change half-year convention when bonus depreciation has been applied.';\n+ begin\n+ // CORRECT: CalcFields in OnValidate runs once per user edit, not in a loop\n+ // This is appropriate for validation logic that needs current flowfield values\n+ CalcFields(Depreciation);\n+ if Depreciation <> 0 then\n+ Error(CannotChangeHalfYearErr);\n+\n+ CalcFields(\"Bonus Depr. Applied Amount\");\n+ if \"Bonus Depr. Applied Amount\" <> 0 then\n+ Error(CannotChangeBonusErr);\n+ end;\n+ }\n+\n+ field(6; \"Depreciation Method\"; Option)\n+ {\n+ OptionCaption = 'Straight-Line,Declining-Balance 1,Declining-Balance 2';\n+ OptionMembers = \"Straight-Line\",\"Declining-Balance 1\",\"Declining-Balance 2\";\n+ Caption = 'Depreciation Method';\n+ }\n+\n+ field(7; \"Starting Date\"; Date)\n+ {\n+ Caption = 'Depreciation Starting Date';\n+ }\n+ }\n+\n+ keys\n+ {\n+ key(PK; \"FA No.\", \"Depreciation Book Code\")\n+ {\n+ Clustered = true;\n+ }\n+ }\n+}\ndiff --git a/src/SalesOrderCard.Page.al b/src/SalesOrderCard.Page.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SalesOrderCard.Page.al\n@@ -0,0 +1,87 @@\n+page 50201 \"Sales Order Card FP\"\n+{\n+ PageType = Card;\n+ SourceTable = \"Sales Header\";\n+ Caption = 'Sales Order Card FP';\n+\n+ layout\n+ {\n+ area(Content)\n+ {\n+ group(General)\n+ {\n+ Caption = 'General';\n+\n+ field(\"No.\"; Rec.\"No.\")\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies the number of the sales order.';\n+ }\n+\n+ field(\"Sell-to Customer No.\"; Rec.\"Sell-to Customer No.\")\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies the number of the customer who will receive the products on the sales order.';\n+ }\n+\n+ field(\"Document Date\"; Rec.\"Document Date\")\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies the date when the sales order was created.';\n+ }\n+\n+ field(\"Total Amount\"; TotalAmount)\n+ {\n+ Caption = 'Total Amount Including VAT';\n+ ApplicationArea = All;\n+ Editable = false;\n+ ToolTip = 'Specifies the total amount including VAT for the sales order.';\n+ }\n+ }\n+ }\n+ }\n+\n+ actions\n+ {\n+ area(Processing)\n+ {\n+ action(RefreshTotals)\n+ {\n+ Caption = 'Refresh Totals';\n+ ApplicationArea = All;\n+ ToolTip = 'Recalculates and refreshes the total amount for the sales order.';\n+ Image = Refresh;\n+\n+ trigger OnAction()\n+ var\n+ TotalRefreshedMsg: Label 'Total refreshed: %1', Comment = '%1 = total amount including VAT';\n+ begin\n+ // CORRECT: Manual refresh action - user-initiated, runs once\n+ Rec.CalcFields(\"Amount Including VAT\");\n+ TotalAmount := Rec.\"Amount Including VAT\";\n+ Message(TotalRefreshedMsg, TotalAmount);\n+ end;\n+ }\n+ }\n+ }\n+\n+ var\n+ TotalAmount: Decimal;\n+\n+ // CORRECT: OnAfterGetCurrRecord fires once per record selection, not per row\n+ // This is the appropriate place to calculate values when user navigates to a record\n+ trigger OnAfterGetCurrRecord()\n+ begin\n+ // Calculate total amount when user selects a sales order\n+ // This runs once when the record is loaded/selected, not in a loop\n+ Rec.CalcFields(\"Amount Including VAT\");\n+ TotalAmount := Rec.\"Amount Including VAT\";\n+ end;\n+\n+ trigger OnNewRecord(BelowxRec: Boolean)\n+ begin\n+ // CORRECT: Initialize values for new record - runs once per new record creation\n+ TotalAmount := 0;\n+ Rec.\"Document Date\" := WorkDate();\n+ end;\n+}\ndiff --git a/src/CustLedgerEntryAggregator.Codeunit.al b/src/CustLedgerEntryAggregator.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/CustLedgerEntryAggregator.Codeunit.al\n@@ -0,0 +1,20 @@\n+codeunit 50202 \"Cust Ledger Entry Aggregator\"\n+{\n+ procedure SumOpenRemainingAmount(CustomerNo: Code[20]): Decimal\n+ var\n+ CustLedgerEntry: Record \"Cust. Ledger Entry\";\n+ Customer: Record Customer;\n+ Total: Decimal;\n+ begin\n+ CustLedgerEntry.SetRange(\"Customer No.\", CustomerNo);\n+ CustLedgerEntry.SetRange(Open, true);\n+ if CustLedgerEntry.FindSet() then\n+ repeat\n+ CustLedgerEntry.CalcFields(\"Remaining Amount\");\n+ Customer.Get(CustLedgerEntry.\"Customer No.\");\n+ if Customer.\"Application Method\" = Customer.\"Application Method\"::Manual then\n+ Total += CustLedgerEntry.\"Remaining Amount\";\n+ until CustLedgerEntry.Next() = 0;\n+ exit(Total);\n+ end;\n+}\n", "expected_comments": [{"file": "src/CustLedgerEntryAggregator.Codeunit.al", "line_start": 13, "line_end": 13, "body": "CalcFields(\"Remaining Amount\") inside a repeat..until loop over \"Cust. Ledger Entry\" (up to 10M rows) issues one SQL query per iteration — classic N+1 against a hot table. — Replace the loop with `CustLedgerEntry.CalcSums(\"Remaining Amount\")` which executes as a single SUM query, or use a SIFT-backed key.", "severity": "high", "domain": "performance"}, {"file": "src/CustLedgerEntryAggregator.Codeunit.al", "line_start": 14, "line_end": 14, "body": "Customer.Get(CustLedgerEntry.\"Customer No.\") inside a repeat..until over Cust. Ledger Entry is an N+1 query: one Customer lookup per ledger row, redundant since every row already has the same Customer No. (the loop is filtered by CustomerNo). — Move the Customer.Get above the loop (single lookup), and add `Customer.SetLoadFields(\"Application Method\")` since only one field is read.", "severity": "high", "domain": "performance"}], "category": "code-review", "description": "False positive performance findings: calcfields_false_positive (30 false positives). Agent flagged these but reviewers rejected them. Enriched with 2 true-positive findings in addition to the false-positive bait.", "expect_findings": true, "source": "vsoadmin"} {"repo": "microsoft/BCApps", "instance_id": "synthetic__performance-002", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "performance"}, "patch": "diff --git a/src/SetupReader.Codeunit.al b/src/SetupReader.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SetupReader.Codeunit.al\n@@ -0,0 +1,67 @@\n+codeunit 50211 \"Setup Reader\"\n+{\n+ procedure GetSetupValues()\n+ var\n+ GLSetup: Record \"General Ledger Setup\";\n+ SalesSetup: Record \"Sales & Receivables Setup\";\n+ InventorySetup: Record \"Inventory Setup\";\n+ PurchSetup: Record \"Purchases & Payables Setup\";\n+ begin\n+ // CORRECT: Setup tables typically have only 1 record per company\n+ // Any access pattern (Get, FindSet, FindFirst) is fine for singleton tables\n+ GLSetup.Get();\n+ SalesSetup.Get();\n+ InventorySetup.Get();\n+ PurchSetup.Get();\n+\n+ if GLSetup.\"Additional Reporting Currency\" <> '' then\n+ ProcessACYSettings(GLSetup);\n+\n+ if SalesSetup.\"Credit Warnings\" <> SalesSetup.\"Credit Warnings\"::\"No Warning\" then\n+ EnableCreditWarnings(SalesSetup);\n+ end;\n+\n+ procedure ValidateCompanySettings(): Boolean\n+ var\n+ CompanyInfo: Record \"Company Information\";\n+ begin\n+ // CORRECT: Company Information is a singleton table (1 record per company)\n+ // Get() is the appropriate method for singleton tables\n+ if not CompanyInfo.Get() then\n+ exit(false);\n+\n+ if CompanyInfo.Name = '' then\n+ exit(false);\n+\n+ if CompanyInfo.\"Country/Region Code\" = '' then\n+ exit(false);\n+\n+ exit(true);\n+ end;\n+\n+ procedure GetUserSetupForCurrentUser(var UserSetup: Record \"User Setup\"): Boolean\n+ begin\n+ // CORRECT: Looking up single user's setup record\n+ // Get() with UserId is appropriate for single-record lookup\n+ UserSetup.Reset();\n+ if UserSetup.Get(UserId) then\n+ exit(true);\n+ exit(false);\n+ end;\n+\n+ local procedure ProcessACYSettings(GLSetup: Record \"General Ledger Setup\")\n+ var\n+ ACYEnabledMsg: Label 'ACY is enabled: %1', Comment = '%1 = additional reporting currency';\n+ begin\n+ // Process additional currency settings\n+ Message(ACYEnabledMsg, GLSetup.\"Additional Reporting Currency\");\n+ end;\n+\n+ local procedure EnableCreditWarnings(SalesSetup: Record \"Sales & Receivables Setup\")\n+ var\n+ CreditWarningsEnabledMsg: Label 'Credit warnings are enabled';\n+ begin\n+ // Enable credit warning processing\n+ Message(CreditWarningsEnabledMsg);\n+ end;\n+}\ndiff --git a/src/TempBufferProcessor.Codeunit.al b/src/TempBufferProcessor.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/TempBufferProcessor.Codeunit.al\n@@ -0,0 +1,61 @@\n+codeunit 50210 \"Temp Buffer Processor\"\n+{\n+ procedure ProcessBufferEntries(var TempBuffer: Record \"Integer\" temporary)\n+ var\n+ ProcessedCount: Integer;\n+ TotalAmount: Decimal;\n+ ProcessedEntriesMsg: Label 'Processed %1 entries with total %2', Comment = '%1 = number of entries, %2 = total amount';\n+ begin\n+ // CORRECT: TempBuffer is temporary — all operations are in-memory, no SQL queries\n+ // Any access pattern (FindSet, Get, loops) on temp tables is performant\n+ ProcessedCount := 0;\n+ TotalAmount := 0;\n+\n+ if TempBuffer.FindSet() then\n+ repeat\n+ // This might look suspicious, but it's CORRECT because:\n+ // 1. TempBuffer is temporary (in-memory)\n+ // 2. No database round trips are happening\n+ // 3. All data is already loaded in memory\n+ TotalAmount += TempBuffer.Number;\n+ ProcessedCount += 1;\n+\n+ // Even modifying temp records in a loop is fine\n+ TempBuffer.Number := TempBuffer.Number * 2;\n+ TempBuffer.Modify();\n+\n+ until TempBuffer.Next() = 0;\n+\n+ Message(ProcessedEntriesMsg, ProcessedCount, TotalAmount);\n+ end;\n+\n+ procedure BuildTempData(var TempBuffer: Record \"Integer\" temporary)\n+ var\n+ i: Integer;\n+ begin\n+ // CORRECT: Building temp data - all operations are in-memory\n+ TempBuffer.Reset();\n+ TempBuffer.DeleteAll();\n+\n+ for i := 1 to 100 do begin\n+ TempBuffer.Init();\n+ TempBuffer.Number := Random(1000);\n+ TempBuffer.Insert();\n+ end;\n+ end;\n+\n+ procedure FindMaxValue(var TempBuffer: Record \"Integer\" temporary): Integer\n+ var\n+ MaxValue: Integer;\n+ begin\n+ // CORRECT: Finding max in temp table - no performance concern\n+ MaxValue := 0;\n+ if TempBuffer.FindSet() then\n+ repeat\n+ if TempBuffer.Number > MaxValue then\n+ MaxValue := TempBuffer.Number;\n+ until TempBuffer.Next() = 0;\n+\n+ exit(MaxValue);\n+ end;\n+}\ndiff --git a/src/CustomerLookup.Codeunit.al b/src/CustomerLookup.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/CustomerLookup.Codeunit.al\n@@ -0,0 +1,20 @@\n+codeunit 50212 \"Customer Lookup\"\n+{\n+ procedure GetCustomerName(CustomerNo: Code[20]): Text[100]\n+ var\n+ Customer: Record Customer;\n+ begin\n+ Customer.SetRange(\"No.\", CustomerNo);\n+ if Customer.FindFirst() then\n+ exit(Customer.Name);\n+ exit('');\n+ end;\n+\n+ procedure HasCustomersInCountry(CountryRegionCode: Code[10]): Boolean\n+ var\n+ Customer: Record Customer;\n+ begin\n+ Customer.SetRange(\"Country/Region Code\", CountryRegionCode);\n+ exit(Customer.Count() > 0);\n+ end;\n+}\n", "expected_comments": [{"file": "src/CustomerLookup.Codeunit.al", "line_start": 8, "line_end": 8, "body": "FindFirst() after SetRange on the full primary key (\"No.\") of Customer (up to 800k rows). This still does a SQL SELECT TOP 1 with a range predicate instead of a direct key lookup. — Replace with `if Customer.Get(CustomerNo) then exit(Customer.Name);` which is a direct PK lookup (CodeCop AA0233).", "severity": "medium", "domain": "performance"}, {"file": "src/CustomerLookup.Codeunit.al", "line_start": 18, "line_end": 18, "body": "Count() > 0 on Customer (up to 800k rows) for a pure existence check. Count() materializes a SQL COUNT(*) over the filtered set instead of stopping at the first matching row. — Replace with `exit(not Customer.IsEmpty());` which stops at the first match and is significantly cheaper on large tables.", "severity": "medium", "domain": "performance"}], "category": "code-review", "description": "False positive performance findings: findset_false_positive (69 false positives). Agent flagged these but reviewers rejected them. Enriched with 2 true-positive findings in addition to the false-positive bait.", "expect_findings": true, "source": "vsoadmin"} {"repo": "microsoft/BCApps", "instance_id": "synthetic__performance-003", "base_commit": "70fd0246a0a4dbc72cb183ca719106722c03be4d", "created_at": "2026-05-15T00:00:00Z", "environment_setup_version": "27.0", "project_paths": [], "metadata": {"area": "performance"}, "patch": "diff --git a/src/MigrationSetupHandler.Codeunit.al b/src/MigrationSetupHandler.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/MigrationSetupHandler.Codeunit.al\n@@ -0,0 +1,18 @@\n+codeunit 50221 \"Migration Setup Handler\"\n+{\n+ procedure CountMigratablePermissionSets(): Integer\n+ var\n+ PermissionSet: Record \"Permission Set\";\n+ begin\n+ PermissionSet.SetFilter(\"Role ID\", '%1|%2', 'D365 BASIC', 'D365 READ');\n+ exit(PermissionSet.Count());\n+ end;\n+\n+ procedure CountObsoleteRegisters(): Integer\n+ var\n+ DateComprRegister: Record \"Date Compr. Register\";\n+ begin\n+ DateComprRegister.SetFilter(\"Ending Date\", '<%1', CalcDate('<-2Y>', Today));\n+ exit(DateComprRegister.Count());\n+ end;\n+}\ndiff --git a/src/PermissionSetListOverview.Page.al b/src/PermissionSetListOverview.Page.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/PermissionSetListOverview.Page.al\n@@ -0,0 +1,84 @@\n+page 50220 \"Permission Set List Overview\"\n+{\n+ PageType = List;\n+ ApplicationArea = All;\n+ UsageCategory = Administration;\n+ SourceTable = \"Aggregate Permission Set\";\n+ Caption = 'Permission Set List Overview';\n+ Editable = false;\n+\n+ layout\n+ {\n+ area(Content)\n+ {\n+ repeater(Permissions)\n+ {\n+ field(\"Role ID\"; Rec.\"Role ID\")\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies the identifier of the permission set.';\n+ }\n+\n+ field(Name; Rec.Name)\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies the display name of the permission set.';\n+ }\n+\n+ field(Scope; Rec.Scope)\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies whether the permission set is defined by the system or by a tenant.';\n+ }\n+\n+ field(\"App Name\"; Rec.\"App Name\")\n+ {\n+ ApplicationArea = All;\n+ ToolTip = 'Specifies the name of the extension that defines the permission set.';\n+ }\n+\n+ field(\"Permission Count\"; PermissionCount)\n+ {\n+ Caption = 'Permission Count';\n+ ApplicationArea = All;\n+ Editable = false;\n+ ToolTip = 'Specifies the number of permissions that belong to the permission set.';\n+ }\n+ }\n+ }\n+ }\n+\n+ actions\n+ {\n+ area(Processing)\n+ {\n+ action(RefreshCounts)\n+ {\n+ Caption = 'Refresh Permission Counts';\n+ ApplicationArea = All;\n+ ToolTip = 'Recalculates the permission count shown for each permission set.';\n+\n+ trigger OnAction()\n+ begin\n+ CurrPage.Update();\n+ end;\n+ }\n+ }\n+ }\n+\n+ var\n+ PermissionCount: Integer;\n+\n+ trigger OnAfterGetRecord()\n+ var\n+ Permission: Record Permission;\n+ begin\n+ Permission.SetRange(\"Role ID\", Rec.\"Role ID\");\n+ PermissionCount := Permission.Count();\n+ end;\n+\n+ trigger OnOpenPage()\n+ begin\n+ Rec.SetFilter(Scope, '%1|%2', Rec.Scope::System, Rec.Scope::Tenant);\n+ end;\n+}\ndiff --git a/src/SalesInvoiceFilter.Codeunit.al b/src/SalesInvoiceFilter.Codeunit.al\nnew file mode 100644\n--- /dev/null\n+++ b/src/SalesInvoiceFilter.Codeunit.al\n@@ -0,0 +1,26 @@\n+codeunit 50222 \"Sales Invoice Filter\"\n+{\n+ procedure ListLinesByDescription(Description: Text[100])\n+ var\n+ SalesInvoiceLine: Record \"Sales Invoice Line\";\n+ begin\n+ SalesInvoiceLine.SetRange(Description, Description);\n+ if SalesInvoiceLine.FindSet() then\n+ repeat\n+ Message('%1 %2', SalesInvoiceLine.\"Document No.\", SalesInvoiceLine.\"Line No.\");\n+ until SalesInvoiceLine.Next() = 0;\n+ end;\n+\n+ procedure SumQuantityByDocument(DocumentNo: Code[20]): Decimal\n+ var\n+ SalesInvoiceLine: Record \"Sales Invoice Line\";\n+ Total: Decimal;\n+ begin\n+ SalesInvoiceLine.SetRange(\"Document No.\", DocumentNo);\n+ if SalesInvoiceLine.FindSet() then\n+ repeat\n+ Total += SalesInvoiceLine.Quantity;\n+ until SalesInvoiceLine.Next() = 0;\n+ exit(Total);\n+ end;\n+}\n", "expected_comments": [{"file": "src/SalesInvoiceFilter.Codeunit.al", "line_start": 7, "line_end": 7, "body": "SetRange on Sales Invoice Line.Description with no SetCurrentKey and no key including Description. Sales Invoice Line is large (up to 3M rows) and the query will table-scan. — Either add `SetCurrentKey` to a key whose leading field matches the filter, or introduce a new key on the source table that covers Description, before filtering.", "severity": "high", "domain": "performance"}, {"file": "src/SalesInvoiceFilter.Codeunit.al", "line_start": 20, "line_end": 20, "body": "FindSet over Sales Invoice Line (3M rows, ~80 fields) loads every field for every row but the loop only reads `Quantity`. — Add `SalesInvoiceLine.SetLoadFields(Quantity);` before SetRange so SQL returns only the Quantity column (plus key fields). Even better, replace the loop with `SalesInvoiceLine.CalcSums(Quantity)` since Quantity is a SumIndexField on this table.", "severity": "medium", "domain": "performance"}, {"file": "src/PermissionSetListOverview.Page.al", "line_start": 77, "line_end": 77, "body": "PermissionCount is computed with Permission.Count() inside OnAfterGetRecord, which fires once per row rendered on this List page (and again while scrolling), so every visible row issues its own SQL COUNT against the Permission table (a per-row N+1 that scales with the number of permission sets shown). — Expose the value as a FlowField on the source table (FieldClass = FlowField, CalcFormula = count(Permission where(\"Role ID\" = field(\"Role ID\")))) so the aggregate is computed by the query engine instead of recomputing it per row.", "severity": "high", "domain": "performance"}], "category": "code-review", "description": "False positive performance findings: index_false_positive (29 false positives). Agent flagged these but reviewers rejected them. Enriched with 2 true-positive findings in addition to the false-positive bait.", "expect_findings": true, "source": "vsoadmin"} diff --git a/docs/code-review.md b/docs/code-review.md index e3c1513f3..59215f557 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -6,7 +6,7 @@ title: Code Review - BC-Bench # Code Review @@ -45,23 +27,21 @@ Unlike the pass/fail categories, code review is scored with **Precision / Recall A gold entry may also declare **`ignored_comments`** — legitimate-but-optional observations (out-of-scope nitpicks, maintainer-judgment calls) that should be neither required nor penalized. Ignored comments are structurally paired against the generated comments and validated by the same LLM judge, in a single judge pass alongside the expected comments. Any generated comment the judge confirms as an ignored match is dropped from scoring entirely: it earns no recall and does not count against precision. Expected always takes precedence, so a comment that could match both is credited as a real find; a comment that does not hold up as an expected match can still be neutralized as ignored rather than counting as a false positive. Most entries leave `ignored_comments` empty, which scores identically to before. -## Configuring Engine Experiments +## Category and runners + +`code-review` is the evaluation contract: it owns the dataset, structured `review.json` output, scorer, result schema, and leaderboard schema. A runner is the system under test. The same entries can be evaluated through the generic GitHub Copilot CLI and Claude Code runners, allowing direct cross-system comparisons under one scorer. -Code Review runs the production BC-ALAgents generate path. A BC-Bench experiment branch can independently select BC-ALAgents and BCQuality sources in `src/bcbench/agent/shared/config.yaml`: +BC PR Review is a separate agent harness fixed to the `code-review` category. It runs the production BC-ALAgents review engine with BCQuality, while generic Copilot and Claude runners continue to use their own prompts and configuration: -```yaml -pr_review: - engine: - repo: microsoft/BC-ALAgents - ref: main - local_path: null - bcquality: - repo: microsoft/BCQuality - ref: main - local_path: null +```text +bcbench evaluate copilot --category code-review +bcbench evaluate claude --category code-review +bcbench evaluate pr-review ``` -`ref` accepts a branch, tag, or commit. Set either `local_path` for an unpushed local checkout; `BC_PR_REVIEW_ROOT` remains the highest-priority BC-ALAgents local override. The `run code-review` and `evaluate code-review` commands also expose `--engine-repo`, `--engine-ref`, `--engine-local-path`, and matching `--bcquality-*` options. Results record the resolved commits for both sources. +The evaluation workflow pins BC-ALAgents to a commit SHA. Engine updates require a new BC-Bench version and must record that SHA in the release notes. + +BC PR Review records wall-clock duration and two structural BCQuality counts: Markdown knowledge files available after filtering and knowledge files removed by the filter. The counts come from the filtered checkout and its validated `_filter-report.json`. The production `all` pipeline does not currently expose a stable structured API-call, token, or credit contract, so BC-Bench intentionally does not infer those values from console transcripts. ## Baseline Leaderboard @@ -99,117 +79,90 @@ pr_review:

No results available yet. Check back soon!

{% endif %} +## Performance Leaderboard + +{% if site.data.code-review.aggregate and site.data.code-review.aggregate.size > 0 %} + + + + + + + + + + + + + + + + + + {% assign performance_results = site.data.code-review.aggregate | sort: "average_duration" %} + {% for agg in performance_results %} + + + + + + + + + + + + + + {% endfor %} + +
AgentModelAvg TimeAvg Total TokensAvg API CallsAvg AI CreditsAvg Premium RequestsComplete UsageAvg Knowledge FilesAvg Knowledge PrunedVer
{{ agg.agent_name }}{{ agg.model }}{{ agg.average_duration | round: 1 }}s{% if agg.average_total_tokens != null %}{{ agg.average_total_tokens | round: 0 }}{% else %}—{% endif %}{% if agg.average_api_calls != null %}{{ agg.average_api_calls | round: 1 }}{% else %}—{% endif %}{% if agg.average_ai_credits != null %}{{ agg.average_ai_credits | round: 4 }}{% else %}—{% endif %}{% if agg.average_premium_requests != null %}{{ agg.average_premium_requests | round: 4 }}{% else %}—{% endif %}{% if agg.structured_usage_complete_rate != null %}{{ agg.structured_usage_complete_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %}{% if agg.average_knowledge_files != null %}{{ agg.average_knowledge_files | round: 1 }}{% else %}—{% endif %}{% if agg.average_knowledge_pruned != null %}{{ agg.average_knowledge_pruned | round: 1 }}{% else %}—{% endif %}{{ agg.benchmark_version }}
+{% else %} +

No performance results available yet. Check back soon!

+{% endif %} + ## Experiment Leaderboard Compares review-knowledge configurations for the same model (see the Baseline Leaderboard above for the plain agent): - **Inline knowledge (pre-#8700)** — the review checklists BCApps shipped inline before adopting BCQuality, injected as custom instructions. -- **PR-review engine** — BC-ALAgents runs against a configured BCQuality revision, with performance and context-filtering metrics captured alongside review quality. {% assign experiment_rows = site.data.code-review.aggregate | where_exp: "agg", "agg.experiment" %} {% if experiment_rows and experiment_rows.size > 0 %} -{% assign experiment_results = experiment_rows | sort: "f1" | reverse %} -
- - -
- -
- - - - - - - - - - - - - - - - {% for agg in experiment_results %} - - - - - - - - - - - - {% endfor %} - -
VariantEngine / BCQualityAgentModelMicro F1 (95% CI)Macro F1 (95% CI)PrecisionRecallVer
{% if agg.experiment.custom_agent == "bc-review-engine" %}PR-review engine{% elsif agg.experiment.custom_instructions %}Inline knowledge (pre-#8700){% else %}Other{% endif %} - {% if agg.experiment.custom_agent == "bc-review-engine" and agg.experiment.plugins %} - {% for plugin in agg.experiment.plugins %} - {% assign plugin_parts = plugin | split: "@" %} - {% if plugin contains "bc-review-engine@" or plugin contains "BCQuality@" %}{{ plugin_parts[0] }}@{{ plugin_parts[1] | slice: 0, 7 }}{% unless forloop.last %}
{% endunless %}{% endif %} - {% endfor %} - {% elsif agg.experiment.custom_agent == "bc-review-engine" or agg.experiment.custom_instructions %}self-contained - {% else %}—{% endif %} - 		{{ agg.agent_name }}{{ agg.model }}{{ agg.f1 | times: 100.0 | round: 1 }}%{% if agg.f1_ci_low %} ({{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.macro_f1 | times: 100.0 | round: 1 }}%{% if agg.macro_f1_ci_low %} ({{ agg.macro_f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.macro_f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.precision | times: 100.0 | round: 1 }}%{{ agg.recall | times: 100.0 | round: 1 }}%{{ agg.benchmark_version }}
-
- - - - + + + + + + + + + + + + + + + + {% assign experiment_results = experiment_rows | sort: "f1" | reverse %} + {% for agg in experiment_results %} + + + + + + + + + + + + {% endfor %} + +
VariantAgentModelMicro F1 (95% CI)Macro F1 (95% CI)PrecisionRecallAvg TimeVer
+ {%- if agg.experiment.custom_instructions -%}Inline knowledge (pre-#8700) + {%- else -%}Other{%- endif -%} + {{ agg.agent_name }}{{ agg.model }}{{ agg.f1 | times: 100.0 | round: 1 }}%{% if agg.f1_ci_low %} ({{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.macro_f1 | times: 100.0 | round: 1 }}%{% if agg.macro_f1_ci_low %} ({{ agg.macro_f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.macro_f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}{{ agg.precision | times: 100.0 | round: 1 }}%{{ agg.recall | times: 100.0 | round: 1 }}%{{ agg.average_duration | round: 1 }}s{{ agg.benchmark_version }}
{% else %}

No experiment results available yet. Check back soon!

{% endif %} @@ -226,8 +179,5 @@ Compares review-knowledge configurations for the same model (see the Baseline Le - **Valid output rate** — fraction of tasks whose output parsed into a structured review. Failures score zero on every other metric. (Reported per run.) - **Micro vs. Macro** — *Micro* sums matched, scorable generated (generated minus ignored), and expected across all tasks (tasks with many comments dominate); *Macro* averages per-task scores (every task counts equally). - **95% CI** — confidence interval bootstrapped over the per-task F1 scores, so the leaderboard reports sampling uncertainty even for a single run. The micro `F1` CI resamples runs; the `Macro F1` CI resamples tasks. -- **Avg Tokens / API Calls / Estimated Credits** — mean PR-review engine usage per evaluated entry. Estimated credits use the engine's configured token prices and are not a currency value. -- **Knowledge Used** — mean number of BCQuality knowledge articles remaining after filtering and available to the reviewer. -- **Knowledge Pruned** — mean number of BCQuality knowledge articles removed by the engine's filtering step before review. [← Back to Home](index.md) diff --git a/notebooks/code-review-coverage.ipynb b/notebooks/code-review-coverage.ipynb index 08ca098d5..ec5cf92fe 100644 --- a/notebooks/code-review-coverage.ipynb +++ b/notebooks/code-review-coverage.ipynb @@ -8,8 +8,8 @@ "# Per-article BCQuality coverage (code-review)\n", "\n", "Ad-hoc analysis of how the code-review gold dataset maps onto BCQuality\n", - "knowledge articles. Every finding is annotated with the article it derives\n", - "from (`ReviewComment.article`), and false-positive-guard entries carry their\n", + "knowledge articles. Every finding is annotated with the articles it derives\n", + "from (`ReviewComment.articles`), and false-positive-guard entries carry their\n", "association at entry level (`metadata.articles`). This notebook aggregates\n", "those annotations via `bcbench.analysis.bcquality_article_coverage`.\n", "\n", @@ -28,7 +28,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "9 articles covered across 17/130 annotated entries; 241 of 250 articles have zero coverage\n" + "9 articles covered across 17/144 annotated entries; 241 of 250 articles have zero coverage\n" ] } ], @@ -408,25 +408,36 @@ " - web-services/version-apis-by-adding-not-mutating-published-versions\n", " - web-services/webhook-eligibility-and-validationtoken-renewal\n", "\n", - "Unannotated entries (113):\n", + "Unannotated entries (127):\n", + " - synthetic__accessibility-setselectionfilter-scope-01\n", " - synthetic__breaking-access-modifier-01\n", + " - synthetic__breaking-event-subscriber-suppress-01\n", " - synthetic__breaking-notification-callback-01\n", + " - synthetic__breaking-protected-var-field-01\n", " - synthetic__breaking-relocation-01\n", + " - synthetic__breaking-scope-creep-unrelated-api-01\n", " - synthetic__caption-clean-01\n", " - synthetic__currrec-clean-01\n", " - synthetic__data-modeling-blocked-validation-skip-01\n", " - synthetic__data-modeling-excluded-from-calculation-01\n", + " - synthetic__data-modeling-tablerelation-restriction-mismatch-01\n", " - synthetic__errh-errortype-internal-01\n", " - synthetic__errh-tryfunction-swallowed-01\n", " - synthetic__error-clean-01\n", + " - synthetic__error-handling-assistedit-cancel-01\n", + " - synthetic__error-handling-case-unreachable-else-01\n", + " - synthetic__error-handling-drilldown-position-01\n", + " - synthetic__error-handling-errorinfo-actionable-boundary-01\n", " - synthetic__error-handling-fieldno-swap-01\n", " - synthetic__error-handling-guiallowed-01\n", " - synthetic__error-handling-silent-skip-01\n", " - synthetic__error-testfield-enabled-01\n", + " - synthetic__events-ishandled-reset-boundary-01\n", " - synthetic__get-clean-01\n", " - synthetic__obsolete-clean-01\n", " - synthetic__perf-batched-commit-checkpoint-01\n", " - synthetic__perf-clean-01\n", + " - synthetic__perf-progress-dialog-deleteall-01\n", " - synthetic__performance-001\n", " - synthetic__performance-002\n", " - synthetic__performance-003\n", @@ -502,9 +513,12 @@ " - synthetic__style-clean-03\n", " - synthetic__style-clean-04\n", " - synthetic__style-duplicate-action-01\n", + " - synthetic__style-field-repurpose-indent-01\n", + " - synthetic__style-showmandatory-flowfield-01\n", " - synthetic__style-this-keyword-01\n", " - synthetic__style-tooltip-mismatch-01\n", " - synthetic__testing-tolerance-clean-01\n", + " - synthetic__testing-ui-handler-assert-after-run-01\n", " - synthetic__upgrade-001\n", " - synthetic__upgrade-002\n", " - synthetic__upgrade-003\n", diff --git a/pyproject.toml b/pyproject.toml index e9eaf5c90..a9f6ecab6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "bcbench" -version = "0.8.1" +version = "0.9.0" description = "Benchmarking tool for Business Central (AL) ecosystem, inspired by SWE-Bench" readme = "README.md" requires-python = ">=3.13,<3.14" @@ -21,7 +21,7 @@ dependencies = [ "typer>=0.9.0", "typing-extensions>=4.0", "pyyaml>=6.0", - "pydantic>=2.0", + "pydantic>=2.12", "textual>=7.0", "numpy>=2.3.5", "scipy>=1.16.3", @@ -40,7 +40,7 @@ default = true where = ["src"] [tool.setuptools.package-data] -bcbench = ["agent/*.yaml"] +bcbench = ["agent/*.yaml", "agent/pr_review/scripts/*.ps1"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/bcbench/agent/__init__.py b/src/bcbench/agent/__init__.py index 984cb7115..89be26b6b 100644 --- a/src/bcbench/agent/__init__.py +++ b/src/bcbench/agent/__init__.py @@ -3,9 +3,6 @@ from bcbench.agent.bcal import BCalBackendConfig, run_bcal_agent from bcbench.agent.claude import run_claude_code from bcbench.agent.copilot import run_copilot_agent +from bcbench.agent.pr_review import run_pr_review_agent -# The AI harnesses are the top-level backends. The code-review category is NOT a fourth -# harness: it runs the Copilot-powered BC-ALAgents review engine, whose backend lives under -# the copilot package (bcbench.agent.copilot.pr_review.run_pr_review_agent) and is reached -# only through the dedicated `code-review` command, not by picking a harness here. -__all__ = ["BCalBackendConfig", "run_bcal_agent", "run_claude_code", "run_copilot_agent"] +__all__ = ["BCalBackendConfig", "run_bcal_agent", "run_claude_code", "run_copilot_agent", "run_pr_review_agent"] diff --git a/src/bcbench/agent/copilot/pr_review/__init__.py b/src/bcbench/agent/copilot/pr_review/__init__.py deleted file mode 100644 index c5871df5e..000000000 --- a/src/bcbench/agent/copilot/pr_review/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bcbench.agent.copilot.pr_review.agent import run_pr_review_agent - -__all__ = ["run_pr_review_agent"] diff --git a/src/bcbench/agent/copilot/pr_review/agent.py b/src/bcbench/agent/copilot/pr_review/agent.py deleted file mode 100644 index 30fa12851..000000000 --- a/src/bcbench/agent/copilot/pr_review/agent.py +++ /dev/null @@ -1,336 +0,0 @@ -"""Run the BC-ALAgents review engine (generate half) as a BC-Bench agent. - -The code-review category runs the engine's own generate shell -(``Invoke-PRReviewShell.ps1 -GenerateOnly``) in local mode against the entry's changes, -so BC-Bench measures the real PROD engine + BCQuality rather than a divergent -re-implementation. The BC-Bench ``--model`` threads straight through to the single -Copilot the engine spawns (``COPILOT_MODEL``). - -The engine writes ``agent-output.txt`` (the harvested findings report); we map it to -``review.json`` in the repo root so the existing code-review scorer runs unchanged. -""" - -import json -import os -import shutil -import subprocess -import time -from collections.abc import Generator -from contextlib import contextmanager -from pathlib import Path -from typing import Any - -import yaml - -from bcbench.agent.copilot.pr_review.metrics import build_pr_review_metrics -from bcbench.agent.copilot.pr_review.review_output import engine_report_to_review_comments, load_engine_report -from bcbench.config import get_config -from bcbench.dataset import BaseDatasetEntry -from bcbench.dataset.codereview import CodeReviewEntry -from bcbench.exceptions import AgentError, AgentTimeoutError -from bcbench.logger import get_logger -from bcbench.operations.git_operations import clone_repo_at_revision, remove_tree -from bcbench.types import AgentMetrics, EvaluationCategory, ExperimentConfiguration - -logger = get_logger(__name__) -_config = get_config() - -_AGENT_OUTPUT_FILE = "agent-output.txt" -_REVIEW_OUTPUT_FILE = "review.json" -_PREPARE_BCQUALITY_SCRIPT = Path(__file__).parent / "scripts" / "Prepare-BCQualityRoot.ps1" - - -def _load_pr_review_settings() -> dict[str, Any]: - config_file = _config.paths.agent_share_dir / "config.yaml" - data = yaml.safe_load(config_file.read_text()) or {} - return data.get("pr_review") or {} - - -def _validate_engine_root(raw: str | Path) -> Path: - root = Path(raw).expanduser() - shell = root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" - if not shell.exists(): - raise AgentError(f"Engine review shell not found at {shell}. Check the configured BC-ALAgents source.") - return root - - -@contextmanager -def _prepare_engine_root( - settings: dict[str, Any], - destination: Path, - engine_ref: str | None = None, - engine_repo: str | None = None, - engine_local_path: str | None = None, -) -> Generator[Path]: - engine_cfg = settings.get("engine") or {} - environment_root = os.environ.get("BC_PR_REVIEW_ROOT") - if environment_root: - yield _validate_engine_root(environment_root) - return - - if engine_local_path and (engine_repo or engine_ref): - raise AgentError("--engine-local-path cannot be combined with --engine-repo or --engine-ref.") - - cli_remote_source = engine_repo is not None or engine_ref is not None - local_path = engine_local_path if engine_local_path is not None else None if cli_remote_source else engine_cfg.get("local_path") - if local_path: - yield _validate_engine_root(local_path) - return - - repo = engine_repo or engine_cfg.get("repo") - ref = engine_ref or engine_cfg.get("ref") - if not repo or not ref: - raise AgentError("Engine source not configured. Set pr_review.engine.repo/ref, use --engine-repo/--engine-ref, or provide BC_PR_REVIEW_ROOT.") - - try: - clone_repo_at_revision(str(repo), str(ref), destination) - yield _validate_engine_root(destination) - finally: - if destination.exists(): - remove_tree(destination) - - -def _resolve_engine_revision(engine_root: Path) -> str: - """Resolve the engine checkout's git revision (with a dirty marker) for provenance.""" - head = subprocess.run(["git", "-C", str(engine_root), "rev-parse", "HEAD"], capture_output=True, text=True, check=False) - if head.returncode != 0 or not head.stdout.strip(): - return "unknown" - sha = head.stdout.strip() - dirty = subprocess.run(["git", "-C", str(engine_root), "status", "--porcelain"], capture_output=True, text=True, check=False) - if dirty.returncode == 0 and dirty.stdout.strip(): - return f"{sha}-dirty" - return sha - - -def _resolve_pwsh() -> str: - pwsh = shutil.which("pwsh") - if not pwsh: - raise AgentError("PowerShell (pwsh) not found in PATH. The BC-ALAgents engine requires PowerShell 7+.") - return pwsh - - -def _resolve_gh_token() -> str: - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - if token: - return token - gh = shutil.which("gh") - if gh: - result = subprocess.run([gh, "auth", "token"], capture_output=True, text=True, check=False) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout.strip() - raise AgentError("No GitHub token available for Copilot CLI auth. Set GH_TOKEN or run `gh auth login`.") - - -def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: - return subprocess.run(["git", *args], cwd=str(cwd), capture_output=True, text=True, check=True) - - -def _commit_patch_as_head(repo_path: Path) -> None: - """Commit the applied working-tree patch so the engine can diff base..HEAD. - - The code-review pipeline applies the entry patch as uncommitted changes (and - marks new files intent-to-add). The engine's local mode diffs a committed - ``BASE_REF...HEAD`` range, so materialize the changes as a head commit on top - of the base commit (which is the current HEAD). - """ - _git(["add", "-A"], repo_path) - status = _git(["status", "--porcelain"], repo_path) - if not status.stdout.strip(): - raise AgentError("No changes to review: the entry patch produced an empty working tree diff.") - _git( - ["-c", "user.name=bcbench", "-c", "user.email=bcbench@local", "commit", "-q", "--no-verify", "-m", "bcbench review head"], - repo_path, - ) - - -def _init_trusted_workspace(path: Path) -> Path: - path.mkdir(parents=True, exist_ok=True) - _git(["init", "-q"], path) - _git(["-c", "user.name=bcbench", "-c", "user.email=bcbench@local", "commit", "-q", "--allow-empty", "-m", "trusted"], path) - return path - - -def _prepare_bcquality_root( - engine_root: Path, - pwsh: str, - dest: Path, - bcquality_ref: str | None, - bcquality_repo: str | None = None, - bcquality_local_path: str | None = None, -) -> tuple[Path, str | None]: - env = {**os.environ} - if bcquality_repo: - env["BCQUALITY_REPO"] = bcquality_repo - if bcquality_ref: - env["BCQUALITY_REF"] = bcquality_ref - args = [pwsh, "-NoProfile", "-File", str(_PREPARE_BCQUALITY_SCRIPT), "-EngineRoot", str(engine_root), "-Root", str(dest)] - if bcquality_local_path: - args += ["-LocalPath", bcquality_local_path] - result = subprocess.run( - args, - capture_output=True, - text=True, - env=env, - check=False, - ) - if result.returncode != 0: - logger.error(f"BCQuality preparation failed:\n{result.stdout}\n{result.stderr}") - raise AgentError(f"Failed to prepare BCQuality root (exit {result.returncode}).") - root: Path | None = None - sha: str | None = None - for line in result.stdout.splitlines(): - if line.startswith("root="): - root = Path(line[len("root=") :].strip()) - elif line.startswith("sha="): - sha = line[len("sha=") :].strip() - if root is None or not root.exists(): - raise AgentError("BCQuality preparation did not report a valid root.") - return root, sha - - -def _resolve_bcquality_source( - settings: dict[str, Any], - bcquality_ref: str | None, - bcquality_repo: str | None, - bcquality_local_path: str | None, -) -> tuple[str | None, str | None, str | None]: - bcquality_cfg = settings.get("bcquality") or {} - if bcquality_local_path and (bcquality_repo or bcquality_ref): - raise AgentError("--bcquality-local-path cannot be combined with --bcquality-repo or --bcquality-ref.") - - remote_override = bcquality_repo is not None or bcquality_ref is not None - resolved_repo = bcquality_repo or bcquality_cfg.get("repo") - resolved_ref = bcquality_ref or bcquality_cfg.get("ref") - resolved_local_path = bcquality_local_path - if resolved_local_path is None and not remote_override: - resolved_local_path = bcquality_cfg.get("local_path") - return resolved_ref, resolved_repo, resolved_local_path - - -def _write_review_json(output_dir: Path, repo_path: Path) -> int: - agent_output = output_dir / _AGENT_OUTPUT_FILE - if not agent_output.exists(): - raise AgentError(f"Engine did not produce {_AGENT_OUTPUT_FILE} in {output_dir}.") - report = load_engine_report(agent_output.read_text(encoding="utf-8")) - if report is None: - raise AgentError(f"Engine {_AGENT_OUTPUT_FILE} was empty or not a valid findings report; refusing to score it as a clean review.") - if not isinstance(report.get("findings"), list): - raise AgentError(f"Engine report in {_AGENT_OUTPUT_FILE} has no findings list (got {type(report.get('findings')).__name__}); refusing to score it as a clean review.") - comments = engine_report_to_review_comments(report) - (repo_path / _REVIEW_OUTPUT_FILE).write_text(json.dumps(comments, indent=2), encoding="utf-8") - return len(comments) - - -def run_pr_review_agent( - entry: BaseDatasetEntry, - model: str, - category: EvaluationCategory, - repo_path: Path, - output_dir: Path, - bcquality_ref: str | None = None, - bcquality_repo: str | None = None, - bcquality_local_path: str | None = None, - engine_ref: str | None = None, - engine_repo: str | None = None, - engine_local_path: str | None = None, - min_severity: str | None = None, -) -> tuple[AgentMetrics | None, ExperimentConfiguration]: - """Run the engine's generate half on a code-review entry and write review.json. - - Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents - PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it - owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific - inputs (BCQuality source, min severity) for the code-review category only. - - Returns: - Tuple of (AgentMetrics, ExperimentConfiguration). - """ - if category is not EvaluationCategory.CODE_REVIEW: - raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.") - if not isinstance(entry, CodeReviewEntry): - raise AgentError(f"The engine agent requires a CodeReviewEntry, got {type(entry).__name__}.") - settings = _load_pr_review_settings() - pwsh = _resolve_pwsh() - gh_token = _resolve_gh_token() - agent_version = str(settings.get("agent_version", "0.0.0")) - severity = min_severity or settings.get("min_severity") or "Low" - bcquality_ref, bcquality_repo, bcquality_local_path = _resolve_bcquality_source( - settings, - bcquality_ref, - bcquality_repo, - bcquality_local_path, - ) - - output_dir.mkdir(parents=True, exist_ok=True) - logger.info(f"Running BC-ALAgents review engine on: {entry.instance_id}") - - with _prepare_engine_root( - settings, - output_dir / "engine", - engine_ref=engine_ref, - engine_repo=engine_repo, - engine_local_path=engine_local_path, - ) as engine_root: - _commit_patch_as_head(repo_path) - trusted_workspace = _init_trusted_workspace(output_dir / "trusted") - bcquality_root, bcquality_sha = _prepare_bcquality_root( - engine_root, - pwsh, - output_dir / "bcquality", - bcquality_ref, - bcquality_repo, - bcquality_local_path, - ) - - shell = engine_root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" - env = { - **os.environ, - "REVIEW_SOURCE": "local", - "BASE_REF": entry.base_commit, - "REVIEW_TARGET_WORKSPACE": str(repo_path), - "REVIEW_WORKSPACE": str(trusted_workspace), - "REVIEW_OUTPUT_DIR": str(output_dir), - "BCQUALITY_ROOT": str(bcquality_root), - "COPILOT_MODEL": model, - "COPILOT_REVIEW_AGENT_VERSION": agent_version, - "COPILOT_REVIEW_LOG_LEVEL": "debug", - "AGENT_MINIMUM_SEVERITY": severity, - "GH_TOKEN": gh_token, - } - - plugins = [f"bc-review-engine@{_resolve_engine_revision(engine_root)}"] - if bcquality_sha: - plugins.append(f"BCQuality@{bcquality_sha}") - config = ExperimentConfiguration( - custom_agent="bc-review-engine", - plugins=plugins, - ) - - start = time.monotonic() - try: - result = subprocess.run( - [pwsh, "-NoProfile", "-File", str(shell), "-GenerateOnly", "-OutputDir", str(output_dir)], - cwd=str(repo_path), - env=env, - capture_output=True, - text=True, - timeout=_config.timeout.agent_execution, - check=True, - ) - logger.debug(f"Engine stdout:\n{result.stdout}") - if result.stderr: - logger.debug(f"Engine stderr:\n{result.stderr}") - count = _write_review_json(output_dir, repo_path) - logger.info(f"Engine review complete for {entry.instance_id}: wrote {count} comment(s) to {_REVIEW_OUTPUT_FILE}") - except subprocess.TimeoutExpired: - logger.exception(f"Engine review timed out after {_config.timeout.agent_execution} seconds") - metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) - raise AgentTimeoutError("Engine review timed out", metrics=metrics, config=config) from None - except subprocess.CalledProcessError as e: - logger.exception(f"Engine review failed (exit {e.returncode}):\n{e.stdout}\n{e.stderr}") - raise AgentError(f"Engine review execution failed: {e}") from None - except Exception: - logger.exception("Unexpected error running engine review") - raise - else: - return build_pr_review_metrics(output_dir, bcquality_root, time.monotonic() - start), config diff --git a/src/bcbench/agent/copilot/pr_review/metrics.py b/src/bcbench/agent/copilot/pr_review/metrics.py deleted file mode 100644 index 9286d0509..000000000 --- a/src/bcbench/agent/copilot/pr_review/metrics.py +++ /dev/null @@ -1,164 +0,0 @@ -import json -import re -from pathlib import Path -from typing import Any - -from bcbench.agent.copilot.metrics import parse_metrics -from bcbench.logger import get_logger -from bcbench.types import AgentMetrics - -logger = get_logger(__name__) - -RUN_METRICS_FILE_NAME = "_run-metrics.json" -FILTER_REPORT_FILE_NAME = "_filter-report.json" -TRANSCRIPT_FILE_NAME = "agent-transcript.log" -METRIC_NUMBER_PATTERN = r"[0-9][0-9,]*(?:\.[0-9]+)?[kKmM]?" -AI_CREDITS_PATTERN = re.compile(rf"(?m)^(?:err:\s*)?AI Credits\s+({METRIC_NUMBER_PATTERN})") -PREMIUM_REQUESTS_PATTERN = re.compile(rf"(?:Requests\s+|Total usage est:\s*)({METRIC_NUMBER_PATTERN})\s+Premium", re.IGNORECASE) -TOKENS_PATTERN = re.compile( - rf"(?m)^(?:err:\s*)?Tokens\s+↑\s*({METRIC_NUMBER_PATTERN})" - rf"(?:\s+\(({METRIC_NUMBER_PATTERN})\s+cached(?:,\s*{METRIC_NUMBER_PATTERN}\s+written)?\))?" - rf"\s+•\s+↓\s*({METRIC_NUMBER_PATTERN})" - rf"(?:\s+\(({METRIC_NUMBER_PATTERN})\s+reasoning\))?" -) - - -def _load_json(path: Path) -> dict[str, Any] | None: - if not path.exists(): - logger.debug(f"Engine perf file not found: {path}") - return None - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - logger.warning(f"Could not read engine perf file {path}: {exc}") - return None - if not isinstance(payload, dict): - logger.warning(f"Engine perf file {path} is not a JSON object; ignoring") - return None - return payload - - -def _as_int(value: object) -> int | None: - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _as_float(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - try: - return float(value) - except (TypeError, ValueError): - return None - - -def _parse_compact_number(value: str) -> float: - normalized = value.replace(",", "").lower() - multiplier = 1.0 - if normalized.endswith("k"): - normalized = normalized[:-1] - multiplier = 1_000.0 - elif normalized.endswith("m"): - normalized = normalized[:-1] - multiplier = 1_000_000.0 - return float(normalized) * multiplier - - -def parse_run_metrics(path: Path) -> dict[str, Any]: - payload = _load_json(path) - if payload is None: - return {} - - result: dict[str, Any] = {} - for source_key, target_key, coerce in ( - ("prompt_tokens", "prompt_tokens", _as_int), - ("completion_tokens", "completion_tokens", _as_int), - ("total_tokens", "total_tokens", _as_int), - ("api_calls", "api_calls", _as_int), - ("estimated_credits", "estimated_credits", _as_float), - ("wall_time_seconds", "wall_time_seconds", _as_float), - ): - value = coerce(payload.get(source_key)) - if value is not None: - result[target_key] = value - - if "total_tokens" not in result and "prompt_tokens" in result and "completion_tokens" in result: - result["total_tokens"] = int(result["prompt_tokens"]) + int(result["completion_tokens"]) - return result - - -def parse_transcript_metrics(path: Path) -> dict[str, int | float]: - if not path.exists(): - logger.debug(f"Engine transcript not found: {path}") - return {} - try: - lines = path.read_text(encoding="utf-8").splitlines(keepends=True) - except OSError as exc: - logger.warning(f"Could not read engine transcript {path}: {exc}") - return {} - - parsed = parse_metrics(lines, session_log_path=path) - result: dict[str, int | float] = {} - if parsed: - if parsed.prompt_tokens is not None: - result["prompt_tokens"] = parsed.prompt_tokens - if parsed.completion_tokens is not None: - result["completion_tokens"] = parsed.completion_tokens - if parsed.turn_count is not None: - result["api_calls"] = parsed.turn_count - - transcript = "".join(lines) - token_matches = list(TOKENS_PATTERN.finditer(transcript)) - if token_matches: - token_match = token_matches[-1] - result["prompt_tokens"] = int(_parse_compact_number(token_match.group(1))) - result["completion_tokens"] = int(_parse_compact_number(token_match.group(3))) - - credit_matches = list(AI_CREDITS_PATTERN.finditer(transcript)) - if not credit_matches: - credit_matches = list(PREMIUM_REQUESTS_PATTERN.finditer(transcript)) - if credit_matches: - result["estimated_credits"] = _parse_compact_number(credit_matches[-1].group(1)) - if "prompt_tokens" in result and "completion_tokens" in result: - result["total_tokens"] = int(result["prompt_tokens"]) + int(result["completion_tokens"]) - return result - - -def _count_filtered_knowledge(bcquality_root: Path) -> int: - return sum(1 for path in bcquality_root.rglob("*.md") if path.is_file() and "knowledge" in {part.lower() for part in path.relative_to(bcquality_root).parts[:-1]}) - - -def parse_filter_report(path: Path, bcquality_root: Path) -> dict[str, int]: - payload = _load_json(path) - if payload is None: - return {} - removed = payload.get("removed") - if not isinstance(removed, list): - return {} - return { - "knowledge_pruned": sum(1 for item in removed if isinstance(item, dict) and item.get("kind") == "knowledge"), - "knowledge_used": _count_filtered_knowledge(bcquality_root), - } - - -def build_pr_review_metrics(output_dir: Path, bcquality_root: Path, execution_time: float) -> AgentMetrics: - transcript = parse_transcript_metrics(output_dir / TRANSCRIPT_FILE_NAME) - run = {**transcript, **parse_run_metrics(output_dir / RUN_METRICS_FILE_NAME)} - filter_report = output_dir / FILTER_REPORT_FILE_NAME - if not filter_report.exists(): - filter_report = bcquality_root / FILTER_REPORT_FILE_NAME - knowledge = parse_filter_report(filter_report, bcquality_root) - return AgentMetrics( - execution_time=execution_time, - prompt_tokens=_as_int(run.get("prompt_tokens")), - completion_tokens=_as_int(run.get("completion_tokens")), - total_tokens=_as_int(run.get("total_tokens")), - api_calls=_as_int(run.get("api_calls")), - estimated_credits=_as_float(run.get("estimated_credits")), - knowledge_used=knowledge.get("knowledge_used"), - knowledge_pruned=knowledge.get("knowledge_pruned"), - ) diff --git a/src/bcbench/agent/pr_review/__init__.py b/src/bcbench/agent/pr_review/__init__.py new file mode 100644 index 000000000..987026334 --- /dev/null +++ b/src/bcbench/agent/pr_review/__init__.py @@ -0,0 +1,3 @@ +from bcbench.agent.pr_review.agent import run_pr_review_agent + +__all__ = ["run_pr_review_agent"] diff --git a/src/bcbench/agent/pr_review/agent.py b/src/bcbench/agent/pr_review/agent.py new file mode 100644 index 000000000..a4f90f194 --- /dev/null +++ b/src/bcbench/agent/pr_review/agent.py @@ -0,0 +1,233 @@ +"""Run the BC-ALAgents review engine as a BC-Bench agent. + +The code-review category runs the production orchestrator in local mode against the +entry's changes. Local mode executes the complete generation, parsing, filtering, and +artifact pipeline but returns before posting, so BC-Bench measures the real production +engine + BCQuality rather than a divergent re-implementation. The BC-Bench ``--model`` +threads straight through to the Copilot process the engine spawns (``COPILOT_MODEL``). + +The engine writes normalized findings to ``al-code-review-findings.json``; we map them +to ``review.json`` in the repo root so the existing code-review scorer runs unchanged. +""" + +import json +import os +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any + +import yaml + +from bcbench.agent.pr_review.metrics import build_pr_review_metrics +from bcbench.agent.pr_review.review_output import engine_report_to_review_comments, load_engine_report +from bcbench.config import get_config +from bcbench.dataset import BaseDatasetEntry +from bcbench.dataset.codereview import CodeReviewEntry +from bcbench.exceptions import AgentError, AgentTimeoutError +from bcbench.logger import get_logger +from bcbench.operations import commit_changes, has_changes, init_repo +from bcbench.types import AgentMetrics, EvaluationCategory, ExperimentConfiguration + +logger = get_logger(__name__) +_config = get_config() + +_FINDINGS_OUTPUT_FILE = "al-code-review-findings.json" +_REVIEW_OUTPUT_FILE = "review.json" +_PREPARE_BCQUALITY_SCRIPT = Path(__file__).parent / "scripts" / "Prepare-BCQualityRoot.ps1" + + +def _load_pr_review_settings() -> dict[str, Any]: + config_file = _config.paths.agent_share_dir / "config.yaml" + return yaml.safe_load(config_file.read_text(encoding="utf-8"))["pr_review"] + + +def _resolve_pr_review_root(engine_path: Path | None) -> Path: + if engine_path is None: + raise AgentError("Engine root not configured. Pass --engine-path or set BC_PR_REVIEW_ROOT.") + root = engine_path.expanduser().resolve() + engine = root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-CopilotPRReview.ps1" + if not engine.exists(): + raise AgentError(f"Engine orchestrator not found at {engine}. Check --engine-path points at a BC-ALAgents checkout.") + return root + + +def _resolve_pwsh() -> str: + pwsh = shutil.which("pwsh") + if not pwsh: + raise AgentError("PowerShell (pwsh) not found in PATH. The BC-ALAgents engine requires PowerShell 7+.") + return pwsh + + +def _commit_patch_as_head(repo_path: Path) -> None: + """Commit the applied working-tree patch so the engine can diff base..HEAD. + + The code-review pipeline applies the entry patch as uncommitted changes (and + marks new files intent-to-add). The engine's local mode diffs a committed + ``BASE_REF...HEAD`` range, so materialize the changes as a head commit on top + of the base commit (which is the current HEAD). + """ + if not has_changes(repo_path): + raise AgentError("No changes to review: the entry patch produced an empty working tree diff.") + commit_changes(repo_path, "bcbench review head", no_verify=True) + + +def _init_trusted_workspace(path: Path) -> Path: + init_repo(path) + commit_changes(path, "trusted", allow_empty=True) + return path + + +def _prepare_bcquality_root( + engine_root: Path, + pwsh: str, + dest: Path, + bcquality_ref: str | None, + bcquality_repo: str | None = None, + bcquality_local_path: Path | None = None, +) -> Path: + env = {**os.environ} + if bcquality_repo: + env["BCQUALITY_REPO"] = bcquality_repo + if bcquality_ref: + env["BCQUALITY_REF"] = bcquality_ref + args = [pwsh, "-NoProfile", "-File", str(_PREPARE_BCQUALITY_SCRIPT), "-EngineRoot", str(engine_root), "-Root", str(dest)] + if bcquality_local_path: + args += ["-LocalPath", str(bcquality_local_path)] + result = subprocess.run( + args, + capture_output=True, + text=True, + encoding="utf-8", + env=env, + check=False, + ) + if result.returncode != 0: + logger.error(f"BCQuality preparation failed:\n{result.stdout}\n{result.stderr}") + raise AgentError(f"Failed to prepare BCQuality root (exit {result.returncode}).") + root: Path | None = None + for line in result.stdout.splitlines(): + if line.startswith("root="): + root = Path(line[len("root=") :].strip()) + if root is None or not root.exists(): + raise AgentError("BCQuality preparation did not report a valid root.") + return root + + +def _write_review_json(output_dir: Path, repo_path: Path) -> int: + findings_output = output_dir / _FINDINGS_OUTPUT_FILE + if not findings_output.exists(): + raise AgentError(f"Engine did not produce {_FINDINGS_OUTPUT_FILE} in {output_dir}.") + report = load_engine_report(findings_output.read_text(encoding="utf-8")) + if report is None: + raise AgentError(f"Engine {_FINDINGS_OUTPUT_FILE} was empty or invalid; refusing to score it as a clean review.") + outcome = report.get("outcome") + if outcome == "failed": + reason = report.get("outcomeReason") or "unknown reason" + raise AgentError(f"Engine review failed: {reason}") + if outcome not in {"completed", "partial", "not-applicable", "no-knowledge"}: + raise AgentError(f"Engine {_FINDINGS_OUTPUT_FILE} has unsupported outcome {outcome!r}.") + if not isinstance(report.get("findings"), list): + raise AgentError(f"Engine report in {_FINDINGS_OUTPUT_FILE} has no findings list (got {type(report.get('findings')).__name__}); refusing to score it as a clean review.") + comments = engine_report_to_review_comments(report) + (repo_path / _REVIEW_OUTPUT_FILE).write_text(json.dumps(comments, indent=2), encoding="utf-8") + return len(comments) + + +def run_pr_review_agent( + entry: BaseDatasetEntry, + model: str, + category: EvaluationCategory, + repo_path: Path, + output_dir: Path, + engine_path: Path | None = None, + bcquality_ref: str | None = None, + bcquality_repo: str | None = None, + bcquality_local_path: Path | None = None, + min_severity: str | None = None, +) -> tuple[AgentMetrics | None, ExperimentConfiguration]: + """Run the engine's complete local review pipeline and write review.json. + + Separate from run_copilot_agent by design: this spawns the PROD BC-ALAgents + PowerShell orchestrator (Copilot is spawned inside the engine, not here), so it + owns none of the copilot-harness prompt/MCP/LSP wiring and takes engine-specific + inputs (BCQuality source, min severity) for the code-review category only. + + Returns: + Tuple of (AgentMetrics, ExperimentConfiguration). + """ + if category is not EvaluationCategory.CODE_REVIEW: + raise AgentError(f"The engine agent only supports the code-review category, got {category.value}.") + if not isinstance(entry, CodeReviewEntry): + raise AgentError(f"The engine agent requires a CodeReviewEntry, got {type(entry).__name__}.") + + repo_path = repo_path.resolve() + output_dir = output_dir.resolve() + settings = _load_pr_review_settings() + engine_root = _resolve_pr_review_root(engine_path) + pwsh = _resolve_pwsh() + severity = min_severity or settings["min_severity"] + bcquality_cfg = settings["bcquality"] + bcquality_repo = bcquality_repo or bcquality_cfg["repo"] + bcquality_ref = bcquality_ref or bcquality_cfg["ref"] + output_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Running BC-ALAgents review engine on: {entry.instance_id}") + + _commit_patch_as_head(repo_path) + trusted_workspace = _init_trusted_workspace(output_dir / "trusted") + bcquality_root = _prepare_bcquality_root( + engine_root, + pwsh, + output_dir / "bcquality", + bcquality_ref, + bcquality_repo, + bcquality_local_path, + ) + + engine = engine_root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-CopilotPRReview.ps1" + env = { + **os.environ, + "REVIEW_SOURCE": "local", + "REVIEW_PHASE": "all", + "BASE_REF": entry.base_commit, + "REVIEW_TARGET_WORKSPACE": str(repo_path), + "REVIEW_WORKSPACE": str(trusted_workspace), + "REVIEW_OUTPUT_DIR": str(output_dir), + "BCQUALITY_ROOT": str(bcquality_root), + "GITHUB_REPOSITORY": entry.repo, + "COPILOT_MODEL": model, + "AGENT_MINIMUM_SEVERITY": severity, + } + + config = ExperimentConfiguration() + + start = time.monotonic() + try: + result = subprocess.run( + [pwsh, "-NoProfile", "-File", str(engine)], + cwd=str(repo_path), + env=env, + capture_output=True, + text=True, + encoding="utf-8", + timeout=_config.timeout.agent_execution, + check=True, + ) + logger.debug(f"Engine stdout:\n{result.stdout}") + if result.stderr: + logger.debug(f"Engine stderr:\n{result.stderr}") + count = _write_review_json(output_dir, repo_path) + logger.info(f"Engine review complete for {entry.instance_id}: wrote {count} comment(s) to {_REVIEW_OUTPUT_FILE}") + except subprocess.TimeoutExpired: + logger.exception(f"Engine review timed out after {_config.timeout.agent_execution} seconds") + metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) + raise AgentTimeoutError("Engine review timed out", metrics=metrics, config=config) from None + except subprocess.CalledProcessError as e: + logger.exception(f"Engine review failed (exit {e.returncode}):\n{e.stdout}\n{e.stderr}") + raise AgentError(f"Engine review execution failed: {e}") from None + except Exception: + logger.exception("Unexpected error running engine review") + raise + else: + return build_pr_review_metrics(output_dir, bcquality_root, time.monotonic() - start), config diff --git a/src/bcbench/agent/pr_review/metrics.py b/src/bcbench/agent/pr_review/metrics.py new file mode 100644 index 000000000..052b8834e --- /dev/null +++ b/src/bcbench/agent/pr_review/metrics.py @@ -0,0 +1,133 @@ +import json +from pathlib import Path +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator + +from bcbench.exceptions import AgentError +from bcbench.types import AgentMetrics + +FILTER_REPORT_FILE_NAME = "_filter-report.json" +RUN_METRICS_FILE_NAME = "_run-metrics.json" +_KNOWLEDGE_LAYERS = {"microsoft", "community", "custom"} +_NonNegativeInt = Annotated[int, Field(ge=0)] +_NonNegativeFloat = Annotated[float, Field(ge=0)] + + +class _FilterRemoval(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + kind: Literal["knowledge", "skill"] + + +class _FilterReport(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + removed: list[_FilterRemoval] + + +class _RunMetrics(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + schema_version: Literal[1] + metrics_source: Literal["copilot-cli-otel", "not-applicable"] + cli_version: str | None + wall_time_seconds: _NonNegativeFloat | None + prompt_tokens: _NonNegativeInt | None + cached_tokens: _NonNegativeInt | None + cache_creation_tokens: _NonNegativeInt | None + completion_tokens: _NonNegativeInt | None + reasoning_tokens: _NonNegativeInt | None + total_tokens: _NonNegativeInt | None + api_calls: _NonNegativeInt | None + failed_api_calls: _NonNegativeInt | None + usage_api_calls: _NonNegativeInt | None + ai_credits: _NonNegativeFloat | None + premium_requests: _NonNegativeFloat | None + models: list[str] + usage_complete: bool + malformed_records: _NonNegativeInt + + @model_validator(mode="after") + def validate_not_applicable_shape(self) -> "_RunMetrics": + if self.metrics_source != "not-applicable": + return self + expected = { + "cli_version": None, + "wall_time_seconds": 0, + "prompt_tokens": 0, + "cached_tokens": 0, + "cache_creation_tokens": 0, + "completion_tokens": 0, + "reasoning_tokens": None, + "total_tokens": 0, + "api_calls": 0, + "failed_api_calls": 0, + "usage_api_calls": 0, + "ai_credits": 0.0, + "premium_requests": None, + "models": [], + "usage_complete": True, + "malformed_records": 0, + } + invalid = [name for name, value in expected.items() if getattr(self, name) != value] + if invalid: + raise ValueError(f"not-applicable metrics have invalid fields: {', '.join(invalid)}") + return self + + +def _load_run_metrics(path: Path) -> _RunMetrics: + if not path.exists(): + raise AgentError(f"Engine run metrics artifact not found at {path}.") + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except (json.JSONDecodeError, OSError) as exc: + raise AgentError(f"Could not read engine run metrics artifact {path}: {exc}") from exc + try: + return _RunMetrics.model_validate(payload) + except ValidationError as exc: + raise AgentError(f"Engine run metrics artifact {path} does not satisfy schema version 1: {exc}") from exc + + +def _load_filter_report(path: Path) -> _FilterReport: + if not path.exists(): + raise AgentError(f"BCQuality filter report not found at {path}.") + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except (json.JSONDecodeError, OSError) as exc: + raise AgentError(f"Could not read BCQuality filter report {path}: {exc}") from exc + try: + return _FilterReport.model_validate(payload) + except ValidationError as exc: + raise AgentError(f"BCQuality filter report {path} has an invalid shape: {exc}") from exc + + +def _count_available_knowledge(bcquality_root: Path) -> int: + def is_knowledge_file(path: Path) -> bool: + parts = path.relative_to(bcquality_root).parts + return len(parts) >= 3 and parts[0].lower() in _KNOWLEDGE_LAYERS and parts[1].lower() == "knowledge" + + return sum(1 for path in bcquality_root.rglob("*.md") if path.is_file() and is_knowledge_file(path)) + + +def build_pr_review_metrics(output_dir: Path, bcquality_root: Path, execution_time: float) -> AgentMetrics: + run = _load_run_metrics(output_dir / RUN_METRICS_FILE_NAME) + report = _load_filter_report(bcquality_root / FILTER_REPORT_FILE_NAME) + return AgentMetrics( + execution_time=execution_time, + prompt_tokens=run.prompt_tokens, + completion_tokens=run.completion_tokens, + cached_tokens=run.cached_tokens, + cache_creation_tokens=run.cache_creation_tokens, + reasoning_tokens=run.reasoning_tokens, + total_tokens=run.total_tokens, + api_calls=run.api_calls, + failed_api_calls=run.failed_api_calls, + usage_api_calls=run.usage_api_calls, + ai_credits=run.ai_credits, + premium_requests=run.premium_requests, + usage_complete=run.usage_complete, + malformed_records=run.malformed_records, + knowledge_files=_count_available_knowledge(bcquality_root), + knowledge_pruned=sum(1 for item in report.removed if item.kind == "knowledge"), + ) diff --git a/src/bcbench/agent/copilot/pr_review/review_output.py b/src/bcbench/agent/pr_review/review_output.py similarity index 72% rename from src/bcbench/agent/copilot/pr_review/review_output.py rename to src/bcbench/agent/pr_review/review_output.py index ab0c86add..5f1def674 100644 --- a/src/bcbench/agent/copilot/pr_review/review_output.py +++ b/src/bcbench/agent/pr_review/review_output.py @@ -1,21 +1,23 @@ -"""Map the engine's findings report onto BC-Bench's review.json schema. +"""Map production-normalized engine findings onto BC-Bench's review.json schema. -The BC-ALAgents engine writes ``agent-output.txt`` (the harvested -``_review-report.json``) in its output directory. Each finding is shaped like:: +The BC-ALAgents engine writes ``al-code-review-findings.json`` after its production +parsing and filtering stages. Each finding is shaped like:: { + "filePath": "", + "lineNumber": , "severity": "Critical|High|Medium|Low", - "location": { "file": "", "line": }, - "message": "", "domain": "", + "issue": "", + "recommendation": "", ... } BC-Bench's code-review scorer instead reads ``review.json`` from the repo root as a flat list of ``{file, line_start, line_end, severity, body}`` objects (see ``bcbench.evaluate.review_parsing.parse_review_output``). This module performs the -one transform between the two so the engine's generate half plugs into the -existing scoring pipeline unchanged. +one transform between the two so the production engine plugs into the existing +scoring pipeline unchanged. """ import json @@ -30,7 +32,7 @@ def load_engine_report(raw_output: str) -> dict[str, Any] | None: - """Parse the engine's ``agent-output.txt`` text into a report dict. + """Parse the engine's normalized findings artifact into a report dict. Returns ``None`` when the text is empty or not a JSON object. """ @@ -39,7 +41,7 @@ def load_engine_report(raw_output: str) -> dict[str, Any] | None: try: report = json.loads(raw_output) except json.JSONDecodeError: - logger.warning("Engine agent-output.txt is not valid JSON") + logger.warning("Engine findings artifact is not valid JSON") return None if not isinstance(report, dict): logger.warning(f"Engine report is not a JSON object (got {type(report).__name__})") @@ -67,14 +69,12 @@ def engine_report_to_review_comments(report: dict[str, Any]) -> list[dict[str, A if not isinstance(finding, dict): continue - location = finding.get("location") - location = location if isinstance(location, dict) else {} - file_value = location.get("file") - line_value = location.get("line") - - # The finding's human text: prefer the rendered message, then the - # structured issue/recommendation the signature is built from. - body = finding.get("message") or finding.get("issue") or finding.get("recommendation") + file_value = finding.get("filePath") + line_value = finding.get("lineNumber") + issue = finding.get("issue") + recommendation = finding.get("recommendation") + body_parts = [value.strip() for value in (issue, recommendation) if isinstance(value, str) and value.strip()] + body = "\n\nRecommendation: ".join(body_parts) if not isinstance(file_value, str) or not file_value.strip(): continue diff --git a/src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 b/src/bcbench/agent/pr_review/scripts/Prepare-BCQualityRoot.ps1 similarity index 66% rename from src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 rename to src/bcbench/agent/pr_review/scripts/Prepare-BCQualityRoot.ps1 index 5c3711169..5196070c1 100644 --- a/src/bcbench/agent/copilot/pr_review/scripts/Prepare-BCQualityRoot.ps1 +++ b/src/bcbench/agent/pr_review/scripts/Prepare-BCQualityRoot.ps1 @@ -17,8 +17,7 @@ original working tree is never modified (the filter deletes files). This is the fast inner loop for optimizing BCQuality structure and re-scoring in BC-Bench. - Emits the resolved root and SHA as `root=` / `sha=` lines on - stdout for the Python caller to parse. + Emits the resolved root as `root=` on stdout for the Python caller. #> [CmdletBinding()] param( @@ -55,11 +54,6 @@ if ($LocalPath) { Copy-Item -LiteralPath $_.FullName -Destination $Root -Recurse -Force } - # Provenance: use the source checkout's HEAD sha when it is a git repo. - $resolvedSha = 'local' - $headSha = (& git -C $src rev-parse HEAD 2>$null) - if ($LASTEXITCODE -eq 0 -and $headSha) { $resolvedSha = "local:$($headSha.Trim())" } - Write-Host "BCQuality local source SHA: $resolvedSha" } else { $repo = $cfg.bcquality.repo @@ -67,26 +61,16 @@ else { Write-Host "Fetching BCQuality from $repo@$ref into $Root" if (Test-Path -LiteralPath $Root) { Remove-Item -LiteralPath $Root -Recurse -Force } - New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Root) | Out-Null - if ($repo -match '^[^/\\]+/[^/\\]+$') { - gh repo clone $repo $Root -- --depth=1 "--revision=$ref" --quiet - if ($LASTEXITCODE -ne 0) { throw "gh clone of BCQuality ref '$repo@$ref' failed (exit $LASTEXITCODE)" } - } - else { - New-Item -ItemType Directory -Force -Path $Root | Out-Null - git -C $Root init -q - git -C $Root remote add origin $repo - git -C $Root fetch --depth=1 origin "$ref" - if ($LASTEXITCODE -ne 0) { throw "git fetch of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } - git -C $Root checkout -q FETCH_HEAD - if ($LASTEXITCODE -ne 0) { throw "git checkout of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } - } + New-Item -ItemType Directory -Force -Path $Root | Out-Null + git -C $Root init -q + git -C $Root remote add origin $repo + git -C $Root fetch --depth=1 origin "$ref" + if ($LASTEXITCODE -ne 0) { throw "git fetch of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } + git -C $Root checkout -q FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw "git checkout of BCQuality ref '$ref' failed (exit $LASTEXITCODE)" } - $resolvedSha = (& git -C $Root rev-parse HEAD).Trim() - Write-Host "BCQuality resolved SHA: $resolvedSha" } & $filter -BCQualityRoot $Root -Config $cfg | Out-Null "root=$Root" -"sha=$resolvedSha" diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index 8b362841e..a6dd44d9e 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -83,6 +83,26 @@ prompt: Extensibility request: {{task}} + code-review-template: | + You are reviewing a Business Central (AL) code repository at {{repo_path}}. + + Review only the staged and unstaged dataset changes in the working tree (`git diff HEAD`). Do not compare commits such as HEAD~1..HEAD or origin/main. + + Important constraints: + - Do NOT modify source code, tests, configuration, or any other repository content. + - The only file you may create or update is `{{repo_path}}/review.json`. + - You MUST write `review.json` before finishing; otherwise the review counts as missing output. + + `review.json` must contain one JSON array. Each finding must be an object with these fields: + - `file`: repository-relative `.al` or `.json` path (string, required) + - `line_start`: 1-based line where the issue starts (positive integer, required) + - `line_end`: 1-based line where the issue ends (positive integer, optional) + - `domain`: review domain (string, optional) + - `body`: concise description of the issue (non-empty string, required) + - `severity`: one of `critical`, `high`, `medium`, or `low` (optional) + + If there are no findings, write an empty array. Write only valid JSON to `review.json`, with no surrounding object, Markdown fence, or commentary. + # controls: # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions//` # - Copilot: copies to repo/.github/ and renames AGENTS.md to copilot-instructions.md @@ -166,23 +186,15 @@ mcp: # command: "npx" # args: ["-y", "@modelcontextprotocol/server-filesystem", "{{repo_path}}"] -# BC-ALAgents review engine settings. The code-review category runs the engine's own -# generate half (Invoke-PRReviewShell.ps1 -GenerateOnly) in local mode instead of a -# bespoke review prompt, so it measures the real PROD engine + BCQuality. These knobs -# live here in the shared config so they are easy to tweak on a private branch. -# engine: BC-ALAgents source. repo/ref are fetched for reproducible CI runs. -# Set local_path for local development without pushing. BC_PR_REVIEW_ROOT -# remains an environment override and takes precedence over these values. -# bcquality: content source for the engine run; all optional (null = engine's pinned -# repo/ref). Set local_path to iterate on a local BCQuality checkout without -# pushing (it is copied and filtered; the original is never modified). -# CLI flags for either source override these values. +# BC-ALAgents review engine settings for the dedicated pr-review runner. This runner uses +# the production orchestrator's complete local, non-posting path to measure BC PR Review +# + BCQuality on the code-review category, including production parsing and filtering. +# Generic Copilot and Claude runners use code-review-template above on the same category. +# Engine and local BCQuality checkout paths are machine inputs supplied through CLI flags. +# BCQuality repo/ref remain experiment settings; repo supports testing forks while null values +# use the engine's pinned content source. CLI flags override these values. pr_review: - engine: - repo: microsoft/BC-ALAgents - ref: main - local_path: null + min_severity: Medium bcquality: repo: null ref: null - local_path: null diff --git a/src/bcbench/cli_options.py b/src/bcbench/cli_options.py index 346668b02..6cd77c1eb 100644 --- a/src/bcbench/cli_options.py +++ b/src/bcbench/cli_options.py @@ -11,6 +11,31 @@ # Note: Defaults are provided in function signatures, not here RepoPath = Annotated[Path, typer.Option(help="Path to repository")] +PRReviewEnginePath = Annotated[ + Path | None, + typer.Option( + "--engine-path", + envvar="BC_PR_REVIEW_ROOT", + help="Path to a local BC-ALAgents checkout", + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + ), +] + +BCQualityLocalPath = Annotated[ + Path | None, + typer.Option( + "--bcquality-local-path", + help="Path to a local BCQuality checkout (copied and filtered; never modified)", + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + ), +] + OutputDir = Annotated[Path, typer.Option(help="Directory to save evaluation results", file_okay=False, dir_okay=True)] RunId = Annotated[str, typer.Option(envvar="GITHUB_RUN_ID", help="Unique identifier for this evaluation run")] @@ -23,17 +48,6 @@ EvaluationCategoryOption = Annotated[EvaluationCategory, typer.Option(help="Category of evaluation to perform")] - -def reject_code_review(category: EvaluationCategory, verb: str) -> None: - """Guard general harness commands (copilot/claude) against the code-review category. - - code-review is not a harness choice: it is always served by the dedicated engine - command, so a general harness must refuse it rather than run a divergent review. - """ - if category is EvaluationCategory.CODE_REVIEW: - raise typer.BadParameter(f"code-review is not available under a general harness; use 'bcbench {verb} code-review' instead.") - - CopilotModelName = Literal[ "claude-sonnet-5", "claude-opus-5", diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 9f6a51fab..6033811b3 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -6,9 +6,9 @@ import typer -from bcbench.agent import BCalBackendConfig, run_bcal_agent, run_claude_code, run_copilot_agent -from bcbench.agent.copilot.pr_review import run_pr_review_agent +from bcbench.agent import BCalBackendConfig, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent from bcbench.cli_options import ( + BCQualityLocalPath, ClaudeCodeModel, ContainerName, ContainerPassword, @@ -16,9 +16,9 @@ CopilotModel, EvaluationCategoryOption, OutputDir, + PRReviewEnginePath, RepoPath, RunId, - reject_code_review, ) from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, NL2ALEntry @@ -42,65 +42,6 @@ def _prepare_run_dir(output_dir: Path, run_id: str) -> Path: return run_dir -def _run_pr_review_evaluation( - entry_id: str, - model: str, - repo_path: Path, - output_dir: Path, - run_id: str, - engine_ref: str | None = None, - engine_repo: str | None = None, - engine_local_path: str | None = None, - bcquality_ref: str | None = None, - bcquality_repo: str | None = None, - bcquality_local_path: str | None = None, - min_severity: str | None = None, -) -> None: - """Evaluate a code-review entry through the BC-ALAgents review engine. - - Backs the dedicated 'evaluate code-review' command: the code-review category always - runs the engine's own generate half (the real PROD path), so callers never drive a - bespoke review prompt. BCQuality source and severity default to the engine config when - not overridden. - """ - category = EvaluationCategory.CODE_REVIEW - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] - run_dir = _prepare_run_dir(output_dir, run_id) - - logger.info(f"Running evaluation on entry {entry_id} with the BC-ALAgents review engine") - - context = EvaluationContext( - entry=entry, - repo_path=repo_path, - result_dir=run_dir, - container=None, - model=model, - agent_name=AgentHarness.PR_REVIEW, - category=category, - ) - - category.pipeline.execute( - context, - lambda ctx: run_pr_review_agent( - entry=ctx.entry, - repo_path=ctx.repo_path, - category=category, - model=ctx.model, - output_dir=ctx.result_dir, - engine_ref=engine_ref, - engine_repo=engine_repo, - engine_local_path=engine_local_path, - bcquality_ref=bcquality_ref, - bcquality_repo=bcquality_repo, - bcquality_local_path=bcquality_local_path, - min_severity=min_severity, - ), - ) - - logger.info("Evaluation complete!") - logger.info(f"Results saved to: {run_dir}") - - @evaluate_app.command("copilot") def evaluate_copilot( entry_id: Annotated[str, typer.Argument(help="Entry ID to run")], @@ -120,7 +61,6 @@ def evaluate_copilot( To only run the agent to generate a patch without building/testing, use 'bcbench run copilot' instead. """ - reject_code_review(category, "evaluate") entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] run_dir = _prepare_run_dir(output_dir, run_id) @@ -176,7 +116,6 @@ def evaluate_claude_code( To only run the agent to generate a patch without building/testing, use 'bcbench run claude' instead. """ - reject_code_review(category, "evaluate") entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] run_dir = _prepare_run_dir(output_dir, run_id) @@ -213,46 +152,66 @@ def evaluate_claude_code( logger.info(f"Results saved to: {run_dir}") -@evaluate_app.command("code-review") -def evaluate_code_review( +@evaluate_app.command("pr-review") +def evaluate_pr_review( entry_id: Annotated[str, typer.Argument(help="Entry ID to run")], - model: CopilotModel = "claude-sonnet-5", + model: CopilotModel = "gpt-5.6-luna", repo_path: RepoPath = _config.paths.testbed_path, output_dir: OutputDir = _config.paths.evaluation_results_path, run_id: RunId = "pr_review_test_run", - engine_ref: Annotated[str | None, typer.Option(help="Override the BC-ALAgents ref (defaults to pr_review.engine.ref)")] = None, - engine_repo: Annotated[str | None, typer.Option(help="Override the BC-ALAgents repo (defaults to pr_review.engine.repo)")] = None, - engine_local_path: Annotated[str | None, typer.Option(help="Use a local BC-ALAgents checkout instead of fetching")] = None, + engine_path: PRReviewEnginePath = None, bcquality_ref: Annotated[str | None, typer.Option(help="Override the BCQuality ref (defaults to the engine's pinned ref)")] = None, - bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to pr_review.bcquality.repo or the engine pin)")] = None, - bcquality_local_path: Annotated[str | None, typer.Option(help="Use a local BCQuality checkout (copied + filtered, never modified) instead of fetching")] = None, + bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to config/engine)")] = None, + bcquality_local_path: BCQualityLocalPath = None, min_severity: Annotated[str | None, typer.Option(help="AGENT_MINIMUM_SEVERITY floor (defaults to config)")] = None, ) -> None: """ - Evaluate the code-review category on a single entry via the BC-ALAgents review engine. + Evaluate BC PR Review on a single code-review entry. - code-review is not a general harness choice: it always runs the engine's own generate - shell in local mode - the real PROD generate path - then scores the resulting - review.json with the standard code-review judge. BC-ALAgents and BCQuality sources can - be configured in config.yaml or overridden with command options. + This production-fidelity runner is fixed to the code-review category, while the same + category can also run through the generic copilot and claude commands for cross-system + comparison. The resulting review.json is scored by the shared code-review pipeline. + Requires a local BC-ALAgents checkout + (--engine-path or BC_PR_REVIEW_ROOT), PowerShell 7+, and an authenticated + Copilot CLI. - To only generate review.json without scoring, use 'bcbench run code-review' instead. + To only generate review.json without scoring, use 'bcbench run pr-review' instead. """ - _run_pr_review_evaluation( - entry_id, - model=model, + category = EvaluationCategory.CODE_REVIEW + entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + run_dir = _prepare_run_dir(output_dir, run_id) + + logger.info(f"Running evaluation on entry {entry_id} with the BC-ALAgents review engine") + + context = EvaluationContext( + entry=entry, repo_path=repo_path, - output_dir=output_dir, - run_id=run_id, - engine_ref=engine_ref, - engine_repo=engine_repo, - engine_local_path=engine_local_path, - bcquality_ref=bcquality_ref, - bcquality_repo=bcquality_repo, - bcquality_local_path=bcquality_local_path, - min_severity=min_severity, + result_dir=run_dir, + container=None, + model=model, + agent_name=AgentHarness.PR_REVIEW, + category=category, ) + category.pipeline.execute( + context, + lambda ctx: run_pr_review_agent( + entry=ctx.entry, + repo_path=ctx.repo_path, + category=category, + model=ctx.model, + output_dir=ctx.result_dir, + engine_path=engine_path, + bcquality_ref=bcquality_ref, + bcquality_repo=bcquality_repo, + bcquality_local_path=bcquality_local_path, + min_severity=min_severity, + ), + ) + + logger.info("Evaluation complete!") + logger.info(f"Results saved to: {run_dir}") + @evaluate_app.command("bcal") def evaluate_bcal( diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 959c477cf..c9cce020b 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -1,22 +1,19 @@ """CLI commands for running agents.""" -from pathlib import Path from typing import Annotated, cast import typer -from bcbench.agent.bcal import BCalBackendConfig, run_bcal_agent -from bcbench.agent.claude import run_claude_code -from bcbench.agent.copilot import run_copilot_agent -from bcbench.agent.copilot.pr_review import run_pr_review_agent +from bcbench.agent import BCalBackendConfig, run_bcal_agent, run_claude_code, run_copilot_agent, run_pr_review_agent from bcbench.cli_options import ( + BCQualityLocalPath, ClaudeCodeModel, ContainerName, CopilotModel, EvaluationCategoryOption, OutputDir, + PRReviewEnginePath, RepoPath, - reject_code_review, ) from bcbench.config import get_config from bcbench.dataset import NL2ALEntry @@ -29,45 +26,6 @@ run_app = typer.Typer(help="Run agents on single dataset entry") -def _run_pr_review( - entry_id: str, - model: str, - repo_path: Path, - output_dir: Path, - engine_ref: str | None = None, - engine_repo: str | None = None, - engine_local_path: str | None = None, - bcquality_ref: str | None = None, - bcquality_repo: str | None = None, - bcquality_local_path: str | None = None, - min_severity: str | None = None, -) -> None: - """Generate review.json for a code-review entry via the BC-ALAgents review engine. - - Backs the dedicated 'run code-review' command: code-review always runs the engine's - real generate half, never a bespoke prompt. BCQuality source and severity default to - the engine config when not overridden. - """ - category = EvaluationCategory.CODE_REVIEW - entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] - category.pipeline.setup_workspace(entry, repo_path) - - run_pr_review_agent( - entry=entry, - repo_path=repo_path, - model=model, - category=category, - output_dir=output_dir, - engine_ref=engine_ref, - engine_repo=engine_repo, - engine_local_path=engine_local_path, - bcquality_ref=bcquality_ref, - bcquality_repo=bcquality_repo, - bcquality_local_path=bcquality_local_path, - min_severity=min_severity, - ) - - @run_app.command("copilot") def run_copilot( entry_id: Annotated[str, typer.Argument(help="Entry ID to run")], @@ -80,14 +38,13 @@ def run_copilot( al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, ) -> None: """ - Run GitHub Copilot CLI on a single entry to generate a patch (without building/testing). + Run GitHub Copilot CLI on a single entry to generate the category output. For full evaluation including building and running tests, use 'bcbench evaluate' instead. Example: uv run bcbench run copilot microsoft__BCApps-5633 --category bug-fix --repo-path /path/to/BCApps """ - reject_code_review(category, "run") entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] category.pipeline.setup_workspace(entry, repo_path) @@ -115,14 +72,13 @@ def run_claude( al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, ) -> None: """ - Run Claude Code on a single entry to generate a patch (without building/testing). + Run Claude Code on a single entry to generate the category output. For full evaluation including building and running tests, use 'bcbench evaluate' instead. Example: uv run bcbench run claude microsoft__BCApps-5633 --category bug-fix --repo-path /path/to/BCApps """ - reject_code_review(category, "run") entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] category.pipeline.setup_workspace(entry, repo_path) @@ -138,40 +94,42 @@ def run_claude( ) -@run_app.command("code-review") -def run_code_review( +@run_app.command("pr-review") +def run_pr_review( entry_id: Annotated[str, typer.Argument(help="Entry ID to run")], - model: CopilotModel = "claude-sonnet-5", + model: CopilotModel = "gpt-5.6-luna", repo_path: RepoPath = _config.paths.testbed_path, output_dir: OutputDir = _config.paths.evaluation_results_path, - engine_ref: Annotated[str | None, typer.Option(help="Override the BC-ALAgents ref (defaults to pr_review.engine.ref)")] = None, - engine_repo: Annotated[str | None, typer.Option(help="Override the BC-ALAgents repo (defaults to pr_review.engine.repo)")] = None, - engine_local_path: Annotated[str | None, typer.Option(help="Use a local BC-ALAgents checkout instead of fetching")] = None, + engine_path: PRReviewEnginePath = None, bcquality_ref: Annotated[str | None, typer.Option(help="Override the BCQuality ref (defaults to the engine's pinned ref)")] = None, - bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to pr_review.bcquality.repo or the engine pin)")] = None, - bcquality_local_path: Annotated[str | None, typer.Option(help="Use a local BCQuality checkout (copied + filtered, never modified) instead of fetching")] = None, + bcquality_repo: Annotated[str | None, typer.Option(help="Override the BCQuality repo, e.g. a private fork (defaults to config/engine)")] = None, + bcquality_local_path: BCQualityLocalPath = None, min_severity: Annotated[str | None, typer.Option(help="AGENT_MINIMUM_SEVERITY floor (defaults to config)")] = None, ) -> None: """ - Run the code-review category on a single entry via the BC-ALAgents review engine. + Run BC PR Review on a single code-review entry. - code-review is not a general harness choice: it always runs the engine's real generate - half (never a bespoke prompt), so it has its own command instead of a copilot/claude - sub-command. Writes review.json in the repo root without scoring; for full evaluation - use 'bcbench evaluate code-review'. BC-ALAgents and BCQuality sources can be - configured in config.yaml or overridden with command options. + This production-fidelity runner is fixed to the code-review category, while the same + category can also run through the generic copilot and claude commands for cross-system + comparison. Writes review.json without scoring; for full evaluation use + 'bcbench evaluate pr-review'. Requires a local BC-ALAgents checkout + (--engine-path or BC_PR_REVIEW_ROOT), PowerShell 7+, and an authenticated + Copilot CLI. Example: - uv run bcbench run code-review synthetic__style-018 --repo-path /path/to/testbed + uv run bcbench run pr-review synthetic__style-018 --repo-path /path/to/testbed """ - _run_pr_review( - entry_id, + category = EvaluationCategory.CODE_REVIEW + entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0] + category.pipeline.setup_workspace(entry, repo_path) + + run_pr_review_agent( + entry=entry, model=model, repo_path=repo_path, + category=category, output_dir=output_dir, - engine_ref=engine_ref, - engine_repo=engine_repo, - engine_local_path=engine_local_path, + engine_path=engine_path, bcquality_ref=bcquality_ref, bcquality_repo=bcquality_repo, bcquality_local_path=bcquality_local_path, diff --git a/src/bcbench/dataset/codereview.py b/src/bcbench/dataset/codereview.py index 4281e4f14..55a6242f4 100644 --- a/src/bcbench/dataset/codereview.py +++ b/src/bcbench/dataset/codereview.py @@ -53,7 +53,9 @@ def from_input(cls, value: str) -> Severity: class ReviewComment(BaseModel): - model_config = ConfigDict(frozen=True) + # Reject unknown keys so a superseded annotation (e.g. the singular `article` this + # model used before) fails loudly instead of being dropped into invisible coverage loss. + model_config = ConfigDict(frozen=True, extra="forbid") file: Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9 ./_-]*\.(al|json)$")] line_start: Annotated[int, Field(ge=1)] @@ -61,10 +63,12 @@ class ReviewComment(BaseModel): domain: str | None = None body: Annotated[str, Field(min_length=1)] severity: Severity | None = None - # BCQuality knowledge article this finding derives from, as `/` - # (e.g. `security/hardcoded-secret`). Optional and backward-compatible; drives - # per-article coverage tracking. Older entries leave it unset (counted as unannotated). - article: ArticleId | None = None + # BCQuality knowledge articles this finding derives from, as `/` + # (e.g. `security/hardcoded-secret`). A single finding can exercise several articles, + # most commonly when it pairs a positive requirement with an explicit false-positive + # boundary. Optional; drives per-article coverage tracking. Entries that leave it + # empty are counted as unannotated. + articles: list[ArticleId] = Field(default_factory=list) @field_validator("severity", mode="before") @classmethod @@ -89,7 +93,7 @@ class CodeReviewEntryMetadata(EntryMetadata): # BCQuality knowledge articles this entry exercises as `/`. Primarily # for false-positive-guard entries (expected_comments=[]) that test an article by - # omission and thus have no per-comment `article` to carry the association. + # omission and thus have no per-comment `articles` to carry the association. articles: list[ArticleId] = Field(default_factory=list) @@ -115,7 +119,7 @@ def _validate_article_annotations(self) -> Self: empty). An article already declared on a comment must not be repeated at entry level, so the two annotation sources cannot silently drift apart. """ - comment_articles = {c.article for c in self.expected_comments if c.article} + comment_articles = {article for c in self.expected_comments for article in c.articles} overlap = comment_articles & set(self.metadata.articles) if overlap: raise ValueError( @@ -134,10 +138,10 @@ def get_expected_output(self) -> str: def declared_articles(self) -> set[ArticleId]: """BCQuality articles this entry is annotated against. - Union of every expected comment's `article` and the entry-level + Union of every expected comment's `articles` and the entry-level `metadata.articles` (which carries the association for false-positive-guard entries whose `expected_comments` is empty). """ - articles = {c.article for c in self.expected_comments if c.article} + articles = {article for c in self.expected_comments for article in c.articles} articles.update(self.metadata.articles) return articles diff --git a/src/bcbench/evaluate/review_parsing.py b/src/bcbench/evaluate/review_parsing.py index 8a531d876..999075d8a 100644 --- a/src/bcbench/evaluate/review_parsing.py +++ b/src/bcbench/evaluate/review_parsing.py @@ -44,11 +44,14 @@ def _to_int(value: object) -> int | None: def _normalize_comment(item: dict[Any, Any]) -> ReviewComment | None: - file_path = item.get("file") or item.get("filePath") or item.get("path") + # Two producers are in play: the pr_review agent emits file/line_start/line_end/body, + # while the BCApps review instructions ask for filePath/lineNumber/issue. BCQuality + # reports name the line "line", so that spelling is kept for a flattened report. + file_path = item.get("file") or item.get("filePath") line_start = _to_int(item.get("line_start") or item.get("lineNumber") or item.get("line")) - line_end = _to_int(item.get("line_end") or item.get("lineEnd") or item.get("endLine")) + line_end = _to_int(item.get("line_end")) domain = item.get("domain") - body = item.get("body") or item.get("issue") or item.get("comment") + body = item.get("body") or item.get("issue") if not isinstance(file_path, str) or not file_path.strip(): return None @@ -107,7 +110,7 @@ def parse_review_output(raw_output: str) -> list[ReviewComment] | None: raw_items = raw elif isinstance(raw, dict) and isinstance(raw.get("findings"), list): raw_items = raw["findings"] - elif isinstance(raw, dict) and any(key in raw for key in ("file", "filePath", "path")): + elif isinstance(raw, dict) and any(key in raw for key in ("file", "filePath")): raw_items = [raw] else: logger.warning(f"Expected JSON array or object with findings[], got {type(raw).__name__}") diff --git a/src/bcbench/operations/__init__.py b/src/bcbench/operations/__init__.py index 3469ab806..0b3d7ac58 100644 --- a/src/bcbench/operations/__init__.py +++ b/src/bcbench/operations/__init__.py @@ -18,6 +18,8 @@ clone_repo_at_revision, commit_changes, fetch_commit_if_missing, + has_changes, + init_repo, stage_and_get_diff, ) from bcbench.operations.hooks_operations import setup_hooks @@ -44,6 +46,8 @@ "copy_symbol_apps", "extract_tests_from_patch", "fetch_commit_if_missing", + "has_changes", + "init_repo", "remove_tree", "resolve_artifact_version_root", "run_tests", diff --git a/src/bcbench/operations/git_operations.py b/src/bcbench/operations/git_operations.py index 3cbcd8905..ab5accf09 100644 --- a/src/bcbench/operations/git_operations.py +++ b/src/bcbench/operations/git_operations.py @@ -79,11 +79,27 @@ def fetch_commit_if_missing(repo_path: Path, commit: str) -> None: logger.info(f"Commit {commit} fetched") -def commit_changes(repo_path: Path, message: str) -> None: +def init_repo(repo_path: Path) -> None: + repo_path.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "init", "-q"], cwd=repo_path, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True) + + +def has_changes(repo_path: Path) -> bool: + result = subprocess.run(["git", "status", "--porcelain"], cwd=repo_path, capture_output=True, encoding="utf-8", text=True, check=True) + return bool(result.stdout.strip()) + + +def commit_changes(repo_path: Path, message: str, *, allow_empty: bool = False, no_verify: bool = False) -> None: logger.info(f"Committing changes: {message}") subprocess.run(["git", "add", "-A"], cwd=repo_path, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True) + commit_args = ["git", "-c", "user.name=bcbench", "-c", "user.email=bcbench@noreply", "commit"] + if allow_empty: + commit_args.append("--allow-empty") + if no_verify: + commit_args.append("--no-verify") + commit_args.extend(["-m", message]) subprocess.run( - ["git", "-c", "user.name=bcbench", "-c", "user.email=bcbench@noreply", "commit", "--allow-empty", "-m", message], + commit_args, cwd=repo_path, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 0204f8924..15eda4c3f 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -11,14 +11,6 @@ logger = get_logger(__name__) -_CODE_REVIEW_METRIC_FIELDS = ( - "total_tokens", - "api_calls", - "estimated_credits", - "knowledge_used", - "knowledge_pruned", -) - class BaseEvaluationResult(BaseModel): """Base class for all evaluation results with shared metrics across categories.""" @@ -63,9 +55,6 @@ def save(self, output_dir: Path, result_file: str) -> None: output_dir.mkdir(parents=True, exist_ok=True) with output_file.open("a", encoding="utf-8") as f: result_dict = self.model_dump(mode="json") - if self.category is not EvaluationCategory.CODE_REVIEW and result_dict["metrics"]: - for field in _CODE_REVIEW_METRIC_FIELDS: - result_dict["metrics"].pop(field, None) # Per-instance JSONL result files are uploaded as workflow artifacts and are the only inputs required by the summarize-results workflow. f.write(json.dumps(result_dict) + "\n") diff --git a/src/bcbench/results/codereview.py b/src/bcbench/results/codereview.py index 0ebf372d5..40d1222be 100644 --- a/src/bcbench/results/codereview.py +++ b/src/bcbench/results/codereview.py @@ -2,7 +2,7 @@ from typing import Any, NamedTuple, Self import numpy as np -from pydantic import Field +from pydantic import Field, SerializerFunctionWrapHandler, model_serializer from rich.console import Group, RenderableType from rich.panel import Panel from rich.table import Table @@ -190,6 +190,30 @@ class CodeReviewResult(JudgeScoredEvaluationResult): f_beta_2: float = Field(default=0.0, ge=0.0, le=1.0) severity_mae: float = 0.0 + @model_serializer(mode="wrap") + def serialize_structured_metrics(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + data = handler(self) + if self.metrics is None: + return data + serialized_metrics = data["metrics"] + for field in ( + "cached_tokens", + "cache_creation_tokens", + "reasoning_tokens", + "total_tokens", + "api_calls", + "failed_api_calls", + "usage_api_calls", + "ai_credits", + "premium_requests", + "usage_complete", + "malformed_records", + "knowledge_files", + "knowledge_pruned", + ): + serialized_metrics[field] = getattr(self.metrics, field) + return data + @classmethod def create( cls, @@ -288,6 +312,9 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary): Macro metrics average per-task scores (each task weighted equally). """ + average_prompt_tokens: float | None = None + average_completion_tokens: float | None = None + generated_comment_count: int = Field(default=0, ge=0) expected_comment_count: int = Field(default=0, ge=0) matched_comment_count: int = Field(default=0, ge=0) @@ -310,39 +337,48 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary): severity_mae: float = 0.0 valid_review_output_rate: float = Field(default=0.0, ge=0.0, le=1.0) + average_cached_tokens: float | None = None + average_cache_creation_tokens: float | None = None + average_reasoning_tokens: float | None = None average_total_tokens: float | None = None average_api_calls: float | None = None - average_estimated_credits: float | None = None - average_knowledge_used: float | None = None + average_failed_api_calls: float | None = None + average_usage_api_calls: float | None = None + average_ai_credits: float | None = None + average_premium_requests: float | None = None + structured_usage_complete_rate: float | None = Field(default=None, ge=0.0, le=1.0) + average_malformed_records: float | None = None + average_knowledge_files: float | None = None average_knowledge_pruned: float | None = None # Per-task F1 keyed by instance_id, retained so the leaderboard can bootstrap a confidence # interval over tasks (meaningful even for a single run) instead of only over runs. instance_results: dict[str, float] = Field(default_factory=dict) - def _perf_markdown(self) -> str: - if all( - value is None - for value in ( - self.average_total_tokens, - self.average_api_calls, - self.average_estimated_credits, - self.average_knowledge_used, - self.average_knowledge_pruned, - ) - ): - return "" - tokens = f"{self.average_total_tokens:.0f}" if self.average_total_tokens is not None else "n/a" - api_calls = f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a" - credits = f"{self.average_estimated_credits:.4f}" if self.average_estimated_credits is not None else "n/a" - used = f"{self.average_knowledge_used:.1f}" if self.average_knowledge_used is not None else "n/a" + def _performance_markdown(self) -> str: + def metric(value: float | None, digits: int = 1) -> str: + return f"{value:.{digits}f}" if value is not None else "n/a" + + usage_complete = f"{self.structured_usage_complete_rate * 100:.1f}%" if self.structured_usage_complete_rate is not None else "n/a" + available = metric(self.average_knowledge_files) pruned = f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a" return ( "## Performance\n" "\n" - "| Avg duration (s) | Avg total tokens | Avg API calls | Avg est. credits | Avg knowledge used | Avg knowledge pruned |\n" - "|-----------------:|-----------------:|--------------:|-----------------:|-------------------:|---------------------:|\n" - f"| {self.average_duration:.1f} | {tokens} | {api_calls} | {credits} | {used} | {pruned} |\n" + "| Avg duration (s) | Avg prompt tokens | Avg cached tokens | Avg cache-creation tokens | Avg completion tokens | Avg reasoning tokens | Avg total tokens |\n" + "|-----------------:|------------------:|------------------:|-------------------------:|----------------------:|---------------------:|-----------------:|\n" + f"| {self.average_duration:.1f} | {metric(self.average_prompt_tokens)} | {metric(self.average_cached_tokens)} | " + f"{metric(self.average_cache_creation_tokens)} | {metric(self.average_completion_tokens)} | {metric(self.average_reasoning_tokens)} | " + f"{metric(self.average_total_tokens)} |\n" + "\n" + "| Avg API calls | Avg failed API calls | Avg calls with usage | Avg AI credits | Avg premium requests | Complete structured usage | Avg malformed records |\n" + "|--------------:|---------------------:|---------------------:|---------------:|---------------------:|--------------------------:|----------------------:|\n" + f"| {metric(self.average_api_calls)} | {metric(self.average_failed_api_calls)} | {metric(self.average_usage_api_calls)} | " + f"{metric(self.average_ai_credits, 4)} | {metric(self.average_premium_requests, 4)} | {usage_complete} | {metric(self.average_malformed_records)} |\n" + "\n" + "| Avg knowledge files | Avg knowledge pruned |\n" + "|--------------------:|---------------------:|\n" + f"| {available} | {pruned} |\n" "\n" ) @@ -383,35 +419,10 @@ def render_github_metrics_markdown(self) -> str: "|-------------:|-------------------------:|\n" f"| {self.severity_mae:.3f} | {valid_rate:.1f}% |\n" "\n" - f"{self._perf_markdown()}" + f"{self._performance_markdown()}" f"{_METRIC_EXPLANATIONS}" ) - def _perf_console_tables(self) -> list[RenderableType]: - if all( - value is None - for value in ( - self.average_total_tokens, - self.average_api_calls, - self.average_estimated_credits, - self.average_knowledge_used, - self.average_knowledge_pruned, - ) - ): - return [] - tokens = f"{self.average_total_tokens:.0f}" if self.average_total_tokens is not None else "n/a" - api_calls = f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a" - credits = f"{self.average_estimated_credits:.4f}" if self.average_estimated_credits is not None else "n/a" - used = f"{self.average_knowledge_used:.1f}" if self.average_knowledge_used is not None else "n/a" - pruned = f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a" - return [ - _build_console_table( - "Performance", - ["Avg duration (s)", "Avg total tokens", "Avg API calls", "Avg est. credits", "Avg knowledge used", "Avg knowledge pruned"], - [f"{self.average_duration:.1f}", tokens, api_calls, credits, used, pruned], - ) - ] - def render_console_metrics(self) -> RenderableType: metric_columns = ["Precision", "Recall", "F1", "Fβ (β=0.5)", "Fβ (β=2)"] @@ -455,7 +466,40 @@ def render_console_metrics(self) -> RenderableType: ["Severity MAE", "Valid review output rate"], [f"{self.severity_mae:.3f}", f"{self.valid_review_output_rate * 100:.1f}%"], ), - *self._perf_console_tables(), + _build_console_table( + "Performance: tokens", + ["Avg duration (s)", "Prompt", "Cached", "Cache creation", "Completion", "Reasoning", "Total"], + [ + f"{self.average_duration:.1f}", + f"{self.average_prompt_tokens:.1f}" if self.average_prompt_tokens is not None else "n/a", + f"{self.average_cached_tokens:.1f}" if self.average_cached_tokens is not None else "n/a", + f"{self.average_cache_creation_tokens:.1f}" if self.average_cache_creation_tokens is not None else "n/a", + f"{self.average_completion_tokens:.1f}" if self.average_completion_tokens is not None else "n/a", + f"{self.average_reasoning_tokens:.1f}" if self.average_reasoning_tokens is not None else "n/a", + f"{self.average_total_tokens:.1f}" if self.average_total_tokens is not None else "n/a", + ], + ), + _build_console_table( + "Performance: requests", + ["API calls", "Failed", "With usage", "AI credits", "Premium requests", "Complete usage", "Malformed"], + [ + f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a", + f"{self.average_failed_api_calls:.1f}" if self.average_failed_api_calls is not None else "n/a", + f"{self.average_usage_api_calls:.1f}" if self.average_usage_api_calls is not None else "n/a", + f"{self.average_ai_credits:.4f}" if self.average_ai_credits is not None else "n/a", + f"{self.average_premium_requests:.4f}" if self.average_premium_requests is not None else "n/a", + f"{self.structured_usage_complete_rate * 100:.1f}%" if self.structured_usage_complete_rate is not None else "n/a", + f"{self.average_malformed_records:.1f}" if self.average_malformed_records is not None else "n/a", + ], + ), + _build_console_table( + "Performance: knowledge", + ["Avg knowledge files", "Avg knowledge pruned"], + [ + f"{self.average_knowledge_files:.1f}" if self.average_knowledge_files is not None else "n/a", + f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a", + ], + ), Panel( _CONSOLE_METRIC_EXPLANATIONS, title="📖 How to read these metrics", @@ -516,6 +560,12 @@ def average_metric(name: str) -> float | None: values = [value for result in code_review_results if result.metrics and (value := getattr(result.metrics, name)) is not None] return sum(values) / len(values) if values else None + usage_completeness = [ + result.metrics.usage_complete and result.metrics.malformed_records == 0 + for result in code_review_results + if result.metrics and result.metrics.usage_complete is not None and result.metrics.malformed_records is not None + ] + return summary.model_copy( update={ "generated_comment_count": generated_total, @@ -537,23 +587,20 @@ def average_metric(name: str) -> float | None: "severity_mae": round(severity_mae, 3), "valid_review_output_rate": round(valid_output_rate, 3), "instance_results": {r.instance_id: round(r.f1, 6) for r in code_review_results}, + "average_prompt_tokens": average_metric("prompt_tokens"), + "average_completion_tokens": average_metric("completion_tokens"), + "average_cached_tokens": average_metric("cached_tokens"), + "average_cache_creation_tokens": average_metric("cache_creation_tokens"), + "average_reasoning_tokens": average_metric("reasoning_tokens"), "average_total_tokens": average_metric("total_tokens"), "average_api_calls": average_metric("api_calls"), - "average_estimated_credits": average_metric("estimated_credits"), - "average_knowledge_used": average_metric("knowledge_used"), + "average_failed_api_calls": average_metric("failed_api_calls"), + "average_usage_api_calls": average_metric("usage_api_calls"), + "average_ai_credits": average_metric("ai_credits"), + "average_premium_requests": average_metric("premium_requests"), + "structured_usage_complete_rate": sum(usage_completeness) / len(usage_completeness) if usage_completeness else None, + "average_malformed_records": average_metric("malformed_records"), + "average_knowledge_files": average_metric("knowledge_files"), "average_knowledge_pruned": average_metric("knowledge_pruned"), } ) - - def to_dict(self) -> dict[str, Any]: - data = super().to_dict() - for key, digits in ( - ("average_total_tokens", 1), - ("average_api_calls", 2), - ("average_estimated_credits", 4), - ("average_knowledge_used", 2), - ("average_knowledge_pruned", 2), - ): - if data[key] is not None: - data[key] = round(float(data[key]), digits) - return data diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index 8a59e3437..69cece34b 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, Field, field_validator from bcbench.logger import get_logger from bcbench.results.metrics import bootstrap_ci, pass_hat_k @@ -147,10 +147,20 @@ class CodeReviewLeaderboardAggregate(JudgeBasedLeaderboardAggregate): macro_precision: float = 0.0 macro_recall: float = 0.0 + average_prompt_tokens: float | None = None + average_completion_tokens: float | None = None + average_cached_tokens: float | None = None + average_cache_creation_tokens: float | None = None + average_reasoning_tokens: float | None = None average_total_tokens: float | None = None average_api_calls: float | None = None - average_estimated_credits: float | None = None - average_knowledge_used: float | None = None + average_failed_api_calls: float | None = None + average_usage_api_calls: float | None = None + average_ai_credits: float | None = None + average_premium_requests: float | None = None + structured_usage_complete_rate: float | None = Field(default=None, ge=0.0, le=1.0) + average_malformed_records: float | None = None + average_knowledge_files: float | None = None average_knowledge_pruned: float | None = None @classmethod @@ -194,10 +204,20 @@ def mean_metric(name: str) -> float | None: "macro_f_beta_2": sum(r.macro_f_beta_2 for r in cr_runs) / n, "macro_precision": sum(r.macro_precision for r in cr_runs) / n, "macro_recall": sum(r.macro_recall for r in cr_runs) / n, + "average_prompt_tokens": mean_metric("average_prompt_tokens"), + "average_completion_tokens": mean_metric("average_completion_tokens"), + "average_cached_tokens": mean_metric("average_cached_tokens"), + "average_cache_creation_tokens": mean_metric("average_cache_creation_tokens"), + "average_reasoning_tokens": mean_metric("average_reasoning_tokens"), "average_total_tokens": mean_metric("average_total_tokens"), "average_api_calls": mean_metric("average_api_calls"), - "average_estimated_credits": mean_metric("average_estimated_credits"), - "average_knowledge_used": mean_metric("average_knowledge_used"), + "average_failed_api_calls": mean_metric("average_failed_api_calls"), + "average_usage_api_calls": mean_metric("average_usage_api_calls"), + "average_ai_credits": mean_metric("average_ai_credits"), + "average_premium_requests": mean_metric("average_premium_requests"), + "structured_usage_complete_rate": mean_metric("structured_usage_complete_rate"), + "average_malformed_records": mean_metric("average_malformed_records"), + "average_knowledge_files": mean_metric("average_knowledge_files"), "average_knowledge_pruned": mean_metric("average_knowledge_pruned"), } ) diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index 689388add..03ee60934 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -119,8 +119,8 @@ def from_json(cls, payload: dict[str, Any]) -> "EvaluationResultSummary": def to_dict(self) -> dict[str, Any]: data = self.model_dump(mode="json") data["average_duration"] = round(data["average_duration"], 1) - data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1) - data["average_completion_tokens"] = round(data["average_completion_tokens"], 1) + data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1) if data["average_prompt_tokens"] is not None else None + data["average_completion_tokens"] = round(data["average_completion_tokens"], 1) if data["average_completion_tokens"] is not None else None data["average_llm_duration"] = round(data["average_llm_duration"], 1) if data["average_llm_duration"] is not None else None return data diff --git a/src/bcbench/types.py b/src/bcbench/types.py index 654472f81..bf5036e0e 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Annotated, Literal, TypedDict -from pydantic import BaseModel, ConfigDict, StringConstraints, model_validator +from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator if TYPE_CHECKING: from bcbench.dataset import BaseDatasetEntry @@ -76,15 +76,26 @@ class AgentMetrics(BaseModel): prompt_tokens: int | None = None completion_tokens: int | None = None - total_tokens: int | None = None - api_calls: int | None = None - estimated_credits: float | None = None - knowledge_used: int | None = None - knowledge_pruned: int | None = None + # Structured usage metrics emitted by agent harnesses + cached_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + cache_creation_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + reasoning_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + total_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + api_calls: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + failed_api_calls: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + usage_api_calls: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + ai_credits: float | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + premium_requests: float | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + usage_complete: bool | None = Field(default=None, exclude_if=lambda value: value is None) + malformed_records: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) # Tool usage statistics from agent logs tool_usage: dict[str, int] | None = None + # BC PR Review's structural BCQuality filter metrics + knowledge_files: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + knowledge_pruned: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + class ExperimentConfiguration(BaseModel): """Configuration for agent experiment execution. @@ -203,16 +214,7 @@ def expected_metrics(self) -> frozenset[str]: case AgentHarness.BCAL: expected = AgentMetrics(execution_time=None) case AgentHarness.PR_REVIEW: - expected = AgentMetrics( - execution_time=None, - prompt_tokens=None, - completion_tokens=None, - total_tokens=None, - api_calls=None, - estimated_credits=None, - knowledge_used=None, - knowledge_pruned=None, - ) + expected = AgentMetrics(execution_time=None, knowledge_files=None, knowledge_pruned=None) case _: raise ValueError(f"Unknown AgentHarness: {self}") diff --git a/tests/test_bcquality_article_coverage.py b/tests/test_bcquality_article_coverage.py index b0035d38d..4537dad35 100644 --- a/tests/test_bcquality_article_coverage.py +++ b/tests/test_bcquality_article_coverage.py @@ -35,15 +35,15 @@ def _entry( ) -def _comment(body: str, *, article: ArticleId | None = None) -> ReviewComment: - return ReviewComment(file="src/A.al", line_start=1, body=body, article=article) +def _comment(body: str, *articles: ArticleId) -> ReviewComment: + return ReviewComment(file="src/A.al", line_start=1, body=body, articles=list(articles)) class TestDeclaredArticles: def test_declared_from_comment_and_metadata(self): entry = _entry( "synthetic__security-001", - comments=[_comment("finding", article="security/secrettext-for-credentials")], + comments=[_comment("finding", "security/secrettext-for-credentials")], articles=["security/permission-set-avoid-wildcard-grants"], ) assert entry.declared_articles() == { @@ -51,6 +51,26 @@ def test_declared_from_comment_and_metadata(self): "security/permission-set-avoid-wildcard-grants", } + def test_comment_may_declare_several_articles(self): + entry = _entry( + "synthetic__error-handling-errorinfo-actionable-boundary-01", + comments=[ + _comment( + "raise ErrorInfo with a navigation action; do not require a Fix-it here", + "error-handling/prefer-errorinfo-for-actionable-errors", + "error-handling/fielderror-vs-testfield", + ) + ], + ) + assert entry.declared_articles() == { + "error-handling/prefer-errorinfo-for-actionable-errors", + "error-handling/fielderror-vs-testfield", + } + + def test_superseded_singular_article_key_is_rejected(self): + with pytest.raises(ValidationError, match="article"): + ReviewComment(file="src/A.al", line_start=1, body="x", article="security/secrettext-for-credentials") # ty: ignore[unknown-argument] + def test_unannotated_entry_declares_nothing(self): entry = _entry("synthetic__security-002", comments=[_comment("finding")]) assert entry.declared_articles() == set() @@ -60,15 +80,15 @@ def test_article_declared_both_places_is_rejected(self): with pytest.raises(ValidationError, match="declared both per-comment"): _entry( "synthetic__security-003", - comments=[_comment("finding", article=shared)], + comments=[_comment("finding", shared)], articles=[shared], ) def test_collect_maps_article_to_sorted_entry_ids(self): shared = "security/validate-user-configurable-urls" entries = [ - _entry("synthetic__security-015", comments=[_comment("ssrf", article=shared)]), - _entry("synthetic__security-011", comments=[_comment("ssrf", article=shared)]), + _entry("synthetic__security-015", comments=[_comment("ssrf", shared)]), + _entry("synthetic__security-011", comments=[_comment("ssrf", shared)]), ] declared = collect_declared_articles(entries) assert declared[shared] == ["synthetic__security-011", "synthetic__security-015"] @@ -77,7 +97,7 @@ def test_collect_maps_article_to_sorted_entry_ids(self): class TestCoverageReport: def test_without_inventory_reports_declared_only(self): entries = [ - _entry("synthetic__security-001", comments=[_comment("x", article="security/secrettext-for-credentials")]), + _entry("synthetic__security-001", comments=[_comment("x", "security/secrettext-for-credentials")]), _entry("synthetic__security-002", comments=[_comment("y")]), ] report = build_coverage_report(entries, inventory=None) @@ -93,8 +113,8 @@ def test_with_inventory_flags_zero_and_unknown(self): "security/permission-set-avoid-wildcard-grants", } entries = [ - _entry("synthetic__security-001", comments=[_comment("x", article="security/secrettext-for-credentials")]), - _entry("synthetic__security-009", comments=[_comment("z", article="security/made-up-slug")]), + _entry("synthetic__security-001", comments=[_comment("x", "security/secrettext-for-credentials")]), + _entry("synthetic__security-009", comments=[_comment("z", "security/made-up-slug")]), ] report = build_coverage_report(entries, inventory=inventory) assert report.inventory_available is True @@ -106,7 +126,7 @@ def test_with_inventory_flags_zero_and_unknown(self): def test_covered_entry_ids_deduplicated_across_entries(self): article = "security/inherent-permissions-minimal-grant" entries = [ - _entry("synthetic__security-013", comments=[_comment("a", article=article), _comment("b", article=article)]), + _entry("synthetic__security-013", comments=[_comment("a", article), _comment("b", article)]), ] report = build_coverage_report(entries, inventory={article}) assert len(report.covered) == 1 diff --git a/tests/test_copilot_prompt.py b/tests/test_copilot_prompt.py index a0bbdf13f..e0311e9e5 100644 --- a/tests/test_copilot_prompt.py +++ b/tests/test_copilot_prompt.py @@ -1,7 +1,11 @@ from pathlib import Path from unittest.mock import patch +import yaml + from bcbench.agent.shared import build_prompt +from bcbench.config import get_config +from bcbench.dataset.codereview import CodeReviewEntry from bcbench.types import EvaluationCategory from tests.conftest import create_dataset_entry, create_problem_statement_dir @@ -130,3 +134,20 @@ def test_build_prompt_test_generation_both_mode(tmp_path: Path): assert "[HAS_PATCH]" in result # gold patch should be indicated assert "[HAS_ISSUE]" in result # problem statement should be indicated assert "Fix payment validation bug" in result # task should be included in both mode + + +def test_build_prompt_code_review_enforces_review_json_contract(tmp_path: Path): + config_path = get_config().paths.agent_share_dir / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + entry = CodeReviewEntry.model_construct(project_paths=[], patch="diff --git a/src/Foo.al b/src/Foo.al") + + prompt = build_prompt(entry, tmp_path, config, EvaluationCategory.CODE_REVIEW) + + assert "staged and unstaged dataset changes" in prompt + assert "git diff HEAD" in prompt + assert "Do NOT modify source code" in prompt + assert f"`{tmp_path}/review.json`" in prompt + assert "one JSON array" in prompt + for field in ("file", "line_start", "line_end", "domain", "body", "severity"): + assert f"`{field}`" in prompt + assert "empty array" in prompt diff --git a/tests/test_git_operations.py b/tests/test_git_operations.py index 51f0b3704..b97d120b8 100644 --- a/tests/test_git_operations.py +++ b/tests/test_git_operations.py @@ -6,7 +6,16 @@ import pytest from bcbench.exceptions import EmptyDiffError -from bcbench.operations.git_operations import checkout_commit, clean_project_paths, clone_repo_at_revision, commit_changes, fetch_commit_if_missing, stage_and_get_diff +from bcbench.operations.git_operations import ( + checkout_commit, + clean_project_paths, + clone_repo_at_revision, + commit_changes, + fetch_commit_if_missing, + has_changes, + init_repo, + stage_and_get_diff, +) class TestCommitChanges: @@ -45,6 +54,16 @@ def test_commit_works_without_global_git_identity(self, temp_git_repo): result = subprocess.run(["git", "log", "--oneline", "-1"], cwd=temp_git_repo, capture_output=True, text=True, check=True) assert "should work without identity" in result.stdout + def test_init_and_empty_commit(self, tmp_path): + repo_path = tmp_path / "repo" + + init_repo(repo_path) + commit_changes(repo_path, "empty", allow_empty=True) + + assert not has_changes(repo_path) + result = subprocess.run(["git", "log", "--format=%s", "-1"], cwd=repo_path, capture_output=True, text=True, check=True) + assert result.stdout.strip() == "empty" + class TestStageAndGetDiff: @pytest.fixture diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index d1e193a92..3b8e430ee 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -1,10 +1,14 @@ import json +import subprocess from pathlib import Path +from unittest.mock import patch import pytest -from bcbench.agent.copilot.pr_review.agent import _prepare_engine_root, _resolve_bcquality_source, _write_review_json +from bcbench.agent.pr_review.agent import _write_review_json, run_pr_review_agent from bcbench.exceptions import AgentError +from bcbench.types import EvaluationCategory +from tests.conftest import create_codereview_entry def _dirs(tmp_path: Path) -> tuple[Path, Path]: @@ -16,120 +20,7 @@ def _dirs(tmp_path: Path) -> tuple[Path, Path]: def _write_output(output_dir: Path, text: str) -> None: - (output_dir / "agent-output.txt").write_text(text, encoding="utf-8") - - -def _write_engine_shell(root: Path) -> None: - shell = root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-PRReviewShell.ps1" - shell.parent.mkdir(parents=True) - shell.write_text("", encoding="utf-8") - - -def test_prepare_engine_root_uses_configured_local_path(tmp_path: Path) -> None: - engine = tmp_path / "local-engine" - _write_engine_shell(engine) - - with _prepare_engine_root( - {"engine": {"repo": "microsoft/BC-ALAgents", "ref": "main", "local_path": str(engine)}}, - tmp_path / "clone", - ) as resolved: - assert resolved == engine - - -def test_prepare_engine_root_environment_override_takes_precedence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - configured = tmp_path / "configured" - override = tmp_path / "override" - _write_engine_shell(configured) - _write_engine_shell(override) - monkeypatch.setenv("BC_PR_REVIEW_ROOT", str(override)) - - with _prepare_engine_root( - {"engine": {"local_path": str(configured)}}, - tmp_path / "clone", - engine_local_path=str(configured), - ) as resolved: - assert resolved == override - - -def test_prepare_engine_root_clones_configured_ref_and_cleans_up(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - destination = tmp_path / "engine" - configured_local = tmp_path / "configured-local" - clone_args: list[object] = [] - _write_engine_shell(configured_local) - - def fake_clone(repo: str, revision: str, target: Path) -> None: - clone_args.extend([repo, revision, target]) - _write_engine_shell(target) - - monkeypatch.delenv("BC_PR_REVIEW_ROOT", raising=False) - monkeypatch.setattr("bcbench.agent.copilot.pr_review.agent.clone_repo_at_revision", fake_clone) - - with _prepare_engine_root( - {"engine": {"repo": "contoso/BC-ALAgents", "ref": "feature/review", "local_path": str(configured_local)}}, - destination, - engine_repo="fabrikam/BC-ALAgents", - engine_ref="experiment/engine", - ) as resolved: - assert resolved == destination - assert destination.exists() - - assert clone_args == ["fabrikam/BC-ALAgents", "experiment/engine", destination] - assert not destination.exists() - - -def test_prepare_engine_root_cleans_up_failed_clone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - destination = tmp_path / "engine" - - def failed_clone(repo: str, revision: str, target: Path) -> None: - target.mkdir(parents=True) - (target / "partial").write_text("", encoding="utf-8") - raise RuntimeError(f"Could not clone {repo}@{revision}") - - monkeypatch.delenv("BC_PR_REVIEW_ROOT", raising=False) - monkeypatch.setattr("bcbench.agent.copilot.pr_review.agent.clone_repo_at_revision", failed_clone) - - with ( - pytest.raises(RuntimeError, match="Could not clone"), - _prepare_engine_root( - {"engine": {"repo": "contoso/BC-ALAgents", "ref": "feature/review"}}, - destination, - ), - ): - pass - - assert not destination.exists() - - -def test_remote_bcquality_override_ignores_configured_local_path() -> None: - resolved = _resolve_bcquality_source( - {"bcquality": {"repo": "microsoft/BCQuality", "ref": "main", "local_path": "C:/local/BCQuality"}}, - bcquality_ref="feature/knowledge", - bcquality_repo=None, - bcquality_local_path=None, - ) - - assert resolved == ("feature/knowledge", "microsoft/BCQuality", None) - - -def test_local_bcquality_override_uses_configured_remote_defaults() -> None: - resolved = _resolve_bcquality_source( - {"bcquality": {"repo": "microsoft/BCQuality", "ref": "main", "local_path": None}}, - bcquality_ref=None, - bcquality_repo=None, - bcquality_local_path="C:/local/BCQuality", - ) - - assert resolved == ("main", "microsoft/BCQuality", "C:/local/BCQuality") - - -def test_conflicting_bcquality_cli_sources_raise() -> None: - with pytest.raises(AgentError, match="cannot be combined"): - _resolve_bcquality_source( - {"bcquality": {}}, - bcquality_ref="feature/knowledge", - bcquality_repo=None, - bcquality_local_path="C:/local/BCQuality", - ) + (output_dir / "al-code-review-findings.json").write_text(text, encoding="utf-8") def test_valid_empty_findings_is_a_clean_review(tmp_path: Path) -> None: @@ -143,7 +34,7 @@ def test_findings_are_mapped(tmp_path: Path) -> None: out, repo = _dirs(tmp_path) report = { "outcome": "completed", - "findings": [{"severity": "High", "location": {"file": "src/Foo.al", "line": 42}, "message": "x", "domain": "ui"}], + "findings": [{"severity": "High", "filePath": "src/Foo.al", "lineNumber": 42, "issue": "x", "domain": "ui"}], } _write_output(out, json.dumps(report)) assert _write_review_json(out, repo) == 1 @@ -159,15 +50,117 @@ def test_missing_agent_output_raises(tmp_path: Path) -> None: def test_invalid_output_raises_instead_of_clean_review(tmp_path: Path, text: str) -> None: out, repo = _dirs(tmp_path) _write_output(out, text) - with pytest.raises(AgentError, match="empty or not a valid"): + with pytest.raises(AgentError, match="empty or invalid"): _write_review_json(out, repo) assert not (repo / "review.json").exists() -@pytest.mark.parametrize("report", [{"outcome": "failed"}, {"outcome": "dispatch", "findings": None}, {"findings": "nope"}]) +@pytest.mark.parametrize( + "report", + [ + {"outcome": "completed"}, + {"outcome": "partial", "findings": None}, + {"outcome": "no-knowledge", "findings": "nope"}, + ], +) def test_malformed_report_raises_instead_of_clean_review(tmp_path: Path, report: dict) -> None: out, repo = _dirs(tmp_path) _write_output(out, json.dumps(report)) with pytest.raises(AgentError, match="no findings list"): _write_review_json(out, repo) assert not (repo / "review.json").exists() + + +def test_failed_engine_outcome_raises_instead_of_clean_review(tmp_path: Path) -> None: + out, repo = _dirs(tmp_path) + _write_output(out, json.dumps({"outcome": "failed", "outcomeReason": "dispatch failed", "findings": []})) + + with pytest.raises(AgentError, match="dispatch failed"): + _write_review_json(out, repo) + + assert not (repo / "review.json").exists() + + +def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GITHUB_REPOSITORY", "microsoft/BC-Bench") + settings = { + "min_severity": "Medium", + "bcquality": {"repo": None, "ref": None}, + } + completed = subprocess.CompletedProcess(args=["pwsh"], returncode=0, stdout="✓", stderr="") + entry = create_codereview_entry(repo="microsoft/BCApps") + bcquality_root = tmp_path / "bcquality" + knowledge_root = bcquality_root / "microsoft" / "knowledge" / "performance" + knowledge_root.mkdir(parents=True) + (knowledge_root / "one.md").write_text("# One", encoding="utf-8") + (bcquality_root / "_filter-report.json").write_text('{"removed": []}', encoding="utf-8") + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "_run-metrics.json").write_text( + json.dumps( + { + "schema_version": 1, + "metrics_source": "copilot-cli-otel", + "cli_version": "1.0.81-0", + "wall_time_seconds": 2.4, + "prompt_tokens": 100, + "cached_tokens": 20, + "cache_creation_tokens": 5, + "completion_tokens": 10, + "reasoning_tokens": 4, + "total_tokens": 110, + "api_calls": 2, + "failed_api_calls": 0, + "usage_api_calls": 2, + "ai_credits": 0.25, + "premium_requests": 0.5, + "models": ["gpt-5.6-luna"], + "usage_complete": True, + "malformed_records": 0, + } + ), + encoding="utf-8", + ) + + with ( + patch("bcbench.agent.pr_review.agent._load_pr_review_settings", return_value=settings), + patch("bcbench.agent.pr_review.agent._resolve_pr_review_root", return_value=tmp_path / "engine"), + patch("bcbench.agent.pr_review.agent._resolve_pwsh", return_value="pwsh"), + patch("bcbench.agent.pr_review.agent._commit_patch_as_head"), + patch("bcbench.agent.pr_review.agent._init_trusted_workspace", return_value=tmp_path / "trusted"), + patch("bcbench.agent.pr_review.agent._prepare_bcquality_root", return_value=bcquality_root), + patch("bcbench.agent.pr_review.agent._write_review_json", return_value=0), + patch("bcbench.agent.pr_review.agent.time.monotonic", side_effect=[10.0, 12.5]), + patch("bcbench.agent.pr_review.agent.subprocess.run", return_value=completed) as run_process, + ): + metrics, config = run_pr_review_agent( + entry=entry, + model="gpt-5.6-luna", + category=EvaluationCategory.CODE_REVIEW, + repo_path=tmp_path / "repo", + output_dir=Path("output"), + engine_path=tmp_path / "engine", + ) + + assert metrics is not None + assert metrics.execution_time == 2.5 + assert metrics.total_tokens == 110 + assert metrics.api_calls == 2 + assert metrics.ai_credits == 0.25 + assert metrics.reasoning_tokens == 4 + assert metrics.premium_requests == 0.5 + assert metrics.knowledge_files == 1 + assert metrics.knowledge_pruned == 0 + assert config.is_empty() + assert run_process.call_args.kwargs["encoding"] == "utf-8" + assert run_process.call_args.kwargs["cwd"] == str((tmp_path / "repo").resolve()) + engine_env = run_process.call_args.kwargs["env"] + assert engine_env["REVIEW_TARGET_WORKSPACE"] == str((tmp_path / "repo").resolve()) + assert engine_env["REVIEW_OUTPUT_DIR"] == str((tmp_path / "output").resolve()) + assert engine_env["REVIEW_WORKSPACE"] == str(tmp_path / "trusted") + assert engine_env["BCQUALITY_ROOT"] == str(tmp_path / "bcquality") + assert engine_env["GITHUB_REPOSITORY"] == "microsoft/BCApps" + assert engine_env["AGENT_MINIMUM_SEVERITY"] == "Medium" + assert run_process.call_args.args[0][-1].endswith("Invoke-CopilotPRReview.ps1") + assert "-GenerateOnly" not in run_process.call_args.args[0] diff --git a/tests/test_pr_review_metrics.py b/tests/test_pr_review_metrics.py index 6e4c9e82f..12460e43e 100644 --- a/tests/test_pr_review_metrics.py +++ b/tests/test_pr_review_metrics.py @@ -1,110 +1,285 @@ import json from pathlib import Path -from bcbench.agent.copilot.pr_review.metrics import ( - FILTER_REPORT_FILE_NAME, - RUN_METRICS_FILE_NAME, - TRANSCRIPT_FILE_NAME, - build_pr_review_metrics, - parse_filter_report, - parse_run_metrics, - parse_transcript_metrics, -) +import pytest + +from bcbench.agent.pr_review.metrics import FILTER_REPORT_FILE_NAME, RUN_METRICS_FILE_NAME, build_pr_review_metrics +from bcbench.exceptions import AgentError + + +def _write_filter_report(root: Path, removed: object) -> None: + (root / FILTER_REPORT_FILE_NAME).write_text(json.dumps({"removed": removed}), encoding="utf-8") -def _write(path: Path, payload: object) -> None: - path.write_text(json.dumps(payload), encoding="utf-8") +def _run_metrics(**overrides: object) -> dict[str, object]: + payload: dict[str, object] = { + "schema_version": 1, + "metrics_source": "copilot-cli-otel", + "cli_version": "1.0.81-0", + "wall_time_seconds": 12.346, + "prompt_tokens": 150, + "cached_tokens": 60, + "cache_creation_tokens": 10, + "completion_tokens": 28, + "reasoning_tokens": 7, + "total_tokens": 178, + "api_calls": 2, + "failed_api_calls": 1, + "usage_api_calls": 2, + "ai_credits": 1.75, + "premium_requests": 1.75, + "models": ["gpt-5.4-mini", "gpt-5.6-sol"], + "usage_complete": True, + "malformed_records": 0, + } + return {**payload, **overrides} + +def _write_run_metrics(root: Path, **overrides: object) -> None: + (root / RUN_METRICS_FILE_NAME).write_text(json.dumps(_run_metrics(**overrides)), encoding="utf-8") -def test_parse_run_metrics(tmp_path: Path) -> None: - path = tmp_path / RUN_METRICS_FILE_NAME - _write( - path, - { - "prompt_tokens": 1200, - "completion_tokens": 300, - "api_calls": 7, - "estimated_credits": 0.33, - }, + +def test_build_metrics_reads_structured_usage_and_filtered_knowledge(tmp_path: Path) -> None: + knowledge = tmp_path / "microsoft" / "knowledge" / "performance" + knowledge.mkdir(parents=True) + (knowledge / "one.md").write_text("# One", encoding="utf-8") + (knowledge / "two.md").write_text("# Two", encoding="utf-8") + (knowledge / "two.good.al").write_text("", encoding="utf-8") + (tmp_path / "skills").mkdir() + (tmp_path / "skills" / "entry.md").write_text("# Entry", encoding="utf-8") + _write_filter_report( + tmp_path, + [ + {"path": "community/knowledge/old.md", "kind": "knowledge", "reason": "layer-disabled"}, + {"path": "community/skills/old.md", "kind": "skill", "reason": "layer-disabled"}, + ], ) + _write_run_metrics(tmp_path) - metrics = parse_run_metrics(path) + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=12.5) - assert metrics == { - "prompt_tokens": 1200, - "completion_tokens": 300, - "total_tokens": 1500, - "api_calls": 7, - "estimated_credits": 0.33, - } + assert metrics.execution_time == 12.5 + assert metrics.prompt_tokens == 150 + assert metrics.cached_tokens == 60 + assert metrics.cache_creation_tokens == 10 + assert metrics.completion_tokens == 28 + assert metrics.reasoning_tokens == 7 + assert metrics.total_tokens == 178 + assert metrics.api_calls == 2 + assert metrics.failed_api_calls == 1 + assert metrics.usage_api_calls == 2 + assert metrics.ai_credits == 1.75 + assert metrics.premium_requests == 1.75 + assert metrics.usage_complete is True + assert metrics.malformed_records == 0 + assert metrics.knowledge_files == 2 + assert metrics.knowledge_pruned == 1 -def test_parse_transcript_metrics_from_engine_artifact(tmp_path: Path) -> None: - path = tmp_path / TRANSCRIPT_FILE_NAME - path.write_text( - """err: --- Start of group: Sending request to the AI model --- -err: --- Start of group: Sending request to the AI model --- -err: AI Credits 138 (1m 20s) -err: Tokens ↑ 1,234,567 (1,000,000 cached) • ↓ 86,543 (500 reasoning)""", - encoding="utf-8", +def test_legal_null_optional_fields_and_multiple_models_are_accepted(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + cli_version=None, + wall_time_seconds=None, + cached_tokens=None, + cache_creation_tokens=None, + reasoning_tokens=None, + ai_credits=None, + premium_requests=None, + models=["gpt-5.4-mini", "gpt-5.6-sol"], ) - metrics = parse_transcript_metrics(path) + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) - assert metrics == { - "prompt_tokens": 1234567, - "completion_tokens": 86543, - "total_tokens": 1321110, - "api_calls": 2, - "estimated_credits": 138, - } + assert metrics.cached_tokens is None + assert metrics.cache_creation_tokens is None + assert metrics.ai_credits is None + assert metrics.reasoning_tokens is None + assert metrics.premium_requests is None + assert metrics.total_tokens == 178 -def test_parse_filter_report_counts_used_and_pruned_knowledge(tmp_path: Path) -> None: - knowledge = tmp_path / "content" / "knowledge" - knowledge.mkdir(parents=True) - (knowledge / "one.md").write_text("# One", encoding="utf-8") - (knowledge / "two.md").write_text("# Two", encoding="utf-8") - report = tmp_path / FILTER_REPORT_FILE_NAME - _write(report, {"removed": [{"kind": "knowledge"}, {"kind": "skill"}, {"kind": "knowledge"}]}) +def test_partial_usage_preserves_exact_counts_and_completeness_metadata(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + prompt_tokens=25, + cached_tokens=None, + cache_creation_tokens=None, + completion_tokens=5, + total_tokens=30, + api_calls=2, + failed_api_calls=1, + usage_api_calls=1, + ai_credits=0.1, + reasoning_tokens=None, + premium_requests=None, + usage_complete=False, + malformed_records=3, + ) - metrics = parse_filter_report(report, tmp_path) + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) - assert metrics == {"knowledge_pruned": 2, "knowledge_used": 2} + assert metrics.prompt_tokens == 25 + assert metrics.total_tokens == 30 + assert metrics.api_calls == 2 + assert metrics.usage_api_calls == 1 + assert metrics.ai_credits == 0.1 + assert metrics.reasoning_tokens is None + assert metrics.premium_requests is None + assert metrics.usage_complete is False + assert metrics.malformed_records == 3 -def test_build_metrics_degrades_when_side_files_are_missing(tmp_path: Path) -> None: - metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=12.5) +def test_missing_run_metrics_raises(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) - assert metrics.execution_time == 12.5 - assert metrics.total_tokens is None - assert metrics.knowledge_used is None - assert metrics.knowledge_pruned is None + with pytest.raises(AgentError, match="run metrics artifact not found"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) -def test_build_metrics_combines_engine_and_knowledge_signals(tmp_path: Path) -> None: - output = tmp_path / "output" - output.mkdir() - bcquality = tmp_path / "bcquality" - knowledge = bcquality / "knowledge" - knowledge.mkdir(parents=True) - (knowledge / "used.md").write_text("# Used", encoding="utf-8") - _write( - output / RUN_METRICS_FILE_NAME, - { - "prompt_tokens": 1000, - "completion_tokens": 200, - "total_tokens": 1200, - "api_calls": 5, - "estimated_credits": 0.25, - }, +def test_invalid_run_metrics_json_raises(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + (tmp_path / RUN_METRICS_FILE_NAME).write_text("not json", encoding="utf-8") + + with pytest.raises(AgentError, match="Could not read engine run metrics artifact"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +@pytest.mark.parametrize( + "overrides", + [ + {"schema_version": 2}, + {"metrics_source": "console-transcript"}, + {"api_calls": "2"}, + {"usage_complete": 1}, + {"reasoning_tokens": "5"}, + {"premium_requests": "1.0"}, + {"cli_version": 79}, + {"models": ["gpt-5.6-sol", 5]}, + {"unexpected": "field"}, + ], +) +def test_invalid_run_metrics_contract_raises(tmp_path: Path, overrides: dict[str, object]) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics(tmp_path, **overrides) + + with pytest.raises(AgentError, match="does not satisfy schema version 1"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_missing_filter_report_raises(tmp_path: Path) -> None: + _write_run_metrics(tmp_path) + + with pytest.raises(AgentError, match="not found"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_missing_run_metrics_key_raises(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + payload = _run_metrics() + del payload["models"] + (tmp_path / RUN_METRICS_FILE_NAME).write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(AgentError, match="does not satisfy schema version 1"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_not_applicable_zero_shape_is_accepted(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + metrics_source="not-applicable", + cli_version=None, + wall_time_seconds=0, + prompt_tokens=0, + cached_tokens=0, + cache_creation_tokens=0, + completion_tokens=0, + reasoning_tokens=None, + total_tokens=0, + api_calls=0, + failed_api_calls=0, + usage_api_calls=0, + ai_credits=0.0, + premium_requests=None, + models=[], + usage_complete=True, + malformed_records=0, ) - _write(bcquality / FILTER_REPORT_FILE_NAME, {"removed": [{"kind": "knowledge"}]}) - metrics = build_pr_review_metrics(output, bcquality, execution_time=8.0) + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=0.25) - assert metrics.total_tokens == 1200 - assert metrics.api_calls == 5 - assert metrics.estimated_credits == 0.25 - assert metrics.knowledge_used == 1 - assert metrics.knowledge_pruned == 1 + assert metrics.execution_time == 0.25 + assert metrics.prompt_tokens == 0 + assert metrics.api_calls == 0 + assert metrics.ai_credits == 0.0 + assert metrics.usage_complete is True + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("cli_version", "1.0.79"), + ("wall_time_seconds", 1.0), + ("prompt_tokens", None), + ("reasoning_tokens", 0), + ("premium_requests", 0.0), + ("models", ["gpt-5.6-sol"]), + ("usage_complete", False), + ("malformed_records", 1), + ], +) +def test_not_applicable_rejects_noncanonical_shape(tmp_path: Path, field: str, value: object) -> None: + _write_filter_report(tmp_path, []) + not_applicable = { + "metrics_source": "not-applicable", + "cli_version": None, + "wall_time_seconds": 0, + "prompt_tokens": 0, + "cached_tokens": 0, + "cache_creation_tokens": 0, + "completion_tokens": 0, + "reasoning_tokens": None, + "total_tokens": 0, + "api_calls": 0, + "failed_api_calls": 0, + "usage_api_calls": 0, + "ai_credits": 0.0, + "premium_requests": None, + "models": [], + "usage_complete": True, + "malformed_records": 0, + field: value, + } + _write_run_metrics(tmp_path, **not_applicable) + + with pytest.raises(AgentError, match="not-applicable metrics have invalid fields"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +@pytest.mark.parametrize( + "payload", + [ + [], + {}, + {"removed": "invalid"}, + {"removed": [{"kind": "unknown"}]}, + {"removed": ["invalid"]}, + ], +) +def test_malformed_filter_report_raises(tmp_path: Path, payload: object) -> None: + (tmp_path / FILTER_REPORT_FILE_NAME).write_text(json.dumps(payload), encoding="utf-8") + _write_run_metrics(tmp_path) + + with pytest.raises(AgentError, match="filter report"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) + + +def test_invalid_filter_report_json_raises(tmp_path: Path) -> None: + (tmp_path / FILTER_REPORT_FILE_NAME).write_text("not json", encoding="utf-8") + _write_run_metrics(tmp_path) + + with pytest.raises(AgentError, match="Could not read"): + build_pr_review_metrics(tmp_path, tmp_path, execution_time=1.0) diff --git a/tests/test_pr_review_metrics_reporting.py b/tests/test_pr_review_metrics_reporting.py index aa155effb..b138a6b97 100644 --- a/tests/test_pr_review_metrics_reporting.py +++ b/tests/test_pr_review_metrics_reporting.py @@ -2,94 +2,200 @@ from pathlib import Path from bcbench.results.codereview import CodeReviewResultSummary -from bcbench.results.leaderboard import CodeReviewLeaderboardAggregate, ExecutionBasedLeaderboardAggregate -from bcbench.results.summary import ExecutionBasedEvaluationResultSummary +from bcbench.results.leaderboard import CodeReviewLeaderboardAggregate from bcbench.types import AgentMetrics from tests.conftest import create_bugfix_result, create_codereview_result -def _metrics(tokens: int, calls: int, credits: float, used: int, pruned: int) -> AgentMetrics: +def _metrics( + *, + duration: float, + scale: int, + usage_complete: bool = True, + malformed_records: int = 0, +) -> AgentMetrics: return AgentMetrics( - execution_time=4.0, - prompt_tokens=tokens - 100, - completion_tokens=100, - total_tokens=tokens, - api_calls=calls, - estimated_credits=credits, - knowledge_used=used, - knowledge_pruned=pruned, + execution_time=duration, + prompt_tokens=900 * scale, + cached_tokens=200 * scale, + cache_creation_tokens=50 * scale, + completion_tokens=100 * scale, + reasoning_tokens=25 * scale, + total_tokens=1000 * scale, + api_calls=10 * scale, + failed_api_calls=scale, + usage_api_calls=9 * scale, + ai_credits=0.5 * scale, + premium_requests=0.25 * scale, + usage_complete=usage_complete, + malformed_records=malformed_records, + knowledge_files=20 * scale, + knowledge_pruned=4 * scale, ) def test_summary_aggregates_pr_review_metrics() -> None: summary = CodeReviewResultSummary.from_results( [ - create_codereview_result(instance_id="proj__review-1", metrics=_metrics(1000, 10, 0.5, 20, 4)), - create_codereview_result(instance_id="proj__review-2", metrics=_metrics(2000, 20, 1.5, 30, 8)), + create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1)), + create_codereview_result(instance_id="proj__review-2", metrics=_metrics(duration=6.0, scale=2)), ], run_id="run", ) + assert summary.average_duration == 5 + assert summary.average_prompt_tokens == 1350 + assert summary.average_cached_tokens == 300 + assert summary.average_cache_creation_tokens == 75 + assert summary.average_completion_tokens == 150 + assert summary.average_reasoning_tokens == 37.5 assert summary.average_total_tokens == 1500 assert summary.average_api_calls == 15 - assert summary.average_estimated_credits == 1 - assert summary.average_knowledge_used == 25 + assert summary.average_failed_api_calls == 1.5 + assert summary.average_usage_api_calls == 13.5 + assert summary.average_ai_credits == 0.75 + assert summary.average_premium_requests == 0.375 + assert summary.structured_usage_complete_rate == 1 + assert summary.average_malformed_records == 0 + assert summary.average_knowledge_files == 30 assert summary.average_knowledge_pruned == 6 +def test_summary_marks_malformed_structured_usage_incomplete() -> None: + summary = CodeReviewResultSummary.from_results( + [ + create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1)), + create_codereview_result(instance_id="proj__review-2", metrics=_metrics(duration=6.0, scale=2, malformed_records=2)), + ], + run_id="run", + ) + + assert summary.structured_usage_complete_rate == 0.5 + assert summary.average_malformed_records == 1 + assert summary.average_total_tokens == 1500 + + +def test_summary_serializes_legal_null_token_metrics() -> None: + metrics = AgentMetrics( + execution_time=4.0, + usage_complete=False, + malformed_records=0, + knowledge_files=20, + knowledge_pruned=4, + ) + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(instance_id="proj__review-1", metrics=metrics)], + run_id="run", + ) + + serialized = summary.to_dict() + + assert serialized["average_prompt_tokens"] is None + assert serialized["average_completion_tokens"] is None + assert serialized["structured_usage_complete_rate"] == 0 + + def test_leaderboard_propagates_pr_review_metrics() -> None: first = CodeReviewResultSummary.from_results( - [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(1000, 10, 0.5, 20, 4))], + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1))], run_id="one", ) second = CodeReviewResultSummary.from_results( - [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(2000, 20, 1.5, 30, 8))], + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=6.0, scale=2, usage_complete=False))], run_id="two", ) aggregate = CodeReviewLeaderboardAggregate.from_runs([first, second]) + assert aggregate.average_duration == 5 + assert aggregate.average_prompt_tokens == 1350 + assert aggregate.average_completion_tokens == 150 + assert aggregate.average_reasoning_tokens == 37.5 assert aggregate.average_total_tokens == 1500 - assert aggregate.average_knowledge_used == 25 + assert aggregate.average_api_calls == 15 + assert aggregate.average_ai_credits == 0.75 + assert aggregate.average_premium_requests == 0.375 + assert aggregate.structured_usage_complete_rate == 0.5 + assert aggregate.average_knowledge_files == 30 assert aggregate.average_knowledge_pruned == 6 def test_github_summary_renders_performance_metrics() -> None: summary = CodeReviewResultSummary.from_results( - [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(1500, 12, 0.75, 24, 5))], + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1))], run_id="run", ) markdown = summary.render_github_metrics_markdown() assert "## Performance" in markdown - assert "Avg knowledge used" in markdown - assert "0.7500" in markdown + assert "Avg total tokens" in markdown + assert "Avg API calls" in markdown + assert "Avg AI credits" in markdown + assert "Avg premium requests" in markdown + assert "Complete structured usage" in markdown + assert "| 10.0 | 1.0 | 9.0 | 0.5000 | 0.2500 | 100.0% | 0.0 |" in markdown + assert "Avg knowledge files" in markdown -def test_github_summary_renders_knowledge_only_metrics() -> None: - metrics = AgentMetrics(execution_time=4.0, knowledge_used=24, knowledge_pruned=5) - summary = CodeReviewResultSummary.from_results( - [create_codereview_result(instance_id="proj__review-1", metrics=metrics)], - run_id="run", - ) +def test_generic_result_does_not_serialize_pr_review_metrics(tmp_path: Path) -> None: + result = create_bugfix_result(metrics=AgentMetrics(execution_time=4.0)) + result.save(tmp_path, "results.jsonl") - markdown = summary.render_github_metrics_markdown() + saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] + + for field in ( + "cached_tokens", + "cache_creation_tokens", + "reasoning_tokens", + "total_tokens", + "api_calls", + "failed_api_calls", + "usage_api_calls", + "ai_credits", + "premium_requests", + "usage_complete", + "malformed_records", + "knowledge_files", + "knowledge_pruned", + ): + assert field not in saved_metrics + + +def test_code_review_result_serializes_structured_metrics(tmp_path: Path) -> None: + result = create_codereview_result(metrics=_metrics(duration=4.0, scale=1)) + result.save(tmp_path, "results.jsonl") - assert "## Performance" in markdown - assert "| 4.0 | n/a | n/a | n/a | 24.0 | 5.0 |" in markdown + saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] + + assert saved_metrics["total_tokens"] == 1000 + assert saved_metrics["api_calls"] == 10 + assert saved_metrics["ai_credits"] == 0.5 + assert saved_metrics["reasoning_tokens"] == 25 + assert saved_metrics["premium_requests"] == 0.25 + assert saved_metrics["usage_complete"] is True + assert saved_metrics["malformed_records"] == 0 + assert saved_metrics["knowledge_files"] == 20 + assert saved_metrics["knowledge_pruned"] == 4 + + +def test_code_review_result_preserves_nullable_structured_metrics(tmp_path: Path) -> None: + result = create_codereview_result( + metrics=AgentMetrics( + execution_time=4.0, + reasoning_tokens=None, + premium_requests=None, + usage_complete=True, + malformed_records=0, + knowledge_files=20, + knowledge_pruned=4, + ) + ) + result.save(tmp_path, "results.jsonl") + saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] -def test_execution_based_models_do_not_serialize_code_review_metrics(tmp_path: Path) -> None: - result = create_bugfix_result(metrics=AgentMetrics(execution_time=4.0)) - summary = ExecutionBasedEvaluationResultSummary.from_results([result], run_id="run") - aggregate = ExecutionBasedLeaderboardAggregate.from_runs([summary]) - result.save(tmp_path, "results.jsonl") - saved_result = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8")) - - assert "total_tokens" not in saved_result["metrics"] - assert "knowledge_used" not in saved_result["metrics"] - assert "average_total_tokens" not in summary.to_dict() - assert "average_knowledge_used" not in summary.to_dict() - assert "average_total_tokens" not in aggregate.model_dump(mode="json") - assert "average_knowledge_used" not in aggregate.model_dump(mode="json") + assert "reasoning_tokens" in saved_metrics + assert saved_metrics["reasoning_tokens"] is None + assert "premium_requests" in saved_metrics + assert saved_metrics["premium_requests"] is None diff --git a/tests/test_pr_review_output.py b/tests/test_pr_review_output.py index 8b56bc702..5cbe6be51 100644 --- a/tests/test_pr_review_output.py +++ b/tests/test_pr_review_output.py @@ -1,6 +1,6 @@ import json -from bcbench.agent.copilot.pr_review.review_output import engine_report_to_review_comments, load_engine_report +from bcbench.agent.pr_review.review_output import engine_report_to_review_comments, load_engine_report from bcbench.evaluate.review_parsing import parse_review_output @@ -22,13 +22,14 @@ def test_load_engine_report_rejects_empty_or_invalid() -> None: assert load_engine_report("[]") is None -def test_maps_nested_location_and_message() -> None: +def test_maps_production_normalized_finding() -> None: report = _report( [ { "severity": "High", - "location": {"file": "src/Foo.al", "line": 42}, - "message": "Missing ToolTip on field.", + "filePath": "src/Foo.al", + "lineNumber": 42, + "issue": "Missing ToolTip on field.", "domain": "ui", } ] @@ -51,8 +52,9 @@ def test_normalizes_path_and_lowercases_severity() -> None: [ { "severity": "CRITICAL", - "location": {"file": ".\\src\\Bar.al", "line": 7}, - "message": "Unchecked Get.", + "filePath": ".\\src\\Bar.al", + "lineNumber": 7, + "issue": "Unchecked Get.", "domain": "error-handling", } ] @@ -62,25 +64,25 @@ def test_normalizes_path_and_lowercases_severity() -> None: assert comment["severity"] == "critical" -def test_falls_back_to_issue_then_recommendation_for_body() -> None: +def test_includes_issue_and_recommendation_in_body() -> None: report = _report( [ - {"location": {"file": "a.al", "line": 1}, "issue": "issue text"}, - {"location": {"file": "b.al", "line": 2}, "recommendation": "rec text"}, + {"filePath": "a.al", "lineNumber": 1, "issue": "issue text", "recommendation": "rec text"}, + {"filePath": "b.al", "lineNumber": 2, "recommendation": "rec only"}, ] ) comments = engine_report_to_review_comments(report) - assert [c["body"] for c in comments] == ["issue text", "rec text"] + assert [c["body"] for c in comments] == ["issue text\n\nRecommendation: rec text", "rec only"] def test_drops_findings_missing_file_line_or_body() -> None: report = _report( [ - {"location": {"line": 5}, "message": "no file"}, - {"location": {"file": "c.al"}, "message": "no line"}, - {"location": {"file": "d.al", "line": 0}, "message": "non-positive line"}, - {"location": {"file": "e.al", "line": 3}, "message": " "}, - {"location": {"file": "f.al", "line": 3}}, + {"lineNumber": 5, "issue": "no file"}, + {"filePath": "c.al", "issue": "no line"}, + {"filePath": "d.al", "lineNumber": 0, "issue": "non-positive line"}, + {"filePath": "e.al", "lineNumber": 3, "issue": " "}, + {"filePath": "f.al", "lineNumber": 3}, ] ) assert engine_report_to_review_comments(report) == [] @@ -97,8 +99,9 @@ def test_output_is_consumable_by_review_parser() -> None: [ { "severity": "Medium", - "location": {"file": "src/Baz.al", "line": 10}, - "message": "Some finding.", + "filePath": "src/Baz.al", + "lineNumber": 10, + "issue": "Some finding.", "domain": "performance", } ] diff --git a/tests/test_review_runners.py b/tests/test_review_runners.py new file mode 100644 index 000000000..d67d486ad --- /dev/null +++ b/tests/test_review_runners.py @@ -0,0 +1,137 @@ +from pathlib import Path +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from bcbench.cli import app +from bcbench.commands import evaluate as evaluate_commands +from bcbench.commands import run as run_commands +from bcbench.dataset.codereview import CodeReviewEntry +from bcbench.evaluate.codereview import CodeReviewPipeline +from bcbench.types import AgentHarness, EvaluationCategory + + +@pytest.mark.parametrize( + ("command", "runner_name"), + [ + (run_commands.run_copilot, "run_copilot_agent"), + (run_commands.run_claude, "run_claude_code"), + ], +) +def test_generic_run_commands_accept_code_review(tmp_path: Path, command, runner_name: str) -> None: + entry = object() + with ( + patch.object(CodeReviewEntry, "load", return_value=[entry]), + patch.object(CodeReviewPipeline, "setup_workspace"), + patch.object(run_commands, runner_name) as agent_runner, + ): + command("synthetic__style-018", EvaluationCategory.CODE_REVIEW, repo_path=tmp_path, output_dir=tmp_path / "out") + + assert agent_runner.call_args.kwargs["entry"] is entry + assert agent_runner.call_args.kwargs["category"] is EvaluationCategory.CODE_REVIEW + + +@pytest.mark.parametrize( + ("command", "agent_name"), + [ + (evaluate_commands.evaluate_copilot, AgentHarness.COPILOT), + (evaluate_commands.evaluate_claude_code, AgentHarness.CLAUDE), + ], +) +def test_generic_evaluate_commands_use_code_review_pipeline(tmp_path: Path, command, agent_name: AgentHarness) -> None: + contexts = [] + with ( + patch.object(CodeReviewEntry, "load", return_value=[object()]), + patch.object(CodeReviewPipeline, "execute", side_effect=lambda context, runner: contexts.append(context)), + ): + command( + "synthetic__style-018", + EvaluationCategory.CODE_REVIEW, + repo_path=tmp_path, + output_dir=tmp_path / "out", + run_id=agent_name.name.lower(), + ) + + assert len(contexts) == 1 + assert contexts[0].agent_name is agent_name + assert contexts[0].category is EvaluationCategory.CODE_REVIEW + + +def test_pr_review_evaluation_is_fixed_to_runner_and_category(tmp_path: Path) -> None: + contexts = [] + with ( + patch.object(CodeReviewEntry, "load", return_value=[object()]), + patch.object(CodeReviewPipeline, "execute", side_effect=lambda context, runner: (contexts.append(context), runner(context))), + patch.object(evaluate_commands, "run_pr_review_agent") as agent_runner, + ): + result = CliRunner().invoke( + app, + [ + "evaluate", + "pr-review", + "synthetic__style-018", + "--repo-path", + str(tmp_path), + "--output-dir", + str(tmp_path / "out"), + "--run-id", + "pr-review", + "--engine-path", + str(tmp_path), + "--bcquality-local-path", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, result.exception + assert len(contexts) == 1 + assert contexts[0].agent_name is AgentHarness.PR_REVIEW + assert contexts[0].category is EvaluationCategory.CODE_REVIEW + assert contexts[0].model == "gpt-5.6-luna" + assert agent_runner.call_args.kwargs["engine_path"] == tmp_path + assert agent_runner.call_args.kwargs["bcquality_local_path"] == tmp_path + + +def test_pr_review_run_is_fixed_to_code_review(tmp_path: Path) -> None: + entry = object() + with ( + patch.object(CodeReviewEntry, "load", return_value=[entry]), + patch.object(CodeReviewPipeline, "setup_workspace"), + patch.object(run_commands, "run_pr_review_agent") as agent_runner, + ): + result = CliRunner().invoke( + app, + [ + "run", + "pr-review", + "synthetic__style-018", + "--repo-path", + str(tmp_path), + "--output-dir", + str(tmp_path / "out"), + "--engine-path", + str(tmp_path), + "--bcquality-local-path", + str(tmp_path), + ], + ) + + assert result.exit_code == 0, result.exception + assert agent_runner.call_args.kwargs["entry"] is entry + assert agent_runner.call_args.kwargs["category"] is EvaluationCategory.CODE_REVIEW + assert agent_runner.call_args.kwargs["model"] == "gpt-5.6-luna" + assert agent_runner.call_args.kwargs["engine_path"] == tmp_path + assert agent_runner.call_args.kwargs["bcquality_local_path"] == tmp_path + + +def test_pr_review_is_public_command() -> None: + runner = CliRunner() + + run_help = runner.invoke(app, ["run", "--help"]) + evaluate_help = runner.invoke(app, ["evaluate", "--help"]) + + assert run_help.exit_code == 0 + assert evaluate_help.exit_code == 0 + assert "pr-review" in run_help.stdout + assert "pr-review" in evaluate_help.stdout diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py new file mode 100644 index 000000000..8e9c8f4b3 --- /dev/null +++ b/tests/test_review_workflows.py @@ -0,0 +1,57 @@ +from pathlib import Path + +import yaml + +WORKFLOWS = Path(__file__).parents[1] / ".github" / "workflows" +ACTIONS = Path(__file__).parents[1] / ".github" / "actions" + + +def _workflow(name: str) -> str: + text = (WORKFLOWS / name).read_text(encoding="utf-8") + assert yaml.safe_load(text) + return text + + +def test_copilot_workflow_routes_code_review_through_copilot() -> None: + workflow = _workflow("copilot-evaluation.yml") + + assert '"code-review"' in workflow + assert "bcbench evaluate copilot" in workflow + assert "bcbench evaluate pr-review" not in workflow + assert "BC_PR_REVIEW_ROOT" not in workflow + assert 'agent: "GitHub Copilot CLI"' in workflow + + +def test_claude_workflow_routes_code_review_through_claude() -> None: + workflow = _workflow("claude-evaluation.yml") + + assert '"code-review"' in workflow + assert "bcbench evaluate claude" in workflow + assert 'agent: "Claude Code"' in workflow + + +def test_pr_review_workflow_is_fixed_to_code_review() -> None: + workflow = _workflow("pr-review-evaluation.yml") + + assert "category: code-review" in workflow + assert "bcbench evaluate pr-review" in workflow + assert "repository: microsoft/BC-ALAgents" in workflow + assert "533dd39dfe29218c09e5e31c39c78bb72fa20aa2" in workflow + assert "ref: main" not in workflow + assert '--engine-path "${{ github.workspace }}/bc-alagents-engine"' in workflow + assert "BC_PR_REVIEW_ROOT:" not in workflow + assert "install-agent-harnesses" in workflow + assert "install-eval-clis" not in workflow + assert "copilot-requests: write" in workflow + assert 'agent: "BC PR Review"' in workflow + assert '"mai-code-1.1-flash"' in workflow + assert "mai-code-1-flash-picker" not in workflow + for input_name in ("model:", "test-run:", "repeat:", "git-ref:"): + assert input_name in workflow + + +def test_agent_harness_action_pins_published_copilot_version() -> None: + action = (ACTIONS / "install-agent-harnesses" / "action.yml").read_text(encoding="utf-8") + + assert "@github/copilot@1.0.79" in action + assert "@github/copilot@1.0.80" not in action diff --git a/uv.lock b/uv.lock index cc0f7cf14..47534e93c 100644 --- a/uv.lock +++ b/uv.lock @@ -245,7 +245,7 @@ wheels = [ [[package]] name = "bcbench" -version = "0.8.1" +version = "0.9.0" source = { editable = "." } dependencies = [ { name = "jinja2" }, @@ -286,7 +286,7 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonschema", specifier = ">=4.0" }, { name = "numpy", specifier = ">=2.3.5" }, - { name = "pydantic", specifier = ">=2.0" }, + { name = "pydantic", specifier = ">=2.12" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.0" }, From 1472fb87090dd7345cd2675c5fd72a8269f52750 Mon Sep 17 00:00:00 2001 From: wenjiefan Date: Fri, 21 Aug 2026 10:25:47 +0200 Subject: [PATCH 3/3] Keep PR review diagnostics in raw artifacts Preserve the original public performance metrics while keeping later producer diagnostics out of persisted results and dashboards. Gate promoted usage values on complete, well-formed telemetry and retain exact AI credit precision. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/code-review.md | 12 +- pyproject.toml | 2 +- src/bcbench/agent/pr_review/metrics.py | 20 +-- src/bcbench/results/codereview.py | 97 ++------------ src/bcbench/results/leaderboard.py | 18 +-- src/bcbench/results/summary.py | 1 - src/bcbench/types.py | 33 +++-- tests/test_evaluation_summary.py | 12 ++ tests/test_pr_review_agent.py | 4 +- tests/test_pr_review_metrics.py | 51 ++++---- tests/test_pr_review_metrics_reporting.py | 148 +++++++--------------- uv.lock | 2 +- 12 files changed, 128 insertions(+), 272 deletions(-) diff --git a/docs/code-review.md b/docs/code-review.md index 59215f557..450d82d01 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -39,9 +39,9 @@ bcbench evaluate claude --category code-review bcbench evaluate pr-review ``` -The evaluation workflow pins BC-ALAgents to a commit SHA. Engine updates require a new BC-Bench version and must record that SHA in the release notes. +The evaluation workflow pins BC-ALAgents to a commit SHA. Engine updates require a new BC-Bench version and must record that SHA in the release notes. GitHub Copilot CLI stays pinned to `1.0.79` because `1.0.80` is not published to npm and `1.0.79` is the version whose structured telemetry contract was validated. -BC PR Review records wall-clock duration and two structural BCQuality counts: Markdown knowledge files available after filtering and knowledge files removed by the filter. The counts come from the filtered checkout and its validated `_filter-report.json`. The production `all` pipeline does not currently expose a stable structured API-call, token, or credit contract, so BC-Bench intentionally does not infer those values from console transcripts. +BC PR Review records wall-clock duration, prompt/completion/total tokens, actual model API calls, exact AI credits, and two structural BCQuality counts: Markdown knowledge files available after filtering and knowledge files removed by the filter. Usage values come from the engine's strictly validated schema-v1 `_run-metrics.json`, never from console transcripts. Additional producer diagnostics remain in that raw artifact rather than being promoted into BC-Bench result and leaderboard schemas. ## Baseline Leaderboard @@ -88,11 +88,11 @@ BC PR Review records wall-clock duration and two structural BCQuality counts: Ma Agent Model Avg Time + Avg Prompt Tokens + Avg Completion Tokens Avg Total Tokens Avg API Calls Avg AI Credits - Avg Premium Requests - Complete Usage Avg Knowledge Files Avg Knowledge Pruned Ver @@ -105,11 +105,11 @@ BC PR Review records wall-clock duration and two structural BCQuality counts: Ma {{ agg.agent_name }} {{ agg.model }} {{ agg.average_duration | round: 1 }}s + {% if agg.average_prompt_tokens != null %}{{ agg.average_prompt_tokens | round: 0 }}{% else %}—{% endif %} + {% if agg.average_completion_tokens != null %}{{ agg.average_completion_tokens | round: 0 }}{% else %}—{% endif %} {% if agg.average_total_tokens != null %}{{ agg.average_total_tokens | round: 0 }}{% else %}—{% endif %} {% if agg.average_api_calls != null %}{{ agg.average_api_calls | round: 1 }}{% else %}—{% endif %} {% if agg.average_ai_credits != null %}{{ agg.average_ai_credits | round: 4 }}{% else %}—{% endif %} - {% if agg.average_premium_requests != null %}{{ agg.average_premium_requests | round: 4 }}{% else %}—{% endif %} - {% if agg.structured_usage_complete_rate != null %}{{ agg.structured_usage_complete_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} {% if agg.average_knowledge_files != null %}{{ agg.average_knowledge_files | round: 1 }}{% else %}—{% endif %} {% if agg.average_knowledge_pruned != null %}{{ agg.average_knowledge_pruned | round: 1 }}{% else %}—{% endif %} {{ agg.benchmark_version }} diff --git a/pyproject.toml b/pyproject.toml index a9f6ecab6..3b0e4935d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "typer>=0.9.0", "typing-extensions>=4.0", "pyyaml>=6.0", - "pydantic>=2.12", + "pydantic>=2.0", "textual>=7.0", "numpy>=2.3.5", "scipy>=1.16.3", diff --git a/src/bcbench/agent/pr_review/metrics.py b/src/bcbench/agent/pr_review/metrics.py index 052b8834e..73eb7462f 100644 --- a/src/bcbench/agent/pr_review/metrics.py +++ b/src/bcbench/agent/pr_review/metrics.py @@ -113,21 +113,15 @@ def is_knowledge_file(path: Path) -> bool: def build_pr_review_metrics(output_dir: Path, bcquality_root: Path, execution_time: float) -> AgentMetrics: run = _load_run_metrics(output_dir / RUN_METRICS_FILE_NAME) report = _load_filter_report(bcquality_root / FILTER_REPORT_FILE_NAME) + usage_values_available = run.malformed_records == 0 + token_values_available = usage_values_available and run.usage_complete return AgentMetrics( execution_time=execution_time, - prompt_tokens=run.prompt_tokens, - completion_tokens=run.completion_tokens, - cached_tokens=run.cached_tokens, - cache_creation_tokens=run.cache_creation_tokens, - reasoning_tokens=run.reasoning_tokens, - total_tokens=run.total_tokens, - api_calls=run.api_calls, - failed_api_calls=run.failed_api_calls, - usage_api_calls=run.usage_api_calls, - ai_credits=run.ai_credits, - premium_requests=run.premium_requests, - usage_complete=run.usage_complete, - malformed_records=run.malformed_records, + prompt_tokens=run.prompt_tokens if token_values_available else None, + completion_tokens=run.completion_tokens if token_values_available else None, + total_tokens=run.total_tokens if token_values_available else None, + api_calls=run.api_calls if usage_values_available else None, + ai_credits=run.ai_credits if usage_values_available else None, knowledge_files=_count_available_knowledge(bcquality_root), knowledge_pruned=sum(1 for item in report.removed if item.kind == "knowledge"), ) diff --git a/src/bcbench/results/codereview.py b/src/bcbench/results/codereview.py index 40d1222be..5d61f6b69 100644 --- a/src/bcbench/results/codereview.py +++ b/src/bcbench/results/codereview.py @@ -1,8 +1,8 @@ from collections.abc import Sequence -from typing import Any, NamedTuple, Self +from typing import NamedTuple, Self import numpy as np -from pydantic import Field, SerializerFunctionWrapHandler, model_serializer +from pydantic import Field from rich.console import Group, RenderableType from rich.panel import Panel from rich.table import Table @@ -190,30 +190,6 @@ class CodeReviewResult(JudgeScoredEvaluationResult): f_beta_2: float = Field(default=0.0, ge=0.0, le=1.0) severity_mae: float = 0.0 - @model_serializer(mode="wrap") - def serialize_structured_metrics(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: - data = handler(self) - if self.metrics is None: - return data - serialized_metrics = data["metrics"] - for field in ( - "cached_tokens", - "cache_creation_tokens", - "reasoning_tokens", - "total_tokens", - "api_calls", - "failed_api_calls", - "usage_api_calls", - "ai_credits", - "premium_requests", - "usage_complete", - "malformed_records", - "knowledge_files", - "knowledge_pruned", - ): - serialized_metrics[field] = getattr(self.metrics, field) - return data - @classmethod def create( cls, @@ -337,17 +313,8 @@ class CodeReviewResultSummary(JudgeBasedEvaluationResultSummary): severity_mae: float = 0.0 valid_review_output_rate: float = Field(default=0.0, ge=0.0, le=1.0) - average_cached_tokens: float | None = None - average_cache_creation_tokens: float | None = None - average_reasoning_tokens: float | None = None average_total_tokens: float | None = None average_api_calls: float | None = None - average_failed_api_calls: float | None = None - average_usage_api_calls: float | None = None - average_ai_credits: float | None = None - average_premium_requests: float | None = None - structured_usage_complete_rate: float | None = Field(default=None, ge=0.0, le=1.0) - average_malformed_records: float | None = None average_knowledge_files: float | None = None average_knowledge_pruned: float | None = None @@ -359,26 +326,14 @@ def _performance_markdown(self) -> str: def metric(value: float | None, digits: int = 1) -> str: return f"{value:.{digits}f}" if value is not None else "n/a" - usage_complete = f"{self.structured_usage_complete_rate * 100:.1f}%" if self.structured_usage_complete_rate is not None else "n/a" - available = metric(self.average_knowledge_files) - pruned = f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a" return ( "## Performance\n" "\n" - "| Avg duration (s) | Avg prompt tokens | Avg cached tokens | Avg cache-creation tokens | Avg completion tokens | Avg reasoning tokens | Avg total tokens |\n" - "|-----------------:|------------------:|------------------:|-------------------------:|----------------------:|---------------------:|-----------------:|\n" - f"| {self.average_duration:.1f} | {metric(self.average_prompt_tokens)} | {metric(self.average_cached_tokens)} | " - f"{metric(self.average_cache_creation_tokens)} | {metric(self.average_completion_tokens)} | {metric(self.average_reasoning_tokens)} | " - f"{metric(self.average_total_tokens)} |\n" - "\n" - "| Avg API calls | Avg failed API calls | Avg calls with usage | Avg AI credits | Avg premium requests | Complete structured usage | Avg malformed records |\n" - "|--------------:|---------------------:|---------------------:|---------------:|---------------------:|--------------------------:|----------------------:|\n" - f"| {metric(self.average_api_calls)} | {metric(self.average_failed_api_calls)} | {metric(self.average_usage_api_calls)} | " - f"{metric(self.average_ai_credits, 4)} | {metric(self.average_premium_requests, 4)} | {usage_complete} | {metric(self.average_malformed_records)} |\n" - "\n" - "| Avg knowledge files | Avg knowledge pruned |\n" - "|--------------------:|---------------------:|\n" - f"| {available} | {pruned} |\n" + "| Avg duration (s) | Avg prompt tokens | Avg completion tokens | Avg total tokens | Avg API calls | Avg AI credits | Avg knowledge files | Avg knowledge pruned |\n" + "|-----------------:|------------------:|----------------------:|-----------------:|--------------:|---------------:|--------------------:|---------------------:|\n" + f"| {self.average_duration:.1f} | {metric(self.average_prompt_tokens)} | {metric(self.average_completion_tokens)} | " + f"{metric(self.average_total_tokens)} | {metric(self.average_api_calls)} | {metric(self.average_ai_credits, 4)} | " + f"{metric(self.average_knowledge_files)} | {metric(self.average_knowledge_pruned)} |\n" "\n" ) @@ -467,35 +422,15 @@ def render_console_metrics(self) -> RenderableType: [f"{self.severity_mae:.3f}", f"{self.valid_review_output_rate * 100:.1f}%"], ), _build_console_table( - "Performance: tokens", - ["Avg duration (s)", "Prompt", "Cached", "Cache creation", "Completion", "Reasoning", "Total"], + "Performance", + ["Avg duration (s)", "Prompt", "Completion", "Total", "API calls", "AI credits", "Knowledge files", "Knowledge pruned"], [ f"{self.average_duration:.1f}", f"{self.average_prompt_tokens:.1f}" if self.average_prompt_tokens is not None else "n/a", - f"{self.average_cached_tokens:.1f}" if self.average_cached_tokens is not None else "n/a", - f"{self.average_cache_creation_tokens:.1f}" if self.average_cache_creation_tokens is not None else "n/a", f"{self.average_completion_tokens:.1f}" if self.average_completion_tokens is not None else "n/a", - f"{self.average_reasoning_tokens:.1f}" if self.average_reasoning_tokens is not None else "n/a", f"{self.average_total_tokens:.1f}" if self.average_total_tokens is not None else "n/a", - ], - ), - _build_console_table( - "Performance: requests", - ["API calls", "Failed", "With usage", "AI credits", "Premium requests", "Complete usage", "Malformed"], - [ f"{self.average_api_calls:.1f}" if self.average_api_calls is not None else "n/a", - f"{self.average_failed_api_calls:.1f}" if self.average_failed_api_calls is not None else "n/a", - f"{self.average_usage_api_calls:.1f}" if self.average_usage_api_calls is not None else "n/a", f"{self.average_ai_credits:.4f}" if self.average_ai_credits is not None else "n/a", - f"{self.average_premium_requests:.4f}" if self.average_premium_requests is not None else "n/a", - f"{self.structured_usage_complete_rate * 100:.1f}%" if self.structured_usage_complete_rate is not None else "n/a", - f"{self.average_malformed_records:.1f}" if self.average_malformed_records is not None else "n/a", - ], - ), - _build_console_table( - "Performance: knowledge", - ["Avg knowledge files", "Avg knowledge pruned"], - [ f"{self.average_knowledge_files:.1f}" if self.average_knowledge_files is not None else "n/a", f"{self.average_knowledge_pruned:.1f}" if self.average_knowledge_pruned is not None else "n/a", ], @@ -560,12 +495,6 @@ def average_metric(name: str) -> float | None: values = [value for result in code_review_results if result.metrics and (value := getattr(result.metrics, name)) is not None] return sum(values) / len(values) if values else None - usage_completeness = [ - result.metrics.usage_complete and result.metrics.malformed_records == 0 - for result in code_review_results - if result.metrics and result.metrics.usage_complete is not None and result.metrics.malformed_records is not None - ] - return summary.model_copy( update={ "generated_comment_count": generated_total, @@ -589,17 +518,9 @@ def average_metric(name: str) -> float | None: "instance_results": {r.instance_id: round(r.f1, 6) for r in code_review_results}, "average_prompt_tokens": average_metric("prompt_tokens"), "average_completion_tokens": average_metric("completion_tokens"), - "average_cached_tokens": average_metric("cached_tokens"), - "average_cache_creation_tokens": average_metric("cache_creation_tokens"), - "average_reasoning_tokens": average_metric("reasoning_tokens"), "average_total_tokens": average_metric("total_tokens"), "average_api_calls": average_metric("api_calls"), - "average_failed_api_calls": average_metric("failed_api_calls"), - "average_usage_api_calls": average_metric("usage_api_calls"), "average_ai_credits": average_metric("ai_credits"), - "average_premium_requests": average_metric("premium_requests"), - "structured_usage_complete_rate": sum(usage_completeness) / len(usage_completeness) if usage_completeness else None, - "average_malformed_records": average_metric("malformed_records"), "average_knowledge_files": average_metric("knowledge_files"), "average_knowledge_pruned": average_metric("knowledge_pruned"), } diff --git a/src/bcbench/results/leaderboard.py b/src/bcbench/results/leaderboard.py index 69cece34b..7d0303761 100644 --- a/src/bcbench/results/leaderboard.py +++ b/src/bcbench/results/leaderboard.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, field_validator from bcbench.logger import get_logger from bcbench.results.metrics import bootstrap_ci, pass_hat_k @@ -149,17 +149,9 @@ class CodeReviewLeaderboardAggregate(JudgeBasedLeaderboardAggregate): average_prompt_tokens: float | None = None average_completion_tokens: float | None = None - average_cached_tokens: float | None = None - average_cache_creation_tokens: float | None = None - average_reasoning_tokens: float | None = None average_total_tokens: float | None = None average_api_calls: float | None = None - average_failed_api_calls: float | None = None - average_usage_api_calls: float | None = None average_ai_credits: float | None = None - average_premium_requests: float | None = None - structured_usage_complete_rate: float | None = Field(default=None, ge=0.0, le=1.0) - average_malformed_records: float | None = None average_knowledge_files: float | None = None average_knowledge_pruned: float | None = None @@ -206,17 +198,9 @@ def mean_metric(name: str) -> float | None: "macro_recall": sum(r.macro_recall for r in cr_runs) / n, "average_prompt_tokens": mean_metric("average_prompt_tokens"), "average_completion_tokens": mean_metric("average_completion_tokens"), - "average_cached_tokens": mean_metric("average_cached_tokens"), - "average_cache_creation_tokens": mean_metric("average_cache_creation_tokens"), - "average_reasoning_tokens": mean_metric("average_reasoning_tokens"), "average_total_tokens": mean_metric("average_total_tokens"), "average_api_calls": mean_metric("average_api_calls"), - "average_failed_api_calls": mean_metric("average_failed_api_calls"), - "average_usage_api_calls": mean_metric("average_usage_api_calls"), "average_ai_credits": mean_metric("average_ai_credits"), - "average_premium_requests": mean_metric("average_premium_requests"), - "structured_usage_complete_rate": mean_metric("structured_usage_complete_rate"), - "average_malformed_records": mean_metric("average_malformed_records"), "average_knowledge_files": mean_metric("average_knowledge_files"), "average_knowledge_pruned": mean_metric("average_knowledge_pruned"), } diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index ba8aed5c3..0c4c3812c 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -126,7 +126,6 @@ def to_dict(self) -> dict[str, Any]: data["average_prompt_tokens"] = round(data["average_prompt_tokens"], 1) if data["average_prompt_tokens"] is not None else None data["average_completion_tokens"] = round(data["average_completion_tokens"], 1) if data["average_completion_tokens"] is not None else None data["average_llm_duration"] = round(data["average_llm_duration"], 1) if data["average_llm_duration"] is not None else None - data["average_ai_credits"] = round(data["average_ai_credits"], 2) if data["average_ai_credits"] is not None else None return data def save(self, output_dir: Path, summary_file: str) -> None: diff --git a/src/bcbench/types.py b/src/bcbench/types.py index f46f0210a..9b664b6f0 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Annotated, Literal, TypedDict -from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator +from pydantic import BaseModel, ConfigDict, StringConstraints, model_validator if TYPE_CHECKING: from bcbench.dataset import BaseDatasetEntry @@ -79,25 +79,15 @@ class AgentMetrics(BaseModel): prompt_tokens: int | None = None completion_tokens: int | None = None - # Structured usage metrics emitted by agent harnesses - cached_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - cache_creation_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - reasoning_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - total_tokens: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - api_calls: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - failed_api_calls: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - usage_api_calls: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - ai_credits: float | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - premium_requests: float | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - usage_complete: bool | None = Field(default=None, exclude_if=lambda value: value is None) - malformed_records: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + total_tokens: int | None = None + # Actual model requests, including nested calls and retries; distinct from conversational turns. + api_calls: int | None = None # Tool usage statistics from agent logs tool_usage: dict[str, int] | None = None - # BC PR Review's structural BCQuality filter metrics - knowledge_files: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) - knowledge_pruned: int | None = Field(default=None, ge=0, exclude_if=lambda value: value is None) + knowledge_files: int | None = None + knowledge_pruned: int | None = None class ExperimentConfiguration(BaseModel): @@ -217,7 +207,16 @@ def expected_metrics(self) -> frozenset[str]: case AgentHarness.BCAL: expected = AgentMetrics(execution_time=None) case AgentHarness.PR_REVIEW: - expected = AgentMetrics(execution_time=None, knowledge_files=None, knowledge_pruned=None) + expected = AgentMetrics( + execution_time=None, + prompt_tokens=None, + completion_tokens=None, + total_tokens=None, + api_calls=None, + ai_credits=None, + knowledge_files=None, + knowledge_pruned=None, + ) case _: raise ValueError(f"Unknown AgentHarness: {self}") diff --git a/tests/test_evaluation_summary.py b/tests/test_evaluation_summary.py index eeeca5ade..6dd1ba64a 100644 --- a/tests/test_evaluation_summary.py +++ b/tests/test_evaluation_summary.py @@ -226,6 +226,18 @@ def test_from_results_leaves_ai_credits_none_when_harness_reports_none(self): assert summary.average_ai_credits is None assert summary.to_dict()["average_ai_credits"] is None + def test_to_dict_preserves_exact_ai_credit_precision(self): + result = create_bugfix_result( + instance_id="test__1", + project="app", + resolved=True, + metrics=AgentMetrics(execution_time=100.0, ai_credits=0.123456), + ) + + summary = ExecutionBasedEvaluationResultSummary.from_results([result], run_id="test_run_123") + + assert summary.to_dict()["average_ai_credits"] == pytest.approx(0.123456) + def test_from_results_calculates_average_tool_usage(self): results = [ create_bugfix_result( diff --git a/tests/test_pr_review_agent.py b/tests/test_pr_review_agent.py index 3b8e430ee..4a881222f 100644 --- a/tests/test_pr_review_agent.py +++ b/tests/test_pr_review_agent.py @@ -145,11 +145,11 @@ def test_engine_environment_uses_target_repository_and_absolute_paths(tmp_path: assert metrics is not None assert metrics.execution_time == 2.5 + assert metrics.prompt_tokens == 100 + assert metrics.completion_tokens == 10 assert metrics.total_tokens == 110 assert metrics.api_calls == 2 assert metrics.ai_credits == 0.25 - assert metrics.reasoning_tokens == 4 - assert metrics.premium_requests == 0.5 assert metrics.knowledge_files == 1 assert metrics.knowledge_pruned == 0 assert config.is_empty() diff --git a/tests/test_pr_review_metrics.py b/tests/test_pr_review_metrics.py index 12460e43e..98ca53430 100644 --- a/tests/test_pr_review_metrics.py +++ b/tests/test_pr_review_metrics.py @@ -39,7 +39,7 @@ def _write_run_metrics(root: Path, **overrides: object) -> None: (root / RUN_METRICS_FILE_NAME).write_text(json.dumps(_run_metrics(**overrides)), encoding="utf-8") -def test_build_metrics_reads_structured_usage_and_filtered_knowledge(tmp_path: Path) -> None: +def test_build_metrics_promotes_public_usage_and_filtered_knowledge(tmp_path: Path) -> None: knowledge = tmp_path / "microsoft" / "knowledge" / "performance" knowledge.mkdir(parents=True) (knowledge / "one.md").write_text("# One", encoding="utf-8") @@ -60,18 +60,10 @@ def test_build_metrics_reads_structured_usage_and_filtered_knowledge(tmp_path: P assert metrics.execution_time == 12.5 assert metrics.prompt_tokens == 150 - assert metrics.cached_tokens == 60 - assert metrics.cache_creation_tokens == 10 assert metrics.completion_tokens == 28 - assert metrics.reasoning_tokens == 7 assert metrics.total_tokens == 178 assert metrics.api_calls == 2 - assert metrics.failed_api_calls == 1 - assert metrics.usage_api_calls == 2 assert metrics.ai_credits == 1.75 - assert metrics.premium_requests == 1.75 - assert metrics.usage_complete is True - assert metrics.malformed_records == 0 assert metrics.knowledge_files == 2 assert metrics.knowledge_pruned == 1 @@ -92,15 +84,11 @@ def test_legal_null_optional_fields_and_multiple_models_are_accepted(tmp_path: P metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) - assert metrics.cached_tokens is None - assert metrics.cache_creation_tokens is None assert metrics.ai_credits is None - assert metrics.reasoning_tokens is None - assert metrics.premium_requests is None assert metrics.total_tokens == 178 -def test_partial_usage_preserves_exact_counts_and_completeness_metadata(tmp_path: Path) -> None: +def test_malformed_records_suppress_all_usage_metrics(tmp_path: Path) -> None: _write_filter_report(tmp_path, []) _write_run_metrics( tmp_path, @@ -121,15 +109,33 @@ def test_partial_usage_preserves_exact_counts_and_completeness_metadata(tmp_path metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) - assert metrics.prompt_tokens == 25 - assert metrics.total_tokens == 30 + assert metrics.prompt_tokens is None + assert metrics.completion_tokens is None + assert metrics.total_tokens is None + assert metrics.api_calls is None + assert metrics.ai_credits is None + + +def test_incomplete_usage_suppresses_tokens_but_preserves_exact_calls_and_credits(tmp_path: Path) -> None: + _write_filter_report(tmp_path, []) + _write_run_metrics( + tmp_path, + prompt_tokens=25, + completion_tokens=5, + total_tokens=30, + api_calls=2, + ai_credits=0.1, + usage_complete=False, + malformed_records=0, + ) + + metrics = build_pr_review_metrics(tmp_path, tmp_path, execution_time=2.0) + + assert metrics.prompt_tokens is None + assert metrics.completion_tokens is None + assert metrics.total_tokens is None assert metrics.api_calls == 2 - assert metrics.usage_api_calls == 1 assert metrics.ai_credits == 0.1 - assert metrics.reasoning_tokens is None - assert metrics.premium_requests is None - assert metrics.usage_complete is False - assert metrics.malformed_records == 3 def test_missing_run_metrics_raises(tmp_path: Path) -> None: @@ -213,9 +219,10 @@ def test_not_applicable_zero_shape_is_accepted(tmp_path: Path) -> None: assert metrics.execution_time == 0.25 assert metrics.prompt_tokens == 0 + assert metrics.completion_tokens == 0 + assert metrics.total_tokens == 0 assert metrics.api_calls == 0 assert metrics.ai_credits == 0.0 - assert metrics.usage_complete is True @pytest.mark.parametrize( diff --git a/tests/test_pr_review_metrics_reporting.py b/tests/test_pr_review_metrics_reporting.py index b138a6b97..bd1f3080f 100644 --- a/tests/test_pr_review_metrics_reporting.py +++ b/tests/test_pr_review_metrics_reporting.py @@ -1,40 +1,25 @@ import json -from pathlib import Path from bcbench.results.codereview import CodeReviewResultSummary from bcbench.results.leaderboard import CodeReviewLeaderboardAggregate from bcbench.types import AgentMetrics -from tests.conftest import create_bugfix_result, create_codereview_result +from tests.conftest import create_codereview_result -def _metrics( - *, - duration: float, - scale: int, - usage_complete: bool = True, - malformed_records: int = 0, -) -> AgentMetrics: +def _metrics(*, duration: float, scale: int) -> AgentMetrics: return AgentMetrics( execution_time=duration, prompt_tokens=900 * scale, - cached_tokens=200 * scale, - cache_creation_tokens=50 * scale, completion_tokens=100 * scale, - reasoning_tokens=25 * scale, total_tokens=1000 * scale, api_calls=10 * scale, - failed_api_calls=scale, - usage_api_calls=9 * scale, ai_credits=0.5 * scale, - premium_requests=0.25 * scale, - usage_complete=usage_complete, - malformed_records=malformed_records, knowledge_files=20 * scale, knowledge_pruned=4 * scale, ) -def test_summary_aggregates_pr_review_metrics() -> None: +def test_summary_aggregates_public_pr_review_metrics() -> None: summary = CodeReviewResultSummary.from_results( [ create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1)), @@ -45,46 +30,17 @@ def test_summary_aggregates_pr_review_metrics() -> None: assert summary.average_duration == 5 assert summary.average_prompt_tokens == 1350 - assert summary.average_cached_tokens == 300 - assert summary.average_cache_creation_tokens == 75 assert summary.average_completion_tokens == 150 - assert summary.average_reasoning_tokens == 37.5 assert summary.average_total_tokens == 1500 assert summary.average_api_calls == 15 - assert summary.average_failed_api_calls == 1.5 - assert summary.average_usage_api_calls == 13.5 assert summary.average_ai_credits == 0.75 - assert summary.average_premium_requests == 0.375 - assert summary.structured_usage_complete_rate == 1 - assert summary.average_malformed_records == 0 assert summary.average_knowledge_files == 30 assert summary.average_knowledge_pruned == 6 -def test_summary_marks_malformed_structured_usage_incomplete() -> None: +def test_summary_preserves_unavailable_usage_as_none() -> None: summary = CodeReviewResultSummary.from_results( - [ - create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1)), - create_codereview_result(instance_id="proj__review-2", metrics=_metrics(duration=6.0, scale=2, malformed_records=2)), - ], - run_id="run", - ) - - assert summary.structured_usage_complete_rate == 0.5 - assert summary.average_malformed_records == 1 - assert summary.average_total_tokens == 1500 - - -def test_summary_serializes_legal_null_token_metrics() -> None: - metrics = AgentMetrics( - execution_time=4.0, - usage_complete=False, - malformed_records=0, - knowledge_files=20, - knowledge_pruned=4, - ) - summary = CodeReviewResultSummary.from_results( - [create_codereview_result(instance_id="proj__review-1", metrics=metrics)], + [create_codereview_result(metrics=AgentMetrics(execution_time=4.0, knowledge_files=20, knowledge_pruned=4))], run_id="run", ) @@ -92,16 +48,18 @@ def test_summary_serializes_legal_null_token_metrics() -> None: assert serialized["average_prompt_tokens"] is None assert serialized["average_completion_tokens"] is None - assert serialized["structured_usage_complete_rate"] == 0 + assert serialized["average_total_tokens"] is None + assert serialized["average_api_calls"] is None + assert serialized["average_ai_credits"] is None -def test_leaderboard_propagates_pr_review_metrics() -> None: +def test_leaderboard_propagates_public_pr_review_metrics() -> None: first = CodeReviewResultSummary.from_results( [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1))], run_id="one", ) second = CodeReviewResultSummary.from_results( - [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=6.0, scale=2, usage_complete=False))], + [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=6.0, scale=2))], run_id="two", ) @@ -110,17 +68,14 @@ def test_leaderboard_propagates_pr_review_metrics() -> None: assert aggregate.average_duration == 5 assert aggregate.average_prompt_tokens == 1350 assert aggregate.average_completion_tokens == 150 - assert aggregate.average_reasoning_tokens == 37.5 assert aggregate.average_total_tokens == 1500 assert aggregate.average_api_calls == 15 assert aggregate.average_ai_credits == 0.75 - assert aggregate.average_premium_requests == 0.375 - assert aggregate.structured_usage_complete_rate == 0.5 assert aggregate.average_knowledge_files == 30 assert aggregate.average_knowledge_pruned == 6 -def test_github_summary_renders_performance_metrics() -> None: +def test_github_summary_renders_only_public_performance_metrics() -> None: summary = CodeReviewResultSummary.from_results( [create_codereview_result(instance_id="proj__review-1", metrics=_metrics(duration=4.0, scale=1))], run_id="run", @@ -129,73 +84,58 @@ def test_github_summary_renders_performance_metrics() -> None: markdown = summary.render_github_metrics_markdown() assert "## Performance" in markdown + assert "Avg prompt tokens" in markdown + assert "Avg completion tokens" in markdown assert "Avg total tokens" in markdown assert "Avg API calls" in markdown assert "Avg AI credits" in markdown - assert "Avg premium requests" in markdown - assert "Complete structured usage" in markdown - assert "| 10.0 | 1.0 | 9.0 | 0.5000 | 0.2500 | 100.0% | 0.0 |" in markdown assert "Avg knowledge files" in markdown + for diagnostic in ("cached", "reasoning", "failed API", "usage", "premium", "malformed"): + assert diagnostic not in markdown -def test_generic_result_does_not_serialize_pr_review_metrics(tmp_path: Path) -> None: - result = create_bugfix_result(metrics=AgentMetrics(execution_time=4.0)) +def test_result_json_excludes_raw_only_diagnostics(tmp_path) -> None: + result = create_codereview_result(metrics=_metrics(duration=4.0, scale=1)) result.save(tmp_path, "results.jsonl") saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] - for field in ( + assert saved_metrics["prompt_tokens"] == 900 + assert saved_metrics["completion_tokens"] == 100 + assert saved_metrics["total_tokens"] == 1000 + assert saved_metrics["api_calls"] == 10 + assert saved_metrics["ai_credits"] == 0.5 + assert saved_metrics["knowledge_files"] == 20 + assert saved_metrics["knowledge_pruned"] == 4 + for diagnostic in ( "cached_tokens", "cache_creation_tokens", "reasoning_tokens", - "total_tokens", - "api_calls", "failed_api_calls", "usage_api_calls", - "ai_credits", "premium_requests", "usage_complete", "malformed_records", - "knowledge_files", - "knowledge_pruned", ): - assert field not in saved_metrics + assert diagnostic not in saved_metrics -def test_code_review_result_serializes_structured_metrics(tmp_path: Path) -> None: - result = create_codereview_result(metrics=_metrics(duration=4.0, scale=1)) - result.save(tmp_path, "results.jsonl") - - saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] - - assert saved_metrics["total_tokens"] == 1000 - assert saved_metrics["api_calls"] == 10 - assert saved_metrics["ai_credits"] == 0.5 - assert saved_metrics["reasoning_tokens"] == 25 - assert saved_metrics["premium_requests"] == 0.25 - assert saved_metrics["usage_complete"] is True - assert saved_metrics["malformed_records"] == 0 - assert saved_metrics["knowledge_files"] == 20 - assert saved_metrics["knowledge_pruned"] == 4 - - -def test_code_review_result_preserves_nullable_structured_metrics(tmp_path: Path) -> None: - result = create_codereview_result( - metrics=AgentMetrics( - execution_time=4.0, - reasoning_tokens=None, - premium_requests=None, - usage_complete=True, - malformed_records=0, - knowledge_files=20, - knowledge_pruned=4, - ) +def test_summary_and_leaderboard_schemas_exclude_raw_only_diagnostics() -> None: + summary = CodeReviewResultSummary.from_results( + [create_codereview_result(metrics=_metrics(duration=4.0, scale=1))], + run_id="run", ) - result.save(tmp_path, "results.jsonl") - - saved_metrics = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metrics"] - - assert "reasoning_tokens" in saved_metrics - assert saved_metrics["reasoning_tokens"] is None - assert "premium_requests" in saved_metrics - assert saved_metrics["premium_requests"] is None + aggregate = CodeReviewLeaderboardAggregate.from_runs([summary]) + + for payload in (summary.model_dump(), aggregate.model_dump()): + for diagnostic in ( + "average_cached_tokens", + "average_cache_creation_tokens", + "average_reasoning_tokens", + "average_failed_api_calls", + "average_usage_api_calls", + "average_premium_requests", + "structured_usage_complete_rate", + "average_malformed_records", + ): + assert diagnostic not in payload diff --git a/uv.lock b/uv.lock index 47534e93c..a433acfe2 100644 --- a/uv.lock +++ b/uv.lock @@ -286,7 +286,7 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonschema", specifier = ">=4.0" }, { name = "numpy", specifier = ">=2.3.5" }, - { name = "pydantic", specifier = ">=2.12" }, + { name = "pydantic", specifier = ">=2.0" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.0" },