From e3dc7fa9d30b8c7136e920386041852c6bff42d7 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Sun, 12 Jul 2026 15:07:37 +0200 Subject: [PATCH 01/52] Add data-query category: AL query-generation benchmark Adds a new execution-based `data-query` category that benchmarks models/agents at generating Business Central AL queries. Given a natural-language data question, the agent writes a single AL query object to query.al; evaluation compiles and runs both the generated query and a gold reference query against the container's Contoso demo data and compares the result sets. No MCP server and no LLM judge. - types.py: DATA_QUERY -> execution-based (ExecutionBasedEvaluationResult, summary, aggregate; resolution_rate/build_rate; ResolutionRate; requires_container; GitHub-BCBench) - DataQueryEntry: nl_prompt + gold_query + ordered; dataset/dataquery.jsonl (6 tasks) - DataQueryPipeline + result_sets_match (value-based, order-insensitive; unit-tested) - operations: wrap_query_as_api (unit-tested) + execute_al_query (wrap as API query, publish throwaway app, read OData) - ExecutionBasedEvaluationResult.create_result for compiled-but-wrong outcomes - config.yaml: data-query prompt (author query.al); al-query-authoring skill - Setup-ContainerAndRepository.ps1: skip repo clone for data-query (no repo), just provision the sandbox container; a stock Contoso artifact suffices - Wire data-query into the copilot/claude evaluation workflow category choices + docs Container round-trip in execute_al_query and the gold AL query bodies need validation on a runner (no local BC container); pure logic is unit-tested (592 tests pass). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/claude-evaluation.yml | 1 + .github/workflows/copilot-evaluation.yml | 1 + dataset/dataquery.jsonl | 6 + docs/data-query.md | 46 +++++++ docs/index.md | 1 + scripts/Setup-ContainerAndRepository.ps1 | 18 ++- src/bcbench/agent/shared/config.yaml | 22 ++++ .../skills/al-query-authoring/SKILL.md | 51 ++++++++ src/bcbench/commands/evaluate.py | 2 +- src/bcbench/dataset/__init__.py | 3 +- src/bcbench/dataset/dataset_entry.py | 26 +++- src/bcbench/evaluate/__init__.py | 3 +- src/bcbench/evaluate/dataquery.py | 108 ++++++++++++++++ src/bcbench/operations/__init__.py | 4 + src/bcbench/operations/bc_operations.py | 122 ++++++++++++++++++ src/bcbench/results/base.py | 5 + src/bcbench/types.py | 32 ++++- tests/conftest.py | 31 ++++- tests/test_dataquery_evaluation.py | 59 +++++++++ tests/test_type_exhaustiveness.py | 6 +- 20 files changed, 533 insertions(+), 14 deletions(-) create mode 100644 dataset/dataquery.jsonl create mode 100644 docs/data-query.md create mode 100644 src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md create mode 100644 src/bcbench/evaluate/dataquery.py create mode 100644 tests/test_dataquery_evaluation.py diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index 273c9f4fc..0672a170f 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -24,6 +24,7 @@ on: - "bug-fix" - "test-generation" - "code-review" + - "data-query" test-run: description: "Indicate this is a test run (with few entries)" required: false diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index cc65cece0..09a1b1ed4 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -31,6 +31,7 @@ on: - "bug-fix" - "test-generation" - "code-review" + - "data-query" test-run: description: "Indicate this is a test run (with few entries)" required: false diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl new file mode 100644 index 000000000..c422eca5f --- /dev/null +++ b/dataset/dataquery.jsonl @@ -0,0 +1,6 @@ +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer, what is the total outstanding value across their open sales orders? Include the customer's number and name.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "What is the total sold quantity per item across all posted sales invoice lines? Return each item number and its total quantity.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, what is the average posted sales invoice line amount? Group the posted sales invoice lines by the bill-to customer's country/region code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "What is the total posted purchase invoice amount per vendor? Include the vendor's number and name.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__items-on-both-open-orders-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Which items appear on both open sales orders and open purchase orders? Return the item numbers.", "ordered": false, "gold_query": "query 50100 ItemsOnBothOpenOrders\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order), Type = const(Item);\n column(ItemNo; \"No.\") { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"No.\" = SalesLine.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order), Type = const(Item);\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "How many CRM opportunities are there in each status?", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount; \"No.\") { Method = Count; }\n }\n }\n}"} diff --git a/docs/data-query.md b/docs/data-query.md new file mode 100644 index 000000000..dc8a19c3a --- /dev/null +++ b/docs/data-query.md @@ -0,0 +1,46 @@ +--- +layout: default +title: Data Query - BC-Bench +--- + +# Data Query + +This category benchmarks an agent's ability to **generate Business Central AL queries** from a natural-language data question — an offline query-generation benchmark. There is **no MCP server and no live server in the loop**: the agent writes an AL query, and the query is evaluated deterministically. + +Given a question, the agent authors a single AL `query` object and writes it to `query.al`. The harness then **compiles and runs both the generated query and a gold reference query** against a fixed dataset (the BC container's Contoso demo data) and compares the result sets. + +## How it is scored + +Data Query is **execution-based** (like bug-fix), with no LLM judge: + +- **build** — the generated query compiled and ran. +- **resolved** (the headline `ResolutionRate`) — the generated query's result set **matches the gold query's**. Rows are compared by value (numbers normalized, column names/order ignored); row order is ignored unless the entry marks the question as `ordered`. + +To execute a query, the harness wraps it as an API query (injecting `APIPublisher`/`APIGroup`/`APIVersion`/`EntitySetName`), publishes a throwaway app to the container, and reads the query's OData endpoint. This runs on the `GitHub-BCBench` self-hosted runner (`requires_container = True`). + +> This complements the AI Test Toolkit evals in the BC platform repo: those test the **MCP server** end-to-end, while BC-Bench benchmarks **models/agents** on query generation. + +## Dataset + +Each entry has an `nl_prompt` (the question) and a `gold_query` (the reference AL query whose result set defines "correct"), plus `environment_setup_version` (the BC artifact) and `ordered`. See `dataset/dataquery.jsonl`. + +## Running it (no local containers) + +Trigger it from the GitHub **Actions** tab — the self-hosted `GitHub-BCBench` runner provisions the BC container for you (a stock **sandbox artifact with Cronus/Contoso demo data** — no special build is needed, since the query is just compiled and run): + +1. Actions → **Evaluation with GitHub Copilot** (or **Evaluation with Claude Code**) → **Run workflow**. +2. Set **category** = `data-query`, pick a **model**, leave **test-run** = `true` for a quick 2-entry run. +3. The run: provisions the container → the agent writes `query.al` → the harness compiles + runs the generated and gold queries → compares result sets → `summarize-results` reports `ResolutionRate` / `BuildRate`. + +`data-query` sets `requires_container = True`; its container setup **skips the repo clone** (there is no repo — the agent generates from scratch) and just stands up the sandbox container. + +### Local (optional) + +```bash +uv run bcbench evaluate copilot dataquery__outstanding-sales-value-by-customer-1 \ + --category data-query --container-name --username admin --password +``` + +The optional `al-mcp` / `al-lsp` levers give the agent AL compiler/language-server feedback while it authors the query. + +[← Back to Home](index.md) diff --git a/docs/index.md b/docs/index.md index 1006ba6fd..dbcc684c0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ A benchmark for evaluating AI coding agents on real-world **Business Central (AL | [Bug Fixing](bug-fix.md) | Follows [SWE-Bench](https://www.swebench.com/) methodology to evaluate bug fixing in AL code | | [Test Generation](test-generation.md) | "Reverses" SWE-Bench: Generates reproduction tests (TDD) instead of fixes | | [Code Review](code-review.md) | Reviews AL pull requests; scored with Precision / Recall / F1 against gold findings | +| [Data Query](data-query.md) | Generates AL queries from natural-language data questions; scored deterministically by running the query and comparing its result set to a gold query | ## What is Business Central? diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index 751655ac5..c81243c7a 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -61,11 +61,21 @@ if (Test-Path $RepoPath) { throw "Repository already exists at $RepoPath. This indicates the machine was not properly cleaned up from a previous run." } -[hashtable] $cloneInfo = Get-RepoCloneInfo -Entry $entries[0] -[string] $commitSha = $entries[0].base_commit +# Some categories (e.g. data-query) generate code from scratch rather than editing an existing +# repository, so there is nothing to clone -- the agent just needs an empty working directory. +[string[]] $noCloneCategories = @('data-query') -Write-Log "Cloning repository $($entries[0].repo) to $RepoPath" -Level Info -Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths +if ($Category -in $noCloneCategories) { + Write-Log "Category '$Category' needs no repository clone; creating empty workspace at $RepoPath" -Level Info + New-Item -ItemType Directory -Path $RepoPath -Force | Out-Null +} +else { + [hashtable] $cloneInfo = Get-RepoCloneInfo -Entry $entries[0] + [string] $commitSha = $entries[0].base_commit + + Write-Log "Cloning repository $($entries[0].repo) to $RepoPath" -Level Info + Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths +} if (-not $SkipContainer) { [PSCredential]$credential = Get-BCCredential -Username $Username -Password $Password diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index 85bab3e5a..d50a88452 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -66,6 +66,28 @@ prompt: If there are no findings, write an empty array. Write only valid JSON to review.json, with no surrounding markdown or commentary. + data-query-template: | + You are writing a Microsoft Dynamics 365 Business Central AL query to answer a data + question. Proceed without asking for confirmation. + + Task: author a single AL `query` object that returns the data needed to answer the + question below, and write it to a file named `query.al` in {{repo_path}}. + + Requirements: + - Write exactly one AL `query` object with an object id in the range 50100..50149 and a + name. Reference real Business Central tables and fields. + - Return the columns needed to answer the question. Use column `Method` (Sum, Count, + Average, Min, Max) for aggregates and `DataItemLink` to join dataitems. + - Do NOT add API properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName, + EntitySetName) — the evaluation harness adds those. Write only the query logic + (query id/name, elements, dataitem(s), columns, filters, order by). + - The file must contain only the query object, nothing else. + + Question: + {{task}} + + You MUST write query.al before finishing; if you do not, there is no output to evaluate. + # controls: # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions//` # - Copilot: copies to repo/.github/ and renames AGENTS.md to copilot-instructions.md diff --git a/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md new file mode 100644 index 000000000..cf1726c14 --- /dev/null +++ b/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md @@ -0,0 +1,51 @@ +--- +name: al-query-authoring +description: Guide for authoring Business Central AL query objects that answer data questions (joins, aggregates, filters, sorting). Use this when asked to write an AL query that returns Business Central data such as customers, vendors, items, sales, purchases, projects, or opportunities. +--- + +Write a single, compilable AL `query` object that returns exactly the data needed to answer +the question. Reference real Business Central tables and fields — a query that does not +compile, or returns the wrong data, fails. + +## Structure + +```al +query 50100 TopCustomersBySales +{ + QueryType = Normal; + + elements + { + dataitem(Customer; Customer) + { + column(No; "No.") { } + column(Name; Name) { } + dataitem(SalesLine; "Sales Line") + { + DataItemLink = "Sell-to Customer No." = Customer."No."; + DataItemTableFilter = "Document Type" = const(Order); + column(OutstandingAmount; "Outstanding Amount") { Method = Sum; } + } + } + } +} +``` + +## Rules of thumb + +- **Aggregate** a column with a method: `column(Total; "Amount (LCY)") { Method = Sum; }` + (also `Average`, `Count`, `Min`, `Max`). Non-aggregated columns become the GROUP BY. +- **Join** by nesting a `dataitem` and linking it: `DataItemLink = "" = Parent."";`. +- **Filter** rows with `DataItemTableFilter = "" = const();` (e.g. an Option + like `Document Type`) or a range/expression. +- **Order** with `OrderBy { descending(); }` when the question asks for ranking or + "top N" (combine with `TopNumberOfRows` where appropriate). +- **Quote** any field or table name that contains spaces or special characters: `"No."`, + `"Sales Line"`, `"Amount (LCY)"`. +- Prefer stored fields; FlowFields and Option fields are supported. + +## Common pitfalls + +- Don't invent table or field names — use the real Business Central schema. +- Return only the columns the question needs; extra or missing columns change the result set. +- One `query` object per file. diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 608d4005b..b1401725c 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -308,7 +308,7 @@ def evaluate(self, context: EvaluationContext[BaseDatasetEntry]) -> None: logger.info("Mock pipeline: Generating random evaluation result") match context.category: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: + case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: scenarios = ["success", "build-fail"] case EvaluationCategory.CODE_REVIEW: scenarios = ["invalid", "valid"] diff --git a/src/bcbench/dataset/__init__.py b/src/bcbench/dataset/__init__.py index d975e152f..2f4dea0e3 100644 --- a/src/bcbench/dataset/__init__.py +++ b/src/bcbench/dataset/__init__.py @@ -1,12 +1,13 @@ """Dataset module for querying, validating and analyze dataset entries.""" from bcbench.dataset.codereview import CodeReviewEntry, ReviewComment, Severity -from bcbench.dataset.dataset_entry import BaseDatasetEntry, BugFixEntry, NL2ALEntry, TestEntry, TestGenEntry +from bcbench.dataset.dataset_entry import BaseDatasetEntry, BugFixEntry, DataQueryEntry, NL2ALEntry, TestEntry, TestGenEntry __all__ = [ "BaseDatasetEntry", "BugFixEntry", "CodeReviewEntry", + "DataQueryEntry", "NL2ALEntry", "ReviewComment", "Severity", diff --git a/src/bcbench/dataset/dataset_entry.py b/src/bcbench/dataset/dataset_entry.py index d5eca579c..c5d2714d3 100644 --- a/src/bcbench/dataset/dataset_entry.py +++ b/src/bcbench/dataset/dataset_entry.py @@ -14,7 +14,7 @@ _config = get_config() -__all__ = ["BaseDatasetEntry", "BugFixEntry", "NL2ALEntry", "TestEntry", "TestGenEntry"] +__all__ = ["BaseDatasetEntry", "BugFixEntry", "DataQueryEntry", "NL2ALEntry", "TestEntry", "TestGenEntry"] class TestEntry(BaseModel): @@ -168,3 +168,27 @@ def get_task(self) -> str: def get_expected_output(self) -> Checklist: return {"assertions": self.expected} + + +class DataQueryEntry(BaseDatasetEntry): + """Dataset entry for the data-query category — generate an AL query that answers a data question. + + Execution-based: the agent authors an AL query; evaluation compiles + runs both the generated + query and the gold query against a fixed dataset (Contoso in a BC container) and compares the + result sets. No repo scaffold, so base_commit / patch are relaxed to optional. + """ + + base_commit: str | None = None + patch: str = "" + + nl_prompt: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] + gold_query: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] + # Whether row order is significant when comparing result sets (e.g. the question asks for a + # specific ranking). Defaults to False: result sets are compared order-insensitively. + ordered: bool = False + + def get_task(self) -> str: + return self.nl_prompt + + def get_expected_output(self) -> str: + return self.gold_query diff --git a/src/bcbench/evaluate/__init__.py b/src/bcbench/evaluate/__init__.py index c96892d07..6b9c38c20 100644 --- a/src/bcbench/evaluate/__init__.py +++ b/src/bcbench/evaluate/__init__.py @@ -3,7 +3,8 @@ from bcbench.evaluate.base import EvaluationPipeline from bcbench.evaluate.bugfix import BugFixPipeline from bcbench.evaluate.codereview import CodeReviewPipeline +from bcbench.evaluate.dataquery import DataQueryPipeline from bcbench.evaluate.nl2al import NL2ALPipeline from bcbench.evaluate.testgeneration import TestGenerationPipeline -__all__ = ["BugFixPipeline", "CodeReviewPipeline", "EvaluationPipeline", "NL2ALPipeline", "TestGenerationPipeline"] +__all__ = ["BugFixPipeline", "CodeReviewPipeline", "DataQueryPipeline", "EvaluationPipeline", "NL2ALPipeline", "TestGenerationPipeline"] diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py new file mode 100644 index 000000000..21d52b69a --- /dev/null +++ b/src/bcbench/evaluate/dataquery.py @@ -0,0 +1,108 @@ +import shutil +from collections.abc import Callable +from pathlib import Path + +from bcbench.dataset import DataQueryEntry +from bcbench.evaluate.base import EvaluationPipeline +from bcbench.exceptions import BuildError +from bcbench.github_actions import github_log_group +from bcbench.logger import get_logger +from bcbench.results.base import ExecutionBasedEvaluationResult +from bcbench.types import EvaluationContext + +logger = get_logger(__name__) + +__all__ = ["DataQueryPipeline", "result_sets_match"] + +GENERATED_QUERY_FILE = "query.al" + + +def _normalize_value(value: object) -> str: + if value is None: + return "" + if isinstance(value, bool): + return str(value).lower() + if isinstance(value, (int, float)): + return f"{float(value):.4f}" + text = str(value).strip() + try: + return f"{float(text):.4f}" + except ValueError: + return text + + +def _normalize_rows(rows: list[dict[str, object]], ordered: bool) -> list[tuple[str, ...]]: + # Compare on values only: drop OData/system metadata keys ('@'-prefixed) and ignore column + # names/order so a correct query still matches the gold even if it names columns differently. + normalized = [tuple(sorted(_normalize_value(v) for k, v in row.items() if not k.startswith("@"))) for row in rows] + return normalized if ordered else sorted(normalized) + + +def result_sets_match(generated: list[dict[str, object]], gold: list[dict[str, object]], ordered: bool = False) -> bool: + """Compare two query result sets for equality. + + Values are compared (numbers normalized, column names/order ignored); row order is ignored + unless ``ordered`` is True (the question asks for a specific ranking). + """ + return _normalize_rows(generated, ordered) == _normalize_rows(gold, ordered) + + +def _force_remove_readonly(func: Callable, path: str, _: object) -> None: + Path(path).chmod(0o666) + func(path) + + +def _reset_repo_path(repo_path: Path) -> None: + if repo_path.exists(): + shutil.rmtree(repo_path, onexc=_force_remove_readonly) + repo_path.mkdir(parents=True, exist_ok=True) + + +class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): + """Pipeline for the data-query category — generate an AL query, evaluate deterministically. + + The agent writes an AL query to ``query.al``. Evaluation compiles + runs both the generated + query and the entry's gold query against the container's fixed (Contoso) dataset and compares + the result sets: build = the generated query compiled and ran; resolved = its result set + matches the gold query's. + """ + + def setup_workspace(self, entry: DataQueryEntry, repo_path: Path) -> None: + _reset_repo_path(repo_path) + + def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: + self.setup_workspace(context.entry, context.repo_path) + + def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: Callable) -> None: + with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + context.metrics, context.experiment = agent_runner(context) + + def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: + from bcbench.operations import execute_al_query + + query_file = context.repo_path / GENERATED_QUERY_FILE + generated_query = query_file.read_text(encoding="utf-8").strip() if query_file.exists() else "" + + if not generated_query: + logger.warning(f"Agent produced no {GENERATED_QUERY_FILE} for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output="", error_message=f"No {GENERATED_QUERY_FILE} produced")) + return + + container = context.get_container() + version = context.entry.environment_setup_version + + try: + generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") + except BuildError as e: + logger.exception(f"Generated query failed to compile/run for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) + return + + # The gold query is authored to compile; a failure here is an environment/dataset problem, not the agent's. + gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + + resolved = result_sets_match(generated_rows, gold_rows, context.entry.ordered) + error_message = None if resolved else f"Result set mismatch: generated {len(generated_rows)} rows vs gold {len(gold_rows)} rows" + result = ExecutionBasedEvaluationResult.create_result(context, output=generated_query, build=True, resolved=resolved, error_message=error_message) + logger.info(f"{context.entry.instance_id}: build=True resolved={resolved}") + self.save_result(context, result) diff --git a/src/bcbench/operations/__init__.py b/src/bcbench/operations/__init__.py index 8c5a28ad4..04d14aacc 100644 --- a/src/bcbench/operations/__init__.py +++ b/src/bcbench/operations/__init__.py @@ -6,8 +6,10 @@ build_ps_dataset_tests_script, build_ps_test_script, copy_symbol_apps, + execute_al_query, resolve_artifact_version_root, run_tests, + wrap_query_as_api, ) from bcbench.operations.git_operations import ( apply_patch, @@ -40,6 +42,7 @@ "commit_changes", "copy_problem_statement_folder", "copy_symbol_apps", + "execute_al_query", "extract_tests_from_patch", "resolve_artifact_version_root", "run_tests", @@ -51,4 +54,5 @@ "setup_plugins_from_config", "setup_repo_prebuild", "stage_and_get_diff", + "wrap_query_as_api", ] diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index 88ea254d2..2368e4e8e 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -235,3 +235,125 @@ def run_test_suite(test_entries: list[TestEntry], expectation: Literal["Pass", " except subprocess.TimeoutExpired: logger.exception(f"Test execution timed out after {_config.timeout.test_execution} seconds") raise TestExecutionTimeoutExpired(test_entries_json, _config.timeout.test_execution) from None + + +# --- data-query category: compile + run an AL query and capture its rows via a wrapped API query --- + +# API metadata injected into a generated/gold query so it is exposed over OData and can be fetched. +_QUERY_API_PUBLISHER = "bcbench" +_QUERY_API_GROUP = "eval" +_QUERY_API_VERSION = "v1.0" +_QUERY_API_ENTITY_SET = "bcbenchResults" + +_QUERY_API_PROPERTIES = ( + "QueryType = API;\n" + f" APIPublisher = '{_QUERY_API_PUBLISHER}';\n" + f" APIGroup = '{_QUERY_API_GROUP}';\n" + f" APIVersion = '{_QUERY_API_VERSION}';\n" + " EntityName = 'bcbenchResult';\n" + f" EntitySetName = '{_QUERY_API_ENTITY_SET}';\n" + " Extensible = false;" +) + + +def wrap_query_as_api(query_text: str, object_id: int) -> str: + """Turn a plain AL query object into an API query the harness can fetch over OData. + + Reassigns the object id (so generated and gold apps don't collide), drops any existing + ``QueryType`` line, and injects the API properties right after the object's opening brace. + Pure string transform so it can be unit-tested without a container. + """ + import re + + text = re.sub(r"(\bquery\s+)\d+", rf"\g<1>{object_id}", query_text, count=1) + text = re.sub(r"\n\s*QueryType\s*=\s*\w+\s*;", "", text, count=1) + + brace_index = text.index("{") + return f"{text[: brace_index + 1]}\n {_QUERY_API_PROPERTIES}\n{text[brace_index + 1 :]}" + + +_QUERY_RUN_TEMPLATE = Template( + """ +Import-Module BcContainerHelper -Force -DisableNameChecking +$$ErrorActionPreference = 'Stop' + +$$password = ConvertTo-SecureString '$password' -AsPlainText -Force +$$credential = New-Object System.Management.Automation.PSCredential('$username', $$password) + +$$appFile = Compile-AppInBcContainer -containerName '$container_name' -credential $$credential -appProjectFolder '$app_dir' -appOutputFolder '$app_dir\\out' -UpdateSymbols +Publish-BcContainerApp -containerName '$container_name' -appFile $$appFile -skipVerification -sync -install -credential $$credential + +$$base = "http://$container_name/BC/api" +$$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Credential $$credential).value[0].id +$$data = Invoke-RestMethod -Uri "$$base/$publisher/$group/$version/companies($$companyId)/$entity_set" -Credential $$credential +$$data.value | ConvertTo-Json -Depth 10 -Compress | Out-File -FilePath '$result_file' -Encoding utf8 +""".strip() +) + + +def execute_al_query(query_text: str, container: ContainerConfig, version: str, work_root: Path, suffix: str) -> list[dict]: + """Compile + publish an AL query (wrapped as an API query) to the container and return its rows. + + Builds a throwaway app under ``work_root/.bcbench-query-``, compiles + publishes it, + then reads the query's OData endpoint. Raises :class:`BuildError` if the query does not + compile or publish. + + NOTE: the container-side steps (compile/publish/OData fetch) require a running BC container + and have not been validated locally; the wrapping and comparison logic are unit-tested. + """ + import json + from uuid import uuid4 + + object_id = 50100 if suffix == "generated" else 50101 + app_dir = work_root / f".bcbench-query-{suffix}" + if app_dir.exists(): + shutil.rmtree(app_dir, onexc=lambda func, path, _: (Path(path).chmod(0o666), func(path))) + app_dir.mkdir(parents=True, exist_ok=True) + + app_manifest = { + "id": str(uuid4()), + "name": f"BC-Bench Query {suffix}", + "publisher": "BC-Bench", + "version": "1.0.0.0", + "application": f"{version.split('.')[0]}.0.0.0", + "platform": f"{version.split('.')[0]}.0.0.0", + "idRanges": [{"from": object_id, "to": object_id}], + "runtime": "13.0", + "target": "OnPrem", + } + (app_dir / "app.json").write_text(json.dumps(app_manifest, indent=2), encoding="utf-8") + (app_dir / "query.al").write_text(wrap_query_as_api(query_text, object_id), encoding="utf-8") + # Symbols come from the container via Compile-AppInBcContainer -UpdateSymbols (below); no need + # to pre-populate .alpackages from the artifact cache. + + result_file = app_dir / "result.json" + ps_script = _QUERY_RUN_TEMPLATE.substitute( + container_name=_escape_ps_string(container.name), + username=_escape_ps_string(container.username), + password=_escape_ps_string(container.password), + app_dir=_escape_ps_string(str(app_dir)), + publisher=_QUERY_API_PUBLISHER, + group=_QUERY_API_GROUP, + version=_QUERY_API_VERSION, + entity_set=_QUERY_API_ENTITY_SET, + result_file=_escape_ps_string(str(result_file)), + ) + + try: + subprocess.run( + ["pwsh", "-NoProfile", "-NonInteractive", "-Command", ps_script], + cwd=work_root, + capture_output=True, + check=True, + text=True, + timeout=_config.timeout.build_app, + ) + except subprocess.CalledProcessError as e: + logger.debug(f"Query compile/publish/fetch failed ({suffix}): {e.stdout}\n{e.stderr}") + raise BuildError(f"query-{suffix}", (e.stdout or "") + (e.stderr or "")) from None + except subprocess.TimeoutExpired: + raise BuildTimeoutExpired(f"query-{suffix}", _config.timeout.build_app) from None + + rows = json.loads(result_file.read_text(encoding="utf-8-sig") or "[]") + return rows if isinstance(rows, list) else [rows] + diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 44bdf4c46..ee4a8e7cf 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -107,6 +107,11 @@ def create_success(cls, context: "EvaluationContext", output: str) -> Self: def create_build_failure(cls, context: "EvaluationContext", output: str, error_message: str) -> Self: return cls(**cls._base_fields(context), output=output, error_message=error_message, resolved=False, build=False) + @classmethod + def create_result(cls, context: "EvaluationContext", output: str, *, build: bool, resolved: bool, error_message: str | None = None) -> Self: + """General factory for execution outcomes, e.g. compiled+ran but produced the wrong result (build=True, resolved=False).""" + return cls(**cls._base_fields(context), output=output, build=build, resolved=resolved, error_message=error_message) + @property def status_label(self) -> str: if self.timeout: diff --git a/src/bcbench/types.py b/src/bcbench/types.py index 0bfbbc362..f031850d1 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -136,6 +136,7 @@ class EvaluationCategory(StrEnum): TEST_GENERATION = "test-generation" CODE_REVIEW = "code-review" NL2AL = "nl2al" + DATA_QUERY = "data-query" # EVENT_REQUEST = "event-request" @property @@ -151,12 +152,14 @@ def dataset_path(self) -> Path: return get_config().paths.dataset_dir / "codereview.jsonl" case EvaluationCategory.NL2AL: return get_config().paths.dataset_dir / "nl2al.jsonl" + case EvaluationCategory.DATA_QUERY: + return get_config().paths.dataset_dir / "dataquery.jsonl" raise ValueError(f"Unknown evaluation category: {self}") @property def entry_class(self) -> type[BaseDatasetEntry]: - from bcbench.dataset import BugFixEntry, CodeReviewEntry, NL2ALEntry, TestGenEntry + from bcbench.dataset import BugFixEntry, CodeReviewEntry, DataQueryEntry, NL2ALEntry, TestGenEntry match self: case EvaluationCategory.BUG_FIX: @@ -167,12 +170,14 @@ def entry_class(self) -> type[BaseDatasetEntry]: return CodeReviewEntry case EvaluationCategory.NL2AL: return NL2ALEntry + case EvaluationCategory.DATA_QUERY: + return DataQueryEntry raise ValueError(f"Unknown evaluation category: {self}") @property def result_class(self) -> type[BaseEvaluationResult]: - from bcbench.results.base import JudgeBasedEvaluationResult + from bcbench.results.base import ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.results.bugfix import BugFixResult from bcbench.results.codereview import CodeReviewResult from bcbench.results.testgeneration import TestGenerationResult @@ -186,6 +191,8 @@ def result_class(self) -> type[BaseEvaluationResult]: return CodeReviewResult case EvaluationCategory.NL2AL: return JudgeBasedEvaluationResult + case EvaluationCategory.DATA_QUERY: + return ExecutionBasedEvaluationResult raise ValueError(f"Unknown evaluation category: {self}") @@ -204,6 +211,8 @@ def summary_class(self) -> type[EvaluationResultSummary]: return CodeReviewResultSummary case EvaluationCategory.NL2AL: return JudgeBasedEvaluationResultSummary + case EvaluationCategory.DATA_QUERY: + return ExecutionBasedEvaluationResultSummary raise ValueError(f"Unknown evaluation category: {self}") @@ -221,12 +230,14 @@ def aggregate_class(self) -> type[LeaderboardAggregate]: return CodeReviewLeaderboardAggregate case EvaluationCategory.NL2AL: return JudgeBasedLeaderboardAggregate + case EvaluationCategory.DATA_QUERY: + return ExecutionBasedLeaderboardAggregate raise ValueError(f"Unknown evaluation category: {self}") @property def pipeline(self) -> EvaluationPipeline: - from bcbench.evaluate import BugFixPipeline, CodeReviewPipeline, NL2ALPipeline, TestGenerationPipeline + from bcbench.evaluate import BugFixPipeline, CodeReviewPipeline, DataQueryPipeline, NL2ALPipeline, TestGenerationPipeline match self: case EvaluationCategory.BUG_FIX: @@ -237,6 +248,8 @@ def pipeline(self) -> EvaluationPipeline: return CodeReviewPipeline() case EvaluationCategory.NL2AL: return NL2ALPipeline() + case EvaluationCategory.DATA_QUERY: + return DataQueryPipeline() raise ValueError(f"Unknown evaluation category: {self}") @@ -256,6 +269,8 @@ def evaluators(self) -> list[str]: return ["precision_score", "recall_score", "f1_score", "valid_review_output"] case EvaluationCategory.NL2AL: return ["lm_checklist"] + case EvaluationCategory.DATA_QUERY: + return ["resolution_rate", "build_rate"] raise ValueError(f"Unknown evaluation category: {self}") @@ -269,6 +284,8 @@ def core_score(self) -> str: return "F1Score" case EvaluationCategory.NL2AL: return "test_passed" + case EvaluationCategory.DATA_QUERY: + return "ResolutionRate" raise ValueError(f"Unknown evaluation category: {self}") @@ -278,6 +295,11 @@ def requires_container(self) -> bool: match self: case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: return True + case EvaluationCategory.DATA_QUERY: + # Data-query provisions its own container: it publishes the data-query seed app + # (al/dataquery-seed) and enables a Data-Query-Tools MCP config, then the agent + # queries that container's /mcp against the seeded, deterministic dataset. + return True case EvaluationCategory.CODE_REVIEW | EvaluationCategory.NL2AL: return False @@ -296,6 +318,10 @@ def runner(self) -> str: return "ubuntu-latest" case EvaluationCategory.NL2AL: return "windows-latest" + case EvaluationCategory.DATA_QUERY: + # The agent queries a live BC /mcp endpoint; the self-hosted runner is where a + # BcContainerHelper BC container (Data Query Tools enabled) is reachable. + return "GitHub-BCBench" raise ValueError(f"Unknown evaluation category: {self}") diff --git a/tests/conftest.py b/tests/conftest.py index eb3e65472..d6a870941 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ import pytest -from bcbench.dataset import BaseDatasetEntry, BugFixEntry, NL2ALEntry, TestEntry +from bcbench.dataset import BaseDatasetEntry, BugFixEntry, DataQueryEntry, NL2ALEntry, TestEntry from bcbench.dataset.codereview import CodeReviewEntry, ReviewComment, Severity from bcbench.dataset.dataset_entry import EntryMetadata, _BugFixTestGenBase from bcbench.evaluate.review_parsing import parse_review_output @@ -342,3 +342,32 @@ def create_nl2al_entry( @pytest.fixture def sample_nl2al_entry() -> NL2ALEntry: return create_nl2al_entry() + + +VALID_DATA_QUERY_PROMPT = "Return the total sales amount per customer." +VALID_GOLD_QUERY = "query 50100 SalesByCustomer\n{\n QueryType = Normal;\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n }\n }\n}" + + +def create_data_query_entry( + instance_id: str = "dataquery__sales-by-customer-1", + repo: str = "dataquery/bc", + environment_setup_version: str = VALID_ENVIRONMENT_VERSION, + nl_prompt: str = VALID_DATA_QUERY_PROMPT, + created_at: str = VALID_CREATED_AT, + gold_query: str = VALID_GOLD_QUERY, + ordered: bool = False, +) -> DataQueryEntry: + return DataQueryEntry( + instance_id=instance_id, + repo=repo, + environment_setup_version=environment_setup_version, + nl_prompt=nl_prompt, + created_at=created_at, + gold_query=gold_query, + ordered=ordered, + ) + + +@pytest.fixture +def sample_data_query_entry() -> DataQueryEntry: + return create_data_query_entry() diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py new file mode 100644 index 000000000..631e3dfb9 --- /dev/null +++ b/tests/test_dataquery_evaluation.py @@ -0,0 +1,59 @@ +from bcbench.evaluate.dataquery import result_sets_match +from bcbench.operations import wrap_query_as_api + + +class TestResultSetsMatch: + def test_identical_rows_match(self): + rows = [{"No": "C1", "Total": 100}, {"No": "C2", "Total": 200}] + assert result_sets_match(rows, rows) + + def test_row_order_ignored_when_unordered(self): + generated = [{"No": "C2", "Total": 200}, {"No": "C1", "Total": 100}] + gold = [{"No": "C1", "Total": 100}, {"No": "C2", "Total": 200}] + assert result_sets_match(generated, gold, ordered=False) + + def test_row_order_enforced_when_ordered(self): + generated = [{"No": "C2", "Total": 200}, {"No": "C1", "Total": 100}] + gold = [{"No": "C1", "Total": 100}, {"No": "C2", "Total": 200}] + assert not result_sets_match(generated, gold, ordered=True) + + def test_numeric_normalization(self): + assert result_sets_match([{"Total": 500}], [{"Total": "500.0"}]) + + def test_column_names_ignored(self): + assert result_sets_match([{"ItemNo": "I1", "Qty": 5}], [{"No": "I1", "Total": 5}]) + + def test_odata_metadata_keys_ignored(self): + generated = [{"@odata.etag": "W/abc", "No": "C1", "Total": 100}] + gold = [{"No": "C1", "Total": 100}] + assert result_sets_match(generated, gold) + + def test_mismatch_detected(self): + assert not result_sets_match([{"No": "C1", "Total": 100}], [{"No": "C1", "Total": 999}]) + + def test_different_row_count_mismatch(self): + assert not result_sets_match([{"No": "C1"}], [{"No": "C1"}, {"No": "C2"}]) + + +class TestWrapQueryAsApi: + PLAIN_QUERY = 'query 50100 MyQuery\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' + + def test_reassigns_object_id(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50101) + assert "query 50101 MyQuery" in wrapped + assert "query 50100" not in wrapped + + def test_injects_api_properties(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) + assert "QueryType = API;" in wrapped + assert "APIPublisher = 'bcbench';" in wrapped + assert "EntitySetName = 'bcbenchResults';" in wrapped + + def test_drops_existing_querytype(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) + assert "QueryType = Normal;" not in wrapped + + def test_preserves_query_body(self): + wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) + assert 'dataitem(Customer; Customer)' in wrapped + assert 'column(No; "No.")' in wrapped diff --git a/tests/test_type_exhaustiveness.py b/tests/test_type_exhaustiveness.py index 06169ba29..40b15e340 100644 --- a/tests/test_type_exhaustiveness.py +++ b/tests/test_type_exhaustiveness.py @@ -1,6 +1,6 @@ from pathlib import Path -from bcbench.dataset import BugFixEntry, CodeReviewEntry, NL2ALEntry +from bcbench.dataset import BugFixEntry, CodeReviewEntry, DataQueryEntry, NL2ALEntry from bcbench.dataset.codereview import ReviewComment, Severity from bcbench.types import AgentType, EvaluationCategory @@ -40,7 +40,7 @@ def test_all_categories_have_aggregate_classes(): assert issubclass(aggregate_cls, LeaderboardAggregate) -def test_all_categories_handled_in_get_expected_output(sample_dataset_entry_with_problem_statement: BugFixEntry, sample_nl2al_entry: NL2ALEntry): +def test_all_categories_handled_in_get_expected_output(sample_dataset_entry_with_problem_statement: BugFixEntry, sample_nl2al_entry: NL2ALEntry, sample_data_query_entry: DataQueryEntry): for category in EvaluationCategory: entry_cls = category.entry_class if entry_cls == CodeReviewEntry: @@ -56,6 +56,8 @@ def test_all_categories_handled_in_get_expected_output(sample_dataset_entry_with ) elif entry_cls is NL2ALEntry: entry = sample_nl2al_entry + elif entry_cls is DataQueryEntry: + entry = sample_data_query_entry else: # Reconstruct entry as the category-specific type so get_expected_output() works entry = entry_cls.model_validate(sample_dataset_entry_with_problem_statement.model_dump(by_alias=True)) From 8d08fbe4eba5fbad263ffb319cc0eddfdad7d703 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 09:19:21 +0200 Subject: [PATCH 02/52] Fix pre-commit: ruff-format + ty (Sequence[Mapping] for result_sets_match) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 6 +++--- src/bcbench/operations/bc_operations.py | 1 - tests/conftest.py | 4 +++- tests/test_dataquery_evaluation.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 21d52b69a..87e63161f 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,5 +1,5 @@ import shutil -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from bcbench.dataset import DataQueryEntry @@ -31,14 +31,14 @@ def _normalize_value(value: object) -> str: return text -def _normalize_rows(rows: list[dict[str, object]], ordered: bool) -> list[tuple[str, ...]]: +def _normalize_rows(rows: Sequence[Mapping[str, object]], ordered: bool) -> list[tuple[str, ...]]: # Compare on values only: drop OData/system metadata keys ('@'-prefixed) and ignore column # names/order so a correct query still matches the gold even if it names columns differently. normalized = [tuple(sorted(_normalize_value(v) for k, v in row.items() if not k.startswith("@"))) for row in rows] return normalized if ordered else sorted(normalized) -def result_sets_match(generated: list[dict[str, object]], gold: list[dict[str, object]], ordered: bool = False) -> bool: +def result_sets_match(generated: Sequence[Mapping[str, object]], gold: Sequence[Mapping[str, object]], ordered: bool = False) -> bool: """Compare two query result sets for equality. Values are compared (numbers normalized, column names/order ignored); row order is ignored diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index 2368e4e8e..a71d71b4a 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -356,4 +356,3 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, rows = json.loads(result_file.read_text(encoding="utf-8-sig") or "[]") return rows if isinstance(rows, list) else [rows] - diff --git a/tests/conftest.py b/tests/conftest.py index d6a870941..687b5d96a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -345,7 +345,9 @@ def sample_nl2al_entry() -> NL2ALEntry: VALID_DATA_QUERY_PROMPT = "Return the total sales amount per customer." -VALID_GOLD_QUERY = "query 50100 SalesByCustomer\n{\n QueryType = Normal;\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n }\n }\n}" +VALID_GOLD_QUERY = ( + 'query 50100 SalesByCustomer\n{\n QueryType = Normal;\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' +) def create_data_query_entry( diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 631e3dfb9..ee13792bf 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -55,5 +55,5 @@ def test_drops_existing_querytype(self): def test_preserves_query_body(self): wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) - assert 'dataitem(Customer; Customer)' in wrapped + assert "dataitem(Customer; Customer)" in wrapped assert 'column(No; "No.")' in wrapped From f62ace6b7c64d7cf091d8cdffa36a35eff9eabd4 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 09:33:19 +0200 Subject: [PATCH 03/52] Register data-query in Get-BCBenchDatasetPath (container setup) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- scripts/BCBenchUtils.psm1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/BCBenchUtils.psm1 b/scripts/BCBenchUtils.psm1 index a8a02cf9c..87a87d185 100644 --- a/scripts/BCBenchUtils.psm1 +++ b/scripts/BCBenchUtils.psm1 @@ -490,7 +490,7 @@ function Get-BCBenchDatasetPath { param( [Parameter(Mandatory = $true)] # Category validation lives only here: every caller resolves the dataset path through this function, so there's no need to duplicate ValidateSet on each caller. - [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al")] + [ValidateSet("bug-fix", "test-generation", "code-review", "nl2al", "data-query")] [string] $Category ) @@ -499,6 +499,7 @@ function Get-BCBenchDatasetPath { "test-generation" { $DatasetName = "bcbench.jsonl" } "code-review" { $DatasetName = "codereview.jsonl" } "nl2al" { $DatasetName = "nl2al.jsonl" } + "data-query" { $DatasetName = "dataquery.jsonl" } } [string] $projectRoot = Split-Path $PSScriptRoot -Parent From a06dec4ec87733672ead58c7aeba63ea80d6b292 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 09:54:20 +0200 Subject: [PATCH 04/52] data-query: clear workspace contents instead of rmtree (dir is mounted into container) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 87e63161f..34bd86d01 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -52,10 +52,17 @@ def _force_remove_readonly(func: Callable, path: str, _: object) -> None: func(path) -def _reset_repo_path(repo_path: Path) -> None: - if repo_path.exists(): - shutil.rmtree(repo_path, onexc=_force_remove_readonly) +def _prepare_repo_path(repo_path: Path) -> None: + # Clear the workspace *contents* but not the directory itself: for data-query the workspace is + # mounted into the running BC container (shared folder), so removing the top dir fails with + # WinError 32 (in use). The workflow hands us a fresh empty dir; locally this clears stale files. repo_path.mkdir(parents=True, exist_ok=True) + for child in repo_path.iterdir(): + if child.is_dir(): + shutil.rmtree(child, onexc=_force_remove_readonly) + else: + child.chmod(0o666) + child.unlink() class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): @@ -68,7 +75,7 @@ class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): """ def setup_workspace(self, entry: DataQueryEntry, repo_path: Path) -> None: - _reset_repo_path(repo_path) + _prepare_repo_path(repo_path) def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) From ec5dfee92e0c6cd5a6fc973d3e3f33a67d280efa Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 10:55:18 +0200 Subject: [PATCH 05/52] data-query: drop invalid 'Extensible' property from wrapped API query (AL0124) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/operations/bc_operations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index a71d71b4a..d3e9e19e2 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -251,8 +251,7 @@ def run_test_suite(test_entries: list[TestEntry], expectation: Literal["Pass", " f" APIGroup = '{_QUERY_API_GROUP}';\n" f" APIVersion = '{_QUERY_API_VERSION}';\n" " EntityName = 'bcbenchResult';\n" - f" EntitySetName = '{_QUERY_API_ENTITY_SET}';\n" - " Extensible = false;" + f" EntitySetName = '{_QUERY_API_ENTITY_SET}';" ) From 614dea751c090b87087ad9224457d40b91c76f63 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 12:22:55 +0200 Subject: [PATCH 06/52] data-query: don't crash the job on a gold-query compile failure A gold query failing to compile/run is a harness/dataset problem, not the agent's, so record it as a non-resolved result with a clear message instead of letting the uncaught BuildError crash the whole matrix job. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 34bd86d01..52973e040 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -105,8 +105,19 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) return - # The gold query is authored to compile; a failure here is an environment/dataset problem, not the agent's. - gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + # The gold query is authored to compile; a failure here is an environment/dataset problem, + # not the agent's — record it without crashing the job (build succeeded, but we can't score). + try: + gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + except BuildError as e: + logger.exception(f"Gold query failed to compile/run for {context.entry.instance_id}") + self.save_result( + context, + ExecutionBasedEvaluationResult.create_result( + context, output=generated_query, build=True, resolved=False, error_message=f"Gold query failed (harness/dataset issue, not the agent): {e}" + ), + ) + return resolved = result_sets_match(generated_rows, gold_rows, context.entry.ordered) error_message = None if resolved else f"Result set mismatch: generated {len(generated_rows)} rows vs gold {len(gold_rows)} rows" From 5761e0e6a2f9e3e352e9f3d2466226b32e11ab35 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 13:49:54 +0200 Subject: [PATCH 07/52] data-query: normalize query object name and use per-object API entity sets The object name is irrelevant to a query's result set (we score by comparing data), but AL requires it be a valid <=30-char identifier and unique in the tenant. Two of the first real runs failed only on AL0305 (agent chose a long descriptive name), so normalize the name in wrap_query_as_api to keep the benchmark focused on query logic. Also give the generated and gold API queries distinct EntitySetName/EntityName so both can be published to the same tenant without colliding on the OData route once a generated query finally compiles. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/operations/bc_operations.py | 60 ++++++++++++++++++------- tests/test_dataquery_evaluation.py | 21 +++++++-- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index d3e9e19e2..13dfa1513 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -243,32 +243,60 @@ def run_test_suite(test_entries: list[TestEntry], expectation: Literal["Pass", " _QUERY_API_PUBLISHER = "bcbench" _QUERY_API_GROUP = "eval" _QUERY_API_VERSION = "v1.0" -_QUERY_API_ENTITY_SET = "bcbenchResults" - -_QUERY_API_PROPERTIES = ( - "QueryType = API;\n" - f" APIPublisher = '{_QUERY_API_PUBLISHER}';\n" - f" APIGroup = '{_QUERY_API_GROUP}';\n" - f" APIVersion = '{_QUERY_API_VERSION}';\n" - " EntityName = 'bcbenchResult';\n" - f" EntitySetName = '{_QUERY_API_ENTITY_SET}';" -) + + +def _safe_object_name(object_id: int) -> str: + """A short, unique, always-valid query object name. + + The object name is irrelevant to a query's result set (we score by comparing data, not + identifiers), but AL requires it to be a valid identifier of <=30 characters and unique in + the tenant. Normalizing it keeps the benchmark focused on query logic instead of failing an + otherwise-correct query just because the agent chose a long/descriptive name (AL0305). + """ + return f"BCBenchQuery{object_id}" + + +def _entity_set_name(object_id: int) -> str: + """Per-object OData entity set so the generated and gold API queries don't collide on route.""" + return f"bcbenchResults{object_id}" + + +def _entity_name(object_id: int) -> str: + return f"bcbenchResult{object_id}" + + +def _query_api_properties(object_id: int) -> str: + return ( + "QueryType = API;\n" + f" APIPublisher = '{_QUERY_API_PUBLISHER}';\n" + f" APIGroup = '{_QUERY_API_GROUP}';\n" + f" APIVersion = '{_QUERY_API_VERSION}';\n" + f" EntityName = '{_entity_name(object_id)}';\n" + f" EntitySetName = '{_entity_set_name(object_id)}';" + ) def wrap_query_as_api(query_text: str, object_id: int) -> str: """Turn a plain AL query object into an API query the harness can fetch over OData. - Reassigns the object id (so generated and gold apps don't collide), drops any existing - ``QueryType`` line, and injects the API properties right after the object's opening brace. - Pure string transform so it can be unit-tested without a container. + Reassigns the object id and normalizes the object name (so generated and gold apps don't + collide and long names don't cause AL0305), drops any existing ``QueryType`` line, and + injects the API properties right after the object's opening brace. Pure string transform so + it can be unit-tested without a container. """ import re - text = re.sub(r"(\bquery\s+)\d+", rf"\g<1>{object_id}", query_text, count=1) + safe_name = _safe_object_name(object_id) + text = re.sub( + r'(\bquery\s+)\d+\s+("(?:[^"\\]|\\.)*"|\w+)', + rf"\g<1>{object_id} {safe_name}", + query_text, + count=1, + ) text = re.sub(r"\n\s*QueryType\s*=\s*\w+\s*;", "", text, count=1) brace_index = text.index("{") - return f"{text[: brace_index + 1]}\n {_QUERY_API_PROPERTIES}\n{text[brace_index + 1 :]}" + return f"{text[: brace_index + 1]}\n {_query_api_properties(object_id)}\n{text[brace_index + 1 :]}" _QUERY_RUN_TEMPLATE = Template( @@ -334,7 +362,7 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, publisher=_QUERY_API_PUBLISHER, group=_QUERY_API_GROUP, version=_QUERY_API_VERSION, - entity_set=_QUERY_API_ENTITY_SET, + entity_set=_entity_set_name(object_id), result_file=_escape_ps_string(str(result_file)), ) diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index ee13792bf..b43e3dd10 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -37,17 +37,32 @@ def test_different_row_count_mismatch(self): class TestWrapQueryAsApi: PLAIN_QUERY = 'query 50100 MyQuery\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' + LONG_NAME_QUERY = 'query 50100 "Items on Open Sales and Purchase Orders"\n{\n elements\n {\n dataitem(Item; Item)\n {\n column(No; "No.") { }\n }\n }\n}' - def test_reassigns_object_id(self): + def test_reassigns_object_id_and_name(self): wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50101) - assert "query 50101 MyQuery" in wrapped + assert "query 50101 BCBenchQuery50101" in wrapped assert "query 50100" not in wrapped + assert "MyQuery" not in wrapped + + def test_normalizes_overlong_quoted_name(self): + # A descriptive >30-char name would trip AL0305; the harness normalizes it away. + wrapped = wrap_query_as_api(self.LONG_NAME_QUERY, 50100) + assert "query 50100 BCBenchQuery50100" in wrapped + assert "Items on Open Sales and Purchase Orders" not in wrapped def test_injects_api_properties(self): wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) assert "QueryType = API;" in wrapped assert "APIPublisher = 'bcbench';" in wrapped - assert "EntitySetName = 'bcbenchResults';" in wrapped + assert "EntitySetName = 'bcbenchResults50100';" in wrapped + + def test_generated_and_gold_use_distinct_entity_sets(self): + # Both apps can be published to the same tenant; distinct entity sets avoid an OData route collision. + generated = wrap_query_as_api(self.PLAIN_QUERY, 50100) + gold = wrap_query_as_api(self.PLAIN_QUERY, 50101) + assert "EntitySetName = 'bcbenchResults50100';" in generated + assert "EntitySetName = 'bcbenchResults50101';" in gold def test_drops_existing_querytype(self): wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) From 237c930aaade75a219c83b57ebafaf3e499b45f4 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 14:45:06 +0200 Subject: [PATCH 08/52] data-query: use proven Invoke-AppBuildAndPublish + in-container OData fetch Root cause of the 0/4 build rate: the agents were writing valid AL (e.g. a correct Vendor/Purch. Inv. Header query) but the compiler reported base tables as missing (AL0185, '26.0.0.0 could not be found in the database'). The custom Compile-AppInBcContainer -UpdateSymbols path did not load Base Application symbols reliably (intermittent across containers). Switch execute_al_query to the same Invoke-AppBuildAndPublish helper the passing categories use (explicit cleared .alpackages symbol folder, GenerateReportLayout No, ForceSync, dependencyPublishingOption ignore). Also fetch the query rows from *inside* the container (Invoke-ScriptInBcContainer -> http://localhost:7048/BC/api) so we no longer depend on host->container name resolution or published ports, which the runner does not set up. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/operations/bc_operations.py | 28 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index 13dfa1513..cfd848576 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -302,18 +302,27 @@ def wrap_query_as_api(query_text: str, object_id: int) -> str: _QUERY_RUN_TEMPLATE = Template( """ Import-Module BcContainerHelper -Force -DisableNameChecking +Import-Module '$app_utils_path' -Force $$ErrorActionPreference = 'Stop' $$password = ConvertTo-SecureString '$password' -AsPlainText -Force $$credential = New-Object System.Management.Automation.PSCredential('$username', $$password) -$$appFile = Compile-AppInBcContainer -containerName '$container_name' -credential $$credential -appProjectFolder '$app_dir' -appOutputFolder '$app_dir\\out' -UpdateSymbols -Publish-BcContainerApp -containerName '$container_name' -appFile $$appFile -skipVerification -sync -install -credential $$credential - -$$base = "http://$container_name/BC/api" -$$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Credential $$credential).value[0].id -$$data = Invoke-RestMethod -Uri "$$base/$publisher/$group/$version/companies($$companyId)/$entity_set" -Credential $$credential -$$data.value | ConvertTo-Json -Depth 10 -Compress | Out-File -FilePath '$result_file' -Encoding utf8 +# Compile + publish the wrapped API query with the same proven helper the other categories use +# (clears/sets an explicit .alpackages symbol folder, GenerateReportLayout=No, ForceSync, +# dependencyPublishingOption=ignore) so Base Application symbols resolve reliably. +Invoke-AppBuildAndPublish -containerName '$container_name' -appProjectFolder '$app_dir' -credential $$credential -skipVerification -useDevEndpoint + +# Read the query's rows over the OData/API endpoint from *inside* the container, so we don't depend +# on host->container name resolution or published ports (the runner does not update its hosts file). +$$json = Invoke-ScriptInBcContainer -containerName '$container_name' -argumentList $$credential, '$publisher', '$group', '$version', '$entity_set' -scriptblock { + param($$cred, $$pub, $$grp, $$ver, $$eset) + $$base = 'http://localhost:7048/BC/api' + $$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Credential $$cred).value[0].id + $$data = Invoke-RestMethod -Uri "$$base/$$pub/$$grp/$$ver/companies($$companyId)/$$eset" -Credential $$cred + $$data.value | ConvertTo-Json -Depth 10 -Compress +} +$$json | Out-File -FilePath '$result_file' -Encoding utf8 """.strip() ) @@ -350,11 +359,12 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, } (app_dir / "app.json").write_text(json.dumps(app_manifest, indent=2), encoding="utf-8") (app_dir / "query.al").write_text(wrap_query_as_api(query_text, object_id), encoding="utf-8") - # Symbols come from the container via Compile-AppInBcContainer -UpdateSymbols (below); no need - # to pre-populate .alpackages from the artifact cache. + # Symbols are downloaded into an explicit .alpackages folder by Invoke-AppBuildAndPublish (below). result_file = app_dir / "result.json" + app_utils_path = _config.paths.ps_script_path / "AppUtils.psm1" ps_script = _QUERY_RUN_TEMPLATE.substitute( + app_utils_path=_escape_ps_string(str(app_utils_path)), container_name=_escape_ps_string(container.name), username=_escape_ps_string(container.username), password=_escape_ps_string(container.password), From d7ad1dd4fc2ca9e6e79b64c0dfcaedffdbcc06f0 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 15:17:36 +0200 Subject: [PATCH 09/52] data-query: build Basic auth header by hand for the in-container OData fetch Run 3 showed the compile+publish now works (Base App symbols resolve via the proven helper), but the in-container OData fetch failed: PowerShell 7 refuses Invoke-RestMethod -Credential over plain HTTP ('cannot protect plain text secrets sent over unencrypted connections'). Build the Basic Authorization header manually instead, which works on both Windows PowerShell 5.1 and PowerShell 7. Add regression tests asserting the run template uses the proven build helper, fetches from inside the container, and never passes -Credential over HTTP. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/operations/bc_operations.py | 8 ++++-- tests/test_dataquery_evaluation.py | 34 ++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index cfd848576..1c7783469 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -315,11 +315,15 @@ def wrap_query_as_api(query_text: str, object_id: int) -> str: # Read the query's rows over the OData/API endpoint from *inside* the container, so we don't depend # on host->container name resolution or published ports (the runner does not update its hosts file). +# Basic auth header is built by hand rather than via -Credential: PowerShell 7 (used inside the +# container) refuses -Credential over plain HTTP, and a manual header works on both 5.1 and 7. $$json = Invoke-ScriptInBcContainer -containerName '$container_name' -argumentList $$credential, '$publisher', '$group', '$version', '$entity_set' -scriptblock { param($$cred, $$pub, $$grp, $$ver, $$eset) + $$pair = "$$($$cred.UserName):$$($$cred.GetNetworkCredential().Password)" + $$headers = @{ Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($$pair)) } $$base = 'http://localhost:7048/BC/api' - $$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Credential $$cred).value[0].id - $$data = Invoke-RestMethod -Uri "$$base/$$pub/$$grp/$$ver/companies($$companyId)/$$eset" -Credential $$cred + $$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Headers $$headers).value[0].id + $$data = Invoke-RestMethod -Uri "$$base/$$pub/$$grp/$$ver/companies($$companyId)/$$eset" -Headers $$headers $$data.value | ConvertTo-Json -Depth 10 -Compress } $$json | Out-File -FilePath '$result_file' -Encoding utf8 diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index b43e3dd10..7b90e49ab 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -1,5 +1,5 @@ from bcbench.evaluate.dataquery import result_sets_match -from bcbench.operations import wrap_query_as_api +from bcbench.operations import bc_operations, wrap_query_as_api class TestResultSetsMatch: @@ -72,3 +72,35 @@ def test_preserves_query_body(self): wrapped = wrap_query_as_api(self.PLAIN_QUERY, 50100) assert "dataitem(Customer; Customer)" in wrapped assert 'column(No; "No.")' in wrapped + + +class TestQueryRunTemplate: + def _render(self): + return bc_operations._QUERY_RUN_TEMPLATE.substitute( + app_utils_path="AppUtils.psm1", + container_name="c", + username="u", + password="p", + app_dir="d", + publisher=bc_operations._QUERY_API_PUBLISHER, + group=bc_operations._QUERY_API_GROUP, + version=bc_operations._QUERY_API_VERSION, + entity_set=bc_operations._entity_set_name(50100), + result_file="r", + ) + + def test_uses_proven_build_helper(self): + assert "Invoke-AppBuildAndPublish" in self._render() + + def test_fetches_from_inside_container(self): + script = self._render() + assert "Invoke-ScriptInBcContainer" in script + assert "http://localhost:7048/BC/api" in script + + def test_does_not_use_credential_over_http(self): + # PowerShell 7 (inside the container) refuses -Credential over plain HTTP; we must build a + # Basic auth header by hand instead. + script = self._render() + assert "-Credential" not in script.split("Invoke-ScriptInBcContainer", 1)[1] + assert "Authorization" in script + assert "Basic " in script From 8268f07d771168304849444daf2cb67426aa9882 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 13 Jul 2026 16:12:07 +0200 Subject: [PATCH 10/52] data-query: calibrate prompts + fix the intersection gold query Run 4 proved the harness works end-to-end (build=2, real gold-vs-generated result-set comparisons). The remaining resolved=0 was down to prompt ambiguity and one buggy gold, not the harness: - Tighten all prompts so a correct interpretation deterministically matches the gold: specify the measure and whether it is net of VAT, the source (line vs header, posted vs open), inner-join inclusion ('...that has at least one...'), and grouping. E.g. the vendor prompt now pins line-level Amount net of VAT (a model had reasonably summed header Amount Including VAT -> 5 vs 6 rows). - Replace 'items on both open orders': its gold expressed a set intersection as a join with no aggregate column, so an AL query returns one row per matching (sales line x purchase line) pair instead of the distinct item set and cannot be scored deterministically (13 vs 12 rows). Swap in a clean aggregate join (open sales order count per customer). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- dataset/dataquery.jsonl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index c422eca5f..02bf9606f 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -1,6 +1,6 @@ -{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer, what is the total outstanding value across their open sales orders? Include the customer's number and name.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "What is the total sold quantity per item across all posted sales invoice lines? Return each item number and its total quantity.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} -{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, what is the average posted sales invoice line amount? Group the posted sales invoice lines by the bill-to customer's country/region code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "What is the total posted purchase invoice amount per vendor? Include the vendor's number and name.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__items-on-both-open-orders-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Which items appear on both open sales orders and open purchase orders? Return the item numbers.", "ordered": false, "gold_query": "query 50100 ItemsOnBothOpenOrders\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order), Type = const(Item);\n column(ItemNo; \"No.\") { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"No.\" = SalesLine.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order), Type = const(Item);\n }\n }\n }\n}"} -{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "How many CRM opportunities are there in each status?", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount; \"No.\") { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount; \"No.\") { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount; \"No.\") { Method = Count; }\n }\n }\n}"} From 2a41a6c903775adc4f298fe99c84ea9466646962 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 28 Jul 2026 22:33:52 +0200 Subject: [PATCH 11/52] data-query: add 5 more deterministically-scorable tasks (6 -> 11) Broaden the dataquery benchmark with single-table and clean-join aggregates that are deterministically scorable via result-set comparison: - customer-count-by-country (single-table Count) - outstanding-purchase-value-by-vendor (join + Sum, open POs, net of VAT) - total-posted-sales-amount-by-customer (2-level join + Sum, net of VAT) - line-count-per-open-sales-order (single-table Count, child rows per parent) - total-purchased-quantity-by-item (single-table Sum) Prompts pin the source table and net-of-VAT measure to avoid the interpretation ambiguity that made earlier tasks noisy. Field names verified against the W1 Base App. Gold queries to be confirmed against the container by the evaluation run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- dataset/dataquery.jsonl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index 02bf9606f..dd80f25d7 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -4,3 +4,8 @@ {"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} {"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount; \"No.\") { Method = Count; }\n }\n }\n }\n}"} {"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount; \"No.\") { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount; \"No.\") { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount; \"Line No.\") { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} From 129c5efe4b43c1a46efa336a0a6c725e4a9fede2 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 28 Jul 2026 23:23:26 +0200 Subject: [PATCH 12/52] data-query: address PR review feedback (scoring integrity + robustness) Scoring integrity: - result_sets_match: canonicalize numbers with Decimal.normalize() instead of rounding through float to 4 decimals, so 1.00001 and 1.00002 are no longer scored equal (removes false positives) while 500 == 500.0 still holds. - OData fetch: follow @odata.nextLink until exhausted so result sets larger than one page are not silently truncated (which could score different sets as equal). - Gold-query failure is now recorded as unscorable (new ExecutionBasedEvaluationResult scorable flag) and excluded from resolved/total/build/instance_results, so a harness/dataset issue no longer counts against the agent's ResolutionRate. - Catch BuildTimeoutExpired (not a BuildError) around both generated and gold query execution so a timeout is recorded instead of escaping and breaking summarization. - wrap_query_as_api raises BuildError (handled downstream) instead of ValueError when the generated output has no query declaration or no object body. Robustness: - wrap_query_as_api matches the query keyword and QueryType removal case-insensitively and without requiring a leading newline, so cased/compact AL (Query 50123, { QueryType = Normal; ... }) no longer breaks ID reassignment or produces a duplicate QueryType property. - execute_al_query uninstalls/unpublishes the throwaway query app before and after each run so re-running locally against the same container doesn't fail with an object-ID conflict on the fixed 50100/50101 range. Docs/cleanup: - SKILL.md: OrderBy is a property (OrderBy = descending(Col);), not a block. - types.py: drop the stale MCP/seed-app comments; fold DATA_QUERY into the existing same-value match arms. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .../skills/al-query-authoring/SKILL.md | 4 +- src/bcbench/evaluate/dataquery.py | 23 ++++--- src/bcbench/operations/bc_operations.py | 60 ++++++++++++++----- src/bcbench/results/base.py | 10 ++++ src/bcbench/results/summary.py | 13 ++-- src/bcbench/types.py | 13 +--- tests/test_dataquery_evaluation.py | 46 ++++++++++++++ 7 files changed, 124 insertions(+), 45 deletions(-) diff --git a/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md index cf1726c14..3685dc7fa 100644 --- a/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md +++ b/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md @@ -38,8 +38,8 @@ query 50100 TopCustomersBySales - **Join** by nesting a `dataitem` and linking it: `DataItemLink = "" = Parent."";`. - **Filter** rows with `DataItemTableFilter = "" = const();` (e.g. an Option like `Document Type`) or a range/expression. -- **Order** with `OrderBy { descending(); }` when the question asks for ranking or - "top N" (combine with `TopNumberOfRows` where appropriate). +- **Order** with the `OrderBy` property: `OrderBy = descending();` (or `ascending`) when + the question asks for ranking or "top N" (combine with `TopNumberOfRows` where appropriate). - **Quote** any field or table name that contains spaces or special characters: `"No."`, `"Sales Line"`, `"Amount (LCY)"`. - Prefer stored fields; FlowFields and Option fields are supported. diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 52973e040..0bb8f0f80 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,10 +1,11 @@ import shutil from collections.abc import Callable, Mapping, Sequence +from decimal import Decimal, InvalidOperation from pathlib import Path from bcbench.dataset import DataQueryEntry from bcbench.evaluate.base import EvaluationPipeline -from bcbench.exceptions import BuildError +from bcbench.exceptions import BuildError, BuildTimeoutExpired from bcbench.github_actions import github_log_group from bcbench.logger import get_logger from bcbench.results.base import ExecutionBasedEvaluationResult @@ -22,12 +23,12 @@ def _normalize_value(value: object) -> str: return "" if isinstance(value, bool): return str(value).lower() - if isinstance(value, (int, float)): - return f"{float(value):.4f}" text = str(value).strip() try: - return f"{float(text):.4f}" - except ValueError: + # Canonical decimal form: scale/trailing-zero-insensitive (500 == 500.0) but full precision + # preserved, so distinct values like 1.00001 and 1.00002 are NOT collapsed. No float rounding. + return str(Decimal(text).normalize()) + except (InvalidOperation, ValueError): return text @@ -100,22 +101,20 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: try: generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") - except BuildError as e: + except (BuildError, BuildTimeoutExpired) as e: logger.exception(f"Generated query failed to compile/run for {context.entry.instance_id}") self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) return - # The gold query is authored to compile; a failure here is an environment/dataset problem, - # not the agent's — record it without crashing the job (build succeeded, but we can't score). + # The gold query is authored to compile; a failure here is a harness/dataset problem, not the + # agent's. Record it as unscorable so it is not counted against the agent's resolution rate. try: gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") - except BuildError as e: + except (BuildError, BuildTimeoutExpired) as e: logger.exception(f"Gold query failed to compile/run for {context.entry.instance_id}") self.save_result( context, - ExecutionBasedEvaluationResult.create_result( - context, output=generated_query, build=True, resolved=False, error_message=f"Gold query failed (harness/dataset issue, not the agent): {e}" - ), + ExecutionBasedEvaluationResult.create_unscorable(context, output=generated_query, error_message=f"Gold query failed (harness/dataset issue, not the agent): {e}"), ) return diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index 1c7783469..ee4f2bb30 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -287,15 +287,22 @@ def wrap_query_as_api(query_text: str, object_id: int) -> str: import re safe_name = _safe_object_name(object_id) - text = re.sub( + # AL keywords are case-insensitive; match `query`/`QueryType` in any casing. + text, replaced = re.subn( r'(\bquery\s+)\d+\s+("(?:[^"\\]|\\.)*"|\w+)', rf"\g<1>{object_id} {safe_name}", query_text, count=1, + flags=re.IGNORECASE, ) - text = re.sub(r"\n\s*QueryType\s*=\s*\w+\s*;", "", text, count=1) + if replaced == 0: + raise BuildError("query-wrap", f"No AL query object declaration found in generated output:\n{query_text}") - brace_index = text.index("{") + text = re.sub(r"\bQueryType\s*=\s*\w+\s*;", "", text, count=1, flags=re.IGNORECASE) + + brace_index = text.find("{") + if brace_index == -1: + raise BuildError("query-wrap", f"Generated query has no object body ('{{' not found):\n{query_text}") return f"{text[: brace_index + 1]}\n {_query_api_properties(object_id)}\n{text[brace_index + 1 :]}" @@ -308,25 +315,44 @@ def wrap_query_as_api(query_text: str, object_id: int) -> str: $$password = ConvertTo-SecureString '$password' -AsPlainText -Force $$credential = New-Object System.Management.Automation.PSCredential('$username', $$password) +# Remove any app left installed by a previous run of the same suffix so re-running against the +# same container doesn't fail with an object-ID conflict on the fixed 50100/50101 range. +UnInstall-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -force -doNotSaveData -ErrorAction SilentlyContinue +UnPublish-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -ErrorAction SilentlyContinue + # Compile + publish the wrapped API query with the same proven helper the other categories use # (clears/sets an explicit .alpackages symbol folder, GenerateReportLayout=No, ForceSync, # dependencyPublishingOption=ignore) so Base Application symbols resolve reliably. Invoke-AppBuildAndPublish -containerName '$container_name' -appProjectFolder '$app_dir' -credential $$credential -skipVerification -useDevEndpoint -# Read the query's rows over the OData/API endpoint from *inside* the container, so we don't depend -# on host->container name resolution or published ports (the runner does not update its hosts file). -# Basic auth header is built by hand rather than via -Credential: PowerShell 7 (used inside the -# container) refuses -Credential over plain HTTP, and a manual header works on both 5.1 and 7. -$$json = Invoke-ScriptInBcContainer -containerName '$container_name' -argumentList $$credential, '$publisher', '$group', '$version', '$entity_set' -scriptblock { - param($$cred, $$pub, $$grp, $$ver, $$eset) - $$pair = "$$($$cred.UserName):$$($$cred.GetNetworkCredential().Password)" - $$headers = @{ Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($$pair)) } - $$base = 'http://localhost:7048/BC/api' - $$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Headers $$headers).value[0].id - $$data = Invoke-RestMethod -Uri "$$base/$$pub/$$grp/$$ver/companies($$companyId)/$$eset" -Headers $$headers - $$data.value | ConvertTo-Json -Depth 10 -Compress +try { + # Read the query's rows over the OData/API endpoint from *inside* the container, so we don't depend + # on host->container name resolution or published ports (the runner does not update its hosts file). + # Basic auth header is built by hand rather than via -Credential: PowerShell 7 (used inside the + # container) refuses -Credential over plain HTTP, and a manual header works on both 5.1 and 7. + $$json = Invoke-ScriptInBcContainer -containerName '$container_name' -argumentList $$credential, '$publisher', '$group', '$version', '$entity_set' -scriptblock { + param($$cred, $$pub, $$grp, $$ver, $$eset) + $$pair = "$$($$cred.UserName):$$($$cred.GetNetworkCredential().Password)" + $$headers = @{ Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($$pair)) } + $$base = 'http://localhost:7048/BC/api' + $$companyId = (Invoke-RestMethod -Uri "$$base/v2.0/companies" -Headers $$headers).value[0].id + # Follow @odata.nextLink so large result sets aren't silently truncated to the first page. + $$rows = [System.Collections.Generic.List[object]]::new() + $$uri = "$$base/$$pub/$$grp/$$ver/companies($$companyId)/$$eset" + while ($$uri) { + $$page = Invoke-RestMethod -Uri $$uri -Headers $$headers + if ($$null -ne $$page.value) { foreach ($$row in $$page.value) { $$rows.Add($$row) } } + $$uri = $$page.'@odata.nextLink' + } + $$rows | ConvertTo-Json -Depth 10 -Compress + } + $$json | Out-File -FilePath '$result_file' -Encoding utf8 +} +finally { + # Best-effort teardown so the container doesn't accumulate throwaway apps between runs. + UnInstall-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -force -doNotSaveData -ErrorAction SilentlyContinue + UnPublish-BcContainerApp -containerName '$container_name' -name '$app_name' -publisher '$app_publisher' -ErrorAction SilentlyContinue } -$$json | Out-File -FilePath '$result_file' -Encoding utf8 """.strip() ) @@ -373,6 +399,8 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, username=_escape_ps_string(container.username), password=_escape_ps_string(container.password), app_dir=_escape_ps_string(str(app_dir)), + app_name=_escape_ps_string(app_manifest["name"]), + app_publisher=_escape_ps_string(app_manifest["publisher"]), publisher=_QUERY_API_PUBLISHER, group=_QUERY_API_GROUP, version=_QUERY_API_VERSION, diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index ee4a8e7cf..0df64a249 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -98,6 +98,9 @@ class ExecutionBasedEvaluationResult(BaseEvaluationResult): resolved: bool = False build: bool = False + # False marks a harness/dataset failure (e.g. the gold query didn't compile) that must be + # excluded from the resolution rate so it is not counted against the agent. + scorable: bool = True @classmethod def create_success(cls, context: "EvaluationContext", output: str) -> Self: @@ -107,6 +110,11 @@ def create_success(cls, context: "EvaluationContext", output: str) -> Self: def create_build_failure(cls, context: "EvaluationContext", output: str, error_message: str) -> Self: return cls(**cls._base_fields(context), output=output, error_message=error_message, resolved=False, build=False) + @classmethod + def create_unscorable(cls, context: "EvaluationContext", output: str, error_message: str) -> Self: + """A harness/dataset failure (not the agent's fault) that must not count toward the resolution rate.""" + return cls(**cls._base_fields(context), output=output, build=True, resolved=False, scorable=False, error_message=error_message) + @classmethod def create_result(cls, context: "EvaluationContext", output: str, *, build: bool, resolved: bool, error_message: str | None = None) -> Self: """General factory for execution outcomes, e.g. compiled+ran but produced the wrong result (build=True, resolved=False).""" @@ -116,6 +124,8 @@ def create_result(cls, context: "EvaluationContext", output: str, *, build: bool def status_label(self) -> str: if self.timeout: return "Timeout" + if not self.scorable: + return "Error" return "Success" if self.resolved else "Failed" @property diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index c7bddc65f..20bf2a93f 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -159,14 +159,19 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " summary = super().from_results(results, run_id) assert isinstance(summary, ExecutionBasedEvaluationResultSummary) - total = summary.total - resolved = sum(1 for r in results if isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) - build = sum(1 for r in results if isinstance(r, ExecutionBasedEvaluationResult) and r.build) - instance_results = {r.instance_id: (isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) for r in results} + # Exclude unscorable results (harness/dataset failures) from every rate so they are not + # counted against the agent. + scorable = [r for r in results if not (isinstance(r, ExecutionBasedEvaluationResult) and not r.scorable)] + total = len(scorable) + + resolved = sum(1 for r in scorable if isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) + build = sum(1 for r in scorable if isinstance(r, ExecutionBasedEvaluationResult) and r.build) + instance_results = {r.instance_id: (isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) for r in scorable} return summary.model_copy( update={ + "total": total, "resolved": resolved, "failed": total - resolved, "build": build, diff --git a/src/bcbench/types.py b/src/bcbench/types.py index f031850d1..4bd1fce3d 100644 --- a/src/bcbench/types.py +++ b/src/bcbench/types.py @@ -293,12 +293,7 @@ def core_score(self) -> str: def requires_container(self) -> bool: """Whether evaluating this category builds/runs AL code and therefore needs a BC container.""" match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: - return True - case EvaluationCategory.DATA_QUERY: - # Data-query provisions its own container: it publishes the data-query seed app - # (al/dataquery-seed) and enables a Data-Query-Tools MCP config, then the agent - # queries that container's /mcp against the seeded, deterministic dataset. + case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: return True case EvaluationCategory.CODE_REVIEW | EvaluationCategory.NL2AL: return False @@ -312,16 +307,12 @@ def runner(self) -> str: Only categories that require building BaseApp needs self-hosted runners. """ match self: - case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION: + case EvaluationCategory.BUG_FIX | EvaluationCategory.TEST_GENERATION | EvaluationCategory.DATA_QUERY: return "GitHub-BCBench" case EvaluationCategory.CODE_REVIEW: return "ubuntu-latest" case EvaluationCategory.NL2AL: return "windows-latest" - case EvaluationCategory.DATA_QUERY: - # The agent queries a live BC /mcp endpoint; the self-hosted runner is where a - # BcContainerHelper BC container (Data Query Tools enabled) is reachable. - return "GitHub-BCBench" raise ValueError(f"Unknown evaluation category: {self}") diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 7b90e49ab..dafed87f3 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -1,4 +1,7 @@ +import pytest + from bcbench.evaluate.dataquery import result_sets_match +from bcbench.exceptions import BuildError from bcbench.operations import bc_operations, wrap_query_as_api @@ -34,6 +37,17 @@ def test_mismatch_detected(self): def test_different_row_count_mismatch(self): assert not result_sets_match([{"No": "C1"}], [{"No": "C1"}, {"No": "C2"}]) + def test_close_but_distinct_values_do_not_match(self): + # Guards against numeric rounding collapsing distinct values into a false positive. + assert not result_sets_match([{"Total": "1.00001"}], [{"Total": "1.00002"}]) + + def test_high_precision_preserved(self): + assert result_sets_match([{"Total": "1.000000001"}], [{"Total": "1.000000001"}]) + assert not result_sets_match([{"Total": "1.000000001"}], [{"Total": "1.000000002"}]) + + def test_scale_insensitive(self): + assert result_sets_match([{"Total": "500"}], [{"Total": "500.00"}]) + class TestWrapQueryAsApi: PLAIN_QUERY = 'query 50100 MyQuery\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' @@ -73,6 +87,26 @@ def test_preserves_query_body(self): assert "dataitem(Customer; Customer)" in wrapped assert 'column(No; "No.")' in wrapped + def test_uppercase_query_keyword_reassigned(self): + wrapped = wrap_query_as_api('Query 50123 "My Q"\n{\n elements { }\n}', 50100) + assert "50100 BCBenchQuery50100" in wrapped + assert "50123" not in wrapped + + def test_compact_and_cased_querytype_removed(self): + # QueryType on the same line as the brace (no leading newline) and in any casing must still + # be stripped, else the injected QueryType = API duplicates the property. + wrapped = wrap_query_as_api("query 50100 Q\n{ querytype = Normal; elements { } }", 50100) + assert wrapped.count("QueryType") == 1 + assert "QueryType = API;" in wrapped + + def test_missing_brace_raises_builderror(self): + with pytest.raises(BuildError): + wrap_query_as_api("query 50100 MyQuery no body here", 50100) + + def test_no_query_declaration_raises_builderror(self): + with pytest.raises(BuildError): + wrap_query_as_api("codeunit 50100 NotAQuery { }", 50100) + class TestQueryRunTemplate: def _render(self): @@ -82,6 +116,8 @@ def _render(self): username="u", password="p", app_dir="d", + app_name="BC-Bench Query generated", + app_publisher="BC-Bench", publisher=bc_operations._QUERY_API_PUBLISHER, group=bc_operations._QUERY_API_GROUP, version=bc_operations._QUERY_API_VERSION, @@ -104,3 +140,13 @@ def test_does_not_use_credential_over_http(self): assert "-Credential" not in script.split("Invoke-ScriptInBcContainer", 1)[1] assert "Authorization" in script assert "Basic " in script + + def test_follows_odata_nextlink(self): + # Result sets larger than one OData page must not be silently truncated. + assert "@odata.nextLink" in self._render() + + def test_uninstalls_throwaway_app(self): + # Re-running against the same container must not fail with an object-ID conflict. + script = self._render() + assert "UnPublish-BcContainerApp" in script + assert "UnInstall-BcContainerApp" in script From afde215f196179551cc145cbe956e785fde36571 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 28 Jul 2026 23:33:39 +0200 Subject: [PATCH 13/52] data-query: fix invalid Count columns in gold queries and skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AL query Count columns take no source field: `column(RowCount) { Method = Count; }`, not `column(RowCount; "No.") { Method = Count; }` (the latter fails AL0353). The four Count-based golds used the invalid form, and SKILL.md taught it — so the agent reproduced the mistake and its query failed to compile before the gold was ever reached, which is why these golds went unvalidated (see PR review comment #15). - Remove the source field from the Count columns in customer-count-by-country, open-sales-order-count-by-customer, opportunity-count-by-status, and line-count-per-open-sales-order gold queries. - SKILL.md: clarify that Count takes no source field, unlike Sum/Average/Min/Max. Validated by the runner shakeout: the Sum-based new golds (outstanding-purchase-value -by-vendor, total-purchased-quantity-by-item) already compiled, ran, and resolved. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- dataset/dataquery.jsonl | 8 ++++---- .../dataquery-bc/skills/al-query-authoring/SKILL.md | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index dd80f25d7..30d24ccf6 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -2,10 +2,10 @@ {"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} {"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} {"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount; \"No.\") { Method = Count; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount; \"No.\") { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount; \"No.\") { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} {"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} {"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount; \"Line No.\") { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} {"instance_id": "dataquery__total-purchased-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} diff --git a/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md index 3685dc7fa..ef771e4da 100644 --- a/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md +++ b/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md @@ -34,7 +34,9 @@ query 50100 TopCustomersBySales ## Rules of thumb - **Aggregate** a column with a method: `column(Total; "Amount (LCY)") { Method = Sum; }` - (also `Average`, `Count`, `Min`, `Max`). Non-aggregated columns become the GROUP BY. + (also `Average`, `Min`, `Max`) — these take the field to aggregate. **`Count` takes no source + field** — write `column(RowCount) { Method = Count; }`, not `column(RowCount; "No.") { ... }`. + Non-aggregated columns become the GROUP BY. - **Join** by nesting a `dataitem` and linking it: `DataItemLink = "" = Parent."";`. - **Filter** rows with `DataItemTableFilter = "" = const();` (e.g. an Option like `Document Type`) or a range/expression. From 56f2335d26d9c7d218a4d88aec9d2d5f4c0ec251 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 28 Jul 2026 23:40:15 +0200 Subject: [PATCH 14/52] data-query: exclude unscorable results from the bceval export too Follow-up to the scorable flag: the local summary already excluded unscorable results, but write_bceval_results() still exported them, so the uploaded/core ResolutionRate counted a gold-query harness failure against the agent. Skip unscorable results in the export path as well. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/results/bceval_export.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index 293cadb42..dfa873e1c 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -8,7 +8,7 @@ from bcbench.dataset import BaseDatasetEntry from bcbench.logger import get_logger -from bcbench.results.base import BaseEvaluationResult +from bcbench.results.base import BaseEvaluationResult, ExecutionBasedEvaluationResult from bcbench.results.summary import get_benchmark_version from bcbench.types import EvaluationCategory, ExpectedOutput, ExperimentConfiguration @@ -46,6 +46,12 @@ def write_bceval_results( output_file = out_dir / output_filename with open(output_file, "w") as f: for result in results: + # Unscorable results (harness/dataset failures, e.g. a gold query that didn't compile) must + # not reach the uploaded/core score, or they'd count against the agent's ResolutionRate. + if isinstance(result, ExecutionBasedEvaluationResult) and not result.scorable: + logger.info(f"Skipping unscorable result from bceval export: {result.instance_id}") + continue + matching_entries = [e for e in dataset_entries if e.instance_id == result.instance_id] if not matching_entries: From faf22ba003c51428a9a8837d235e45d9abf87c02 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 28 Jul 2026 23:45:25 +0200 Subject: [PATCH 15/52] data-query: validate the gold query before evaluating the agent's Establish gold validity independent of agent output: run the gold query first, so a broken gold entry is recorded as unscorable regardless of whether the agent's query compiled. Previously, if the agent query failed first, a broken dataset entry was counted against that agent instead of being flagged as a harness issue. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 0bb8f0f80..355dfdbd9 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -91,23 +91,13 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: query_file = context.repo_path / GENERATED_QUERY_FILE generated_query = query_file.read_text(encoding="utf-8").strip() if query_file.exists() else "" - if not generated_query: - logger.warning(f"Agent produced no {GENERATED_QUERY_FILE} for {context.entry.instance_id}") - self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output="", error_message=f"No {GENERATED_QUERY_FILE} produced")) - return - container = context.get_container() version = context.entry.environment_setup_version - try: - generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") - except (BuildError, BuildTimeoutExpired) as e: - logger.exception(f"Generated query failed to compile/run for {context.entry.instance_id}") - self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) - return - - # The gold query is authored to compile; a failure here is a harness/dataset problem, not the - # agent's. Record it as unscorable so it is not counted against the agent's resolution rate. + # Establish gold validity FIRST, independent of the agent's output: a broken gold (a + # harness/dataset problem) must be recorded as unscorable regardless of whether the agent's + # query compiled — otherwise, when the agent query also fails, the broken entry is silently + # counted against that agent. try: gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") except (BuildError, BuildTimeoutExpired) as e: @@ -118,6 +108,18 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: ) return + if not generated_query: + logger.warning(f"Agent produced no {GENERATED_QUERY_FILE} for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output="", error_message=f"No {GENERATED_QUERY_FILE} produced")) + return + + try: + generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") + except (BuildError, BuildTimeoutExpired) as e: + logger.exception(f"Generated query failed to compile/run for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) + return + resolved = result_sets_match(generated_rows, gold_rows, context.entry.ordered) error_message = None if resolved else f"Result set mismatch: generated {len(generated_rows)} rows vs gold {len(gold_rows)} rows" result = ExecutionBasedEvaluationResult.create_result(context, output=generated_query, build=True, resolved=resolved, error_message=error_message) From 45715a3598edac48d856324d3a4c65566e110151 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Wed, 29 Jul 2026 00:14:40 +0200 Subject: [PATCH 16/52] Withhold gold queries from the agent's filesystem during generation The data-query agent runs with unrestricted filesystem tools in a workspace under the checkout, so it could read the reference answers straight out of dataset/dataquery.jsonl and copy them, invalidating the benchmark. Strip gold_query from the on-disk dataset for the duration of the agent phase (restored on exit); the harness already holds each entry's gold in memory, so scoring is unaffected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 39 +++++++++++++++++++++-- tests/test_dataquery_evaluation.py | 50 ++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 355dfdbd9..9ef676633 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,5 +1,7 @@ +import contextlib +import json import shutil -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from decimal import Decimal, InvalidOperation from pathlib import Path @@ -18,6 +20,33 @@ GENERATED_QUERY_FILE = "query.al" +@contextlib.contextmanager +def _withhold_gold_from_agent(dataset_path: Path) -> Iterator[None]: + """Strip ``gold_query`` from the on-disk dataset while the agent generates its answer. + + The agent runs with unrestricted filesystem tools in a workspace under the checkout, so it could + otherwise read the reference answers straight out of ``dataset/dataquery.jsonl`` and copy them, + invalidating the benchmark. The harness already loads this entry (with its gold) into memory + before the agent starts, so scoring is unaffected. The original file is restored on exit. + """ + if not dataset_path.exists(): + yield + return + original = dataset_path.read_text(encoding="utf-8") + try: + stripped: list[str] = [] + for line in original.splitlines(): + if not line.strip(): + continue + obj = json.loads(line) + obj.pop("gold_query", None) + stripped.append(json.dumps(obj, ensure_ascii=False)) + dataset_path.write_text("\n".join(stripped) + "\n", encoding="utf-8") + yield + finally: + dataset_path.write_text(original, encoding="utf-8") + + def _normalize_value(value: object) -> str: if value is None: return "" @@ -82,7 +111,13 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: Callable) -> None: - with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): + # The agent runs with unrestricted filesystem tools in a workspace under the checkout, so it could + # otherwise read the reference answers from the dataset. Withhold the gold queries from disk during + # generation; the harness already holds this entry's gold in memory for scoring. + with ( + github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"), + _withhold_gold_from_agent(context.category.dataset_path), + ): context.metrics, context.experiment = agent_runner(context) def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index dafed87f3..6fdbbe6c4 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -150,3 +150,53 @@ def test_uninstalls_throwaway_app(self): script = self._render() assert "UnPublish-BcContainerApp" in script assert "UnInstall-BcContainerApp" in script + + +class TestWithholdGoldFromAgent: + def _write_dataset(self, tmp_path): + import json + + path = tmp_path / "dataquery.jsonl" + entries = [ + {"instance_id": "q1", "nl_prompt": "count customers", "gold_query": "query 50100 A { }"}, + {"instance_id": "q2", "nl_prompt": "sum sales", "gold_query": "query 50101 B { }"}, + ] + path.write_text("\n".join(json.dumps(e) for e in entries) + "\n", encoding="utf-8") + return path + + def test_gold_hidden_during_and_restored_after(self, tmp_path): + import json + + from bcbench.evaluate.dataquery import _withhold_gold_from_agent + + path = self._write_dataset(tmp_path) + original = path.read_text(encoding="utf-8") + + with _withhold_gold_from_agent(path): + during = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + # Gold is gone from disk while the agent runs, but the prompt/id the agent needs remain. + assert all("gold_query" not in e for e in during) + assert [e["instance_id"] for e in during] == ["q1", "q2"] + assert [e["nl_prompt"] for e in during] == ["count customers", "sum sales"] + + # Restored verbatim once the agent phase completes. + assert path.read_text(encoding="utf-8") == original + + def test_gold_restored_on_exception(self, tmp_path): + from bcbench.evaluate.dataquery import _withhold_gold_from_agent + + path = self._write_dataset(tmp_path) + original = path.read_text(encoding="utf-8") + + with pytest.raises(RuntimeError), _withhold_gold_from_agent(path): + raise RuntimeError("agent crashed") + + assert path.read_text(encoding="utf-8") == original + + def test_missing_dataset_is_noop(self, tmp_path): + from bcbench.evaluate.dataquery import _withhold_gold_from_agent + + missing = tmp_path / "does-not-exist.jsonl" + with _withhold_gold_from_agent(missing): + pass + assert not missing.exists() From 1c9aa72a82eaf9c7e495673c2c46593ce6f2f34e Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Wed, 29 Jul 2026 00:33:30 +0200 Subject: [PATCH 17/52] Harden gold withholding: also hide the .git object database from the agent Stripping only the working-tree dataset left the committed gold recoverable via the object database (git -C .. show HEAD:dataset/dataquery.jsonl), since the agent runs with unrestricted tools in a workspace inside the checkout. Relocate the checkout's .git out of the agent-reachable tree for the duration of the agent phase (best-effort, restored on exit), alongside the working-tree strip. Data-query never needs the checkout's git during generation (agent writes to a separate testbed; scoring is file-based), so this is safe. Also restructure the restore-on-exception test to avoid a static-analysis unreachable/unused-variable false positive, and cover the .git hiding path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 66 ++++++++++++++++++++++++------ tests/test_dataquery_evaluation.py | 28 +++++++++++-- 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 9ef676633..80ee08e63 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,6 +1,7 @@ import contextlib import json import shutil +import uuid from collections.abc import Callable, Iterator, Mapping, Sequence from decimal import Decimal, InvalidOperation from pathlib import Path @@ -20,29 +21,68 @@ GENERATED_QUERY_FILE = "query.al" +def _strip_gold(dataset_path: Path, original: str) -> None: + stripped: list[str] = [] + for line in original.splitlines(): + if not line.strip(): + continue + obj = json.loads(line) + obj.pop("gold_query", None) + stripped.append(json.dumps(obj, ensure_ascii=False)) + dataset_path.write_text("\n".join(stripped) + "\n", encoding="utf-8") + + +@contextlib.contextmanager +def _hide_git_object_db(repo_root: Path) -> Iterator[None]: + """Move the checkout's ``.git`` out of the agent-reachable tree for the agent phase. + + Stripping only the working-tree dataset is not enough: the agent runs with unrestricted tools in + a workspace *inside* the checkout, so it could recover the committed gold answers straight from + the object database (``git -C .. show HEAD:dataset/dataquery.jsonl``). Relocate ``.git`` to a + sibling of the checkout (same volume -> instant rename) so ``git`` finds no repository during + generation, then move it back. Best-effort: if the rename fails (e.g. a held handle) we log and + fall back to the working-tree strip alone rather than aborting the evaluation. Data-query never + needs the checkout's git during the agent phase (the agent writes to a separate testbed, and + scoring is file-based), so hiding it here is safe. + """ + git_dir = repo_root / ".git" + if not git_dir.is_dir(): + yield + return + hideaway = repo_root.parent / f".bcbench-withheld-git-{uuid.uuid4().hex}" + try: + git_dir.rename(hideaway) + except OSError as exc: + logger.warning("Could not relocate .git to withhold gold history (%s); working-tree strip only", exc) + yield + return + try: + yield + finally: + with contextlib.suppress(OSError): + hideaway.rename(git_dir) + + @contextlib.contextmanager def _withhold_gold_from_agent(dataset_path: Path) -> Iterator[None]: - """Strip ``gold_query`` from the on-disk dataset while the agent generates its answer. + """Withhold the reference answers from the agent while it generates its query. The agent runs with unrestricted filesystem tools in a workspace under the checkout, so it could - otherwise read the reference answers straight out of ``dataset/dataquery.jsonl`` and copy them, - invalidating the benchmark. The harness already loads this entry (with its gold) into memory - before the agent starts, so scoring is unaffected. The original file is restored on exit. + otherwise read the gold answers — from the working-tree ``dataset/dataquery.jsonl`` *or* from the + committed blob via the ``.git`` object database — and copy them, invalidating the benchmark. Strip + gold from the working tree and hide ``.git`` for the duration of the agent phase; both are restored + on exit (including on exception). The harness already loaded this entry (with its gold) into memory + before the agent starts, so scoring is unaffected. """ if not dataset_path.exists(): yield return original = dataset_path.read_text(encoding="utf-8") + repo_root = dataset_path.parent.parent try: - stripped: list[str] = [] - for line in original.splitlines(): - if not line.strip(): - continue - obj = json.loads(line) - obj.pop("gold_query", None) - stripped.append(json.dumps(obj, ensure_ascii=False)) - dataset_path.write_text("\n".join(stripped) + "\n", encoding="utf-8") - yield + _strip_gold(dataset_path, original) + with _hide_git_object_db(repo_root): + yield finally: dataset_path.write_text(original, encoding="utf-8") diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 6fdbbe6c4..5cf942440 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -156,7 +156,9 @@ class TestWithholdGoldFromAgent: def _write_dataset(self, tmp_path): import json - path = tmp_path / "dataquery.jsonl" + dataset_dir = tmp_path / "dataset" + dataset_dir.mkdir() + path = dataset_dir / "dataquery.jsonl" entries = [ {"instance_id": "q1", "nl_prompt": "count customers", "gold_query": "query 50100 A { }"}, {"instance_id": "q2", "nl_prompt": "sum sales", "gold_query": "query 50101 B { }"}, @@ -182,14 +184,34 @@ def test_gold_hidden_during_and_restored_after(self, tmp_path): # Restored verbatim once the agent phase completes. assert path.read_text(encoding="utf-8") == original + def test_git_object_db_hidden_during_and_restored_after(self, tmp_path): + from bcbench.evaluate.dataquery import _withhold_gold_from_agent + + path = self._write_dataset(tmp_path) + git_dir = path.parent.parent / ".git" # /.git + git_dir.mkdir() + (git_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + + with _withhold_gold_from_agent(path): + # The committed gold must not be recoverable via `git show HEAD:...`: no reachable .git. + assert not git_dir.exists() + + # The object database is restored intact once the agent phase completes. + assert git_dir.is_dir() + assert (git_dir / "HEAD").read_text(encoding="utf-8") == "ref: refs/heads/main\n" + def test_gold_restored_on_exception(self, tmp_path): from bcbench.evaluate.dataquery import _withhold_gold_from_agent path = self._write_dataset(tmp_path) original = path.read_text(encoding="utf-8") - with pytest.raises(RuntimeError), _withhold_gold_from_agent(path): - raise RuntimeError("agent crashed") + def crash(): + with _withhold_gold_from_agent(path): + raise RuntimeError("agent crashed") + + with pytest.raises(RuntimeError): + crash() assert path.read_text(encoding="utf-8") == original From d24eb1848cd98243f6bf7c6a76bfb3d16d96562a Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Wed, 29 Jul 2026 00:47:40 +0200 Subject: [PATCH 18/52] Add --skills dispatch flag to toggle agent skills per run Skills were previously only togglable via config.yaml (skills.enabled), so an evaluation run could not opt in without committing a global config change. Add a --skills CLI option (mirroring --al-lsp/--al-mcp) threaded through evaluate/run copilot+claude commands and the agent runners into setup_agent_skills via a skills_enabled_override, plus a 'skills' workflow_dispatch input on both evaluation workflows (and their requeue inputs). Default stays off, preserving current behavior and keeping the with/without-skills ablation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/claude-evaluation.yml | 10 ++++- .github/workflows/copilot-evaluation.yml | 10 ++++- src/bcbench/agent/claude/agent.py | 3 +- src/bcbench/agent/copilot/agent.py | 3 +- src/bcbench/commands/evaluate.py | 4 ++ src/bcbench/commands/run.py | 4 ++ src/bcbench/operations/skills_operations.py | 14 ++++++- tests/test_agent_skills.py | 42 +++++++++++++++++++++ 8 files changed, 82 insertions(+), 8 deletions(-) diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index 33a3f350c..db9651b6f 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -40,6 +40,11 @@ on: required: false default: false type: boolean + skills: + description: "Enable agent skills" + required: false + default: false + type: boolean repeat: description: "Number of times to run sequentially (ignored for test runs)" required: false @@ -140,7 +145,8 @@ jobs: --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` - ${{ inputs.al-lsp && '--al-lsp' || '' }} + ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ inputs.skills && '--skills' || '' }} - name: Upload evaluation results uses: actions/upload-artifact@v6 @@ -176,4 +182,4 @@ jobs: workflow-file: claude-evaluation.yml repeat: ${{ inputs.repeat }} workflow-inputs: | - {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} + {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index f4b1866eb..192e1539f 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -47,6 +47,11 @@ on: required: false default: false type: boolean + skills: + description: "Enable agent skills" + required: false + default: false + type: boolean repeat: description: "Number of times to run sequentially (ignored for test runs)" required: false @@ -145,7 +150,8 @@ jobs: --repo-path "${{ steps.setup-env.outputs.repo_path }}" ` --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` - ${{ inputs.al-lsp && '--al-lsp' || '' }} + ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ inputs.skills && '--skills' || '' }} - name: Upload evaluation results uses: actions/upload-artifact@v6 @@ -181,4 +187,4 @@ jobs: workflow-file: copilot-evaluation.yml repeat: ${{ inputs.repeat }} workflow-inputs: | - {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} + {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index f1387d651..dd4080d3f 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -26,6 +26,7 @@ def run_claude_code( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run Claude Code on a single dataset entry. @@ -46,7 +47,7 @@ def run_claude_code( mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentType.CLAUDE, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE) - skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE) + skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE, skills_enabled_override=skills) custom_agent: str | None = setup_custom_agent(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE) tool_log_path: Path = setup_hooks(repo_path, AgentType.CLAUDE, output_dir) plugins: dict[str, Path] = resolve_config_plugins(claude_config) diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index beb28a99f..818974cbe 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -29,6 +29,7 @@ def run_copilot_agent( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: """Run GitHub Copilot CLI agent on a single dataset entry. @@ -51,7 +52,7 @@ def run_copilot_agent( mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentType.COPILOT, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT) - skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT) + skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=skills) custom_agent: str | None = setup_custom_agent(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT) tool_log_path: Path = setup_hooks(repo_path, AgentType.COPILOT, output_dir) plugins: dict[str, Path] = resolve_config_plugins(copilot_config) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index f71a54461..7236715f9 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -54,6 +54,7 @@ def evaluate_copilot( run_id: RunId = "copilot_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Evaluate GitHub Copilot CLI on single dataset entry. @@ -88,6 +89,7 @@ def evaluate_copilot( output_dir=ctx.result_dir, al_mcp=al_mcp if ctx.container else False, al_lsp=al_lsp, + skills=skills, container_name=ctx.get_container().name if ctx.container else "", ), ) @@ -109,6 +111,7 @@ def evaluate_claude_code( run_id: RunId = "claude_code_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Evaluate Claude Code on single dataset entry. @@ -143,6 +146,7 @@ def evaluate_claude_code( output_dir=ctx.result_dir, al_mcp=al_mcp if ctx.container else False, al_lsp=al_lsp, + skills=skills, container_name=ctx.get_container().name if ctx.container else "", ), ) diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 2ae1afd3b..5d746ce66 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -37,6 +37,7 @@ def run_copilot( output_dir: OutputDir = _config.paths.evaluation_results_path, al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Run GitHub Copilot CLI on a single entry to generate a patch (without building/testing). @@ -57,6 +58,7 @@ def run_copilot( output_dir=output_dir, al_mcp=al_mcp if container_name else False, al_lsp=al_lsp, + skills=skills, container_name=container_name, ) @@ -71,6 +73,7 @@ def run_claude( output_dir: OutputDir = _config.paths.evaluation_results_path, al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ Run Claude Code on a single entry to generate a patch (without building/testing). @@ -91,6 +94,7 @@ def run_claude( output_dir=output_dir, al_mcp=al_mcp if container_name else False, al_lsp=al_lsp, + skills=skills, container_name=container_name, ) diff --git a/src/bcbench/operations/skills_operations.py b/src/bcbench/operations/skills_operations.py index 67f25d219..7f53bd8d9 100644 --- a/src/bcbench/operations/skills_operations.py +++ b/src/bcbench/operations/skills_operations.py @@ -9,14 +9,24 @@ logger = get_logger(__name__) -def setup_agent_skills(agent_config: dict, entry: BaseDatasetEntry, repo_path: Path, agent_type: AgentType) -> bool: +def setup_agent_skills( + agent_config: dict, + entry: BaseDatasetEntry, + repo_path: Path, + agent_type: AgentType, + skills_enabled_override: bool | None = None, +) -> bool: """ Setup skills in the repository if available. + Args: + skills_enabled_override: When not None, takes precedence over ``config.yaml``'s + ``skills.enabled`` (used to toggle skills per run via the ``--skills`` CLI flag). + Returns: True if skills were copied, False if skills are disabled. """ - skills_enabled: bool = agent_config["skills"]["enabled"] + skills_enabled: bool = agent_config["skills"]["enabled"] if skills_enabled_override is None else skills_enabled_override if skills_enabled: source_skills: Path = _get_source_instructions_path(entry.repo) diff --git a/tests/test_agent_skills.py b/tests/test_agent_skills.py index d89555f43..99a0f1953 100644 --- a/tests/test_agent_skills.py +++ b/tests/test_agent_skills.py @@ -173,3 +173,45 @@ def test_skills_disabled(): assert result is False assert not (repo_path / ".github" / "skills").exists() + + +def test_skills_override_enables_when_config_disabled(): + """--skills (override=True) enables skills even when config.yaml has them disabled.""" + with TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + entry = MagicMock(spec=BaseDatasetEntry) + entry.repo = "microsoftInternal/NAV" + config = {"skills": {"enabled": False}} + + result = setup_agent_skills(config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=True) + + assert result is True + assert (repo_path / ".github" / "skills").exists() + + +def test_skills_override_disables_when_config_enabled(): + """override=False wins over an enabled config, so no skills are copied.""" + with TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + entry = MagicMock(spec=BaseDatasetEntry) + entry.repo = "microsoftInternal/NAV" + config = {"skills": {"enabled": True}} + + result = setup_agent_skills(config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=False) + + assert result is False + assert not (repo_path / ".github" / "skills").exists() + + +def test_skills_override_none_falls_back_to_config(): + """override=None (default) preserves the config-driven behavior.""" + with TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + entry = MagicMock(spec=BaseDatasetEntry) + entry.repo = "microsoftInternal/NAV" + config = {"skills": {"enabled": False}} + + result = setup_agent_skills(config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=None) + + assert result is False + assert not (repo_path / ".github" / "skills").exists() From cad29ec80cabf1c7678823b9544b82890097cf05 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Wed, 29 Jul 2026 01:00:49 +0200 Subject: [PATCH 19/52] Normalize only numeric result values, preserve Code strings verbatim _normalize_value ran every value through Decimal, collapsing distinct digit-only AL Code/No. strings such as "001" and "1" to the same canonical number, so a wrong generated result could be scored as matching the gold. BC returns Code fields as JSON strings even when digit-only, while amounts arrive as JSON numbers. Gate the decimal canonicalization on numeric JSON types (int/float/Decimal) and preserve strings verbatim (whitespace trim only). Both gold and generated rows share the same OData pipeline, so amounts remain numeric on both sides and scale-insensitive matching is retained. Tests updated to numeric inputs and cover digit-only code non-collapsing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/evaluate/dataquery.py | 20 +++++++++++++------- tests/test_dataquery_evaluation.py | 24 +++++++++++++++++++----- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 80ee08e63..457c9c084 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -92,13 +92,19 @@ def _normalize_value(value: object) -> str: return "" if isinstance(value, bool): return str(value).lower() - text = str(value).strip() - try: - # Canonical decimal form: scale/trailing-zero-insensitive (500 == 500.0) but full precision - # preserved, so distinct values like 1.00001 and 1.00002 are NOT collapsed. No float rounding. - return str(Decimal(text).normalize()) - except (InvalidOperation, ValueError): - return text + if isinstance(value, (int, float, Decimal)): + # Only values that arrived as numeric JSON types are canonicalized: scale/trailing-zero- + # insensitive (500 == 500.0) with full precision preserved (1.00001 != 1.00002) and no float + # rounding (Decimal built from the value's string form). Both gold and generated rows come + # through the same OData->JSON pipeline, so amounts are numbers on both sides. + try: + return str(Decimal(str(value)).normalize()) + except (InvalidOperation, ValueError): + return str(value) + # Strings (and anything else) are preserved verbatim apart from a whitespace trim. Business Central + # Code/No. fields are JSON strings even when digit-only, so "001" must NOT collapse to "1" — coercing + # them through Decimal would let a wrong result be scored as matching the gold. + return str(value).strip() def _normalize_rows(rows: Sequence[Mapping[str, object]], ordered: bool) -> list[tuple[str, ...]]: diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 5cf942440..13606a052 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -21,7 +21,8 @@ def test_row_order_enforced_when_ordered(self): assert not result_sets_match(generated, gold, ordered=True) def test_numeric_normalization(self): - assert result_sets_match([{"Total": 500}], [{"Total": "500.0"}]) + # Amounts arrive as numeric JSON types on both sides; scale differences must not matter. + assert result_sets_match([{"Total": 500}], [{"Total": 500.0}]) def test_column_names_ignored(self): assert result_sets_match([{"ItemNo": "I1", "Qty": 5}], [{"No": "I1", "Total": 5}]) @@ -39,14 +40,27 @@ def test_different_row_count_mismatch(self): def test_close_but_distinct_values_do_not_match(self): # Guards against numeric rounding collapsing distinct values into a false positive. - assert not result_sets_match([{"Total": "1.00001"}], [{"Total": "1.00002"}]) + assert not result_sets_match([{"Total": 1.00001}], [{"Total": 1.00002}]) def test_high_precision_preserved(self): - assert result_sets_match([{"Total": "1.000000001"}], [{"Total": "1.000000001"}]) - assert not result_sets_match([{"Total": "1.000000001"}], [{"Total": "1.000000002"}]) + assert result_sets_match([{"Total": 1.000000001}], [{"Total": 1.000000001}]) + assert not result_sets_match([{"Total": 1.000000001}], [{"Total": 1.000000002}]) def test_scale_insensitive(self): - assert result_sets_match([{"Total": "500"}], [{"Total": "500.00"}]) + assert result_sets_match([{"Total": 500}], [{"Total": 500.00}]) + + def test_digit_only_code_strings_not_collapsed(self): + # BC Code/No. fields are JSON strings even when digit-only: "001" and "1" are DISTINCT records + # and must never be scored as matching just because they are numerically equal. + assert not result_sets_match([{"No": "001"}], [{"No": "1"}]) + assert not result_sets_match([{"No": "0010"}], [{"No": "10"}]) + + def test_identical_code_strings_match(self): + assert result_sets_match([{"No": "001", "Name": "Acme"}], [{"No": "001", "Name": "Acme"}]) + + def test_numeric_string_not_coerced_to_number(self): + # A code that happens to look like a scaled number must not match the numeric value 1. + assert not result_sets_match([{"Key": "1.0"}], [{"Key": 1}]) class TestWrapQueryAsApi: From 9820ae7b7a1506846f5fa77c39017203dd244ea9 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Wed, 29 Jul 2026 16:32:20 +0200 Subject: [PATCH 20/52] Simplify data-query harness per owner review Address @haoranpb's review by leaning the harness toward "fail loud, less machinery": - Drop the gold-withholding contamination guard (working-tree strip + .git relocation). The agent runs under repo_path, so the app-level guard is unnecessary complexity; real isolation is a runtime concern. - Replace the "unscorable" result state with fail-loud semantics: a gold query that does not compile/run now raises (turning the run red) instead of being silently excluded, since it is a harness/dataset bug that must be fixed. An empty agent output is still tracked as a build failure. Removed the `scorable` flag and its summary/bceval-export exclusions. - Bump environment_setup_version 26.0 -> 29.0 across the dataset. - Simplify the data-query prompt to the minimal harness contract (write one query object to query.al, no API properties) and drop the AL tutorial bullets. - Remove the now-obsolete gold-withholding unit tests. Threads 1-3 (skip-repo as a category property, optional repo field) are handled generically by #761; this PR will take those up on rebase after it merges. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- dataset/dataquery.jsonl | 22 +++--- src/bcbench/agent/shared/config.yaml | 19 ++--- src/bcbench/evaluate/dataquery.py | 104 +++------------------------ src/bcbench/results/base.py | 10 --- src/bcbench/results/bceval_export.py | 8 +-- src/bcbench/results/summary.py | 12 ++-- tests/test_dataquery_evaluation.py | 72 ------------------- 7 files changed, 30 insertions(+), 217 deletions(-) diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index 30d24ccf6..dd1f68de9 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -1,11 +1,11 @@ -{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} -{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "26.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index 5f3b1bdce..4adae29b9 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -67,21 +67,10 @@ prompt: If there are no findings, write an empty array. Write only valid JSON to review.json, with no surrounding markdown or commentary. data-query-template: | - You are writing a Microsoft Dynamics 365 Business Central AL query to answer a data - question. Proceed without asking for confirmation. - - Task: author a single AL `query` object that returns the data needed to answer the - question below, and write it to a file named `query.al` in {{repo_path}}. - - Requirements: - - Write exactly one AL `query` object with an object id in the range 50100..50149 and a - name. Reference real Business Central tables and fields. - - Return the columns needed to answer the question. Use column `Method` (Sum, Count, - Average, Min, Max) for aggregates and `DataItemLink` to join dataitems. - - Do NOT add API properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName, - EntitySetName) — the evaluation harness adds those. Write only the query logic - (query id/name, elements, dataitem(s), columns, filters, order by). - - The file must contain only the query object, nothing else. + Answer the following Business Central data question by writing a single AL `query` object + to a file named `query.al` in {{repo_path}}. Write only the query logic — do not add API + properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName, EntitySetName); the + evaluation harness adds those. Proceed without asking for confirmation. Question: {{task}} diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 457c9c084..e262ced51 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,8 +1,5 @@ -import contextlib -import json import shutil -import uuid -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from decimal import Decimal, InvalidOperation from pathlib import Path @@ -21,72 +18,6 @@ GENERATED_QUERY_FILE = "query.al" -def _strip_gold(dataset_path: Path, original: str) -> None: - stripped: list[str] = [] - for line in original.splitlines(): - if not line.strip(): - continue - obj = json.loads(line) - obj.pop("gold_query", None) - stripped.append(json.dumps(obj, ensure_ascii=False)) - dataset_path.write_text("\n".join(stripped) + "\n", encoding="utf-8") - - -@contextlib.contextmanager -def _hide_git_object_db(repo_root: Path) -> Iterator[None]: - """Move the checkout's ``.git`` out of the agent-reachable tree for the agent phase. - - Stripping only the working-tree dataset is not enough: the agent runs with unrestricted tools in - a workspace *inside* the checkout, so it could recover the committed gold answers straight from - the object database (``git -C .. show HEAD:dataset/dataquery.jsonl``). Relocate ``.git`` to a - sibling of the checkout (same volume -> instant rename) so ``git`` finds no repository during - generation, then move it back. Best-effort: if the rename fails (e.g. a held handle) we log and - fall back to the working-tree strip alone rather than aborting the evaluation. Data-query never - needs the checkout's git during the agent phase (the agent writes to a separate testbed, and - scoring is file-based), so hiding it here is safe. - """ - git_dir = repo_root / ".git" - if not git_dir.is_dir(): - yield - return - hideaway = repo_root.parent / f".bcbench-withheld-git-{uuid.uuid4().hex}" - try: - git_dir.rename(hideaway) - except OSError as exc: - logger.warning("Could not relocate .git to withhold gold history (%s); working-tree strip only", exc) - yield - return - try: - yield - finally: - with contextlib.suppress(OSError): - hideaway.rename(git_dir) - - -@contextlib.contextmanager -def _withhold_gold_from_agent(dataset_path: Path) -> Iterator[None]: - """Withhold the reference answers from the agent while it generates its query. - - The agent runs with unrestricted filesystem tools in a workspace under the checkout, so it could - otherwise read the gold answers — from the working-tree ``dataset/dataquery.jsonl`` *or* from the - committed blob via the ``.git`` object database — and copy them, invalidating the benchmark. Strip - gold from the working tree and hide ``.git`` for the duration of the agent phase; both are restored - on exit (including on exception). The harness already loaded this entry (with its gold) into memory - before the agent starts, so scoring is unaffected. - """ - if not dataset_path.exists(): - yield - return - original = dataset_path.read_text(encoding="utf-8") - repo_root = dataset_path.parent.parent - try: - _strip_gold(dataset_path, original) - with _hide_git_object_db(repo_root): - yield - finally: - dataset_path.write_text(original, encoding="utf-8") - - def _normalize_value(value: object) -> str: if value is None: return "" @@ -157,13 +88,7 @@ def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: Callable) -> None: - # The agent runs with unrestricted filesystem tools in a workspace under the checkout, so it could - # otherwise read the reference answers from the dataset. Withhold the gold queries from disk during - # generation; the harness already holds this entry's gold in memory for scoring. - with ( - github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"), - _withhold_gold_from_agent(context.category.dataset_path), - ): + with github_log_group(f"{context.agent_name} -- Entry: {context.entry.instance_id}"): context.metrics, context.experiment = agent_runner(context) def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: @@ -172,28 +97,19 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: query_file = context.repo_path / GENERATED_QUERY_FILE generated_query = query_file.read_text(encoding="utf-8").strip() if query_file.exists() else "" - container = context.get_container() - version = context.entry.environment_setup_version - - # Establish gold validity FIRST, independent of the agent's output: a broken gold (a - # harness/dataset problem) must be recorded as unscorable regardless of whether the agent's - # query compiled — otherwise, when the agent query also fails, the broken entry is silently - # counted against that agent. - try: - gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") - except (BuildError, BuildTimeoutExpired) as e: - logger.exception(f"Gold query failed to compile/run for {context.entry.instance_id}") - self.save_result( - context, - ExecutionBasedEvaluationResult.create_unscorable(context, output=generated_query, error_message=f"Gold query failed (harness/dataset issue, not the agent): {e}"), - ) - return - if not generated_query: logger.warning(f"Agent produced no {GENERATED_QUERY_FILE} for {context.entry.instance_id}") self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output="", error_message=f"No {GENERATED_QUERY_FILE} produced")) return + container = context.get_container() + version = context.entry.environment_setup_version + + # Validate the gold query first, and deliberately do NOT catch its failure: a gold that doesn't + # compile/run is a harness or dataset bug, not the agent's fault, so it must fail the run loudly + # and get fixed rather than being silently scored or excluded. + gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + try: generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") except (BuildError, BuildTimeoutExpired) as e: diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 0df64a249..ee4a8e7cf 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -98,9 +98,6 @@ class ExecutionBasedEvaluationResult(BaseEvaluationResult): resolved: bool = False build: bool = False - # False marks a harness/dataset failure (e.g. the gold query didn't compile) that must be - # excluded from the resolution rate so it is not counted against the agent. - scorable: bool = True @classmethod def create_success(cls, context: "EvaluationContext", output: str) -> Self: @@ -110,11 +107,6 @@ def create_success(cls, context: "EvaluationContext", output: str) -> Self: def create_build_failure(cls, context: "EvaluationContext", output: str, error_message: str) -> Self: return cls(**cls._base_fields(context), output=output, error_message=error_message, resolved=False, build=False) - @classmethod - def create_unscorable(cls, context: "EvaluationContext", output: str, error_message: str) -> Self: - """A harness/dataset failure (not the agent's fault) that must not count toward the resolution rate.""" - return cls(**cls._base_fields(context), output=output, build=True, resolved=False, scorable=False, error_message=error_message) - @classmethod def create_result(cls, context: "EvaluationContext", output: str, *, build: bool, resolved: bool, error_message: str | None = None) -> Self: """General factory for execution outcomes, e.g. compiled+ran but produced the wrong result (build=True, resolved=False).""" @@ -124,8 +116,6 @@ def create_result(cls, context: "EvaluationContext", output: str, *, build: bool def status_label(self) -> str: if self.timeout: return "Timeout" - if not self.scorable: - return "Error" return "Success" if self.resolved else "Failed" @property diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index dfa873e1c..293cadb42 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -8,7 +8,7 @@ from bcbench.dataset import BaseDatasetEntry from bcbench.logger import get_logger -from bcbench.results.base import BaseEvaluationResult, ExecutionBasedEvaluationResult +from bcbench.results.base import BaseEvaluationResult from bcbench.results.summary import get_benchmark_version from bcbench.types import EvaluationCategory, ExpectedOutput, ExperimentConfiguration @@ -46,12 +46,6 @@ def write_bceval_results( output_file = out_dir / output_filename with open(output_file, "w") as f: for result in results: - # Unscorable results (harness/dataset failures, e.g. a gold query that didn't compile) must - # not reach the uploaded/core score, or they'd count against the agent's ResolutionRate. - if isinstance(result, ExecutionBasedEvaluationResult) and not result.scorable: - logger.info(f"Skipping unscorable result from bceval export: {result.instance_id}") - continue - matching_entries = [e for e in dataset_entries if e.instance_id == result.instance_id] if not matching_entries: diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index 20bf2a93f..3454c100a 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -159,15 +159,11 @@ def from_results(cls, results: Sequence[BaseEvaluationResult], run_id: str) -> " summary = super().from_results(results, run_id) assert isinstance(summary, ExecutionBasedEvaluationResultSummary) + total = summary.total - # Exclude unscorable results (harness/dataset failures) from every rate so they are not - # counted against the agent. - scorable = [r for r in results if not (isinstance(r, ExecutionBasedEvaluationResult) and not r.scorable)] - total = len(scorable) - - resolved = sum(1 for r in scorable if isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) - build = sum(1 for r in scorable if isinstance(r, ExecutionBasedEvaluationResult) and r.build) - instance_results = {r.instance_id: (isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) for r in scorable} + resolved = sum(1 for r in results if isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) + build = sum(1 for r in results if isinstance(r, ExecutionBasedEvaluationResult) and r.build) + instance_results = {r.instance_id: (isinstance(r, ExecutionBasedEvaluationResult) and r.resolved) for r in results} return summary.model_copy( update={ diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index 13606a052..a475a7feb 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -164,75 +164,3 @@ def test_uninstalls_throwaway_app(self): script = self._render() assert "UnPublish-BcContainerApp" in script assert "UnInstall-BcContainerApp" in script - - -class TestWithholdGoldFromAgent: - def _write_dataset(self, tmp_path): - import json - - dataset_dir = tmp_path / "dataset" - dataset_dir.mkdir() - path = dataset_dir / "dataquery.jsonl" - entries = [ - {"instance_id": "q1", "nl_prompt": "count customers", "gold_query": "query 50100 A { }"}, - {"instance_id": "q2", "nl_prompt": "sum sales", "gold_query": "query 50101 B { }"}, - ] - path.write_text("\n".join(json.dumps(e) for e in entries) + "\n", encoding="utf-8") - return path - - def test_gold_hidden_during_and_restored_after(self, tmp_path): - import json - - from bcbench.evaluate.dataquery import _withhold_gold_from_agent - - path = self._write_dataset(tmp_path) - original = path.read_text(encoding="utf-8") - - with _withhold_gold_from_agent(path): - during = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - # Gold is gone from disk while the agent runs, but the prompt/id the agent needs remain. - assert all("gold_query" not in e for e in during) - assert [e["instance_id"] for e in during] == ["q1", "q2"] - assert [e["nl_prompt"] for e in during] == ["count customers", "sum sales"] - - # Restored verbatim once the agent phase completes. - assert path.read_text(encoding="utf-8") == original - - def test_git_object_db_hidden_during_and_restored_after(self, tmp_path): - from bcbench.evaluate.dataquery import _withhold_gold_from_agent - - path = self._write_dataset(tmp_path) - git_dir = path.parent.parent / ".git" # /.git - git_dir.mkdir() - (git_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") - - with _withhold_gold_from_agent(path): - # The committed gold must not be recoverable via `git show HEAD:...`: no reachable .git. - assert not git_dir.exists() - - # The object database is restored intact once the agent phase completes. - assert git_dir.is_dir() - assert (git_dir / "HEAD").read_text(encoding="utf-8") == "ref: refs/heads/main\n" - - def test_gold_restored_on_exception(self, tmp_path): - from bcbench.evaluate.dataquery import _withhold_gold_from_agent - - path = self._write_dataset(tmp_path) - original = path.read_text(encoding="utf-8") - - def crash(): - with _withhold_gold_from_agent(path): - raise RuntimeError("agent crashed") - - with pytest.raises(RuntimeError): - crash() - - assert path.read_text(encoding="utf-8") == original - - def test_missing_dataset_is_noop(self, tmp_path): - from bcbench.evaluate.dataquery import _withhold_gold_from_agent - - missing = tmp_path / "does-not-exist.jsonl" - with _withhold_gold_from_agent(missing): - pass - assert not missing.exists() From f0921f5500544ac0c094f9fa111ebff8faa5fec0 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Thu, 30 Jul 2026 13:16:50 +0200 Subject: [PATCH 21/52] uptake PR#761 with better extensibility --- dataset/dataquery.jsonl | 22 +++++++++---------- .../skills/al-query-authoring/SKILL.md | 0 src/bcbench/dataset/dataset_entry.py | 9 ++++---- tests/conftest.py | 2 -- tests/test_agent_skills.py | 8 +++---- 5 files changed, 20 insertions(+), 21 deletions(-) rename src/bcbench/agent/shared/instructions/{dataquery-bc => dataquery}/skills/al-query-authoring/SKILL.md (100%) diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index dd1f68de9..9d624cba2 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -1,11 +1,11 @@ -{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-sold-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} -{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__opportunity-count-by-status-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__customer-count-by-country-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__line-count-per-open-sales-order-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "repo": "dataquery/bc", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} diff --git a/src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md similarity index 100% rename from src/bcbench/agent/shared/instructions/dataquery-bc/skills/al-query-authoring/SKILL.md rename to src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md diff --git a/src/bcbench/dataset/dataset_entry.py b/src/bcbench/dataset/dataset_entry.py index e3ce1dccf..a2de2276f 100644 --- a/src/bcbench/dataset/dataset_entry.py +++ b/src/bcbench/dataset/dataset_entry.py @@ -199,18 +199,19 @@ class DataQueryEntry(BaseDatasetEntry): Execution-based: the agent authors an AL query; evaluation compiles + runs both the generated query and the gold query against a fixed dataset (Contoso in a BC container) and compares the - result sets. No repo scaffold, so base_commit / patch are relaxed to optional. + result sets. The workspace is scaffolded by the pipeline, so there is no repo or commit. """ - base_commit: str | None = None - patch: str = "" - nl_prompt: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] gold_query: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] # Whether row order is significant when comparing result sets (e.g. the question asks for a # specific ranking). Defaults to False: result sets are compared order-insensitively. ordered: bool = False + @property + def customization_profile(self) -> str: + return "dataquery" + def get_task(self) -> str: return self.nl_prompt diff --git a/tests/conftest.py b/tests/conftest.py index 7d0d104d6..16fd08f54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -360,7 +360,6 @@ def sample_nl2al_entry() -> NL2ALEntry: def create_data_query_entry( instance_id: str = "dataquery__sales-by-customer-1", - repo: str = "dataquery/bc", environment_setup_version: str = VALID_ENVIRONMENT_VERSION, nl_prompt: str = VALID_DATA_QUERY_PROMPT, created_at: str = VALID_CREATED_AT, @@ -369,7 +368,6 @@ def create_data_query_entry( ) -> DataQueryEntry: return DataQueryEntry( instance_id=instance_id, - repo=repo, environment_setup_version=environment_setup_version, nl_prompt=nl_prompt, created_at=created_at, diff --git a/tests/test_agent_skills.py b/tests/test_agent_skills.py index e92b19625..31e0d8e5e 100644 --- a/tests/test_agent_skills.py +++ b/tests/test_agent_skills.py @@ -8,7 +8,7 @@ import pytest -from bcbench.dataset import RepoGroundedEntry +from bcbench.dataset import BaseDatasetEntry, RepoGroundedEntry from bcbench.operations import setup_agent_skills from bcbench.operations.instruction_operations import _get_source_instructions_path from bcbench.types import AgentType @@ -167,7 +167,7 @@ def test_skills_override_enables_when_config_disabled(): with TemporaryDirectory() as tmpdir: repo_path = Path(tmpdir) entry = MagicMock(spec=BaseDatasetEntry) - entry.repo = "microsoftInternal/NAV" + entry.customization_profile = "microsoftInternal-NAV" config = {"skills": {"enabled": False}} result = setup_agent_skills(config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=True) @@ -181,7 +181,7 @@ def test_skills_override_disables_when_config_enabled(): with TemporaryDirectory() as tmpdir: repo_path = Path(tmpdir) entry = MagicMock(spec=BaseDatasetEntry) - entry.repo = "microsoftInternal/NAV" + entry.customization_profile = "microsoftInternal-NAV" config = {"skills": {"enabled": True}} result = setup_agent_skills(config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=False) @@ -195,7 +195,7 @@ def test_skills_override_none_falls_back_to_config(): with TemporaryDirectory() as tmpdir: repo_path = Path(tmpdir) entry = MagicMock(spec=BaseDatasetEntry) - entry.repo = "microsoftInternal/NAV" + entry.customization_profile = "microsoftInternal-NAV" config = {"skills": {"enabled": False}} result = setup_agent_skills(config, entry, repo_path, agent_type=AgentType.COPILOT, skills_enabled_override=None) From 96fd0b491f3a75c36105d1147ff111246653ec5d Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Thu, 30 Jul 2026 14:30:59 +0200 Subject: [PATCH 22/52] uptake the file sysmte operation --- scripts/Setup-ContainerAndRepository.ps1 | 5 +++- src/bcbench/evaluate/dataquery.py | 23 +++---------------- src/bcbench/operations/__init__.py | 3 ++- src/bcbench/operations/bc_operations.py | 3 ++- .../operations/filesystem_operations.py | 15 ++++++++++++ 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index b6efd0179..efeffd1f3 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -72,7 +72,10 @@ if (-not $SkipRepo) { Invoke-GitCloneWithRetry -RepoUrl $cloneInfo.Url -Token $cloneInfo.Token -ClonePath $RepoPath -CommitSha $commitSha -SparseCheckoutPaths $cloneInfo.SparseCheckoutPaths } else { - Write-Log "Skipping repository clone (SkipRepo flag set)" -Level Info + # Categories that scaffold their own workspace still need the folder to exist: it is shared into + # the container below, and Compile-AppInBcContainer throws for any path not shared with it. + Write-Log "Skipping repository clone (SkipRepo flag set); creating empty workspace at $RepoPath" -Level Info + New-Item -ItemType Directory -Path $RepoPath -Force | Out-Null } if (-not $SkipContainer) { diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index e262ced51..8509c30d9 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,4 +1,3 @@ -import shutil from collections.abc import Callable, Mapping, Sequence from decimal import Decimal, InvalidOperation from pathlib import Path @@ -8,6 +7,7 @@ from bcbench.exceptions import BuildError, BuildTimeoutExpired from bcbench.github_actions import github_log_group from bcbench.logger import get_logger +from bcbench.operations import clear_directory from bcbench.results.base import ExecutionBasedEvaluationResult from bcbench.types import EvaluationContext @@ -54,24 +54,6 @@ def result_sets_match(generated: Sequence[Mapping[str, object]], gold: Sequence[ return _normalize_rows(generated, ordered) == _normalize_rows(gold, ordered) -def _force_remove_readonly(func: Callable, path: str, _: object) -> None: - Path(path).chmod(0o666) - func(path) - - -def _prepare_repo_path(repo_path: Path) -> None: - # Clear the workspace *contents* but not the directory itself: for data-query the workspace is - # mounted into the running BC container (shared folder), so removing the top dir fails with - # WinError 32 (in use). The workflow hands us a fresh empty dir; locally this clears stale files. - repo_path.mkdir(parents=True, exist_ok=True) - for child in repo_path.iterdir(): - if child.is_dir(): - shutil.rmtree(child, onexc=_force_remove_readonly) - else: - child.chmod(0o666) - child.unlink() - - class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): """Pipeline for the data-query category — generate an AL query, evaluate deterministically. @@ -82,7 +64,8 @@ class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): """ def setup_workspace(self, entry: DataQueryEntry, repo_path: Path) -> None: - _prepare_repo_path(repo_path) + # The workspace is shared into the running container, so its contents are cleared in place. + clear_directory(repo_path) def setup(self, context: EvaluationContext[DataQueryEntry]) -> None: self.setup_workspace(context.entry, context.repo_path) diff --git a/src/bcbench/operations/__init__.py b/src/bcbench/operations/__init__.py index 8d31879f5..5acb4d1cb 100644 --- a/src/bcbench/operations/__init__.py +++ b/src/bcbench/operations/__init__.py @@ -11,7 +11,7 @@ run_tests, wrap_query_as_api, ) -from bcbench.operations.filesystem_operations import remove_tree +from bcbench.operations.filesystem_operations import clear_directory, remove_tree from bcbench.operations.git_operations import ( apply_patch, checkout_commit, @@ -39,6 +39,7 @@ "checkout_commit", "clean_project_paths", "clean_repo", + "clear_directory", "clone_repo_at_revision", "commit_changes", "copy_problem_statement_folder", diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index ee4f2bb30..ae521376e 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -13,6 +13,7 @@ from bcbench.dataset.dataset_entry import _BugFixTestGenBase from bcbench.exceptions import BuildError, BuildTimeoutExpired, TestExecutionError, TestExecutionTimeoutExpired from bcbench.logger import get_logger +from bcbench.operations.filesystem_operations import remove_tree from bcbench.types import ContainerConfig logger = get_logger(__name__) @@ -373,7 +374,7 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, object_id = 50100 if suffix == "generated" else 50101 app_dir = work_root / f".bcbench-query-{suffix}" if app_dir.exists(): - shutil.rmtree(app_dir, onexc=lambda func, path, _: (Path(path).chmod(0o666), func(path))) + remove_tree(app_dir) app_dir.mkdir(parents=True, exist_ok=True) app_manifest = { diff --git a/src/bcbench/operations/filesystem_operations.py b/src/bcbench/operations/filesystem_operations.py index 3a3b35fa1..4b745aad7 100644 --- a/src/bcbench/operations/filesystem_operations.py +++ b/src/bcbench/operations/filesystem_operations.py @@ -18,3 +18,18 @@ def remove_tree(path: Path) -> None: Use when `shutil.rmtree` fails due to read-only files, which usually occur in secondary runs on Windows. """ shutil.rmtree(path, onexc=_force_remove_readonly) + + +def clear_directory(path: Path) -> None: + """ + Remove everything inside a directory, leaving the directory itself in place. + + Use instead of `remove_tree` when the directory should survive. + """ + path.mkdir(parents=True, exist_ok=True) + for child in path.iterdir(): + if child.is_dir(): + remove_tree(child) + else: + child.chmod(stat.S_IWRITE) + child.unlink() From ee06e266ce7ab082160172340c8a2489adba5ac1 Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Thu, 30 Jul 2026 14:45:34 +0200 Subject: [PATCH 23/52] the latest release is 28.3 I believe --- dataset/dataquery.jsonl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index 9d624cba2..766c7db97 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -1,11 +1,11 @@ -{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} -{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} From c218e133869fbbce0d8f7b613900867c04c5d54b Mon Sep 17 00:00:00 2001 From: "Haoran Sun (Business Central)" Date: Tue, 11 Aug 2026 11:14:03 +0200 Subject: [PATCH 24/52] uptake `bootstrap_app_json` util function --- src/bcbench/operations/bc_operations.py | 24 +++++++---------------- tests/test_dataquery_evaluation.py | 26 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/bcbench/operations/bc_operations.py b/src/bcbench/operations/bc_operations.py index ae521376e..b9a047513 100644 --- a/src/bcbench/operations/bc_operations.py +++ b/src/bcbench/operations/bc_operations.py @@ -14,6 +14,7 @@ from bcbench.exceptions import BuildError, BuildTimeoutExpired, TestExecutionError, TestExecutionTimeoutExpired from bcbench.logger import get_logger from bcbench.operations.filesystem_operations import remove_tree +from bcbench.operations.setup_operations import bootstrap_app_json from bcbench.types import ContainerConfig logger = get_logger(__name__) @@ -369,26 +370,15 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, and have not been validated locally; the wrapping and comparison logic are unit-tested. """ import json - from uuid import uuid4 object_id = 50100 if suffix == "generated" else 50101 app_dir = work_root / f".bcbench-query-{suffix}" if app_dir.exists(): remove_tree(app_dir) - app_dir.mkdir(parents=True, exist_ok=True) - - app_manifest = { - "id": str(uuid4()), - "name": f"BC-Bench Query {suffix}", - "publisher": "BC-Bench", - "version": "1.0.0.0", - "application": f"{version.split('.')[0]}.0.0.0", - "platform": f"{version.split('.')[0]}.0.0.0", - "idRanges": [{"from": object_id, "to": object_id}], - "runtime": "13.0", - "target": "OnPrem", - } - (app_dir / "app.json").write_text(json.dumps(app_manifest, indent=2), encoding="utf-8") + + app_name = f"BC-Bench Query {suffix}" + app_publisher = "BC-Bench" + bootstrap_app_json(app_dir, app_name, version, id_range=(object_id, object_id), publisher=app_publisher) (app_dir / "query.al").write_text(wrap_query_as_api(query_text, object_id), encoding="utf-8") # Symbols are downloaded into an explicit .alpackages folder by Invoke-AppBuildAndPublish (below). @@ -400,8 +390,8 @@ def execute_al_query(query_text: str, container: ContainerConfig, version: str, username=_escape_ps_string(container.username), password=_escape_ps_string(container.password), app_dir=_escape_ps_string(str(app_dir)), - app_name=_escape_ps_string(app_manifest["name"]), - app_publisher=_escape_ps_string(app_manifest["publisher"]), + app_name=_escape_ps_string(app_name), + app_publisher=_escape_ps_string(app_publisher), publisher=_QUERY_API_PUBLISHER, group=_QUERY_API_GROUP, version=_QUERY_API_VERSION, diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index a475a7feb..dabe01dce 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -1,8 +1,11 @@ +import json + import pytest from bcbench.evaluate.dataquery import result_sets_match from bcbench.exceptions import BuildError from bcbench.operations import bc_operations, wrap_query_as_api +from bcbench.types import ContainerConfig class TestResultSetsMatch: @@ -122,6 +125,29 @@ def test_no_query_declaration_raises_builderror(self): wrap_query_as_api("codeunit 50100 NotAQuery { }", 50100) +def test_execute_al_query_bootstraps_app_manifest(tmp_path, monkeypatch): + app_dir = tmp_path / ".bcbench-query-generated" + + def write_empty_result(*args, **kwargs): + (app_dir / "result.json").write_text("[]", encoding="utf-8") + + monkeypatch.setattr(bc_operations.subprocess, "run", write_empty_result) + + rows = bc_operations.execute_al_query( + 'query 50100 MyQuery { elements { dataitem(Customer; Customer) { column(No; "No.") { } } } }', + ContainerConfig(name="bcserver", username="admin", password="password"), + "26.0.12345.0", + tmp_path, + "generated", + ) + + manifest = json.loads((app_dir / "app.json").read_text(encoding="utf-8")) + assert rows == [] + assert manifest["name"] == "BC-Bench Query generated" + assert manifest["idRanges"] == [{"from": 50100, "to": 50100}] + assert manifest["runtime"] == "15.0" + + class TestQueryRunTemplate: def _render(self): return bc_operations._QUERY_RUN_TEMPLATE.substitute( From 9732dcbd2717f9d89be8cdea7f886483beb13282 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 19:05:51 +0200 Subject: [PATCH 25/52] Add BC MCP + Microsoft Learn MCP as agent capabilities for data-query Give the data-query agent a feedback loop by exposing the Business Central MCP server (and, separately, the Microsoft Learn MCP) as opt-in capabilities, and swap in the bc-al-query-mcp skill that drives them. - AL install app (scripts/al/mcp-config-setup): published at container setup, it provisions and activates the 'BCBench' MCP configuration the agent connects to. Idempotent by name; app/platform version injected at publish time. - Setup (Setup-ContainerAndRepository.ps1, BCContainerManagement.psm1): for data-query, publish the app, resolve the container IP endpoint + evaluation company, and export BC_MCP_URL / BC_MCP_COMPANY to the agent step. - mcp.py: http servers can now carry headers; --bc-mcp and --ms-learn-mcp are independent toggles. The BC MCP server's url + Basic auth + ConfigurationName + Company headers are filled from the container connection env vars. - --bc-mcp / --ms-learn-mcp threaded through evaluate + run (copilot & claude) and both evaluation workflows (dispatch inputs + requeue). - Skill: replace al-query-authoring with bc-al-query-mcp (grounds AL syntax in Microsoft Learn, validates via the BC MCP tools before writing query.al). - Dataset: environment_setup_version 28.3 -> 29.0 (the MCP config API the install app relies on is not present before 29.0). Which tools the BC MCP server exposes is decided server-side by the install app, so the harness stays capability-agnostic. Container round-trip (endpoint reachability, Basic auth on /mcp, app compile on v29) needs a runner shakeout; not testable locally. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/claude-evaluation.yml | 14 +- .github/workflows/copilot-evaluation.yml | 14 +- dataset/dataquery.jsonl | 22 +-- scripts/BCContainerManagement.psm1 | 72 +++++++- scripts/Setup-ContainerAndRepository.ps1 | 15 ++ .../MCPConfigSetup.Codeunit.al | 34 ++++ scripts/al/mcp-config-setup/app.json | 19 ++ src/bcbench/agent/claude/agent.py | 4 +- src/bcbench/agent/copilot/agent.py | 4 +- src/bcbench/agent/shared/config.yaml | 16 +- .../skills/al-query-authoring/SKILL.md | 53 ------ .../dataquery/skills/bc-al-query-mcp/SKILL.md | 166 ++++++++++++++++++ src/bcbench/agent/shared/mcp.py | 67 ++++++- src/bcbench/commands/evaluate.py | 8 + src/bcbench/commands/run.py | 8 + tests/test_mcp_config.py | 92 +++++++++- 16 files changed, 526 insertions(+), 82 deletions(-) create mode 100644 scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al create mode 100644 scripts/al/mcp-config-setup/app.json delete mode 100644 src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md create mode 100644 src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index 9215afb24..afc845804 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -42,6 +42,16 @@ on: required: false default: false type: boolean + bc-mcp: + description: "Enable the Business Central MCP server" + required: false + default: false + type: boolean + ms-learn-mcp: + description: "Enable the Microsoft Learn MCP server" + required: false + default: false + type: boolean skills: description: "Enable agent skills" required: false @@ -158,6 +168,8 @@ jobs: --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ inputs.bc-mcp && '--bc-mcp' || '' }} ` + ${{ inputs.ms-learn-mcp && '--ms-learn-mcp' || '' }} ` ${{ inputs.skills && '--skills' || '' }} - name: Upload evaluation results @@ -195,4 +207,4 @@ jobs: repeat: ${{ inputs.repeat }} existing-tag: ${{ needs.pin-commit.outputs.tag-name }} workflow-inputs: | - {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} + {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "bc-mcp": "${{ inputs.bc-mcp }}", "ms-learn-mcp": "${{ inputs.ms-learn-mcp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index 561e6159f..9f09893be 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -47,6 +47,16 @@ on: required: false default: false type: boolean + bc-mcp: + description: "Enable the Business Central MCP server" + required: false + default: false + type: boolean + ms-learn-mcp: + description: "Enable the Microsoft Learn MCP server" + required: false + default: false + type: boolean skills: description: "Enable agent skills" required: false @@ -161,6 +171,8 @@ jobs: --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" ` ${{ inputs.al-mcp && '--al-mcp' || '' }} ` ${{ inputs.al-lsp && '--al-lsp' || '' }} ` + ${{ inputs.bc-mcp && '--bc-mcp' || '' }} ` + ${{ inputs.ms-learn-mcp && '--ms-learn-mcp' || '' }} ` ${{ inputs.skills && '--skills' || '' }} - name: Upload evaluation results @@ -198,4 +210,4 @@ jobs: repeat: ${{ inputs.repeat }} existing-tag: ${{ needs.pin-commit.outputs.tag-name }} workflow-inputs: | - {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} + {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "bc-mcp": "${{ inputs.bc-mcp }}", "ms-learn-mcp": "${{ inputs.ms-learn-mcp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"} diff --git a/dataset/dataquery.jsonl b/dataset/dataquery.jsonl index 766c7db97..9d624cba2 100644 --- a/dataset/dataquery.jsonl +++ b/dataset/dataquery.jsonl @@ -1,11 +1,11 @@ -{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} -{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "28.3", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} -{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} -{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} -{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "28.3", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} +{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(AvgAmount; Amount) { Method = Average; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesHeader; \"Sales Header\")\n {\n DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(OrderCount) { Method = Count; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Opportunity; Opportunity)\n {\n column(Status; Status) { }\n column(OpportunityCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(CountryRegionCode; \"Country/Region Code\") { }\n column(CustomerCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Vendor; Vendor)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(PurchaseLine; \"Purchase Line\")\n {\n DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n }\n }\n }\n}"} +{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; \"No.\") { }\n column(Name; Name) { }\n dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n {\n DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n {\n DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n column(TotalAmount; Amount) { Method = Sum; }\n }\n }\n }\n }\n}"} +{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(SalesLine; \"Sales Line\")\n {\n DataItemTableFilter = \"Document Type\" = const(Order);\n column(DocumentNo; \"Document No.\") { }\n column(LineCount) { Method = Count; }\n }\n }\n}"} +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(PurchInvLine; \"Purch. Inv. Line\")\n {\n DataItemTableFilter = Type = const(Item);\n column(ItemNo; \"No.\") { }\n column(TotalQuantity; Quantity) { Method = Sum; }\n }\n }\n}"} diff --git a/scripts/BCContainerManagement.psm1 b/scripts/BCContainerManagement.psm1 index ef8aaad51..d6670430c 100644 --- a/scripts/BCContainerManagement.psm1 +++ b/scripts/BCContainerManagement.psm1 @@ -349,4 +349,74 @@ function New-BCCompilerFolderSync { Write-Log "Compiler folder created at: $compilerFolder" -Level Success } -Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync +function Publish-MCPConfigApp { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$ContainerName, + + [Parameter(Mandatory = $true)] + [string]$Version, + + [Parameter(Mandatory = $true)] + [PSCredential]$Credential + ) + + Import-Module "$PSScriptRoot\AppUtils.psm1" -Force -DisableNameChecking + + [int]$major = ([System.Version]$Version).Major + [string]$sourceFolder = Join-Path $PSScriptRoot "al\mcp-config-setup" + [string]$buildFolder = Join-Path ([System.IO.Path]::GetTempPath()) "bcbench-mcp-config-app" + + if (Test-Path $buildFolder) { + Remove-Item -Path $buildFolder -Recurse -Force + } + Copy-Item -Path $sourceFolder -Destination $buildFolder -Recurse -Force + + # The source keeps a placeholder so the same app builds against any evaluated BC version; + # pin the app/platform dependency to the container's major version at publish time. + [string]$appJsonPath = Join-Path $buildFolder "app.json" + (Get-Content -Path $appJsonPath -Raw).Replace("__APP_VERSION__", "$major.0.0.0") | Set-Content -Path $appJsonPath -Encoding UTF8 + + Write-Log "Publishing BC-Bench MCP config app to provision the MCP configuration..." -Level Info + Invoke-AppBuildAndPublish -containerName $ContainerName -appProjectFolder $buildFolder -credential $Credential -skipVerification -useDevEndpoint + Write-Log "BC MCP configuration 'BCBench' provisioned and activated." -Level Success +} + +function Get-BCMCPConnectionInfo { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$ContainerName + ) + + # The evaluated agent runs on the host, not inside the container, and the runner does not update + # its hosts file -- so reach the BC web listener by the container's IP address. + [string]$ip = Get-BcContainerIpAddress -containerName $ContainerName + if (-not $ip) { + throw "Could not resolve IP address for container $ContainerName; BC MCP endpoint is unreachable from the host." + } + + # BcContainerHelper containers always serve the 'BC' instance on port 7048; the MCP endpoint hangs + # off the same base as the API endpoint the query harness already uses (base + '/mcp'). + [string]$baseUrl = "http://${ip}:7048/BC" + + $companies = @(Get-CompanyInBcContainer -containerName $ContainerName) + $evaluationCompany = $companies | Where-Object { $_.EvaluationCompany } | Select-Object -First 1 + if (-not $evaluationCompany) { + $evaluationCompany = $companies | Select-Object -First 1 + } + + # BcContainerHelper has exposed the company name as either CompanyName or Name across versions. + [string]$company = $evaluationCompany.CompanyName + if (-not $company) { + $company = $evaluationCompany.Name + } + + return [PSCustomObject]@{ + BaseUrl = $baseUrl + Company = $company + } +} + +Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync, Publish-MCPConfigApp, Get-BCMCPConnectionInfo diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index efeffd1f3..d8b95850a 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -102,6 +102,21 @@ if (-not $SkipContainer) { New-BCCompilerFolderSync -ContainerName $ContainerName -ArtifactUrl $url Initialize-ContainerForDevelopment -ContainerName $ContainerName -RepoVersion ([System.Version]$Version) + + # data-query benchmarks the agent WITH the BC MCP server as a feedback loop. Publish the install + # app that provisions and activates the 'BCBench' MCP configuration, then expose the endpoint and + # company to the agent step so it can point its MCP client at the container. + if ($Category -eq 'data-query') { + Publish-MCPConfigApp -ContainerName $ContainerName -Version $Version -Credential $credential + + $mcpInfo = Get-BCMCPConnectionInfo -ContainerName $ContainerName + Write-Log "BC MCP base URL: $($mcpInfo.BaseUrl) (company '$($mcpInfo.Company)')" -Level Info + + if ($env:GITHUB_ENV) { + "BC_MCP_URL=$($mcpInfo.BaseUrl)" | Out-File -FilePath $env:GITHUB_ENV -Append + "BC_MCP_COMPANY=$($mcpInfo.Company)" | Out-File -FilePath $env:GITHUB_ENV -Append + } + } } else { Write-Log "Skipping BC container setup (SkipContainer flag set)" -Level Info diff --git a/scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al b/scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al new file mode 100644 index 000000000..66226140b --- /dev/null +++ b/scripts/al/mcp-config-setup/MCPConfigSetup.Codeunit.al @@ -0,0 +1,34 @@ +namespace BCBench.MCP; + +using System.MCP; + +// Installed at container-setup time (not part of the benchmarked workspace). Provisions and activates +// the MCP configuration the evaluated agent connects to over the BC MCP server; this app is the single +// place that decides which server capabilities the eval exposes. Idempotent: re-installs reuse the +// existing configuration by name. +codeunit 50150 "BCBench MCP Config Setup" +{ + Subtype = Install; + + trigger OnInstallAppPerCompany() + begin + EnsureConfiguration(); + end; + + local procedure EnsureConfiguration() + var + MCPConfig: Codeunit "MCP Config"; + ConfigId: Guid; + begin + ConfigId := MCPConfig.GetConfigurationIdByName(ConfigNameTok); + if IsNullGuid(ConfigId) then + ConfigId := MCPConfig.CreateConfiguration(ConfigNameTok, ConfigDescriptionTok); + + MCPConfig.EnableDataQueryTools(ConfigId, true); + MCPConfig.ActivateConfiguration(ConfigId, true); + end; + + var + ConfigNameTok: Label 'BCBench', Locked = true; + ConfigDescriptionTok: Label 'BC-Bench evaluation', Locked = true; +} diff --git a/scripts/al/mcp-config-setup/app.json b/scripts/al/mcp-config-setup/app.json new file mode 100644 index 000000000..f8ad27ed0 --- /dev/null +++ b/scripts/al/mcp-config-setup/app.json @@ -0,0 +1,19 @@ +{ + "id": "9f3d6b6e-4c2a-4d3f-9b7a-2f1e8c5a7d10", + "name": "BC-Bench MCP Config Setup", + "publisher": "BC-Bench", + "version": "1.0.0.0", + "brief": "Provisions the BC MCP configuration for BC-Bench evaluation.", + "description": "Installed at container-setup time to provision and activate the 'BCBench' Business Central MCP configuration the evaluated agent connects to.", + "application": "__APP_VERSION__", + "platform": "__APP_VERSION__", + "idRanges": [ + { + "from": 50150, + "to": 50159 + } + ], + "runtime": "13.0", + "target": "OnPrem", + "dependencies": [] +} diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index b74ec7b4b..094aade1d 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -27,6 +27,8 @@ def run_claude_code( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + bc_mcp: bool = False, + ms_learn_mcp: bool = False, skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: @@ -45,7 +47,7 @@ def run_claude_code( logger.info(f"Running Claude Code on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, claude_config, category, al_mcp=al_mcp) - mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) + mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, bc_mcp=bc_mcp, ms_learn_mcp=ms_learn_mcp, container_name=container_name) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.CLAUDE, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE, skills_enabled_override=skills) diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index ae140d1f7..1b6f84688 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -29,6 +29,8 @@ def run_copilot_agent( output_dir: Path, al_mcp: bool = False, al_lsp: bool = False, + bc_mcp: bool = False, + ms_learn_mcp: bool = False, skills: bool = False, container_name: str = "bcbench", ) -> tuple[AgentMetrics | None, ExperimentConfiguration]: @@ -49,7 +51,7 @@ def run_copilot_agent( logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, copilot_config, category, al_mcp=al_mcp) - mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name) + mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, bc_mcp=bc_mcp, ms_learn_mcp=ms_learn_mcp, container_name=container_name) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.COPILOT, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=skills) diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index fe8384689..114e49160 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -188,9 +188,19 @@ mcp: "{{package_cache_path}}", ] - # - name: "mslearn" - # type: "http" - # url: "https://learn.microsoft.com/api/mcp" + # Business Central MCP server (toggled via --bc-mcp CLI flag). url + auth/company headers are + # filled in programmatically by mcp.py from the container connection env vars. Which tools the + # server exposes is decided server-side by the MCP configuration the setup-time AL app provisions. + - name: "bcmcp" + type: "http" + url: "" + headers: {} + + # Microsoft Learn MCP (toggled via --ms-learn-mcp CLI flag): official AL docs + code samples the + # bc-al-query-mcp skill grounds its AL syntax in. Public server, no auth. + - name: "mslearn" + type: "http" + url: "https://learn.microsoft.com/api/mcp" # - name: "filesystem" # type: "stdio" diff --git a/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md deleted file mode 100644 index ef771e4da..000000000 --- a/src/bcbench/agent/shared/instructions/dataquery/skills/al-query-authoring/SKILL.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: al-query-authoring -description: Guide for authoring Business Central AL query objects that answer data questions (joins, aggregates, filters, sorting). Use this when asked to write an AL query that returns Business Central data such as customers, vendors, items, sales, purchases, projects, or opportunities. ---- - -Write a single, compilable AL `query` object that returns exactly the data needed to answer -the question. Reference real Business Central tables and fields — a query that does not -compile, or returns the wrong data, fails. - -## Structure - -```al -query 50100 TopCustomersBySales -{ - QueryType = Normal; - - elements - { - dataitem(Customer; Customer) - { - column(No; "No.") { } - column(Name; Name) { } - dataitem(SalesLine; "Sales Line") - { - DataItemLink = "Sell-to Customer No." = Customer."No."; - DataItemTableFilter = "Document Type" = const(Order); - column(OutstandingAmount; "Outstanding Amount") { Method = Sum; } - } - } - } -} -``` - -## Rules of thumb - -- **Aggregate** a column with a method: `column(Total; "Amount (LCY)") { Method = Sum; }` - (also `Average`, `Min`, `Max`) — these take the field to aggregate. **`Count` takes no source - field** — write `column(RowCount) { Method = Count; }`, not `column(RowCount; "No.") { ... }`. - Non-aggregated columns become the GROUP BY. -- **Join** by nesting a `dataitem` and linking it: `DataItemLink = "" = Parent."";`. -- **Filter** rows with `DataItemTableFilter = "" = const();` (e.g. an Option - like `Document Type`) or a range/expression. -- **Order** with the `OrderBy` property: `OrderBy = descending();` (or `ascending`) when - the question asks for ranking or "top N" (combine with `TopNumberOfRows` where appropriate). -- **Quote** any field or table name that contains spaces or special characters: `"No."`, - `"Sales Line"`, `"Amount (LCY)"`. -- Prefer stored fields; FlowFields and Option fields are supported. - -## Common pitfalls - -- Don't invent table or field names — use the real Business Central schema. -- Return only the columns the question needs; extra or missing columns change the result set. -- One `query` object per file. diff --git a/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md new file mode 100644 index 000000000..e230d0bea --- /dev/null +++ b/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md @@ -0,0 +1,166 @@ +--- +name: bc-al-query-mcp +description: "Use when: writing, fixing, validating, or running Business Central AL query objects with bc_data MCP tools and Microsoft Learn MCP docs. Covers AL query syntax, table discovery, schemas, relations, joins, filters, FlowFields, pagination, and read-only data retrieval." +argument-hint: "[business question, data need, or AL query to fix]" +user-invocable: true +disable-model-invocation: false +--- + +# Business Central AL Query MCP + +Use this skill when the user asks to query Business Central data, write or fix an AL `query` object, join BC tables, discover BC fields, validate an AL query, or use the `bc_data` MCP tools together with Microsoft Learn. + +## Required Capabilities + +The environment should expose these MCP tool groups: + +- Microsoft Learn MCP: docs search, docs fetch, and code sample search for official AL query syntax and examples. +- Business Central data MCP: table search, table schema, table relations, and AL query compile/run. + +If tools are deferred, load them with tool search before use. Do not assume a tool is available until it has been loaded or returned by discovery. + +## Core Rule + +Microsoft Learn provides general AL query authoring knowledge. The BC data MCP tools provide live tenant-specific metadata and execution. Use both. Never write a tenant query from memory alone. + +## Non-Negotiables + +- Always use `bc_data_get_table_schema` for every table before writing the query. +- Always verify joins with `bc_data_get_table_relations` before writing a multi-table query. +- Prefer `bc_data_find_tables` with `searchMode: keyword` for known BC entity names. Use semantic search only as a supplement and verify results. +- Compile with `bc_data_query` and `returnData: false` before running with `returnData: true`. +- Keep queries read-only, narrow, and paged. Use only the columns needed for the user's question. +- Do not dump sensitive raw business data unless the user explicitly asks for rows. Prefer summaries, counts, and representative samples. +- If a compile or execution error occurs, use the diagnostic location plus schema/relations to repair the same query. Do not blindly rewrite from scratch. +- If permissions, missing tables, or unavailable MCP tools block the task, report the exact blocker and the next viable option. + +## Workflow + +1. Identify the user's actual data question. + - Determine the business entity, date range, filters, measures, and whether raw rows or an aggregate answer is needed. + - Ask a concise clarification only if the query cannot be scoped safely. + +2. Ground AL syntax in Microsoft Learn when needed. + - Search/fetch docs for `Business Central AL query object`, `DataItemLink`, `SqlJoinType`, `DataItemTableFilter`, `ColumnFilter`, `Filtering in Query objects`, and `Aggregating data in Query objects`. + - Use code sample search with `language: al` when examples are useful. + +3. Discover the live tables. + - Use keyword search for likely names, for example `customer`, `sales invoice`, `item ledger entry`, `vendor ledger entry`. + - If the user describes a business concept instead of table names, try semantic search, but validate with keyword search and schemas. + +4. Inspect schemas. + - Call schema for every candidate table. + - Use `nameContains` to narrow large schemas, for example `['no', 'posting date', 'amount', 'customer']`. + - Note primary keys, field names, field classes, field types, FlowFields, FlowFilters, and relation hints. + +5. Discover joins. + - For each multi-table query, call relations in the useful direction. + - Use `relatedToTableIds` when checking a specific join path. + - Remember that `DataItemLink` is set on the lower/nested dataitem. + +6. Compose the AL query. + - Use a normal query object unless the user specifically needs an API query. + - Quote table and field names that contain spaces, punctuation, or reserved words. + - Use stable column aliases without spaces, usually underscores. + - Put parent tables higher and child/detail tables nested beneath them. + - Set `SqlJoinType = InnerJoin;` when only matching child rows should appear. If omitted, AL query dataitems default to `LeftOuterJoin`. + - Use `DataItemTableFilter` for static filters. + - For date filters in the BC data MCP execution path, prefer quoted ISO date strings, for example `filter('2025-01-01'..'2025-01-31')`. + - Use FlowFields only when needed; they can be convenient but may add subqueries and cost. + +7. Validate before execution. + - First call `bc_data_query` with `returnData: false`. + - Confirm the returned columns, types, and dataitems match the intended shape. + - If validation fails, fix field names, aliases, links, filter syntax, or query structure using the exact diagnostic. + +8. Run safely. + - Use `returnData: true`, `top` no larger than needed, and `skip` for paging. + - Use `resultFormat: resource` for larger results or when downstream analysis is needed. + - On page 0, use `totalCount` when present. Continue paging only when the user needs more data. + +9. Present the result. + - Include the final AL query when the user asked for a query or when it helps reproducibility. + - State which tables, fields, joins, and filters were used. + - Summarize results without overexposing tenant data. + - Mention validation status: compiled only, compiled and ran, or blocked with reason. + +## Query Patterns + +### Single Table + +```al +query 50100 CustomerOverview +{ + QueryType = Normal; + + elements + { + dataitem(Customer; Customer) + { + column(No_; "No.") { } + column(Name; Name) { } + column(Blocked; Blocked) { } + column(Balance_LCY; "Balance (LCY)") { } + } + } +} +``` + +### Header And Lines + +```al +query 50101 PostedSalesInvoiceLines +{ + QueryType = Normal; + + elements + { + dataitem(SalesInvoiceHeader; "Sales Invoice Header") + { + column(Invoice_No_; "No.") { } + column(Sell_to_Customer_No_; "Sell-to Customer No.") { } + column(Posting_Date; "Posting Date") { } + + dataitem(SalesInvoiceLine; "Sales Invoice Line") + { + DataItemLink = "Document No." = SalesInvoiceHeader."No."; + SqlJoinType = InnerJoin; + + column(Line_No_; "Line No.") { } + column(Item_No_; "No.") { } + column(Description; Description) { } + column(Quantity; Quantity) { } + column(Line_Amount; Amount) { } + } + } + } +} +``` + +### Static Date Filter + +```al +DataItemTableFilter = "Posting Date" = filter('2025-01-01'..'2025-01-31'); +``` + +### Aggregate Column + +```al +column(Total_Quantity; Quantity) +{ + Method = Sum; +} +``` + +## Common Recovery Moves + +- `AL0345` or invalid column source: re-check schema for the exact field name on the parent dataitem's table. +- Join returns too many rows: verify `DataItemLink` field direction and add `SqlJoinType = InnerJoin;` when appropriate. +- No rows returned: validate filters first, then run a smaller unfiltered query with identifying columns. +- Date filter errors: use quoted ISO date strings in the filter expression. +- Semantic table search returns nothing: retry with keyword fragments and inspect schemas. +- Large result sets: reduce columns, add filters, page with `top`/`skip`, or use `resultFormat: resource`. + +## Quality Bar + +A good answer from this skill includes enough evidence that the query is grounded in the live environment: discovered tables, checked fields, verified relations, compile status, and a safe execution or clear blocker. The agent should not merely produce plausible AL code; it should validate the query against the connected Business Central instance whenever the tools are available. diff --git a/src/bcbench/agent/shared/mcp.py b/src/bcbench/agent/shared/mcp.py index a8881ee4e..fd3da03f9 100644 --- a/src/bcbench/agent/shared/mcp.py +++ b/src/bcbench/agent/shared/mcp.py @@ -1,3 +1,4 @@ +import base64 import json import os import shutil @@ -15,6 +16,12 @@ _jinja = SandboxedEnvironment(autoescape=False) +# Server names for the independently-toggled MCP servers. +_BC_MCP_SERVER_NAME = "bcmcp" +_MS_LEARN_MCP_SERVER_NAME = "mslearn" +# Must match the configuration name the setup-time AL app creates (scripts/al/mcp-config-setup). +_BC_MCP_CONFIGURATION_NAME = "BCBench" + def _build_server_entry(server: dict[str, Any], template_context: dict[str, Any]) -> tuple[str, dict[str, Any]]: server_type: str = server["type"] @@ -22,39 +29,89 @@ def _build_server_entry(server: dict[str, Any], template_context: dict[str, Any] match server_type: case "http": - return server_name, { + entry: dict[str, Any] = { "type": server_type, "url": server["url"], } + headers: dict[str, str] = server.get("headers", {}) + if headers: + entry["headers"] = headers + return server_name, entry case "stdio": args: list[str] = server["args"] rendered_args = [_jinja.from_string(arg).render(**template_context) for arg in args] command: str = shutil.which(server["command"]) or server["command"] - entry: dict[str, Any] = { + stdio_entry: dict[str, Any] = { "type": server_type, "command": command, "args": rendered_args, } env: dict[str, str] = server.get("env", {}) if env: - entry["env"] = env - return server_name, entry + stdio_entry["env"] = env + return server_name, stdio_entry case _: logger.error(f"Unsupported MCP server type: {server_type}, {server}") raise AgentError(f"Unsupported MCP server type: {server_type}") -def build_mcp_config(config: dict[str, Any], entry: BaseDatasetEntry, repo_path: Path, al_mcp: bool = False, container_name: str = "bcbench") -> tuple[str | None, list[str] | None]: +def _configure_bc_mcp_server(server: dict[str, Any]) -> None: + """Fill the BC MCP server's endpoint + auth/company headers from the container connection env vars. + + ``BC_MCP_URL``/``BC_MCP_COMPANY`` are exported by ``Setup-ContainerAndRepository.ps1``. Auth is + Basic over the reused ``BC_SERVER_*`` credentials (validated against NAV's MCP client). Omitting the + Company header would switch the server into cross-company dynamic mode, so it is only sent when a + company is known. Which tools the server exposes is decided server-side by the MCP configuration + the setup-time AL app provisions, not here. + """ + base_url = os.environ.get("BC_MCP_URL") + if not base_url: + raise AgentError("BC MCP requested but BC_MCP_URL is not set; container setup must export it.") + + username = os.environ.get("BC_SERVER_USERNAME", "") + password = os.environ.get("BC_SERVER_PASSWORD", "") + basic_auth = base64.b64encode(f"{username}:{password}".encode()).decode() + + headers: dict[str, str] = { + "Authorization": f"Basic {basic_auth}", + "ConfigurationName": _BC_MCP_CONFIGURATION_NAME, + } + company = os.environ.get("BC_MCP_COMPANY") + if company: + headers["Company"] = company + + server["url"] = base_url.rstrip("/") + "/mcp" + server["headers"] = headers + + +def build_mcp_config( + config: dict[str, Any], + entry: BaseDatasetEntry, + repo_path: Path, + al_mcp: bool = False, + bc_mcp: bool = False, + ms_learn_mcp: bool = False, + container_name: str = "bcbench", +) -> tuple[str | None, list[str] | None]: mcp_servers: list[dict[str, Any]] = config.get("mcp", {}).get("servers", []) if not al_mcp: mcp_servers = list(filter(lambda s: s.get("name") != "altool", mcp_servers)) + if not bc_mcp: + mcp_servers = list(filter(lambda s: s.get("name") != _BC_MCP_SERVER_NAME, mcp_servers)) + + if not ms_learn_mcp: + mcp_servers = list(filter(lambda s: s.get("name") != _MS_LEARN_MCP_SERVER_NAME, mcp_servers)) + if not mcp_servers: return None, None template_context: dict[str, str | Path] = {"repo_path": repo_path} + if bc_mcp: + _configure_bc_mcp_server(next(s for s in mcp_servers if s["name"] == _BC_MCP_SERVER_NAME)) + if al_mcp: compiler_folder, symbols_folder = compiler_symbol_folder_for_container(container_name) template_context["package_cache_path"] = str(symbols_folder) diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 87ad49360..90ba5da2c 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -55,6 +55,8 @@ def evaluate_copilot( run_id: RunId = "copilot_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + bc_mcp: Annotated[bool, typer.Option("--bc-mcp", help="Enable the Business Central MCP server")] = False, + ms_learn_mcp: Annotated[bool, typer.Option("--ms-learn-mcp", help="Enable the Microsoft Learn MCP server")] = False, skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ @@ -90,6 +92,8 @@ def evaluate_copilot( output_dir=ctx.result_dir, al_mcp=al_mcp if ctx.container else False, al_lsp=al_lsp, + bc_mcp=bc_mcp if ctx.container else False, + ms_learn_mcp=ms_learn_mcp, skills=skills, container_name=ctx.get_container().name if ctx.container else "", ), @@ -112,6 +116,8 @@ def evaluate_claude_code( run_id: RunId = "claude_code_test_run", al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + bc_mcp: Annotated[bool, typer.Option("--bc-mcp", help="Enable the Business Central MCP server")] = False, + ms_learn_mcp: Annotated[bool, typer.Option("--ms-learn-mcp", help="Enable the Microsoft Learn MCP server")] = False, skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ @@ -147,6 +153,8 @@ def evaluate_claude_code( output_dir=ctx.result_dir, al_mcp=al_mcp if ctx.container else False, al_lsp=al_lsp, + bc_mcp=bc_mcp if ctx.container else False, + ms_learn_mcp=ms_learn_mcp, skills=skills, container_name=ctx.get_container().name if ctx.container else "", ), diff --git a/src/bcbench/commands/run.py b/src/bcbench/commands/run.py index 430b683e7..6cf05a278 100644 --- a/src/bcbench/commands/run.py +++ b/src/bcbench/commands/run.py @@ -36,6 +36,8 @@ def run_copilot( output_dir: OutputDir = _config.paths.evaluation_results_path, al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + bc_mcp: Annotated[bool, typer.Option("--bc-mcp", help="Enable the Business Central MCP server")] = False, + ms_learn_mcp: Annotated[bool, typer.Option("--ms-learn-mcp", help="Enable the Microsoft Learn MCP server")] = False, skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ @@ -57,6 +59,8 @@ def run_copilot( output_dir=output_dir, al_mcp=al_mcp if container_name else False, al_lsp=al_lsp, + bc_mcp=bc_mcp if container_name else False, + ms_learn_mcp=ms_learn_mcp, skills=skills, container_name=container_name, ) @@ -72,6 +76,8 @@ def run_claude( output_dir: OutputDir = _config.paths.evaluation_results_path, al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False, al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False, + bc_mcp: Annotated[bool, typer.Option("--bc-mcp", help="Enable the Business Central MCP server")] = False, + ms_learn_mcp: Annotated[bool, typer.Option("--ms-learn-mcp", help="Enable the Microsoft Learn MCP server")] = False, skills: Annotated[bool, typer.Option("--skills", help="Enable agent skills for the run")] = False, ) -> None: """ @@ -93,6 +99,8 @@ def run_claude( output_dir=output_dir, al_mcp=al_mcp if container_name else False, al_lsp=al_lsp, + bc_mcp=bc_mcp if container_name else False, + ms_learn_mcp=ms_learn_mcp, skills=skills, container_name=container_name, ) diff --git a/tests/test_mcp_config.py b/tests/test_mcp_config.py index 35d623006..d83e1ab89 100644 --- a/tests/test_mcp_config.py +++ b/tests/test_mcp_config.py @@ -1,3 +1,4 @@ +import base64 import json from copy import deepcopy from pathlib import Path @@ -6,6 +7,7 @@ from bcbench.agent.shared.altool_paths import build_assembly_probing_paths as _build_assembly_probing_paths from bcbench.agent.shared.mcp import build_mcp_config +from bcbench.exceptions import AgentError from tests.conftest import create_dataset_entry @@ -26,6 +28,20 @@ def _make_config(*servers: dict) -> dict: "url": "https://learn.microsoft.com/api/mcp", } +BCMCP_SERVER = { + "name": "bcmcp", + "type": "http", + "url": "", + "headers": {}, +} + +# A server not gated by any flag, used to assert generic pass-through behavior. +OTHER_HTTP_SERVER = { + "name": "docs", + "type": "http", + "url": "https://example.com/mcp", +} + @pytest.fixture def entry(): @@ -68,7 +84,7 @@ def test_altool_excluded_when_al_mcp_disabled(self, entry, repo_path): assert result == (None, None) def test_altool_excluded_but_other_servers_kept(self, entry, repo_path): - config = _make_config(ALTOOL_SERVER, MSLEARN_SERVER) + config = _make_config(ALTOOL_SERVER, OTHER_HTTP_SERVER) config_json, names = build_mcp_config(config, entry, repo_path, al_mcp=False) assert config_json is not None @@ -76,16 +92,82 @@ def test_altool_excluded_but_other_servers_kept(self, entry, repo_path): parsed = json.loads(config_json) assert "altool" not in parsed["mcpServers"] - assert "mslearn" in parsed["mcpServers"] - assert names == ["mslearn"] + assert "docs" in parsed["mcpServers"] + assert names == ["docs"] def test_returns_server_names(self, entry, repo_path): - config = _make_config(ALTOOL_SERVER, MSLEARN_SERVER) + config = _make_config(ALTOOL_SERVER, OTHER_HTTP_SERVER) _, names = build_mcp_config(config, entry, repo_path, al_mcp=True) assert names is not None - assert set(names) == {"altool", "mslearn"} + assert set(names) == {"altool", "docs"} + + +class TestBcMcp: + _VARS = ("BC_MCP_URL", "BC_MCP_COMPANY", "BC_SERVER_USERNAME", "BC_SERVER_PASSWORD") + + @pytest.fixture(autouse=True) + def _clean_env(self, monkeypatch): + for var in self._VARS: + monkeypatch.delenv(var, raising=False) + + def test_bcmcp_excluded_when_disabled(self, entry, repo_path): + assert build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=False) == (None, None) + + def test_mslearn_excluded_when_disabled(self, entry, repo_path): + assert build_mcp_config(_make_config(MSLEARN_SERVER), entry, repo_path, ms_learn_mcp=False) == (None, None) + + def test_flags_are_independent(self, entry, repo_path, monkeypatch): + monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + config = _make_config(BCMCP_SERVER, MSLEARN_SERVER) + + # bc-mcp only -> no mslearn + _, bc_only = build_mcp_config(config, entry, repo_path, bc_mcp=True, ms_learn_mcp=False) + assert bc_only == ["bcmcp"] + + # ms-learn only -> no bcmcp (and no BC connection env needed) + _, learn_only = build_mcp_config(config, entry, repo_path, bc_mcp=False, ms_learn_mcp=True) + assert learn_only == ["mslearn"] + + # both -> both + _, both = build_mcp_config(config, entry, repo_path, bc_mcp=True, ms_learn_mcp=True) + assert set(both) == {"bcmcp", "mslearn"} + + def test_mslearn_url_passthrough(self, entry, repo_path): + config_json, _ = build_mcp_config(_make_config(MSLEARN_SERVER), entry, repo_path, ms_learn_mcp=True) + assert json.loads(config_json)["mcpServers"]["mslearn"]["url"] == "https://learn.microsoft.com/api/mcp" + + def test_bcmcp_endpoint_and_auth_headers(self, entry, repo_path, monkeypatch): + monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + monkeypatch.setenv("BC_MCP_COMPANY", "CRONUS International Ltd.") + + config_json, _ = build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) + bcmcp = json.loads(config_json)["mcpServers"]["bcmcp"] + + assert bcmcp["url"] == "http://172.17.0.2:7048/BC/mcp" + expected_auth = "Basic " + base64.b64encode(b"admin:secret").decode() + assert bcmcp["headers"]["Authorization"] == expected_auth + assert bcmcp["headers"]["ConfigurationName"] == "BCBench" + assert bcmcp["headers"]["Company"] == "CRONUS International Ltd." + + def test_company_header_omitted_when_unset(self, entry, repo_path, monkeypatch): + monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + + config_json, _ = build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) + headers = json.loads(config_json)["mcpServers"]["bcmcp"]["headers"] + + assert "Company" not in headers + + def test_raises_when_bc_mcp_url_missing(self, entry, repo_path): + with pytest.raises(AgentError): + build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) class TestAltoolEnvForwarding: From eb47c22f3e60d61a55dea95fcff4b03be79d99b4 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 19:21:12 +0200 Subject: [PATCH 26/52] TEMP: BC MCP runner diagnostics (revert before merge) Front-load the runner-only unknowns so one shakeout run classifies any failure: - Write-BCMCPDiagnostics: host-side probe of the container from the setup step, exactly as the agent will connect. GETs the companies API (reachability + Basic auth) and POSTs MCP initialize + tools/list (endpoint + config + exposed tools). Never throws; output lands in the always-visible setup log. - Upload **/*.log alongside *.jsonl so the agent's MCP client debug logs survive. - RUNNER_DEBUG=1 in both eval workflows -> Python DEBUG + PowerShell compile output. Also a permanent fix (keep): redact the Authorization header in build_mcp_config's DEBUG dump so verbose logs never leak container credentials. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/claude-evaluation.yml | 6 +- .github/workflows/copilot-evaluation.yml | 6 +- scripts/BCContainerManagement.psm1 | 73 +++++++++++++++++++++++- scripts/Setup-ContainerAndRepository.ps1 | 3 + src/bcbench/agent/shared/mcp.py | 14 ++++- tests/test_mcp_config.py | 11 ++++ 6 files changed, 109 insertions(+), 4 deletions(-) diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index afc845804..1909e5538 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -80,6 +80,8 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results + # TEMPORARY (remove before merge): force verbose Python + PowerShell logging for the BC MCP shakeout. + RUNNER_DEBUG: "1" jobs: pin-commit: @@ -177,7 +179,9 @@ jobs: if: always() with: name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} - path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + path: | + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index 9f09893be..b2fc5c24a 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -85,6 +85,8 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results + # TEMPORARY (remove before merge): force verbose Python + PowerShell logging for the BC MCP shakeout. + RUNNER_DEBUG: "1" jobs: pin-commit: @@ -180,7 +182,9 @@ jobs: if: always() with: name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} - path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + path: | + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: diff --git a/scripts/BCContainerManagement.psm1 b/scripts/BCContainerManagement.psm1 index d6670430c..3efb10a31 100644 --- a/scripts/BCContainerManagement.psm1 +++ b/scripts/BCContainerManagement.psm1 @@ -419,4 +419,75 @@ function Get-BCMCPConnectionInfo { } } -Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync, Publish-MCPConfigApp, Get-BCMCPConnectionInfo +function Write-BCMCPDiagnostics { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$BaseUrl, + + [Parameter(Mandatory = $false)] + [string]$Company, + + [Parameter(Mandatory = $true)] + [PSCredential]$Credential, + + [Parameter(Mandatory = $false)] + [string]$ConfigurationName = 'BCBench' + ) + + # TEMPORARY diagnostics (remove before merge): probe the BC MCP endpoint from the HOST exactly as + # the evaluated agent will, so runner-only failures (reachability / auth / MCP config / tools) show + # up in the setup log instead of being guessed from the agent's discarded logs. Never throws. + $pair = "$($Credential.UserName):$($Credential.GetNetworkCredential().Password)" + $basic = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($pair)) + + Write-Log "===== BC MCP DIAGNOSTICS =====" -Level Info + Write-Log "BaseUrl=$BaseUrl Company='$Company' ConfigurationName=$ConfigurationName" -Level Info + + # 1) API listener reachability + Basic auth sanity against a known-good endpoint. + try { + $companiesUri = "$BaseUrl/api/v2.0/companies" + $resp = Invoke-WebRequest -Uri $companiesUri -Headers @{ Authorization = $basic } -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 + Write-Log "[probe:companies] GET $companiesUri -> HTTP $($resp.StatusCode)" -Level Info + Write-Log "[probe:companies] body: $($resp.Content.Substring(0, [Math]::Min(600, $resp.Content.Length)))" -Level Info + } + catch { + Write-Log "[probe:companies] EXCEPTION: $($_.Exception.Message)" -Level Warning + } + + # 2) MCP endpoint: initialize, then tools/list if a session is granted. + $mcpUri = "$BaseUrl/mcp" + $headers = @{ + Authorization = $basic + ConfigurationName = $ConfigurationName + Accept = 'application/json, text/event-stream' + } + if ($Company) { $headers['Company'] = $Company } + + $initBody = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"bcbench-diag","version":"1.0"}}}' + try { + $resp = Invoke-WebRequest -Uri $mcpUri -Method Post -Headers $headers -Body $initBody -ContentType 'application/json' -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 + Write-Log "[probe:mcp-initialize] POST $mcpUri -> HTTP $($resp.StatusCode)" -Level Info + + $sessionId = $resp.Headers['Mcp-Session-Id'] + if ($sessionId -is [array]) { $sessionId = $sessionId[0] } + Write-Log "[probe:mcp-initialize] Mcp-Session-Id=$sessionId" -Level Info + Write-Log "[probe:mcp-initialize] body: $($resp.Content.Substring(0, [Math]::Min(1200, $resp.Content.Length)))" -Level Info + + if ($sessionId) { + $sessionHeaders = $headers.Clone() + $sessionHeaders['Mcp-Session-Id'] = $sessionId + Invoke-WebRequest -Uri $mcpUri -Method Post -Headers $sessionHeaders -Body '{"jsonrpc":"2.0","method":"notifications/initialized"}' -ContentType 'application/json' -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 | Out-Null + $toolsResp = Invoke-WebRequest -Uri $mcpUri -Method Post -Headers $sessionHeaders -Body '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' -ContentType 'application/json' -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 + Write-Log "[probe:mcp-tools] tools/list -> HTTP $($toolsResp.StatusCode)" -Level Info + Write-Log "[probe:mcp-tools] body: $($toolsResp.Content.Substring(0, [Math]::Min(2000, $toolsResp.Content.Length)))" -Level Info + } + } + catch { + Write-Log "[probe:mcp] EXCEPTION: $($_.Exception.Message)" -Level Warning + } + + Write-Log "===== END BC MCP DIAGNOSTICS =====" -Level Info +} + +Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync, Publish-MCPConfigApp, Get-BCMCPConnectionInfo, Write-BCMCPDiagnostics diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index d8b95850a..fee267a9a 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -112,6 +112,9 @@ if (-not $SkipContainer) { $mcpInfo = Get-BCMCPConnectionInfo -ContainerName $ContainerName Write-Log "BC MCP base URL: $($mcpInfo.BaseUrl) (company '$($mcpInfo.Company)')" -Level Info + # TEMPORARY: probe the endpoint from the host before the agent runs (remove before merge). + Write-BCMCPDiagnostics -BaseUrl $mcpInfo.BaseUrl -Company $mcpInfo.Company -Credential $credential + if ($env:GITHUB_ENV) { "BC_MCP_URL=$($mcpInfo.BaseUrl)" | Out-File -FilePath $env:GITHUB_ENV -Append "BC_MCP_COMPANY=$($mcpInfo.Company)" | Out-File -FilePath $env:GITHUB_ENV -Append diff --git a/src/bcbench/agent/shared/mcp.py b/src/bcbench/agent/shared/mcp.py index fd3da03f9..312e35b55 100644 --- a/src/bcbench/agent/shared/mcp.py +++ b/src/bcbench/agent/shared/mcp.py @@ -23,6 +23,18 @@ _BC_MCP_CONFIGURATION_NAME = "BCBench" +def _redact_mcp_config(mcp_config: dict[str, Any]) -> dict[str, Any]: + """Deep copy with any Authorization header masked, so DEBUG logs never leak container credentials.""" + import copy + + redacted = copy.deepcopy(mcp_config) + for server in redacted.get("mcpServers", {}).values(): + headers = server.get("headers") + if isinstance(headers, dict) and "Authorization" in headers: + headers["Authorization"] = "Basic ***REDACTED***" + return redacted + + def _build_server_entry(server: dict[str, Any], template_context: dict[str, Any]) -> tuple[str, dict[str, Any]]: server_type: str = server["type"] server_name: str = server["name"] @@ -139,6 +151,6 @@ def build_mcp_config( mcp_config = {"mcpServers": dict(map(lambda s: _build_server_entry(s, template_context), mcp_servers))} logger.info(f"Using MCP servers: {mcp_server_names}") - logger.debug(f"MCP configuration: {json.dumps(mcp_config, indent=2)}") + logger.debug(f"MCP configuration: {json.dumps(_redact_mcp_config(mcp_config), indent=2)}") return json.dumps(mcp_config, separators=(",", ":")), mcp_server_names diff --git a/tests/test_mcp_config.py b/tests/test_mcp_config.py index d83e1ab89..6388d6b03 100644 --- a/tests/test_mcp_config.py +++ b/tests/test_mcp_config.py @@ -169,6 +169,17 @@ def test_raises_when_bc_mcp_url_missing(self, entry, repo_path): with pytest.raises(AgentError): build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) + def test_redaction_masks_authorization_header(self): + from bcbench.agent.shared.mcp import _redact_mcp_config + + config = {"mcpServers": {"bcmcp": {"type": "http", "url": "u", "headers": {"Authorization": "Basic sekret", "Company": "Contoso"}}}} + redacted = _redact_mcp_config(config) + + assert redacted["mcpServers"]["bcmcp"]["headers"]["Authorization"] == "Basic ***REDACTED***" + assert redacted["mcpServers"]["bcmcp"]["headers"]["Company"] == "Contoso" + # Original is untouched (deep copy). + assert config["mcpServers"]["bcmcp"]["headers"]["Authorization"] == "Basic sekret" + class TestAltoolEnvForwarding: _MANAGED_VARS = ( From 63a62f9640eb4c3f9f1b42ab5069ba94c71f8ec7 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 19:38:10 +0200 Subject: [PATCH 27/52] TEMP: pull BC 29 sandbox artifact from insider feed (revert before merge) Data Query tools only exist in BC 29, which is not GA on the public artifact feed yet, so resolve the sandbox artifact from bcinsider (with insider EULA acceptance) and pass accept_insiderEula when building the container. Throwaway alongside the BC MCP diagnostics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- scripts/BCContainerManagement.psm1 | 2 ++ scripts/Setup-ContainerAndRepository.ps1 | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/BCContainerManagement.psm1 b/scripts/BCContainerManagement.psm1 index 3efb10a31..b2fb304b5 100644 --- a/scripts/BCContainerManagement.psm1 +++ b/scripts/BCContainerManagement.psm1 @@ -302,6 +302,8 @@ function New-BCContainerSync { shortcuts = 'None' memoryLimit = "16G" isolation = "hyperv" + # TEMPORARY (remove before merge): required to build a container from an insider (BC 29) artifact. + accept_insiderEula = $true } if ($AcceptEula) { diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index fee267a9a..4ed97572e 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -91,8 +91,9 @@ if (-not $SkipContainer) { Write-Log "Creating container $ContainerName for version $Version..." -Level Info - # Get BC artifact URL - [string] $url = Get-BCArtifactUrl -version $Version -Country $Country + # TEMPORARY (remove before merge): Data Query tools only exist in BC 29, which is not GA on the + # public feed yet, so pull the sandbox artifact from the insider feed. + [string] $url = Get-BCArtifactUrl -version $Version -Country $Country -select Latest -storageAccount bcinsider -accept_insiderEula Write-Log "Retrieved artifact URL: $url" -Level Info # Create container synchronously with NAV folder shared From c7181468db67960c579393a9b9fbb2e859942f65 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 20:08:33 +0200 Subject: [PATCH 28/52] Build the MCP config app inside the shared repo folder Compile-AppInBcContainer only accepts a project folder that is shared with the container; the app was being built under TEMP, which is not mounted, so publish failed with 'appProjectFolder ... is not shared with the container'. Build under BuildRoot (\, the mounted folder) like the query harness does, and clean it up after publish so nothing leaks into the agent workspace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- scripts/BCContainerManagement.psm1 | 21 ++++++++++++++++----- scripts/Setup-ContainerAndRepository.ps1 | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/BCContainerManagement.psm1 b/scripts/BCContainerManagement.psm1 index b2fb304b5..452f594b9 100644 --- a/scripts/BCContainerManagement.psm1 +++ b/scripts/BCContainerManagement.psm1 @@ -361,14 +361,20 @@ function Publish-MCPConfigApp { [string]$Version, [Parameter(Mandatory = $true)] - [PSCredential]$Credential + [PSCredential]$Credential, + + [Parameter(Mandatory = $true)] + [string]$BuildRoot ) Import-Module "$PSScriptRoot\AppUtils.psm1" -Force -DisableNameChecking [int]$major = ([System.Version]$Version).Major [string]$sourceFolder = Join-Path $PSScriptRoot "al\mcp-config-setup" - [string]$buildFolder = Join-Path ([System.IO.Path]::GetTempPath()) "bcbench-mcp-config-app" + # Build inside BuildRoot: Compile-AppInBcContainer only accepts a project folder that is shared with + # the container, and BuildRoot ($RepoPath) is the folder mounted into it. Cleaned up after publish so + # nothing leaks into the agent's workspace. + [string]$buildFolder = Join-Path $BuildRoot ".bcbench-mcp-config-app" if (Test-Path $buildFolder) { Remove-Item -Path $buildFolder -Recurse -Force @@ -380,9 +386,14 @@ function Publish-MCPConfigApp { [string]$appJsonPath = Join-Path $buildFolder "app.json" (Get-Content -Path $appJsonPath -Raw).Replace("__APP_VERSION__", "$major.0.0.0") | Set-Content -Path $appJsonPath -Encoding UTF8 - Write-Log "Publishing BC-Bench MCP config app to provision the MCP configuration..." -Level Info - Invoke-AppBuildAndPublish -containerName $ContainerName -appProjectFolder $buildFolder -credential $Credential -skipVerification -useDevEndpoint - Write-Log "BC MCP configuration 'BCBench' provisioned and activated." -Level Success + try { + Write-Log "Publishing BC-Bench MCP config app to provision the MCP configuration..." -Level Info + Invoke-AppBuildAndPublish -containerName $ContainerName -appProjectFolder $buildFolder -credential $Credential -skipVerification -useDevEndpoint + Write-Log "BC MCP configuration 'BCBench' provisioned and activated." -Level Success + } + finally { + Remove-Item -Path $buildFolder -Recurse -Force -ErrorAction SilentlyContinue + } } function Get-BCMCPConnectionInfo { diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index 4ed97572e..cfbc1f3f8 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -108,7 +108,7 @@ if (-not $SkipContainer) { # app that provisions and activates the 'BCBench' MCP configuration, then expose the endpoint and # company to the agent step so it can point its MCP client at the container. if ($Category -eq 'data-query') { - Publish-MCPConfigApp -ContainerName $ContainerName -Version $Version -Credential $credential + Publish-MCPConfigApp -ContainerName $ContainerName -Version $Version -Credential $credential -BuildRoot $RepoPath $mcpInfo = Get-BCMCPConnectionInfo -ContainerName $ContainerName Write-Log "BC MCP base URL: $($mcpInfo.BaseUrl) (company '$($mcpInfo.Company)')" -Level Info From 5693f082ff476952d21e76a3c984f374d6518c40 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 21:11:08 +0200 Subject: [PATCH 29/52] Pivot data-query to answer DATA, not just write a query (+ contamination canary) The query-generation contract let the model write a correct AL query from memory, so the BC MCP feedback loop was never exercised (verified in run 32511827133: 4/4 resolved but zero bc_data_query tool calls). Ask for the actual DATA instead: - Prompt: retrieve the real data with the BC data tools and write the rows to answer.json, plus the query used to query.al. The answer can't be fabricated, so a correct result requires genuinely querying the environment. - evaluate: run the gold query for the expected rows and compare them to the agent's answer.json rows (result_sets_match). No longer compiles/runs the agent's query; query.al is kept only as an inspection artifact. Gold still fails loud. - _load_answer_rows tolerates a bare array, a single object, or an OData value wrapper. TEMPORARY (revert before merge): a canary block in the prompt asks the agent to try reading ../dataset/dataquery.jsonl and `git show HEAD:dataset/dataquery.jsonl` and report to canary.txt, so we can settle from the logs whether the agent can reach the gold (the cwd sits inside the BC-Bench checkout whose root holds the dataset). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/config.yaml | 21 ++++++--- src/bcbench/evaluate/dataquery.py | 67 ++++++++++++++++++++-------- tests/test_dataquery_evaluation.py | 31 ++++++++++++- 3 files changed, 95 insertions(+), 24 deletions(-) diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index 114e49160..a91152225 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -104,15 +104,26 @@ prompt: If there are no findings, write an empty array. Write only valid JSON to `review.json`, with no surrounding object, Markdown fence, or commentary. data-query-template: | - Answer the following Business Central data question by writing a single AL `query` object - to a file named `query.al` in {{repo_path}}. Write only the query logic — do not add API - properties (QueryType, APIPublisher, APIGroup, APIVersion, EntityName, EntitySetName); the - evaluation harness adds those. Proceed without asking for confirmation. + Answer the following Business Central data question using the ACTUAL data from the connected + environment. You cannot answer from general knowledge — you must retrieve the real data with the + available Business Central data tools and report exactly what they return. If the data tools are + not immediately listed, discover them first (they may be provided as deferred/searchable tools). + + Write two files in {{repo_path}}: + - `answer.json`: a JSON array of the result rows that answer the question, one JSON object per row. + - `query.al`: the single AL query object you used to obtain the data. Question: {{task}} - You MUST write query.al before finishing; if you do not, there is no output to evaluate. + You MUST write answer.json before finishing; if you do not, there is no output to evaluate. + + --- + TEMPORARY DIAGNOSTIC (does not affect your answer — do it first, then continue): attempt to read + the file `../dataset/dataquery.jsonl` relative to your working directory, and separately attempt to + run `git show HEAD:dataset/dataquery.jsonl`. Write a short report to `canary.txt` in {{repo_path}} + stating, for EACH attempt, whether it SUCCEEDED or was DENIED/blocked, and if it succeeded paste the + first 200 characters you read. # controls: # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions//` diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 8509c30d9..8ae046474 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -1,10 +1,10 @@ +import json from collections.abc import Callable, Mapping, Sequence from decimal import Decimal, InvalidOperation from pathlib import Path from bcbench.dataset import DataQueryEntry from bcbench.evaluate.base import EvaluationPipeline -from bcbench.exceptions import BuildError, BuildTimeoutExpired from bcbench.github_actions import github_log_group from bcbench.logger import get_logger from bcbench.operations import clear_directory @@ -16,6 +16,33 @@ __all__ = ["DataQueryPipeline", "result_sets_match"] GENERATED_QUERY_FILE = "query.al" +ANSWER_FILE = "answer.json" + + +def _load_answer_rows(answer_file: Path) -> list[Mapping[str, object]]: + """Parse the agent's answer.json into a list of row objects. + + Accepts a bare JSON array, a single object (one row), or an object wrapping the rows under a + common key (``value``/``rows``/``data``/``results``) so a copied OData payload still works. + """ + try: + data = json.loads(answer_file.read_text(encoding="utf-8-sig") or "[]") + except json.JSONDecodeError as e: + raise ValueError(f"{ANSWER_FILE} is not valid JSON: {e}") from None + + if isinstance(data, dict): + wrapped = next((data[k] for k in ("value", "rows", "data", "results") if isinstance(data.get(k), list)), None) + data = wrapped if wrapped is not None else [data] + + if not isinstance(data, list): + raise TypeError(f"{ANSWER_FILE} must be a JSON array of row objects") + + rows: list[Mapping[str, object]] = [] + for row in data: + if not isinstance(row, dict): + raise TypeError(f"{ANSWER_FILE} rows must be JSON objects, got {type(row).__name__}") + rows.append(row) + return rows def _normalize_value(value: object) -> str: @@ -57,10 +84,12 @@ def result_sets_match(generated: Sequence[Mapping[str, object]], gold: Sequence[ class DataQueryPipeline(EvaluationPipeline[DataQueryEntry]): """Pipeline for the data-query category — generate an AL query, evaluate deterministically. - The agent writes an AL query to ``query.al``. Evaluation compiles + runs both the generated - query and the entry's gold query against the container's fixed (Contoso) dataset and compares - the result sets: build = the generated query compiled and ran; resolved = its result set - matches the gold query's. + The agent answers a data question by retrieving the ACTUAL data with the BC data tools and writing + the rows to ``answer.json`` (plus the ``query.al`` it used, kept only for inspection). Evaluation + runs the entry's gold query against the container's fixed (Contoso) dataset and compares the gold + rows to the agent's rows: build = the agent produced a well-formed answer.json; resolved = its rows + match the gold query's. The data can't be answered from model knowledge, so a correct answer requires + genuinely querying the environment. """ def setup_workspace(self, entry: DataQueryEntry, repo_path: Path) -> None: @@ -78,30 +107,32 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: from bcbench.operations import execute_al_query query_file = context.repo_path / GENERATED_QUERY_FILE + # query.al is only an inspection artifact now; scoring is on the data the agent retrieved. generated_query = query_file.read_text(encoding="utf-8").strip() if query_file.exists() else "" - - if not generated_query: - logger.warning(f"Agent produced no {GENERATED_QUERY_FILE} for {context.entry.instance_id}") - self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output="", error_message=f"No {GENERATED_QUERY_FILE} produced")) - return + answer_file = context.repo_path / ANSWER_FILE container = context.get_container() version = context.entry.environment_setup_version - # Validate the gold query first, and deliberately do NOT catch its failure: a gold that doesn't - # compile/run is a harness or dataset bug, not the agent's fault, so it must fail the run loudly - # and get fixed rather than being silently scored or excluded. + # Compute the expected data by running the gold query, and deliberately do NOT catch its failure: + # a gold that doesn't compile/run is a harness or dataset bug, not the agent's fault, so it must + # fail the run loudly and get fixed rather than being silently scored. gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + if not answer_file.exists(): + logger.warning(f"Agent produced no {ANSWER_FILE} for {context.entry.instance_id}") + self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=f"No {ANSWER_FILE} produced")) + return + try: - generated_rows = execute_al_query(generated_query, container, version, context.repo_path, "generated") - except (BuildError, BuildTimeoutExpired) as e: - logger.exception(f"Generated query failed to compile/run for {context.entry.instance_id}") + agent_rows = _load_answer_rows(answer_file) + except (ValueError, TypeError) as e: + logger.warning(f"Unusable {ANSWER_FILE} for {context.entry.instance_id}: {e}") self.save_result(context, ExecutionBasedEvaluationResult.create_build_failure(context, output=generated_query, error_message=str(e))) return - resolved = result_sets_match(generated_rows, gold_rows, context.entry.ordered) - error_message = None if resolved else f"Result set mismatch: generated {len(generated_rows)} rows vs gold {len(gold_rows)} rows" + resolved = result_sets_match(agent_rows, gold_rows, context.entry.ordered) + error_message = None if resolved else f"Result set mismatch: answer {len(agent_rows)} rows vs gold {len(gold_rows)} rows" result = ExecutionBasedEvaluationResult.create_result(context, output=generated_query, build=True, resolved=resolved, error_message=error_message) logger.info(f"{context.entry.instance_id}: build=True resolved={resolved}") self.save_result(context, result) diff --git a/tests/test_dataquery_evaluation.py b/tests/test_dataquery_evaluation.py index dabe01dce..355a476f7 100644 --- a/tests/test_dataquery_evaluation.py +++ b/tests/test_dataquery_evaluation.py @@ -2,7 +2,7 @@ import pytest -from bcbench.evaluate.dataquery import result_sets_match +from bcbench.evaluate.dataquery import _load_answer_rows, result_sets_match from bcbench.exceptions import BuildError from bcbench.operations import bc_operations, wrap_query_as_api from bcbench.types import ContainerConfig @@ -66,6 +66,35 @@ def test_numeric_string_not_coerced_to_number(self): assert not result_sets_match([{"Key": "1.0"}], [{"Key": 1}]) +class TestLoadAnswerRows: + def _write(self, tmp_path, content: str): + p = tmp_path / "answer.json" + p.write_text(content, encoding="utf-8") + return p + + def test_bare_array(self, tmp_path): + rows = _load_answer_rows(self._write(tmp_path, '[{"No": "C1", "Total": 100}]')) + assert rows == [{"No": "C1", "Total": 100}] + + def test_single_object_becomes_one_row(self, tmp_path): + assert _load_answer_rows(self._write(tmp_path, '{"Total": 42}')) == [{"Total": 42}] + + def test_odata_value_wrapper_unwrapped(self, tmp_path): + rows = _load_answer_rows(self._write(tmp_path, '{"@odata.context": "x", "value": [{"No": "C1"}]}')) + assert rows == [{"No": "C1"}] + + def test_empty_file_is_empty_list(self, tmp_path): + assert _load_answer_rows(self._write(tmp_path, "")) == [] + + def test_invalid_json_raises(self, tmp_path): + with pytest.raises(ValueError, match="not valid JSON"): + _load_answer_rows(self._write(tmp_path, "{not json")) + + def test_non_object_rows_raise(self, tmp_path): + with pytest.raises(TypeError, match="must be JSON objects"): + _load_answer_rows(self._write(tmp_path, "[1, 2, 3]")) + + class TestWrapQueryAsApi: PLAIN_QUERY = 'query 50100 MyQuery\n{\n QueryType = Normal;\n\n elements\n {\n dataitem(Customer; Customer)\n {\n column(No; "No.") { }\n }\n }\n}' LONG_NAME_QUERY = 'query 50100 "Items on Open Sales and Purchase Orders"\n{\n elements\n {\n dataitem(Item; Item)\n {\n column(No; "No.") { }\n }\n }\n}' From 1774a7504f96e37d54e662c4f8eb949244b4ae84 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 22:39:06 +0200 Subject: [PATCH 30/52] Bake gold data, bump build timeout, and remove BC MCP diagnostics 1. Remove the shakeout diagnostics (kept only the v29 insider hack): drop Write-BCMCPDiagnostics + its setup call, RUNNER_DEBUG=1, the *.log artifact upload, and the canary block in the data-query prompt. The auth-header redaction and the shared-folder app fix stay. 2. Bake gold data: DataQueryEntry gains gold_rows (precomputed expected rows). evaluate compares the agent's answer.json to gold_rows and only falls back to running the gold query live when they are not baked. New `bcbench dataset bake-dataquery-gold` command + bake-dataquery-gold.yml workflow populate gold_rows once against a container and commit them, so evals stop recompiling/publishing the gold query per run (the source of the 300s query-gold timeouts on insider 29). 3. Bump build_app timeout 300 -> 500s (headroom for the live/bake path). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/bake-dataquery-gold.yml | 76 +++++++++++++++++++++++ .github/workflows/claude-evaluation.yml | 6 +- .github/workflows/copilot-evaluation.yml | 6 +- scripts/BCContainerManagement.psm1 | 73 +--------------------- scripts/Setup-ContainerAndRepository.ps1 | 3 - src/bcbench/agent/shared/config.yaml | 7 --- src/bcbench/commands/dataset.py | 36 ++++++++++- src/bcbench/config.py | 2 +- src/bcbench/dataset/dataset_entry.py | 16 +++-- src/bcbench/evaluate/dataquery.py | 24 ++++--- 10 files changed, 141 insertions(+), 108 deletions(-) create mode 100644 .github/workflows/bake-dataquery-gold.yml diff --git a/.github/workflows/bake-dataquery-gold.yml b/.github/workflows/bake-dataquery-gold.yml new file mode 100644 index 000000000..b9bed2786 --- /dev/null +++ b/.github/workflows/bake-dataquery-gold.yml @@ -0,0 +1,76 @@ +name: Bake Data-Query Gold Rows + +# Runs each data-query gold query once against a BC container and stores the resulting rows as +# gold_rows in dataset/dataquery.jsonl, then commits them back to the branch. Evaluation then compares +# the agent's answer to these baked rows instead of recompiling/running the gold query live per run. + +on: + workflow_dispatch: + inputs: + git-ref: + description: "Branch to bake and commit to" + required: false + default: "" + +permissions: + contents: write + +env: + CATEGORY: data-query + +jobs: + get-entries: + uses: ./.github/workflows/get-entries.yml + with: + category: data-query + + bake: + needs: get-entries + runs-on: ${{ needs.get-entries.outputs.runner }} + if: needs.get-entries.outputs.entries != '[]' + timeout-minutes: 120 + environment: + name: ado-read + deployment: false + permissions: + contents: write + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ inputs.git-ref || github.ref_name }} + + # Provision a single BC container (all data-query entries share the same version + dataset). + - name: Setup BC Container + id: setup-env + timeout-minutes: 40 + uses: ./.github/actions/setup-bc-container-repo + with: + instance-id: ${{ fromJson(needs.get-entries.outputs.entries)[0] }} + category: data-query + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-container: "false" + skip-repo: "true" + + - name: Setup Python with UV + uses: ./.github/actions/setup-python-uv + + - name: Bake gold rows + run: uv run bcbench dataset bake-dataquery-gold --repo-path "${{ steps.setup-env.outputs.repo_path }}" + + - name: Commit baked dataset + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if (git diff --quiet -- dataset/dataquery.jsonl) { + Write-Output "No gold_rows changes to commit." + } + else { + git add dataset/dataquery.jsonl + git commit -m "Bake data-query gold rows (run ${{ github.run_id }})" + git push origin HEAD:${{ inputs.git-ref || github.ref_name }} + } + shell: pwsh diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index 1909e5538..afc845804 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -80,8 +80,6 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results - # TEMPORARY (remove before merge): force verbose Python + PowerShell logging for the BC MCP shakeout. - RUNNER_DEBUG: "1" jobs: pin-commit: @@ -179,9 +177,7 @@ jobs: if: always() with: name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} - path: | - ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl - ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log + path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index b2fc5c24a..9f09893be 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -85,8 +85,6 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results - # TEMPORARY (remove before merge): force verbose Python + PowerShell logging for the BC MCP shakeout. - RUNNER_DEBUG: "1" jobs: pin-commit: @@ -182,9 +180,7 @@ jobs: if: always() with: name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} - path: | - ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl - ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log + path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: diff --git a/scripts/BCContainerManagement.psm1 b/scripts/BCContainerManagement.psm1 index 452f594b9..a59bf5892 100644 --- a/scripts/BCContainerManagement.psm1 +++ b/scripts/BCContainerManagement.psm1 @@ -432,75 +432,4 @@ function Get-BCMCPConnectionInfo { } } -function Write-BCMCPDiagnostics { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$BaseUrl, - - [Parameter(Mandatory = $false)] - [string]$Company, - - [Parameter(Mandatory = $true)] - [PSCredential]$Credential, - - [Parameter(Mandatory = $false)] - [string]$ConfigurationName = 'BCBench' - ) - - # TEMPORARY diagnostics (remove before merge): probe the BC MCP endpoint from the HOST exactly as - # the evaluated agent will, so runner-only failures (reachability / auth / MCP config / tools) show - # up in the setup log instead of being guessed from the agent's discarded logs. Never throws. - $pair = "$($Credential.UserName):$($Credential.GetNetworkCredential().Password)" - $basic = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes($pair)) - - Write-Log "===== BC MCP DIAGNOSTICS =====" -Level Info - Write-Log "BaseUrl=$BaseUrl Company='$Company' ConfigurationName=$ConfigurationName" -Level Info - - # 1) API listener reachability + Basic auth sanity against a known-good endpoint. - try { - $companiesUri = "$BaseUrl/api/v2.0/companies" - $resp = Invoke-WebRequest -Uri $companiesUri -Headers @{ Authorization = $basic } -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 - Write-Log "[probe:companies] GET $companiesUri -> HTTP $($resp.StatusCode)" -Level Info - Write-Log "[probe:companies] body: $($resp.Content.Substring(0, [Math]::Min(600, $resp.Content.Length)))" -Level Info - } - catch { - Write-Log "[probe:companies] EXCEPTION: $($_.Exception.Message)" -Level Warning - } - - # 2) MCP endpoint: initialize, then tools/list if a session is granted. - $mcpUri = "$BaseUrl/mcp" - $headers = @{ - Authorization = $basic - ConfigurationName = $ConfigurationName - Accept = 'application/json, text/event-stream' - } - if ($Company) { $headers['Company'] = $Company } - - $initBody = '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"bcbench-diag","version":"1.0"}}}' - try { - $resp = Invoke-WebRequest -Uri $mcpUri -Method Post -Headers $headers -Body $initBody -ContentType 'application/json' -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 - Write-Log "[probe:mcp-initialize] POST $mcpUri -> HTTP $($resp.StatusCode)" -Level Info - - $sessionId = $resp.Headers['Mcp-Session-Id'] - if ($sessionId -is [array]) { $sessionId = $sessionId[0] } - Write-Log "[probe:mcp-initialize] Mcp-Session-Id=$sessionId" -Level Info - Write-Log "[probe:mcp-initialize] body: $($resp.Content.Substring(0, [Math]::Min(1200, $resp.Content.Length)))" -Level Info - - if ($sessionId) { - $sessionHeaders = $headers.Clone() - $sessionHeaders['Mcp-Session-Id'] = $sessionId - Invoke-WebRequest -Uri $mcpUri -Method Post -Headers $sessionHeaders -Body '{"jsonrpc":"2.0","method":"notifications/initialized"}' -ContentType 'application/json' -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 | Out-Null - $toolsResp = Invoke-WebRequest -Uri $mcpUri -Method Post -Headers $sessionHeaders -Body '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' -ContentType 'application/json' -SkipHttpErrorCheck -UseBasicParsing -TimeoutSec 60 - Write-Log "[probe:mcp-tools] tools/list -> HTTP $($toolsResp.StatusCode)" -Level Info - Write-Log "[probe:mcp-tools] body: $($toolsResp.Content.Substring(0, [Math]::Min(2000, $toolsResp.Content.Length)))" -Level Info - } - } - catch { - Write-Log "[probe:mcp] EXCEPTION: $($_.Exception.Message)" -Level Warning - } - - Write-Log "===== END BC MCP DIAGNOSTICS =====" -Level Info -} - -Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync, Publish-MCPConfigApp, Get-BCMCPConnectionInfo, Write-BCMCPDiagnostics +Export-ModuleMember -Function Test-Database, Set-AppVersion, Move-AppIntoDevScope, Initialize-ContainerForDevelopment, Test-ContainerExists, New-BCContainerSync, New-BCCompilerFolderSync, Publish-MCPConfigApp, Get-BCMCPConnectionInfo diff --git a/scripts/Setup-ContainerAndRepository.ps1 b/scripts/Setup-ContainerAndRepository.ps1 index cfbc1f3f8..b05203cbf 100644 --- a/scripts/Setup-ContainerAndRepository.ps1 +++ b/scripts/Setup-ContainerAndRepository.ps1 @@ -113,9 +113,6 @@ if (-not $SkipContainer) { $mcpInfo = Get-BCMCPConnectionInfo -ContainerName $ContainerName Write-Log "BC MCP base URL: $($mcpInfo.BaseUrl) (company '$($mcpInfo.Company)')" -Level Info - # TEMPORARY: probe the endpoint from the host before the agent runs (remove before merge). - Write-BCMCPDiagnostics -BaseUrl $mcpInfo.BaseUrl -Company $mcpInfo.Company -Credential $credential - if ($env:GITHUB_ENV) { "BC_MCP_URL=$($mcpInfo.BaseUrl)" | Out-File -FilePath $env:GITHUB_ENV -Append "BC_MCP_COMPANY=$($mcpInfo.Company)" | Out-File -FilePath $env:GITHUB_ENV -Append diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index a91152225..6195c1230 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -118,13 +118,6 @@ prompt: You MUST write answer.json before finishing; if you do not, there is no output to evaluate. - --- - TEMPORARY DIAGNOSTIC (does not affect your answer — do it first, then continue): attempt to read - the file `../dataset/dataquery.jsonl` relative to your working directory, and separately attempt to - run `git show HEAD:dataset/dataquery.jsonl`. Write a short report to `canary.txt` in {{repo_path}} - stating, for EACH attempt, whether it SUCCEEDED or was DENIED/blocked, and if it succeeded paste the - first 200 characters you read. - # controls: # 1. whether to copy custom instructions from `src/bcbench/agent/shared/instructions//` # - Copilot: copies to repo/.github/ and renames AGENTS.md to copilot-instructions.md diff --git a/src/bcbench/commands/dataset.py b/src/bcbench/commands/dataset.py index 11a27e2b3..bdd9e60f3 100644 --- a/src/bcbench/commands/dataset.py +++ b/src/bcbench/commands/dataset.py @@ -1,11 +1,13 @@ """CLI commands for dataset operations.""" import json +from pathlib import Path from typing import Annotated import typer -from bcbench.cli_options import EvaluationCategoryOption +from bcbench.cli_options import ContainerName, ContainerPassword, ContainerUsername, EvaluationCategoryOption +from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, CodeReviewEntry, RepoGroundedEntry from bcbench.dataset.dataset_entry import NL2ALEntry, _BugFixTestGenBase from bcbench.github_actions import write_step_outputs @@ -13,6 +15,7 @@ from bcbench.types import EvaluationCategory logger = get_logger(__name__) +_config = get_config() dataset_app = typer.Typer(help="Query and analyze dataset") @@ -182,6 +185,37 @@ def version( write_step_outputs({github_output: entry.environment_setup_version}) +@dataset_app.command("bake-dataquery-gold") +def bake_dataquery_gold( + container_name: ContainerName = "", + username: ContainerUsername = "", + password: ContainerPassword = "", + repo_path: Annotated[Path, typer.Option(help="Container-shared work folder for building the gold query app")] = _config.paths.testbed_path, + version: Annotated[str, typer.Option(help="BC version override; defaults to each entry's environment_setup_version")] = "", +) -> None: + """Run each data-query gold query once and store its result rows as gold_rows in the dataset. + + Populates the baked expected data so evaluation no longer compiles/runs the gold query live. + Requires a running BC container (repo_path must be shared with it). + """ + from bcbench.operations import execute_al_query + from bcbench.types import ContainerConfig + + dataset_path = EvaluationCategory.DATA_QUERY.dataset_path + entries = [json.loads(line) for line in dataset_path.read_text(encoding="utf-8").splitlines() if line.strip()] + container = ContainerConfig(container_name, username, password) + + for entry in entries: + entry_version = version or entry["environment_setup_version"] + logger.info(f"Baking gold rows for {entry['instance_id']} (v{entry_version})") + rows = execute_al_query(entry["gold_query"], container, entry_version, repo_path, "gold") + entry["gold_rows"] = rows + logger.info(f" -> {len(rows)} rows") + + dataset_path.write_text("\n".join(json.dumps(e, ensure_ascii=False) for e in entries) + "\n", encoding="utf-8") + logger.info(f"Baked gold rows for {len(entries)} entries into {dataset_path}") + + def _modified_instance_ids_from_diff(diff_output: str) -> list[str]: instance_ids = [] diff --git a/src/bcbench/config.py b/src/bcbench/config.py index c06cf4b77..9663fef51 100644 --- a/src/bcbench/config.py +++ b/src/bcbench/config.py @@ -87,7 +87,7 @@ def default(cls) -> TimeoutConfig: """Get default timeout configuration.""" return cls( build_baseapp=30 * 60, # 30 minutes for BaseApp compilation - build_app=5 * 60, # 5 minutes for application compilation + build_app=500, # application compilation/publish (headroom for insider builds) test_execution=3 * 60, # 3 minutes for test execution agent_execution=60 * 60, # 60 minutes for coding agent (claude and copilot) execution # Total bcal CLI budget per instance. diff --git a/src/bcbench/dataset/dataset_entry.py b/src/bcbench/dataset/dataset_entry.py index 606d510fb..256bdbde9 100644 --- a/src/bcbench/dataset/dataset_entry.py +++ b/src/bcbench/dataset/dataset_entry.py @@ -4,7 +4,7 @@ import re from abc import abstractmethod from pathlib import Path -from typing import Annotated, Literal, Self +from typing import Annotated, Any, Literal, Self from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -195,15 +195,21 @@ def get_expected_output(self) -> Checklist: class DataQueryEntry(BaseDatasetEntry): - """Dataset entry for the data-query category — generate an AL query that answers a data question. + """Dataset entry for the data-query category — answer a BC data question using the data tools. - Execution-based: the agent authors an AL query; evaluation compiles + runs both the generated - query and the gold query against a fixed dataset (Contoso in a BC container) and compares the - result sets. The workspace is scaffolded by the pipeline, so there is no repo or commit. + Execution-based: the agent retrieves the actual data (writing the rows to answer.json, plus the + query it used to query.al); evaluation compares those rows to the entry's expected rows. The + expected rows are the baked ``gold_rows`` (or, if not baked, computed by running ``gold_query`` + against the fixed Contoso container). The workspace is scaffolded by the pipeline, so there is no + repo or commit. """ nl_prompt: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] gold_query: Annotated[str, Field(min_length=1, pattern=r"^[^\x00]*$")] + # Precomputed expected result rows for gold_query, "baked" once against the fixed container dataset + # so evaluation compares the agent's data to these without recompiling/running the gold query live. + # Empty means not yet baked; evaluation then falls back to running the gold query. + gold_rows: list[dict[str, Any]] = Field(default_factory=list) # Whether row order is significant when comparing result sets (e.g. the question asks for a # specific ranking). Defaults to False: result sets are compared order-insensitively. ordered: bool = False diff --git a/src/bcbench/evaluate/dataquery.py b/src/bcbench/evaluate/dataquery.py index 8ae046474..0b1ad63a2 100644 --- a/src/bcbench/evaluate/dataquery.py +++ b/src/bcbench/evaluate/dataquery.py @@ -104,20 +104,12 @@ def run_agent(self, context: EvaluationContext[DataQueryEntry], agent_runner: Ca context.metrics, context.experiment = agent_runner(context) def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: - from bcbench.operations import execute_al_query - query_file = context.repo_path / GENERATED_QUERY_FILE # query.al is only an inspection artifact now; scoring is on the data the agent retrieved. generated_query = query_file.read_text(encoding="utf-8").strip() if query_file.exists() else "" answer_file = context.repo_path / ANSWER_FILE - container = context.get_container() - version = context.entry.environment_setup_version - - # Compute the expected data by running the gold query, and deliberately do NOT catch its failure: - # a gold that doesn't compile/run is a harness or dataset bug, not the agent's fault, so it must - # fail the run loudly and get fixed rather than being silently scored. - gold_rows = execute_al_query(context.entry.gold_query, container, version, context.repo_path, "gold") + gold_rows = self._gold_rows(context) if not answer_file.exists(): logger.warning(f"Agent produced no {ANSWER_FILE} for {context.entry.instance_id}") @@ -136,3 +128,17 @@ def evaluate(self, context: EvaluationContext[DataQueryEntry]) -> None: result = ExecutionBasedEvaluationResult.create_result(context, output=generated_query, build=True, resolved=resolved, error_message=error_message) logger.info(f"{context.entry.instance_id}: build=True resolved={resolved}") self.save_result(context, result) + + def _gold_rows(self, context: EvaluationContext[DataQueryEntry]) -> Sequence[Mapping[str, object]]: + """The expected rows: use the baked gold_rows if present, else compute them live (fail loud). + + A gold query that doesn't compile/run is a harness/dataset bug, not the agent's fault, so the + live fallback deliberately does NOT catch its failure — it must fail the run loudly. + """ + if context.entry.gold_rows: + return list(context.entry.gold_rows) + + from bcbench.operations import execute_al_query + + logger.info(f"No baked gold_rows for {context.entry.instance_id}; running gold query live") + return execute_al_query(context.entry.gold_query, context.get_container(), context.entry.environment_setup_version, context.repo_path, "gold") From 27c570f0ae4819e1804c598327b0cba0fc586533 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Fri, 21 Aug 2026 23:44:20 +0200 Subject: [PATCH 31/52] Capture Copilot tool usage from the JSON event stream tool_usage was always None for data-query: the pre-tool-use hook never sees sub-agent or MCP tool calls (the agent delegates BC data work to a task subagent, where bc_data_query runs), so nothing was logged. Parse tool usage from the authoritative copilot --output-format=json stream instead, counting tool.execution_start events by toolName (with lsp: sub-labels for parity). Verified locally that a subagent's inner tool call surfaces in the same stream alongside the task delegation. The hook remains a fallback when the stream carries no tool events. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/copilot/agent.py | 8 +++--- src/bcbench/agent/copilot/metrics.py | 21 +++++++++++++++ tests/test_copilot_metrics_parsing.py | 37 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index 1b6f84688..38ce409fd 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -122,9 +122,11 @@ def run_copilot_agent( if final_response: logger.info(final_response) - tool_usage: dict[str, int] | None = parse_tool_usage_from_hooks(tool_log_path) - if metrics and tool_usage: - metrics = metrics.model_copy(update={"tool_usage": tool_usage}) + # Tool usage now comes from the JSON event stream (tool.execution_start), which — unlike the + # pre-tool-use hook — also captures sub-agent and MCP tool calls. Fall back to the hook only if + # the stream carried none. + if metrics and not metrics.tool_usage and (hook_tool_usage := parse_tool_usage_from_hooks(tool_log_path)): + metrics = metrics.model_copy(update={"tool_usage": hook_tool_usage}) except subprocess.TimeoutExpired: logger.exception(f"Copilot CLI timed out after {_config.timeout.agent_execution} seconds") metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) diff --git a/src/bcbench/agent/copilot/metrics.py b/src/bcbench/agent/copilot/metrics.py index e268335c8..677cb0bb6 100644 --- a/src/bcbench/agent/copilot/metrics.py +++ b/src/bcbench/agent/copilot/metrics.py @@ -1,4 +1,5 @@ import json +from collections import Counter from collections.abc import Sequence from bcbench.logger import get_logger @@ -21,11 +22,25 @@ def _milliseconds_to_seconds(value: object) -> float | None: return None if milliseconds is None else milliseconds / 1000.0 +def _tool_label(data: dict) -> str | None: + """Tool name for a tool.execution_start event, sub-labelling LSP ops (lsp:) like the hook did.""" + tool_name = data.get("toolName") + if not isinstance(tool_name, str) or not tool_name: + return None + if tool_name == "lsp": + arguments = data.get("arguments") + if isinstance(arguments, dict) and isinstance(arguments.get("operation"), str): + return f"lsp:{arguments['operation']}" + return tool_name + + def parse_output(output_lines: Sequence[str]) -> tuple[AgentMetrics | None, str | None]: """Parse metrics and the agent's final response from `copilot --output-format=json` (JSONL) stdout. Relevant events (CLI 1.0.80): model.call_start: one per request sent to the model, so counting them yields the turn count. + tool.execution_start: one per tool invocation (including sub-agent and MCP tool calls, which + the pre-tool-use hook never sees), so counting them by ``toolName`` yields tool usage. session.usage_checkpoint: `data.totalNanoAiu` is cumulative for the session, so the last one wins. result: terminal event whose `usage` sits at the event root rather than under `data`. @@ -36,6 +51,7 @@ def parse_output(output_lines: Sequence[str]) -> tuple[AgentMetrics | None, str llm_duration: float | None = None ai_credits: float | None = None turn_count = 0 + tool_usage: Counter[str] = Counter() response: str | None = None final_response: str | None = None @@ -56,6 +72,10 @@ def parse_output(output_lines: Sequence[str]) -> tuple[AgentMetrics | None, str match event.get("type"): case "model.call_start": turn_count += 1 + case "tool.execution_start": + data = event.get("data") + if isinstance(data, dict) and (label := _tool_label(data)): + tool_usage[label] += 1 case "assistant.message": data = event.get("data") if not isinstance(data, dict): @@ -89,6 +109,7 @@ def parse_output(output_lines: Sequence[str]) -> tuple[AgentMetrics | None, str llm_duration=llm_duration, ai_credits=ai_credits, turn_count=turn_count or None, + tool_usage=dict(tool_usage) or None, ) else: logger.warning("No metrics found in Copilot JSON output") diff --git a/tests/test_copilot_metrics_parsing.py b/tests/test_copilot_metrics_parsing.py index 61fbd8977..204091289 100644 --- a/tests/test_copilot_metrics_parsing.py +++ b/tests/test_copilot_metrics_parsing.py @@ -78,3 +78,40 @@ def test_parse_output_without_metrics(): assert metrics is None assert response == "done" + + +def test_parse_output_counts_tool_usage_from_stream(): + output_lines = [ + _json_line({"type": "model.call_start", "data": {"turnId": "0"}}), + _json_line({"type": "tool.execution_start", "data": {"toolName": "bc_data_query", "arguments": {}}}), + _json_line({"type": "tool.execution_start", "data": {"toolName": "bc_data_query", "arguments": {}}}), + _json_line({"type": "tool.execution_start", "data": {"toolName": "task", "arguments": {}}}), + # A sub-agent's inner tool call surfaces in the same stream and must be counted too. + _json_line({"type": "tool.execution_start", "data": {"toolName": "view", "arguments": {"path": "x"}}}), + ] + + metrics, _ = parse_output(output_lines) + + assert metrics is not None + assert metrics.tool_usage == {"bc_data_query": 2, "task": 1, "view": 1} + + +def test_parse_output_sublabels_lsp_operations(): + output_lines = [ + _json_line({"type": "model.call_start", "data": {"turnId": "0"}}), + _json_line({"type": "tool.execution_start", "data": {"toolName": "lsp", "arguments": {"operation": "hover"}}}), + _json_line({"type": "tool.execution_start", "data": {"toolName": "lsp", "arguments": {"operation": "hover"}}}), + _json_line({"type": "tool.execution_start", "data": {"toolName": "lsp", "arguments": {"operation": "findReferences"}}}), + ] + + metrics, _ = parse_output(output_lines) + + assert metrics is not None + assert metrics.tool_usage == {"lsp:hover": 2, "lsp:findReferences": 1} + + +def test_parse_output_tool_usage_none_when_no_tools(): + metrics, _ = parse_output([_json_line({"type": "model.call_start", "data": {"turnId": "0"}})]) + + assert metrics is not None + assert metrics.tool_usage is None From eb4abc8474eb5ef1fbb2e976b618d747da92ea25 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Sat, 22 Aug 2026 11:05:07 +0200 Subject: [PATCH 32/52] Scrub BC connection vars from the agent env; temp-raise gold timeout The data-query agent was reaching the real data by shelling out (260 powershell calls vs 1 bc_data_query in the full run), reading BC_SERVER_*/BC_MCP_* from its environment and hitting BC's API directly -- bypassing the MCP server the benchmark is meant to exercise. Launch both CLI agents with agent_subprocess_env(), which drops BC_SERVER_*, BC_MCP_* and BC_CONTAINER_NAME from the process environment. MCP servers are unaffected: altool receives credentials through its MCP-config env block and the BC MCP server through its auth header, so connectivity via the MCP path is preserved while the direct-API side-door is closed. Also temporarily raise build_app 500 -> 900s: live gold compile/publish still blew 500s on 2/11 insider-29 entries. This is a stopgap until gold data is baked. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/claude/agent.py | 8 ++----- src/bcbench/agent/copilot/agent.py | 14 ++++++------ src/bcbench/agent/shared/__init__.py | 3 ++- src/bcbench/agent/shared/env.py | 18 +++++++++++++++ src/bcbench/config.py | 2 +- tests/test_agent_env.py | 34 ++++++++++++++++++++++++++++ 6 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 src/bcbench/agent/shared/env.py create mode 100644 tests/test_agent_env.py diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 094aade1d..2e4eab96e 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -1,5 +1,4 @@ import json -import os import shutil import subprocess from pathlib import Path @@ -7,7 +6,7 @@ import yaml from bcbench.agent.claude.metrics import parse_metrics -from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins +from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry from bcbench.exceptions import AgentError, AgentTimeoutError @@ -104,10 +103,7 @@ def run_claude_code( result = subprocess.run( cmd_args, cwd=str(repo_path), - env={ - **os.environ, - "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", - }, + env=agent_subprocess_env({"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"}), timeout=_config.timeout.agent_execution, check=True, capture_output=True, diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index 38ce409fd..6d9b37c62 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -1,6 +1,5 @@ """GitHub Copilot CLI Agent implementation.""" -import os import subprocess import sys from pathlib import Path @@ -8,7 +7,7 @@ import yaml from bcbench.agent.copilot.metrics import parse_output -from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins +from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins from bcbench.config import get_config from bcbench.copilot_cli import find_copilot from bcbench.dataset import BaseDatasetEntry @@ -102,11 +101,12 @@ def run_copilot_agent( result = subprocess.run( cmd_args, cwd=str(repo_path), - env={ - **os.environ, - "GITHUB_COPILOT_PROMPT_MODE_REPO_HOOKS": "true", - "GITHUB_COPILOT_PROMPT_MODE_WORKSPACE_MCP": "true", - }, + env=agent_subprocess_env( + { + "GITHUB_COPILOT_PROMPT_MODE_REPO_HOOKS": "true", + "GITHUB_COPILOT_PROMPT_MODE_WORKSPACE_MCP": "true", + } + ), capture_output=True, timeout=_config.timeout.agent_execution, check=True, diff --git a/src/bcbench/agent/shared/__init__.py b/src/bcbench/agent/shared/__init__.py index 50581fef4..d009d9b6c 100644 --- a/src/bcbench/agent/shared/__init__.py +++ b/src/bcbench/agent/shared/__init__.py @@ -1,9 +1,10 @@ """Shared code for CLI-based agents (Claude, Copilot).""" +from bcbench.agent.shared.env import agent_subprocess_env from bcbench.agent.shared.hooks_parser import parse_tool_usage_from_hooks from bcbench.agent.shared.lsp import build_al_lsp_plugin from bcbench.agent.shared.mcp import build_mcp_config from bcbench.agent.shared.plugin import resolve_config_plugins from bcbench.agent.shared.prompt import build_prompt -__all__ = ["build_al_lsp_plugin", "build_mcp_config", "build_prompt", "parse_tool_usage_from_hooks", "resolve_config_plugins"] +__all__ = ["agent_subprocess_env", "build_al_lsp_plugin", "build_mcp_config", "build_prompt", "parse_tool_usage_from_hooks", "resolve_config_plugins"] diff --git a/src/bcbench/agent/shared/env.py b/src/bcbench/agent/shared/env.py new file mode 100644 index 000000000..6989664c7 --- /dev/null +++ b/src/bcbench/agent/shared/env.py @@ -0,0 +1,18 @@ +import os + +# BC container connection details/credentials the harness uses to build the MCP config and to reach the +# container. They must NOT leak into a launched agent's own process environment: otherwise the agent can +# read the credentials and query BC's API directly from a shell, bypassing the MCP server the benchmark +# is meant to exercise. MCP servers still receive what they need through the MCP configuration (an +# embedded env block for altool, an auth header for the BC MCP server), so withholding these from the +# agent process closes the direct-API side-door without breaking MCP connectivity. +_WITHHELD_ENV_PREFIXES = ("BC_SERVER_", "BC_MCP_") +_WITHHELD_ENV_VARS = frozenset({"BC_CONTAINER_NAME"}) + + +def agent_subprocess_env(overrides: dict[str, str] | None = None) -> dict[str, str]: + """``os.environ`` for a launched agent, with the BC container connection vars removed.""" + env = {k: v for k, v in os.environ.items() if not k.startswith(_WITHHELD_ENV_PREFIXES) and k not in _WITHHELD_ENV_VARS} + if overrides: + env.update(overrides) + return env diff --git a/src/bcbench/config.py b/src/bcbench/config.py index 9663fef51..24e6bf0b3 100644 --- a/src/bcbench/config.py +++ b/src/bcbench/config.py @@ -87,7 +87,7 @@ def default(cls) -> TimeoutConfig: """Get default timeout configuration.""" return cls( build_baseapp=30 * 60, # 30 minutes for BaseApp compilation - build_app=500, # application compilation/publish (headroom for insider builds) + build_app=900, # TEMPORARY (revert with baking): live gold compile/publish blows 500s on insider 29 test_execution=3 * 60, # 3 minutes for test execution agent_execution=60 * 60, # 60 minutes for coding agent (claude and copilot) execution # Total bcal CLI budget per instance. diff --git a/tests/test_agent_env.py b/tests/test_agent_env.py new file mode 100644 index 000000000..7846dd7ef --- /dev/null +++ b/tests/test_agent_env.py @@ -0,0 +1,34 @@ +from bcbench.agent.shared.env import agent_subprocess_env + + +def test_scrubs_bc_connection_vars(monkeypatch): + monkeypatch.setenv("BC_SERVER_URL", "http://bcbench-sales") + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") + monkeypatch.setenv("BC_MCP_COMPANY", "CRONUS") + monkeypatch.setenv("BC_CONTAINER_NAME", "bcbench-sales") + + env = agent_subprocess_env() + + assert not any(k.startswith(("BC_SERVER_", "BC_MCP_")) for k in env) + assert "BC_CONTAINER_NAME" not in env + + +def test_preserves_other_vars(monkeypatch): + monkeypatch.setenv("PATH", "/usr/bin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + + env = agent_subprocess_env() + + assert env["PATH"] == "/usr/bin" + assert "BC_SERVER_PASSWORD" not in env + + +def test_overrides_are_applied(monkeypatch): + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + + env = agent_subprocess_env({"FLAG": "on"}) + + assert env["FLAG"] == "on" + assert "BC_SERVER_PASSWORD" not in env From 1f8515fb2a430d5af03dd4ec63c35f5dc9f4c83f Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Sat, 22 Aug 2026 11:38:25 +0200 Subject: [PATCH 33/52] TEMP: re-add agent *.log upload to diagnose BC MCP tool use (revert before merge) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/claude-evaluation.yml | 5 ++++- .github/workflows/copilot-evaluation.yml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index afc845804..ed9ecd1e5 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -177,7 +177,10 @@ jobs: if: always() with: name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} - path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + # TEMPORARY (remove before merge): include agent *.log to diagnose BC MCP tool use. + path: | + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index 9f09893be..66e387952 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -180,7 +180,10 @@ jobs: if: always() with: name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} - path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + # TEMPORARY (remove before merge): include agent *.log to diagnose BC MCP tool use. + path: | + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl + ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: From 50dcda33021177092f2da48d4eb721e4f3cbde88 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 11:31:33 +0200 Subject: [PATCH 34/52] Add localhost MCP gateway to isolate agent from BC /api (Phase A) Front the BC MCP endpoint with an in-process localhost gateway that path-restricts to /mcp and injects Basic auth / Company / ConfigurationName upstream. The agent's MCP config now carries only a credential-free http://127.0.0.1:/BC/mcp URL, so scraping the launched process command line yields no BC credentials and /api stays unreachable through the proxy. Phase A keeps the agent on the host (proxy runs in-process, no container); this validates the proxy contract (auth injection + path filter + SSE streaming) before Phase B moves the agent into an isolated container. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/claude/agent.py | 17 +- src/bcbench/agent/copilot/agent.py | 17 +- src/bcbench/agent/shared/__init__.py | 3 +- src/bcbench/agent/shared/config.yaml | 8 +- src/bcbench/agent/shared/mcp.py | 40 ++--- src/bcbench/agent/shared/mcp_gateway.py | 203 ++++++++++++++++++++++++ tests/test_mcp_config.py | 50 ++---- tests/test_mcp_gateway.py | 139 ++++++++++++++++ 8 files changed, 403 insertions(+), 74 deletions(-) create mode 100644 src/bcbench/agent/shared/mcp_gateway.py create mode 100644 tests/test_mcp_gateway.py diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index 2e4eab96e..ee9ed3e7a 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -6,7 +6,7 @@ import yaml from bcbench.agent.claude.metrics import parse_metrics -from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins +from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins, start_bc_mcp_gateway from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry from bcbench.exceptions import AgentError, AgentTimeoutError @@ -46,7 +46,17 @@ def run_claude_code( logger.info(f"Running Claude Code on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, claude_config, category, al_mcp=al_mcp) - mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, bc_mcp=bc_mcp, ms_learn_mcp=ms_learn_mcp, container_name=container_name) + bc_gateway = start_bc_mcp_gateway(bc_mcp) + mcp_config_json, mcp_server_names = build_mcp_config( + claude_config, + entry, + repo_path, + al_mcp=al_mcp, + bc_mcp=bc_mcp, + ms_learn_mcp=ms_learn_mcp, + container_name=container_name, + bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, + ) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.CLAUDE, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE) skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, harness=AgentHarness.CLAUDE, skills_enabled_override=skills) @@ -139,3 +149,6 @@ def run_claude_code( raise else: return metrics, config + finally: + if bc_gateway is not None: + bc_gateway.stop() diff --git a/src/bcbench/agent/copilot/agent.py b/src/bcbench/agent/copilot/agent.py index 6d9b37c62..78a68aac8 100644 --- a/src/bcbench/agent/copilot/agent.py +++ b/src/bcbench/agent/copilot/agent.py @@ -7,7 +7,7 @@ import yaml from bcbench.agent.copilot.metrics import parse_output -from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins +from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins, start_bc_mcp_gateway from bcbench.config import get_config from bcbench.copilot_cli import find_copilot from bcbench.dataset import BaseDatasetEntry @@ -50,7 +50,17 @@ def run_copilot_agent( logger.info(f"Running GitHub Copilot CLI on: {entry.instance_id}") prompt: str = build_prompt(entry, repo_path, copilot_config, category, al_mcp=al_mcp) - mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, bc_mcp=bc_mcp, ms_learn_mcp=ms_learn_mcp, container_name=container_name) + bc_gateway = start_bc_mcp_gateway(bc_mcp) + mcp_config_json, mcp_server_names = build_mcp_config( + copilot_config, + entry, + repo_path, + al_mcp=al_mcp, + bc_mcp=bc_mcp, + ms_learn_mcp=ms_learn_mcp, + container_name=container_name, + bc_mcp_gateway_url=bc_gateway.base_url if bc_gateway else None, + ) lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentHarness.COPILOT, al_lsp=al_lsp, container_name=container_name) instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT) skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, harness=AgentHarness.COPILOT, skills_enabled_override=skills) @@ -139,3 +149,6 @@ def run_copilot_agent( raise else: return metrics, config + finally: + if bc_gateway is not None: + bc_gateway.stop() diff --git a/src/bcbench/agent/shared/__init__.py b/src/bcbench/agent/shared/__init__.py index d009d9b6c..37607c960 100644 --- a/src/bcbench/agent/shared/__init__.py +++ b/src/bcbench/agent/shared/__init__.py @@ -4,7 +4,8 @@ from bcbench.agent.shared.hooks_parser import parse_tool_usage_from_hooks from bcbench.agent.shared.lsp import build_al_lsp_plugin from bcbench.agent.shared.mcp import build_mcp_config +from bcbench.agent.shared.mcp_gateway import start_bc_mcp_gateway from bcbench.agent.shared.plugin import resolve_config_plugins from bcbench.agent.shared.prompt import build_prompt -__all__ = ["agent_subprocess_env", "build_al_lsp_plugin", "build_mcp_config", "build_prompt", "parse_tool_usage_from_hooks", "resolve_config_plugins"] +__all__ = ["agent_subprocess_env", "build_al_lsp_plugin", "build_mcp_config", "build_prompt", "parse_tool_usage_from_hooks", "resolve_config_plugins", "start_bc_mcp_gateway"] diff --git a/src/bcbench/agent/shared/config.yaml b/src/bcbench/agent/shared/config.yaml index 6195c1230..7caa2ea7f 100644 --- a/src/bcbench/agent/shared/config.yaml +++ b/src/bcbench/agent/shared/config.yaml @@ -192,13 +192,13 @@ mcp: "{{package_cache_path}}", ] - # Business Central MCP server (toggled via --bc-mcp CLI flag). url + auth/company headers are - # filled in programmatically by mcp.py from the container connection env vars. Which tools the - # server exposes is decided server-side by the MCP configuration the setup-time AL app provisions. + # Business Central MCP server (toggled via --bc-mcp CLI flag). The url is filled in + # programmatically by mcp.py to point at a localhost gateway (mcp_gateway.py) that injects auth + # upstream, so no credentials appear here. Which tools the server exposes is decided server-side by + # the MCP configuration the setup-time AL app provisions. - name: "bcmcp" type: "http" url: "" - headers: {} # Microsoft Learn MCP (toggled via --ms-learn-mcp CLI flag): official AL docs + code samples the # bc-al-query-mcp skill grounds its AL syntax in. Public server, no auth. diff --git a/src/bcbench/agent/shared/mcp.py b/src/bcbench/agent/shared/mcp.py index 312e35b55..a309214bc 100644 --- a/src/bcbench/agent/shared/mcp.py +++ b/src/bcbench/agent/shared/mcp.py @@ -1,4 +1,3 @@ -import base64 import json import os import shutil @@ -19,8 +18,6 @@ # Server names for the independently-toggled MCP servers. _BC_MCP_SERVER_NAME = "bcmcp" _MS_LEARN_MCP_SERVER_NAME = "mslearn" -# Must match the configuration name the setup-time AL app creates (scripts/al/mcp-config-setup). -_BC_MCP_CONFIGURATION_NAME = "BCBench" def _redact_mcp_config(mcp_config: dict[str, Any]) -> dict[str, Any]: @@ -67,33 +64,19 @@ def _build_server_entry(server: dict[str, Any], template_context: dict[str, Any] raise AgentError(f"Unsupported MCP server type: {server_type}") -def _configure_bc_mcp_server(server: dict[str, Any]) -> None: - """Fill the BC MCP server's endpoint + auth/company headers from the container connection env vars. +def _configure_bc_mcp_server(server: dict[str, Any], gateway_base_url: str | None) -> None: + """Point the BC MCP server at the local credential-free gateway. - ``BC_MCP_URL``/``BC_MCP_COMPANY`` are exported by ``Setup-ContainerAndRepository.ps1``. Auth is - Basic over the reused ``BC_SERVER_*`` credentials (validated against NAV's MCP client). Omitting the - Company header would switch the server into cross-company dynamic mode, so it is only sent when a - company is known. Which tools the server exposes is decided server-side by the MCP configuration - the setup-time AL app provisions, not here. + The gateway (``mcp_gateway.py``) fronts the real BC MCP endpoint: it injects the Basic auth / + Company / ConfigurationName headers upstream and rejects any non-``/mcp`` path. So the agent's MCP + config carries only a ``http://127.0.0.1:/.../mcp`` URL with no credentials -- nothing the + agent can replay against BC's ``/api`` or scrape from the launched process command line. """ - base_url = os.environ.get("BC_MCP_URL") - if not base_url: - raise AgentError("BC MCP requested but BC_MCP_URL is not set; container setup must export it.") + if not gateway_base_url: + raise AgentError("BC MCP requested but the local MCP gateway URL is unavailable.") - username = os.environ.get("BC_SERVER_USERNAME", "") - password = os.environ.get("BC_SERVER_PASSWORD", "") - basic_auth = base64.b64encode(f"{username}:{password}".encode()).decode() - - headers: dict[str, str] = { - "Authorization": f"Basic {basic_auth}", - "ConfigurationName": _BC_MCP_CONFIGURATION_NAME, - } - company = os.environ.get("BC_MCP_COMPANY") - if company: - headers["Company"] = company - - server["url"] = base_url.rstrip("/") + "/mcp" - server["headers"] = headers + server["url"] = gateway_base_url.rstrip("/") + "/mcp" + server.pop("headers", None) def build_mcp_config( @@ -104,6 +87,7 @@ def build_mcp_config( bc_mcp: bool = False, ms_learn_mcp: bool = False, container_name: str = "bcbench", + bc_mcp_gateway_url: str | None = None, ) -> tuple[str | None, list[str] | None]: mcp_servers: list[dict[str, Any]] = config.get("mcp", {}).get("servers", []) @@ -122,7 +106,7 @@ def build_mcp_config( template_context: dict[str, str | Path] = {"repo_path": repo_path} if bc_mcp: - _configure_bc_mcp_server(next(s for s in mcp_servers if s["name"] == _BC_MCP_SERVER_NAME)) + _configure_bc_mcp_server(next(s for s in mcp_servers if s["name"] == _BC_MCP_SERVER_NAME), bc_mcp_gateway_url) if al_mcp: compiler_folder, symbols_folder = compiler_symbol_folder_for_container(container_name) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py new file mode 100644 index 000000000..99e4108b4 --- /dev/null +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -0,0 +1,203 @@ +"""A localhost MCP gateway that fronts the BC MCP endpoint for a benchmarked agent. + +Why this exists: the agent must reach BC *only* through the MCP server, never the raw OData ``/api`` +or the SQL database. The BC container serves ``/api`` and ``/mcp`` on the same port, so a plain +firewall cannot separate them, and putting the Basic credentials in the agent's MCP config leaks them +onto the agent process command line (recoverable via ``Get-CimInstance Win32_Process``), which the +agent could replay against ``/api``. + +This gateway closes both holes: it path-restricts to ``/mcp`` (everything else -> 403) and injects the +Basic auth / Company / ConfigurationName headers itself, so the agent's MCP config carries only a +credential-free ``http://127.0.0.1:/.../mcp`` URL. The upstream endpoint and credentials come +from the harness environment (``BC_MCP_URL`` / ``BC_SERVER_*`` / ``BC_MCP_COMPANY``), which is never +scrubbed for the harness itself. +""" + +import base64 +import os +import threading +from http.client import HTTPConnection +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +from bcbench.exceptions import AgentError +from bcbench.logger import get_logger + +logger = get_logger(__name__) + +# Must match the configuration name the setup-time AL app creates (scripts/al/mcp-config-setup). +_CONFIGURATION_NAME = "BCBench" + +# Connection-level headers that must not be forwarded across a proxy hop (RFC 7230 6.1), plus the +# framing/credential headers this gateway sets itself. +_HOP_BY_HOP = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } +) +_STRIPPED_REQUEST_HEADERS = _HOP_BY_HOP | {"host", "content-length", "accept-encoding", "authorization", "company", "configurationname"} + +_UPSTREAM_TIMEOUT_SECONDS = 600 +_STREAM_CHUNK_BYTES = 8192 + + +class BcMcpGateway: + def __init__(self, upstream_url: str, username: str, password: str, company: str | None) -> None: + split = urlsplit(upstream_url) + if not split.hostname: + raise AgentError(f"BC MCP upstream URL is malformed: {upstream_url!r}") + + self._origin_host: str = split.hostname + self._origin_port: int = split.port or (443 if split.scheme == "https" else 80) + base_path: str = split.path.rstrip("/") + self._mcp_path: str = f"{base_path}/mcp" + self._base_path: str = base_path + + injected: dict[str, str] = { + "Authorization": f"Basic {base64.b64encode(f'{username}:{password}'.encode()).decode()}", + "ConfigurationName": _CONFIGURATION_NAME, + } + if company: + injected["Company"] = company + self._injected_headers: dict[str, str] = injected + + self._server: ThreadingHTTPServer | None = None + self._thread: threading.Thread | None = None + self._lock = threading.Lock() + self._forwarded_count = 0 + self.base_url: str | None = None + + @property + def forwarded_count(self) -> int: + with self._lock: + return self._forwarded_count + + def _note_forwarded(self) -> None: + with self._lock: + self._forwarded_count += 1 + + def start(self) -> "BcMcpGateway": + gateway = self + server = ThreadingHTTPServer(("127.0.0.1", 0), _build_handler(gateway)) + self._server = server + port = server.server_address[1] + self.base_url = f"http://127.0.0.1:{port}{self._base_path}" + self._thread = threading.Thread(target=server.serve_forever, name="bc-mcp-gateway", daemon=True) + self._thread.start() + return self + + def stop(self) -> None: + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + if self._thread is not None: + self._thread.join(timeout=5) + self._thread = None + logger.info(f"BC MCP gateway forwarded {self.forwarded_count} request(s) to the BC MCP endpoint") + + +def _build_handler(gateway: BcMcpGateway) -> type[BaseHTTPRequestHandler]: + class _ProxyHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log + pass + + def _path_allowed(self) -> bool: + path_only = self.path.split("?", 1)[0] + return path_only == gateway._mcp_path or path_only.startswith(gateway._mcp_path + "/") + + def _handle(self) -> None: + if not self._path_allowed(): + self.send_error(403, "Forbidden") + return + + length = self.headers.get("Content-Length") + body: bytes | None = self.rfile.read(int(length)) if length else None + + request_headers: dict[str, str] = {k: v for k, v in self.headers.items() if k.lower() not in _STRIPPED_REQUEST_HEADERS} + request_headers["Host"] = f"{gateway._origin_host}:{gateway._origin_port}" + request_headers.update(gateway._injected_headers) + + connection = HTTPConnection(gateway._origin_host, gateway._origin_port, timeout=_UPSTREAM_TIMEOUT_SECONDS) + try: + connection.request(self.command, self.path, body=body, headers=request_headers) + response = connection.getresponse() + gateway._note_forwarded() + self._relay(response) + except Exception: + logger.exception("BC MCP gateway failed to reach the upstream endpoint") + self.send_error(502, "Bad Gateway") + finally: + connection.close() + + def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse + self.send_response_only(response.status) + content_length: str | None = None + for key, value in response.getheaders(): + lowered = key.lower() + if lowered == "content-length": + content_length = value + continue + if lowered in _HOP_BY_HOP: + continue + self.send_header(key, value) + + if content_length is not None: + self.send_header("Content-Length", content_length) + self.end_headers() + remaining = int(content_length) + while remaining > 0: + chunk = response.read(min(_STREAM_CHUNK_BYTES, remaining)) + if not chunk: + break + self.wfile.write(chunk) + remaining -= len(chunk) + else: + # No content length -> stream (e.g. SSE) with our own chunked framing, flushing each + # block so server-sent events reach the agent as they arrive. + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + while True: + chunk = response.read(_STREAM_CHUNK_BYTES) + if not chunk: + break + self.wfile.write(b"%X\r\n" % len(chunk)) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + do_GET = _handle + do_POST = _handle + do_DELETE = _handle + + return _ProxyHandler + + +def start_bc_mcp_gateway(enabled: bool) -> BcMcpGateway | None: + """Start a localhost MCP gateway in front of the BC container, or return None when disabled.""" + if not enabled: + return None + + upstream = os.environ.get("BC_MCP_URL") + if not upstream: + raise AgentError("BC MCP requested but BC_MCP_URL is not set; container setup must export it.") + + gateway = BcMcpGateway( + upstream_url=upstream, + username=os.environ.get("BC_SERVER_USERNAME", ""), + password=os.environ.get("BC_SERVER_PASSWORD", ""), + company=os.environ.get("BC_MCP_COMPANY"), + ).start() + logger.info(f"BC MCP gateway listening at {gateway.base_url}/mcp (credential-free; path-restricted to /mcp)") + return gateway diff --git a/tests/test_mcp_config.py b/tests/test_mcp_config.py index 6388d6b03..435a22093 100644 --- a/tests/test_mcp_config.py +++ b/tests/test_mcp_config.py @@ -1,4 +1,3 @@ -import base64 import json from copy import deepcopy from pathlib import Path @@ -105,12 +104,7 @@ def test_returns_server_names(self, entry, repo_path): class TestBcMcp: - _VARS = ("BC_MCP_URL", "BC_MCP_COMPANY", "BC_SERVER_USERNAME", "BC_SERVER_PASSWORD") - - @pytest.fixture(autouse=True) - def _clean_env(self, monkeypatch): - for var in self._VARS: - monkeypatch.delenv(var, raising=False) + _GATEWAY_URL = "http://127.0.0.1:54321/BC" def test_bcmcp_excluded_when_disabled(self, entry, repo_path): assert build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=False) == (None, None) @@ -118,54 +112,36 @@ def test_bcmcp_excluded_when_disabled(self, entry, repo_path): def test_mslearn_excluded_when_disabled(self, entry, repo_path): assert build_mcp_config(_make_config(MSLEARN_SERVER), entry, repo_path, ms_learn_mcp=False) == (None, None) - def test_flags_are_independent(self, entry, repo_path, monkeypatch): - monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") - monkeypatch.setenv("BC_SERVER_USERNAME", "admin") - monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + def test_flags_are_independent(self, entry, repo_path): config = _make_config(BCMCP_SERVER, MSLEARN_SERVER) # bc-mcp only -> no mslearn - _, bc_only = build_mcp_config(config, entry, repo_path, bc_mcp=True, ms_learn_mcp=False) + _, bc_only = build_mcp_config(config, entry, repo_path, bc_mcp=True, ms_learn_mcp=False, bc_mcp_gateway_url=self._GATEWAY_URL) assert bc_only == ["bcmcp"] - # ms-learn only -> no bcmcp (and no BC connection env needed) + # ms-learn only -> no bcmcp (and no gateway needed) _, learn_only = build_mcp_config(config, entry, repo_path, bc_mcp=False, ms_learn_mcp=True) assert learn_only == ["mslearn"] # both -> both - _, both = build_mcp_config(config, entry, repo_path, bc_mcp=True, ms_learn_mcp=True) + _, both = build_mcp_config(config, entry, repo_path, bc_mcp=True, ms_learn_mcp=True, bc_mcp_gateway_url=self._GATEWAY_URL) assert set(both) == {"bcmcp", "mslearn"} def test_mslearn_url_passthrough(self, entry, repo_path): config_json, _ = build_mcp_config(_make_config(MSLEARN_SERVER), entry, repo_path, ms_learn_mcp=True) assert json.loads(config_json)["mcpServers"]["mslearn"]["url"] == "https://learn.microsoft.com/api/mcp" - def test_bcmcp_endpoint_and_auth_headers(self, entry, repo_path, monkeypatch): - monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") - monkeypatch.setenv("BC_SERVER_USERNAME", "admin") - monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") - monkeypatch.setenv("BC_MCP_COMPANY", "CRONUS International Ltd.") - - config_json, _ = build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) + def test_bcmcp_points_at_gateway_without_credentials(self, entry, repo_path): + config_json, _ = build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True, bc_mcp_gateway_url=self._GATEWAY_URL) bcmcp = json.loads(config_json)["mcpServers"]["bcmcp"] - assert bcmcp["url"] == "http://172.17.0.2:7048/BC/mcp" - expected_auth = "Basic " + base64.b64encode(b"admin:secret").decode() - assert bcmcp["headers"]["Authorization"] == expected_auth - assert bcmcp["headers"]["ConfigurationName"] == "BCBench" - assert bcmcp["headers"]["Company"] == "CRONUS International Ltd." - - def test_company_header_omitted_when_unset(self, entry, repo_path, monkeypatch): - monkeypatch.setenv("BC_MCP_URL", "http://172.17.0.2:7048/BC") - monkeypatch.setenv("BC_SERVER_USERNAME", "admin") - monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") - - config_json, _ = build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) - headers = json.loads(config_json)["mcpServers"]["bcmcp"]["headers"] - - assert "Company" not in headers + assert bcmcp["url"] == "http://127.0.0.1:54321/BC/mcp" + # The gateway injects auth upstream, so the agent config carries no credentials or headers. + assert "headers" not in bcmcp + assert "Authorization" not in config_json + assert "Basic" not in config_json - def test_raises_when_bc_mcp_url_missing(self, entry, repo_path): + def test_raises_when_gateway_url_missing(self, entry, repo_path): with pytest.raises(AgentError): build_mcp_config(_make_config(BCMCP_SERVER), entry, repo_path, bc_mcp=True) diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py new file mode 100644 index 000000000..e531a45ed --- /dev/null +++ b/tests/test_mcp_gateway.py @@ -0,0 +1,139 @@ +import base64 +import json +import threading +from http.client import HTTPConnection +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +import pytest + +from bcbench.agent.shared.mcp_gateway import start_bc_mcp_gateway +from bcbench.exceptions import AgentError + + +class _UpstreamHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log + pass + + def _record(self) -> None: + self.server.last_headers = dict(self.headers.items()) + self.server.last_path = self.path + self.server.last_method = self.command + length = self.headers.get("Content-Length") + self.server.last_body = self.rfile.read(int(length)) if length else b"" + + def do_POST(self) -> None: + self._record() + payload = json.dumps({"jsonrpc": "2.0", "id": 1, "result": {"ok": True}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Mcp-Session-Id", "sess-123") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: + self._record() + # Stream an SSE response with no Content-Length, ended by closing the connection. + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(b"event: message\ndata: one\n\n") + self.wfile.flush() + self.wfile.write(b"event: message\ndata: two\n\n") + self.wfile.flush() + + +@pytest.fixture +def upstream(): + server = ThreadingHTTPServer(("127.0.0.1", 0), _UpstreamHandler) + server.last_headers = {} + server.last_path = None + server.last_method = None + server.last_body = b"" + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield server + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.fixture +def gateway(upstream, monkeypatch): + port = upstream.server_address[1] + monkeypatch.setenv("BC_MCP_URL", f"http://127.0.0.1:{port}/BC") + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + monkeypatch.setenv("BC_MCP_COMPANY", "CRONUS International Ltd.") + gw = start_bc_mcp_gateway(enabled=True) + yield gw + gw.stop() + + +def _request(base_url: str, method: str, path: str, body: bytes | None = None): + split = urlsplit(base_url) + conn = HTTPConnection(split.hostname, split.port, timeout=10) + try: + conn.request(method, path, body=body) + response = conn.getresponse() + return response.status, dict(response.getheaders()), response.read() + finally: + conn.close() + + +class TestBcMcpGateway: + def test_disabled_returns_none(self): + assert start_bc_mcp_gateway(enabled=False) is None + + def test_raises_without_upstream_url(self, monkeypatch): + monkeypatch.delenv("BC_MCP_URL", raising=False) + with pytest.raises(AgentError): + start_bc_mcp_gateway(enabled=True) + + def test_base_url_mirrors_upstream_path(self, gateway): + assert gateway.base_url.endswith("/BC") + assert gateway.base_url.startswith("http://127.0.0.1:") + + def test_forwards_mcp_post_and_injects_credentials(self, gateway, upstream): + status, _headers, body = _request(gateway.base_url, "POST", "/BC/mcp", body=b'{"jsonrpc":"2.0"}') + + assert status == 200 + assert json.loads(body)["result"] == {"ok": True} + # The upstream saw the injected credentials/headers, not the (credential-free) agent request. + expected_auth = "Basic " + base64.b64encode(b"admin:secret").decode() + assert upstream.last_headers["Authorization"] == expected_auth + assert upstream.last_headers["ConfigurationName"] == "BCBench" + assert upstream.last_headers["Company"] == "CRONUS International Ltd." + assert upstream.last_path == "/BC/mcp" + assert upstream.last_body == b'{"jsonrpc":"2.0"}' + + def test_passes_through_response_headers(self, gateway): + _status, headers, _body = _request(gateway.base_url, "POST", "/BC/mcp", body=b"{}") + assert headers.get("Mcp-Session-Id") == "sess-123" + + def test_streams_sse_response(self, gateway): + status, headers, body = _request(gateway.base_url, "GET", "/BC/mcp") + assert status == 200 + assert headers["Content-Type"] == "text/event-stream" + assert b"data: one" in body + assert b"data: two" in body + + def test_rejects_non_mcp_path(self, gateway, upstream): + status, _headers, _body = _request(gateway.base_url, "GET", "/BC/api/v2.0/companies") + assert status == 403 + # A blocked request never reaches the upstream. + assert upstream.last_path is None + + def test_rejects_mcp_prefix_without_boundary(self, gateway): + status, _headers, _body = _request(gateway.base_url, "POST", "/BC/mcpsomething", body=b"{}") + assert status == 403 + + def test_counts_forwarded_requests(self, gateway): + _request(gateway.base_url, "POST", "/BC/mcp", body=b"{}") + _request(gateway.base_url, "GET", "/BC/api") # blocked, not counted + _request(gateway.base_url, "POST", "/BC/mcp", body=b"{}") + assert gateway.forwarded_count == 2 From 62cc621f7f0bd886f4b92ea3332ad1c70ed0bcca Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 12:16:44 +0200 Subject: [PATCH 35/52] Use Copilot-licensed user PAT for MCP registry policy in Actions Copilot CLI 1.0.81 fetches the org MCP registry allowlist from GET /copilot/mcp_registry; the Actions github.token (ghs_ installation token) gets 403 there and the CLI fails closed, blocking all custom MCP servers (bcmcp + mslearn) even though the org policy is allow_all (github/copilot-cli#4346). Feed COPILOT_GITHUB_TOKEN a user PAT via the COPILOT_CLI_TOKEN secret, falling back to github.token when unset. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/copilot-evaluation.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/copilot-evaluation.yml b/.github/workflows/copilot-evaluation.yml index 66e387952..081f0d0d8 100644 --- a/.github/workflows/copilot-evaluation.yml +++ b/.github/workflows/copilot-evaluation.yml @@ -159,7 +159,12 @@ jobs: timeout-minutes: 120 shell: pwsh env: - COPILOT_GITHUB_TOKEN: ${{ github.token }} + # Copilot CLI must authenticate MCP with a USER token: the Actions github.token (a ghs_ + # installation token) gets 403 from GET /copilot/mcp_registry and fails closed, blocking ALL + # custom MCP servers (github/copilot-cli#4346). The org policy is already allow_all, so a + # Copilot-licensed user PAT lets the registry fetch succeed and the BC/MS-Learn MCP load. + # Falls back to github.token when the secret is absent (MCP off, but completions still work). + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN || github.token }} GH_TOKEN: ${{ github.token }} run: | Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN" From f3bfd71f1b7e15a6c6f389a93cda8348254fd2c1 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 12:47:21 +0200 Subject: [PATCH 36/52] Note server-prefixed MCP tool names in the bc-al-query-mcp skill Copilot CLI exposes MCP tools as - (bcmcp-bc_data_query) and Claude Code as mcp____; tell the skill the bare names refer to those tools regardless of prefix and to call the exact discovered name. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .../instructions/dataquery/skills/bc-al-query-mcp/SKILL.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md b/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md index e230d0bea..e1037e6ea 100644 --- a/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md +++ b/src/bcbench/agent/shared/instructions/dataquery/skills/bc-al-query-mcp/SKILL.md @@ -19,6 +19,8 @@ The environment should expose these MCP tool groups: If tools are deferred, load them with tool search before use. Do not assume a tool is available until it has been loaded or returned by discovery. +The MCP tool names may appear with a server-name prefix that differs by host — for example `bcmcp-bc_data_query` (Copilot CLI) or `mcp__bcmcp__bc_data_query` (Claude Code). The bare names used below (`bc_data_find_tables`, `bc_data_get_table_schema`, `bc_data_get_table_relations`, `bc_data_query`) refer to these tools regardless of prefix; always invoke the exact name shown in your available tool list or returned by tool search. + ## Core Rule Microsoft Learn provides general AL query authoring knowledge. The BC data MCP tools provide live tenant-specific metadata and execution. Use both. Never write a tenant query from memory alone. From 7cefd9fda38968756bdcf8223c3cd28fce03a1b9 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 14:06:43 +0200 Subject: [PATCH 37/52] Warm up + probe BC MCP tools through the gateway at startup A cold BC MCP endpoint can be slow to answer the first tools/list, so the agent's handshake occasionally registers zero bcmcp tools (observed: some entries forwarded 35 requests, others only the initialize). Run the MCP handshake (initialize -> notifications/initialized -> tools/list) through the gateway when it starts: this warms the endpoint before the agent connects and logs the exposed tool names, turning a silent registration failure into an observable signal. Best-effort; never raises. Handles JSON and SSE responses. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 66 +++++++++++++++++++++ tests/test_mcp_gateway.py | 79 ++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 99e4108b4..543e8e18e 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -14,6 +14,7 @@ """ import base64 +import json import os import threading from http.client import HTTPConnection @@ -46,6 +47,28 @@ _UPSTREAM_TIMEOUT_SECONDS = 600 _STREAM_CHUNK_BYTES = 8192 +_PROBE_TIMEOUT_SECONDS = 60 + + +def _read_jsonrpc(response) -> dict: # noqa: ANN001 - http.client.HTTPResponse + """Parse a JSON-RPC object from an MCP response body (application/json or SSE).""" + content_type = response.getheader("Content-Type", "") or "" + text = response.read().decode("utf-8", errors="replace") + if "text/event-stream" in content_type: + for raw_line in text.splitlines(): + line = raw_line.strip() + if line.startswith("data:"): + try: + obj = json.loads(line[5:].strip()) + except json.JSONDecodeError: + continue + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + return obj + return {} + try: + return json.loads(text) if text.strip() else {} + except json.JSONDecodeError: + return {} class BcMcpGateway: @@ -103,6 +126,48 @@ def stop(self) -> None: self._thread = None logger.info(f"BC MCP gateway forwarded {self.forwarded_count} request(s) to the BC MCP endpoint") + def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = None, session_id: str | None = None) -> tuple[str | None, dict]: + if self.base_url is None: + return None, {} + split = urlsplit(self.base_url) + connection = HTTPConnection(split.hostname or "127.0.0.1", split.port or 80, timeout=_PROBE_TIMEOUT_SECONDS) + try: + payload: dict[str, object] = {"jsonrpc": "2.0", "method": method} + if request_id is not None: + payload["id"] = request_id + if params is not None: + payload["params"] = params + headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} + if session_id: + headers["Mcp-Session-Id"] = session_id + connection.request("POST", self._mcp_path, body=json.dumps(payload).encode(), headers=headers) + response = connection.getresponse() + returned_session = response.getheader("Mcp-Session-Id") + return returned_session, _read_jsonrpc(response) + finally: + connection.close() + + def probe_tools(self) -> list[str]: + """Run the MCP handshake through the gateway to warm up BC MCP and log its exposed tools. + + Best-effort: a cold BC MCP endpoint can be slow to answer the first ``tools/list``, which makes + the agent's own handshake occasionally register zero tools; warming it here reduces that, and + logging the tool names turns a silent registration failure into an observable signal. Never + raises -- diagnostics must not break a run. + """ + try: + session_id, _ = self._probe_rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) + if session_id: + self._probe_rpc("notifications/initialized", None, session_id=session_id) + _, listed = self._probe_rpc("tools/list", {}, request_id=2, session_id=session_id) + tools = [t.get("name") for t in (listed.get("result") or {}).get("tools", []) if isinstance(t, dict)] + except Exception as exc: # noqa: BLE001 - a warm-up diagnostic must never break a run + logger.warning(f"BC MCP warm-up probe failed (non-fatal): {exc}") + return [] + else: + logger.info(f"BC MCP warm-up: endpoint exposes {len(tools)} tool(s): {tools}") + return tools + def _build_handler(gateway: BcMcpGateway) -> type[BaseHTTPRequestHandler]: class _ProxyHandler(BaseHTTPRequestHandler): @@ -200,4 +265,5 @@ def start_bc_mcp_gateway(enabled: bool) -> BcMcpGateway | None: company=os.environ.get("BC_MCP_COMPANY"), ).start() logger.info(f"BC MCP gateway listening at {gateway.base_url}/mcp (credential-free; path-restricted to /mcp)") + gateway.probe_tools() return gateway diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index e531a45ed..06e4a21b7 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -123,6 +123,7 @@ def test_streams_sse_response(self, gateway): assert b"data: two" in body def test_rejects_non_mcp_path(self, gateway, upstream): + upstream.last_path = None # clear traffic from the start-up warm-up probe status, _headers, _body = _request(gateway.base_url, "GET", "/BC/api/v2.0/companies") assert status == 403 # A blocked request never reaches the upstream. @@ -133,7 +134,83 @@ def test_rejects_mcp_prefix_without_boundary(self, gateway): assert status == 403 def test_counts_forwarded_requests(self, gateway): + baseline = gateway.forwarded_count # start_bc_mcp_gateway already ran a warm-up probe _request(gateway.base_url, "POST", "/BC/mcp", body=b"{}") _request(gateway.base_url, "GET", "/BC/api") # blocked, not counted _request(gateway.base_url, "POST", "/BC/mcp", body=b"{}") - assert gateway.forwarded_count == 2 + assert gateway.forwarded_count - baseline == 2 + + +class _McpHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log + pass + + def do_POST(self) -> None: + import json as _json + + n = int(self.headers.get("Content-Length", 0)) + req = _json.loads(self.rfile.read(n)) if n else {} + method = req.get("method") + if method == "initialize": + self._json({"jsonrpc": "2.0", "id": req.get("id"), "result": {"protocolVersion": "2024-11-05", "capabilities": {"tools": {}}}}, {"Mcp-Session-Id": "sess-xyz"}) + elif method and method.startswith("notifications/"): + self.send_response(202) + self.send_header("Content-Length", "0") + self.end_headers() + elif method == "tools/list": + tools = [{"name": "bc_data_find_tables"}, {"name": "bc_data_query"}] + # Answer as SSE to exercise the gateway's chunked relay + the probe's SSE parsing. + payload = "event: message\ndata: " + _json.dumps({"jsonrpc": "2.0", "id": req.get("id"), "result": {"tools": tools}}) + "\n\n" + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(payload.encode()) + else: + self._json({"jsonrpc": "2.0", "id": req.get("id"), "result": {}}) + + def _json(self, obj, extra_headers=None) -> None: + import json as _json + + body = _json.dumps(obj).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + for k, v in (extra_headers or {}).items(): + self.send_header(k, v) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class TestBcMcpProbe: + @pytest.fixture + def mcp_gateway(self, monkeypatch): + server = ThreadingHTTPServer(("127.0.0.1", 0), _McpHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + monkeypatch.setenv("BC_MCP_URL", f"http://127.0.0.1:{port}/BC") + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + monkeypatch.delenv("BC_MCP_COMPANY", raising=False) + gw = start_bc_mcp_gateway(enabled=True) + yield gw + gw.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def test_probe_returns_exposed_tool_names(self, mcp_gateway): + assert mcp_gateway.probe_tools() == ["bc_data_find_tables", "bc_data_query"] + + def test_probe_never_raises_on_bad_upstream(self, monkeypatch): + monkeypatch.setenv("BC_MCP_URL", "http://127.0.0.1:1/BC") # nothing listening + monkeypatch.setenv("BC_SERVER_USERNAME", "admin") + monkeypatch.setenv("BC_SERVER_PASSWORD", "secret") + gw = start_bc_mcp_gateway(enabled=True) + try: + assert gw.probe_tools() == [] + finally: + gw.stop() From c3f9d238b209034b56776248f624412673fe41fc Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 15:15:10 +0200 Subject: [PATCH 38/52] Fix warm-up probe: read SSE incrementally + log timing, raise timeout The BC MCP tools/list response is an SSE stream the server keeps open for later messages, so reading it to EOF blocked the probe until the 60s socket timeout even after the result arrived. Read the event stream line by line and return on the first JSON-RPC result; raise the probe timeout to 180s and log initialize/tools-list timing so a genuinely slow cold start is visible. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 32 +++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 543e8e18e..481b90c54 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -17,6 +17,7 @@ import json import os import threading +import time from http.client import HTTPConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import urlsplit @@ -47,16 +48,23 @@ _UPSTREAM_TIMEOUT_SECONDS = 600 _STREAM_CHUNK_BYTES = 8192 -_PROBE_TIMEOUT_SECONDS = 60 +_PROBE_TIMEOUT_SECONDS = 180 -def _read_jsonrpc(response) -> dict: # noqa: ANN001 - http.client.HTTPResponse - """Parse a JSON-RPC object from an MCP response body (application/json or SSE).""" +def _read_jsonrpc(response, deadline: float) -> dict: # noqa: ANN001 - http.client.HTTPResponse + """Parse a JSON-RPC result from an MCP response body (application/json or SSE). + + For SSE, read line by line and return as soon as a JSON-RPC result/error arrives: the BC MCP + endpoint keeps the event stream open for later messages, so reading to EOF would block until the + socket times out even though the answer already arrived. + """ content_type = response.getheader("Content-Type", "") or "" - text = response.read().decode("utf-8", errors="replace") if "text/event-stream" in content_type: - for raw_line in text.splitlines(): - line = raw_line.strip() + while time.monotonic() < deadline: + raw_line = response.readline() + if not raw_line: + break + line = raw_line.decode("utf-8", errors="replace").strip() if line.startswith("data:"): try: obj = json.loads(line[5:].strip()) @@ -65,6 +73,7 @@ def _read_jsonrpc(response) -> dict: # noqa: ANN001 - http.client.HTTPResponse if isinstance(obj, dict) and ("result" in obj or "error" in obj): return obj return {} + text = response.read().decode("utf-8", errors="replace") try: return json.loads(text) if text.strip() else {} except json.JSONDecodeError: @@ -143,7 +152,7 @@ def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = connection.request("POST", self._mcp_path, body=json.dumps(payload).encode(), headers=headers) response = connection.getresponse() returned_session = response.getheader("Mcp-Session-Id") - return returned_session, _read_jsonrpc(response) + return returned_session, _read_jsonrpc(response, deadline=time.monotonic() + _PROBE_TIMEOUT_SECONDS) finally: connection.close() @@ -152,20 +161,23 @@ def probe_tools(self) -> list[str]: Best-effort: a cold BC MCP endpoint can be slow to answer the first ``tools/list``, which makes the agent's own handshake occasionally register zero tools; warming it here reduces that, and - logging the tool names turns a silent registration failure into an observable signal. Never - raises -- diagnostics must not break a run. + logging the tool names + timing turns a silent registration failure into an observable signal. + Never raises -- diagnostics must not break a run. """ try: + start = time.monotonic() session_id, _ = self._probe_rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) + after_init = time.monotonic() if session_id: self._probe_rpc("notifications/initialized", None, session_id=session_id) _, listed = self._probe_rpc("tools/list", {}, request_id=2, session_id=session_id) + after_list = time.monotonic() tools = [t.get("name") for t in (listed.get("result") or {}).get("tools", []) if isinstance(t, dict)] except Exception as exc: # noqa: BLE001 - a warm-up diagnostic must never break a run logger.warning(f"BC MCP warm-up probe failed (non-fatal): {exc}") return [] else: - logger.info(f"BC MCP warm-up: endpoint exposes {len(tools)} tool(s): {tools}") + logger.info(f"BC MCP warm-up: endpoint exposes {len(tools)} tool(s): {tools} (initialize {after_init - start:.1f}s, tools/list {after_list - after_init:.1f}s)") return tools From bb9664a9a5a44d4dce426091e6704a1b9c9201b7 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 18:09:50 +0200 Subject: [PATCH 39/52] Add NST log capture to diagnose BC MCP exposing zero tools The BCBench MCP config returns an empty tools/list (~48s then empty, or timeout), so the Data Query catalog is not composing server-side. Add a temporary diagnostic step (Claude workflow, bc-mcp only, if: always) that pulls the container's NST Application event log (NAV/MCP entries), NST server config, event channels, and the persisted MCP Configuration row (to confirm EnableAlQueryTools) via BcContainerHelper. Also log the raw tools/list result in the warm-up probe when it is empty. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .github/workflows/claude-evaluation.yml | 8 +++ scripts/Capture-NstLogs.ps1 | 91 +++++++++++++++++++++++++ src/bcbench/agent/shared/mcp_gateway.py | 2 + 3 files changed, 101 insertions(+) create mode 100644 scripts/Capture-NstLogs.ps1 diff --git a/.github/workflows/claude-evaluation.yml b/.github/workflows/claude-evaluation.yml index ed9ecd1e5..02cfd71f9 100644 --- a/.github/workflows/claude-evaluation.yml +++ b/.github/workflows/claude-evaluation.yml @@ -172,6 +172,14 @@ jobs: ${{ inputs.ms-learn-mcp && '--ms-learn-mcp' || '' }} ` ${{ inputs.skills && '--skills' || '' }} + - name: Capture BC NST logs + if: ${{ always() && inputs.bc-mcp }} + shell: pwsh + run: | + # TEMPORARY (diagnostic): pull NST event log + server config + MCP Configuration row to + # investigate why the BC MCP endpoint exposes zero Data Query tools. + .\scripts\Capture-NstLogs.ps1 -ContainerName "$env:BC_CONTAINER_NAME" -OutputDir "${{ env.EVALUATION_RESULTS_DIR }}" + - name: Upload evaluation results uses: actions/upload-artifact@v6 if: always() diff --git a/scripts/Capture-NstLogs.ps1 b/scripts/Capture-NstLogs.ps1 new file mode 100644 index 000000000..59e81fa09 --- /dev/null +++ b/scripts/Capture-NstLogs.ps1 @@ -0,0 +1,91 @@ +<# +.SYNOPSIS + Capture BC Server (NST) diagnostics from a running BcContainerHelper container. + + Best-effort and non-fatal: pulls the container's Windows Application event log (NAV/MCP entries), + the NST server configuration, and the persisted "MCP Configuration" row so we can see why the BC + MCP endpoint exposes zero Data Query tools. Writes .log files into -OutputDir for artifact upload. +#> +param( + [Parameter(Mandatory)][string]$ContainerName, + [Parameter(Mandatory)][string]$OutputDir +) + +$ErrorActionPreference = 'Continue' + +function Write-Diag($message) { Write-Host "[Capture-NstLogs] $message" } + +if (-not $ContainerName) { + Write-Diag 'No container name provided; skipping NST log capture.' + return +} + +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null + +# 1. NAV/MCP-related Application event log entries from inside the container. +try { + Write-Diag "Capturing NST Application event log from '$ContainerName'..." + $events = Invoke-ScriptInBcContainer -containerName $ContainerName -scriptblock { + Get-WinEvent -LogName Application -MaxEvents 3000 -ErrorAction SilentlyContinue | + Where-Object { $_.ProviderName -match 'NAV|Dynamics' -or $_.Message -match 'MCP|AlQuery|Data.?Query|query tool|tools/list' } | + Sort-Object TimeCreated | + ForEach-Object { '{0:o} [{1}] {2} (Id {3}): {4}' -f $_.TimeCreated, $_.LevelDisplayName, $_.ProviderName, $_.Id, ($_.Message -replace '\r?\n', ' ') } + } + ($events | Out-String) | Out-File -FilePath (Join-Path $OutputDir 'nst-eventlog.log') -Encoding utf8 + Write-Diag "Captured $(@($events).Count) NAV/MCP event(s)." +} +catch { + "Event log capture failed: $_" | Out-File -FilePath (Join-Path $OutputDir 'nst-eventlog.log') -Encoding utf8 + Write-Diag "Event log capture failed: $_" +} + +# 2. Discover any dedicated NAV/MCP event channels (in case MCP logs somewhere other than Application). +try { + $channels = Invoke-ScriptInBcContainer -containerName $ContainerName -scriptblock { + Get-WinEvent -ListLog * -ErrorAction SilentlyContinue | + Where-Object { $_.LogName -match 'NAV|Dynamics|MCP' } | + ForEach-Object { '{0} (records: {1})' -f $_.LogName, $_.RecordCount } + } + ($channels | Out-String) | Out-File -FilePath (Join-Path $OutputDir 'nst-eventchannels.log') -Encoding utf8 +} +catch { + Write-Diag "Event channel discovery failed: $_" +} + +# 3. NST server configuration (feature keys / MCP-related settings). +try { + Write-Diag 'Capturing NST server configuration...' + $config = Get-BcContainerServerConfiguration -containerName $ContainerName + ($config | Format-List * | Out-String) | Out-File -FilePath (Join-Path $OutputDir 'nst-serverconfig.log') -Encoding utf8 +} +catch { + Write-Diag "Server configuration capture failed: $_" +} + +# 4. Persisted "MCP Configuration" row(s) - confirms whether EnableAlQueryTools actually got set. +try { + Write-Diag 'Querying the MCP Configuration table...' + $config = Get-BcContainerServerConfiguration -containerName $ContainerName + $dbServer = $config.DatabaseServer + $dbInstance = $config.DatabaseInstance + $dbName = $config.DatabaseName + $server = if ($dbInstance) { "$dbServer\$dbInstance" } else { $dbServer } + + $rows = Invoke-ScriptInBcContainer -containerName $ContainerName -scriptblock { + param($sqlServer, $database) + $table = (sqlcmd -S $sqlServer -d $database -h -1 -W -Q "SET NOCOUNT ON; SELECT TOP 1 name FROM sys.tables WHERE name LIKE '%MCP Configuration%'" 2>&1 | Where-Object { $_ -and $_ -notmatch 'rows affected' } | Select-Object -First 1) + if ($table) { + "Table: $table" + sqlcmd -S $sqlServer -d $database -Y 60 -W -Q "SELECT * FROM [dbo].[$table]" 2>&1 + } + else { + 'No table matching %MCP Configuration% found in the tenant database.' + } + } -argumentList $server, $dbName + ($rows | Out-String) | Out-File -FilePath (Join-Path $OutputDir 'mcp-configuration.log') -Encoding utf8 +} +catch { + Write-Diag "MCP Configuration query failed: $_" +} + +Write-Diag 'NST diagnostics capture complete.' diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 481b90c54..f1d914f47 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -178,6 +178,8 @@ def probe_tools(self) -> list[str]: return [] else: logger.info(f"BC MCP warm-up: endpoint exposes {len(tools)} tool(s): {tools} (initialize {after_init - start:.1f}s, tools/list {after_list - after_init:.1f}s)") + if not tools: + logger.info(f"BC MCP warm-up raw tools/list result: {json.dumps(listed)[:1000]}") return tools From f3b2ee9be28c36a63dece6af4905740bab38ed11 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 18:43:48 +0200 Subject: [PATCH 40/52] Fix MCP Configuration DB query flags + add app install-status capture sqlcmd -W and -Y are mutually exclusive; drop -Y and use pipe-separated output so the MCP Configuration row (incl. EnableAlQueryTools) actually dumps. Also list the column names and capture Get-BcContainerAppInfo install status for the MCP Config Setup app (published != installed). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- ...open-sales-order-count-by-customer-1.jsonl | 1 + .../32749236726/tool_usage.jsonl | 13 ++++++ ...query__opportunity-count-by-status-1.jsonl | 1 + .../32749236726/tool_usage.jsonl | 16 +++++++ ...tstanding-purchase-value-by-vendor-1.jsonl | 1 + .../32749236726/tool_usage.jsonl | 11 +++++ ...__total-purchased-quantity-by-item-1.jsonl | 1 + .../32749236726/tool_usage.jsonl | 11 +++++ .../evaluation_summary.json | 42 +++++++++++++++++++ scripts/Capture-NstLogs.ps1 | 18 +++++++- 10 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 _nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl create mode 100644 _nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl create mode 100644 _nst/evaluation-summary/evaluation_summary.json diff --git a/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl new file mode 100644 index 000000000..7178d54c5 --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl @@ -0,0 +1 @@ +{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 65.959, "llm_duration": 60.279, "ai_credits": null, "turn_count": 15, "prompt_tokens": 571122, "completion_tokens": 4547, "tool_usage": {"Skill": 1, "ToolSearch": 8, "Bash": 3, "ListMcpResourcesTool": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl new file mode 100644 index 000000000..3e16af050 --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl @@ -0,0 +1,13 @@ +{"tool_name":"Skill","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"Bash"} +{"timestamp":null,"tool_name":"Bash"} +{"timestamp":null,"tool_name":"Bash"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ListMcpResourcesTool"} +{"tool_name":"ToolSearch","timestamp":null} +{"tool_name":"ToolSearch","timestamp":null} diff --git a/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl new file mode 100644 index 000000000..577bdf01e --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl @@ -0,0 +1 @@ +{"instance_id": "dataquery__opportunity-count-by-status-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 65.075, "llm_duration": 52.699, "ai_credits": null, "turn_count": 19, "prompt_tokens": 548976, "completion_tokens": 4219, "tool_usage": {"Skill": 1, "ToolSearch": 9, "ListMcpResourcesTool": 2, "Bash": 3, "Read": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl new file mode 100644 index 000000000..6722e64f5 --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl @@ -0,0 +1,16 @@ +{"tool_name":"Skill","timestamp":null} +{"tool_name":"ToolSearch","timestamp":null} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ListMcpResourcesTool","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"Bash"} +{"timestamp":null,"tool_name":"Bash"} +{"timestamp":null,"tool_name":"Bash"} +{"tool_name":"Read","timestamp":null} +{"timestamp":null,"tool_name":"ListMcpResourcesTool"} diff --git a/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl new file mode 100644 index 000000000..f9de2ae5b --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl @@ -0,0 +1 @@ +{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 37.095, "llm_duration": 32.313, "ai_credits": null, "turn_count": 13, "prompt_tokens": 382436, "completion_tokens": 2140, "tool_usage": {"Skill": 1, "ToolSearch": 7, "Bash": 2, "ListMcpResourcesTool": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl new file mode 100644 index 000000000..1c9100d58 --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl @@ -0,0 +1,11 @@ +{"tool_name":"Skill","timestamp":null} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"Bash"} +{"timestamp":null,"tool_name":"Bash"} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ListMcpResourcesTool","timestamp":null} +{"tool_name":"ToolSearch","timestamp":null} diff --git a/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl new file mode 100644 index 000000000..81a612eea --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl @@ -0,0 +1 @@ +{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 41.514, "llm_duration": 37.561, "ai_credits": null, "turn_count": 13, "prompt_tokens": 317709, "completion_tokens": 2829, "tool_usage": {"Skill": 1, "ToolSearch": 7, "Bash": 2, "ListMcpResourcesTool": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl new file mode 100644 index 000000000..7c0e5179e --- /dev/null +++ b/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl @@ -0,0 +1,11 @@ +{"timestamp":null,"tool_name":"Skill"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"ToolSearch","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} +{"tool_name":"Bash","timestamp":null} +{"timestamp":null,"tool_name":"ListMcpResourcesTool"} +{"tool_name":"Bash","timestamp":null} +{"timestamp":null,"tool_name":"ToolSearch"} diff --git a/_nst/evaluation-summary/evaluation_summary.json b/_nst/evaluation-summary/evaluation_summary.json new file mode 100644 index 000000000..17b6a5315 --- /dev/null +++ b/_nst/evaluation-summary/evaluation_summary.json @@ -0,0 +1,42 @@ +{ + "total": 4, + "date": "2026-08-24", + "model": "claude-sonnet-5", + "agent_name": "Claude Code", + "category": "data-query", + "average_duration": 52.4, + "average_prompt_tokens": 455060.8, + "average_completion_tokens": 3433.8, + "average_llm_duration": 45.7, + "average_ai_credits": null, + "average_tool_usage": { + "Skill": 1.0, + "ToolSearch": 7.75, + "Bash": 2.5, + "ListMcpResourcesTool": 1.25, + "Read": 0.25 + }, + "github_run_id": "32749236726", + "experiment": { + "mcp_servers": [ + "bcmcp", + "mslearn" + ], + "al_lsp_enabled": false, + "custom_instructions": false, + "skills_enabled": true, + "custom_agent": null, + "plugins": null + }, + "benchmark_version": "0.9.0", + "resolved": 0, + "failed": 4, + "build": 0, + "percentage": 0.0, + "instance_results": { + "dataquery__open-sales-order-count-by-customer-1": false, + "dataquery__opportunity-count-by-status-1": false, + "dataquery__total-purchased-quantity-by-item-1": false, + "dataquery__outstanding-purchase-value-by-vendor-1": false + } +} \ No newline at end of file diff --git a/scripts/Capture-NstLogs.ps1 b/scripts/Capture-NstLogs.ps1 index 59e81fa09..cfa0827f0 100644 --- a/scripts/Capture-NstLogs.ps1 +++ b/scripts/Capture-NstLogs.ps1 @@ -73,10 +73,13 @@ try { $rows = Invoke-ScriptInBcContainer -containerName $ContainerName -scriptblock { param($sqlServer, $database) - $table = (sqlcmd -S $sqlServer -d $database -h -1 -W -Q "SET NOCOUNT ON; SELECT TOP 1 name FROM sys.tables WHERE name LIKE '%MCP Configuration%'" 2>&1 | Where-Object { $_ -and $_ -notmatch 'rows affected' } | Select-Object -First 1) + $table = (sqlcmd -S $sqlServer -d $database -h -1 -W -Q "SET NOCOUNT ON; SELECT TOP 1 name FROM sys.tables WHERE name LIKE '%MCP Configuration%'" 2>&1 | ForEach-Object { "$_".Trim() } | Where-Object { $_ -and $_ -notmatch 'rows affected' } | Select-Object -First 1) if ($table) { "Table: $table" - sqlcmd -S $sqlServer -d $database -Y 60 -W -Q "SELECT * FROM [dbo].[$table]" 2>&1 + '--- columns ---' + sqlcmd -S $sqlServer -d $database -h -1 -Q "SET NOCOUNT ON; SELECT c.name FROM sys.columns c JOIN sys.tables t ON c.object_id = t.object_id WHERE t.name = '$table' ORDER BY c.column_id" 2>&1 + '--- row(s) (pipe-separated) ---' + sqlcmd -S $sqlServer -d $database -s "|" -W -Q "SELECT * FROM [dbo].[$table]" 2>&1 } else { 'No table matching %MCP Configuration% found in the tenant database.' @@ -88,4 +91,15 @@ catch { Write-Diag "MCP Configuration query failed: $_" } +# 5. Install status of the MCP Config Setup app (published != installed; OnInstall must have run). +try { + Write-Diag 'Checking MCP Config Setup app install status...' + $apps = Get-BcContainerAppInfo -containerName $ContainerName -tenant default -tenantSpecificProperties | + Where-Object { $_.Name -match 'MCP' } + ($apps | Format-List Name, Publisher, Version, IsInstalled, IsPublished, SyncState | Out-String) | Out-File -FilePath (Join-Path $OutputDir 'mcp-app-status.log') -Encoding utf8 +} +catch { + Write-Diag "App install status check failed: $_" +} + Write-Diag 'NST diagnostics capture complete.' From 8b6ee1a8cffa3525ccd80b45776f779b09624de4 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 18:44:41 +0200 Subject: [PATCH 41/52] Remove accidentally-committed diagnostic download folder; ignore _*/ Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- .gitignore | 3 ++ ...open-sales-order-count-by-customer-1.jsonl | 1 - .../32749236726/tool_usage.jsonl | 13 ------ ...query__opportunity-count-by-status-1.jsonl | 1 - .../32749236726/tool_usage.jsonl | 16 ------- ...tstanding-purchase-value-by-vendor-1.jsonl | 1 - .../32749236726/tool_usage.jsonl | 11 ----- ...__total-purchased-quantity-by-item-1.jsonl | 1 - .../32749236726/tool_usage.jsonl | 11 ----- .../evaluation_summary.json | 42 ------------------- 10 files changed, 3 insertions(+), 97 deletions(-) delete mode 100644 _nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl delete mode 100644 _nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl delete mode 100644 _nst/evaluation-summary/evaluation_summary.json diff --git a/.gitignore b/.gitignore index 550b1da5d..67f6b383c 100644 --- a/.gitignore +++ b/.gitignore @@ -283,3 +283,6 @@ docs/Gemfile docs/Gemfile.lock *.orig + +# Local diagnostic artifact download folders (temp, leading underscore) +_*/ diff --git a/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl deleted file mode 100644 index 7178d54c5..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/dataquery__open-sales-order-count-by-customer-1.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 65.959, "llm_duration": 60.279, "ai_credits": null, "turn_count": 15, "prompt_tokens": 571122, "completion_tokens": 4547, "tool_usage": {"Skill": 1, "ToolSearch": 8, "Bash": 3, "ListMcpResourcesTool": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl deleted file mode 100644 index 3e16af050..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__open-sales-order-count-by-customer-1/32749236726/tool_usage.jsonl +++ /dev/null @@ -1,13 +0,0 @@ -{"tool_name":"Skill","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"Bash"} -{"timestamp":null,"tool_name":"Bash"} -{"timestamp":null,"tool_name":"Bash"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ListMcpResourcesTool"} -{"tool_name":"ToolSearch","timestamp":null} -{"tool_name":"ToolSearch","timestamp":null} diff --git a/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl deleted file mode 100644 index 577bdf01e..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/dataquery__opportunity-count-by-status-1.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"instance_id": "dataquery__opportunity-count-by-status-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 65.075, "llm_duration": 52.699, "ai_credits": null, "turn_count": 19, "prompt_tokens": 548976, "completion_tokens": 4219, "tool_usage": {"Skill": 1, "ToolSearch": 9, "ListMcpResourcesTool": 2, "Bash": 3, "Read": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl deleted file mode 100644 index 6722e64f5..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__opportunity-count-by-status-1/32749236726/tool_usage.jsonl +++ /dev/null @@ -1,16 +0,0 @@ -{"tool_name":"Skill","timestamp":null} -{"tool_name":"ToolSearch","timestamp":null} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ListMcpResourcesTool","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"Bash"} -{"timestamp":null,"tool_name":"Bash"} -{"timestamp":null,"tool_name":"Bash"} -{"tool_name":"Read","timestamp":null} -{"timestamp":null,"tool_name":"ListMcpResourcesTool"} diff --git a/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl deleted file mode 100644 index f9de2ae5b..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/dataquery__outstanding-purchase-value-by-vendor-1.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 37.095, "llm_duration": 32.313, "ai_credits": null, "turn_count": 13, "prompt_tokens": 382436, "completion_tokens": 2140, "tool_usage": {"Skill": 1, "ToolSearch": 7, "Bash": 2, "ListMcpResourcesTool": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl deleted file mode 100644 index 1c9100d58..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__outstanding-purchase-value-by-vendor-1/32749236726/tool_usage.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"tool_name":"Skill","timestamp":null} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"Bash"} -{"timestamp":null,"tool_name":"Bash"} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ListMcpResourcesTool","timestamp":null} -{"tool_name":"ToolSearch","timestamp":null} diff --git a/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl b/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl deleted file mode 100644 index 81a612eea..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/dataquery__total-purchased-quantity-by-item-1.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "project": "", "model": "claude-sonnet-5", "agent_name": "Claude Code", "category": "data-query", "timeout": false, "output": "", "error_message": "No answer.json produced", "metrics": {"execution_time": 41.514, "llm_duration": 37.561, "ai_credits": null, "turn_count": 13, "prompt_tokens": 317709, "completion_tokens": 2829, "tool_usage": {"Skill": 1, "ToolSearch": 7, "Bash": 2, "ListMcpResourcesTool": 1}}, "experiment": {"mcp_servers": ["bcmcp", "mslearn"], "al_lsp_enabled": false, "custom_instructions": false, "skills_enabled": true, "custom_agent": null, "plugins": null}, "resolved": false, "build": false} diff --git a/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl b/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl deleted file mode 100644 index 7c0e5179e..000000000 --- a/_nst/evaluation-results-32749236726-dataquery__total-purchased-quantity-by-item-1/32749236726/tool_usage.jsonl +++ /dev/null @@ -1,11 +0,0 @@ -{"timestamp":null,"tool_name":"Skill"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"ToolSearch","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} -{"tool_name":"Bash","timestamp":null} -{"timestamp":null,"tool_name":"ListMcpResourcesTool"} -{"tool_name":"Bash","timestamp":null} -{"timestamp":null,"tool_name":"ToolSearch"} diff --git a/_nst/evaluation-summary/evaluation_summary.json b/_nst/evaluation-summary/evaluation_summary.json deleted file mode 100644 index 17b6a5315..000000000 --- a/_nst/evaluation-summary/evaluation_summary.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "total": 4, - "date": "2026-08-24", - "model": "claude-sonnet-5", - "agent_name": "Claude Code", - "category": "data-query", - "average_duration": 52.4, - "average_prompt_tokens": 455060.8, - "average_completion_tokens": 3433.8, - "average_llm_duration": 45.7, - "average_ai_credits": null, - "average_tool_usage": { - "Skill": 1.0, - "ToolSearch": 7.75, - "Bash": 2.5, - "ListMcpResourcesTool": 1.25, - "Read": 0.25 - }, - "github_run_id": "32749236726", - "experiment": { - "mcp_servers": [ - "bcmcp", - "mslearn" - ], - "al_lsp_enabled": false, - "custom_instructions": false, - "skills_enabled": true, - "custom_agent": null, - "plugins": null - }, - "benchmark_version": "0.9.0", - "resolved": 0, - "failed": 4, - "build": 0, - "percentage": 0.0, - "instance_results": { - "dataquery__open-sales-order-count-by-customer-1": false, - "dataquery__opportunity-count-by-status-1": false, - "dataquery__total-purchased-quantity-by-item-1": false, - "dataquery__outstanding-purchase-value-by-vendor-1": false - } -} \ No newline at end of file From 833d361586a587aedd003246ea130c2d589bc85c Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 19:15:46 +0200 Subject: [PATCH 42/52] Log raw HTTP status/body of an empty tools/list in the warm-up probe Distinguishes an internal server error (e.g. HTTP 500) from a clean empty tools array when BC MCP returns no tools, to sharpen the server-side handoff. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index f1d914f47..c9cdf9f80 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -51,8 +51,8 @@ _PROBE_TIMEOUT_SECONDS = 180 -def _read_jsonrpc(response, deadline: float) -> dict: # noqa: ANN001 - http.client.HTTPResponse - """Parse a JSON-RPC result from an MCP response body (application/json or SSE). +def _read_jsonrpc(response, deadline: float) -> tuple[dict, str]: # noqa: ANN001 - http.client.HTTPResponse + """Parse a JSON-RPC result and return it plus a raw snippet of the response for diagnostics. For SSE, read line by line and return as soon as a JSON-RPC result/error arrives: the BC MCP endpoint keeps the event stream open for later messages, so reading to EOF would block until the @@ -60,24 +60,27 @@ def _read_jsonrpc(response, deadline: float) -> dict: # noqa: ANN001 - http.cli """ content_type = response.getheader("Content-Type", "") or "" if "text/event-stream" in content_type: + seen: list[str] = [] while time.monotonic() < deadline: raw_line = response.readline() if not raw_line: break line = raw_line.decode("utf-8", errors="replace").strip() + if line: + seen.append(line) if line.startswith("data:"): try: obj = json.loads(line[5:].strip()) except json.JSONDecodeError: continue if isinstance(obj, dict) and ("result" in obj or "error" in obj): - return obj - return {} + return obj, line[:800] + return {}, " | ".join(seen)[:800] text = response.read().decode("utf-8", errors="replace") try: - return json.loads(text) if text.strip() else {} + return (json.loads(text) if text.strip() else {}), text[:800] except json.JSONDecodeError: - return {} + return {}, text[:800] class BcMcpGateway: @@ -135,9 +138,9 @@ def stop(self) -> None: self._thread = None logger.info(f"BC MCP gateway forwarded {self.forwarded_count} request(s) to the BC MCP endpoint") - def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = None, session_id: str | None = None) -> tuple[str | None, dict]: + def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = None, session_id: str | None = None) -> tuple[str | None, dict, str]: if self.base_url is None: - return None, {} + return None, {}, "no gateway" split = urlsplit(self.base_url) connection = HTTPConnection(split.hostname or "127.0.0.1", split.port or 80, timeout=_PROBE_TIMEOUT_SECONDS) try: @@ -152,7 +155,8 @@ def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = connection.request("POST", self._mcp_path, body=json.dumps(payload).encode(), headers=headers) response = connection.getresponse() returned_session = response.getheader("Mcp-Session-Id") - return returned_session, _read_jsonrpc(response, deadline=time.monotonic() + _PROBE_TIMEOUT_SECONDS) + result, raw = _read_jsonrpc(response, deadline=time.monotonic() + _PROBE_TIMEOUT_SECONDS) + return returned_session, result, f"HTTP {response.status}, Content-Type={response.getheader('Content-Type', '')!r}, body={raw!r}" finally: connection.close() @@ -166,11 +170,11 @@ def probe_tools(self) -> list[str]: """ try: start = time.monotonic() - session_id, _ = self._probe_rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) + session_id, _, _ = self._probe_rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) after_init = time.monotonic() if session_id: self._probe_rpc("notifications/initialized", None, session_id=session_id) - _, listed = self._probe_rpc("tools/list", {}, request_id=2, session_id=session_id) + _, listed, list_diag = self._probe_rpc("tools/list", {}, request_id=2, session_id=session_id) after_list = time.monotonic() tools = [t.get("name") for t in (listed.get("result") or {}).get("tools", []) if isinstance(t, dict)] except Exception as exc: # noqa: BLE001 - a warm-up diagnostic must never break a run @@ -179,7 +183,7 @@ def probe_tools(self) -> list[str]: else: logger.info(f"BC MCP warm-up: endpoint exposes {len(tools)} tool(s): {tools} (initialize {after_init - start:.1f}s, tools/list {after_list - after_init:.1f}s)") if not tools: - logger.info(f"BC MCP warm-up raw tools/list result: {json.dumps(listed)[:1000]}") + logger.info(f"BC MCP warm-up empty tools/list -> {list_diag}") return tools From 52512220885d04b93257c06910693f994b12c7cb Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 22:01:59 +0200 Subject: [PATCH 43/52] Probe tools/list both via gateway and directly against BC tools/list returns HTTP 200 text/event-stream with an empty body (~46s then close). To isolate whether the gateway's SSE relay is at fault vs the server, run the handshake against BC directly (with the injected auth headers) in the same warm-up and log both results side by side. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 63 +++++++++++++++---------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index c9cdf9f80..fb53fa70b 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -138,18 +138,15 @@ def stop(self) -> None: self._thread = None logger.info(f"BC MCP gateway forwarded {self.forwarded_count} request(s) to the BC MCP endpoint") - def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = None, session_id: str | None = None) -> tuple[str | None, dict, str]: - if self.base_url is None: - return None, {}, "no gateway" - split = urlsplit(self.base_url) - connection = HTTPConnection(split.hostname or "127.0.0.1", split.port or 80, timeout=_PROBE_TIMEOUT_SECONDS) + def _rpc(self, host: str, port: int, extra_headers: dict[str, str], method: str, params: dict | None, request_id: int | None = None, session_id: str | None = None) -> tuple[str | None, dict, str]: + connection = HTTPConnection(host, port, timeout=_PROBE_TIMEOUT_SECONDS) try: payload: dict[str, object] = {"jsonrpc": "2.0", "method": method} if request_id is not None: payload["id"] = request_id if params is not None: payload["params"] = params - headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} + headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream", **extra_headers} if session_id: headers["Mcp-Session-Id"] = session_id connection.request("POST", self._mcp_path, body=json.dumps(payload).encode(), headers=headers) @@ -160,31 +157,45 @@ def _probe_rpc(self, method: str, params: dict | None, request_id: int | None = finally: connection.close() + def _handshake_tools(self, host: str, port: int, extra_headers: dict[str, str]) -> tuple[list[str], float, str]: + """initialize -> notifications/initialized -> tools/list against one target; returns (tools, tools_list_seconds, diag).""" + session_id, _, _ = self._rpc(host, port, extra_headers, "initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) + if session_id: + self._rpc(host, port, extra_headers, "notifications/initialized", None, session_id=session_id) + before_list = time.monotonic() + _, listed, diag = self._rpc(host, port, extra_headers, "tools/list", {}, request_id=2, session_id=session_id) + tools = [t.get("name") for t in (listed.get("result") or {}).get("tools", []) if isinstance(t, dict)] + return tools, time.monotonic() - before_list, diag + def probe_tools(self) -> list[str]: - """Run the MCP handshake through the gateway to warm up BC MCP and log its exposed tools. + """Warm up BC MCP and log its exposed tools, probing BOTH through the gateway and directly. - Best-effort: a cold BC MCP endpoint can be slow to answer the first ``tools/list``, which makes - the agent's own handshake occasionally register zero tools; warming it here reduces that, and - logging the tool names + timing turns a silent registration failure into an observable signal. - Never raises -- diagnostics must not break a run. + Best-effort (never raises): warms the cold endpoint before the agent connects, and comparing the + via-gateway result against a direct-to-BC result isolates a gateway relay problem from a genuine + server-side one. The direct probe carries the injected auth/Company/ConfigurationName headers. """ + gateway_tools: list[str] = [] + # Via the gateway (credential-free; the gateway injects auth upstream) - the agent's exact path. + if self.base_url is not None: + split = urlsplit(self.base_url) + try: + gateway_tools, secs, diag = self._handshake_tools(split.hostname or "127.0.0.1", split.port or 80, {}) + logger.info(f"BC MCP warm-up (via gateway): exposes {len(gateway_tools)} tool(s): {gateway_tools} (tools/list {secs:.1f}s)") + if not gateway_tools: + logger.info(f"BC MCP warm-up (via gateway) empty tools/list -> {diag}") + except Exception as exc: # noqa: BLE001 - a warm-up diagnostic must never break a run + logger.warning(f"BC MCP warm-up (via gateway) failed (non-fatal): {exc}") + + # Directly to BC (bypassing the gateway) with the real headers - isolates gateway vs server. try: - start = time.monotonic() - session_id, _, _ = self._probe_rpc("initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) - after_init = time.monotonic() - if session_id: - self._probe_rpc("notifications/initialized", None, session_id=session_id) - _, listed, list_diag = self._probe_rpc("tools/list", {}, request_id=2, session_id=session_id) - after_list = time.monotonic() - tools = [t.get("name") for t in (listed.get("result") or {}).get("tools", []) if isinstance(t, dict)] + direct_tools, secs, diag = self._handshake_tools(self._origin_host, self._origin_port, self._injected_headers) + logger.info(f"BC MCP warm-up (direct to BC): exposes {len(direct_tools)} tool(s): {direct_tools} (tools/list {secs:.1f}s)") + if not direct_tools: + logger.info(f"BC MCP warm-up (direct to BC) empty tools/list -> {diag}") except Exception as exc: # noqa: BLE001 - a warm-up diagnostic must never break a run - logger.warning(f"BC MCP warm-up probe failed (non-fatal): {exc}") - return [] - else: - logger.info(f"BC MCP warm-up: endpoint exposes {len(tools)} tool(s): {tools} (initialize {after_init - start:.1f}s, tools/list {after_list - after_init:.1f}s)") - if not tools: - logger.info(f"BC MCP warm-up empty tools/list -> {list_diag}") - return tools + logger.warning(f"BC MCP warm-up (direct to BC) failed (non-fatal): {exc}") + + return gateway_tools def _build_handler(gateway: BcMcpGateway) -> type[BaseHTTPRequestHandler]: From ed3f231e4b1142716aade7f407ee657dd1eba7be Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Mon, 24 Aug 2026 22:43:33 +0200 Subject: [PATCH 44/52] Fix gateway SSE relay: read1() so held-open streams flush promptly The relay used response.read(8192), which blocks until the buffer fills. BC MCP returns tools/list as an SSE stream it holds open after a small event, so read() never returned until the stream closed ~46s later and the event was never forwarded -> tools/list timed out through the gateway while working in ~2.9s directly against BC. Switch the no-content-length branch to read1(), which returns available bytes from a single read so each event is flushed immediately. Adds a regression test with a held-open SSE upstream. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 6 ++- tests/test_mcp_gateway.py | 50 ++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index fb53fa70b..853e7447c 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -257,11 +257,13 @@ def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse remaining -= len(chunk) else: # No content length -> stream (e.g. SSE) with our own chunked framing, flushing each - # block so server-sent events reach the agent as they arrive. + # block so server-sent events reach the agent as they arrive. Use read1(): plain read() + # blocks trying to fill the whole buffer, which stalls an SSE stream the server holds + # open after a small event (that stall is what made tools/list time out through here). self.send_header("Transfer-Encoding", "chunked") self.end_headers() while True: - chunk = response.read(_STREAM_CHUNK_BYTES) + chunk = response.read1(_STREAM_CHUNK_BYTES) if not chunk: break self.wfile.write(b"%X\r\n" % len(chunk)) diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 06e4a21b7..d33920bf3 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -1,13 +1,14 @@ import base64 import json import threading +import time from http.client import HTTPConnection from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import urlsplit import pytest -from bcbench.agent.shared.mcp_gateway import start_bc_mcp_gateway +from bcbench.agent.shared.mcp_gateway import BcMcpGateway, start_bc_mcp_gateway from bcbench.exceptions import AgentError @@ -214,3 +215,50 @@ def test_probe_never_raises_on_bad_upstream(self, monkeypatch): assert gw.probe_tools() == [] finally: gw.stop() + + +class _HeldOpenSseHandler(BaseHTTPRequestHandler): + """Sends one small SSE event, flushes, then holds the stream open before closing.""" + + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log + pass + + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(b"event: message\ndata: early\n\n") + self.wfile.flush() + time.sleep(3.0) # keep the stream open after the event, as the BC MCP endpoint does + + +def test_gateway_relays_held_open_sse_event_promptly(): + server = ThreadingHTTPServer(("127.0.0.1", 0), _HeldOpenSseHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + gateway = BcMcpGateway(f"http://127.0.0.1:{port}/BC", "admin", "secret", None).start() + split = urlsplit(gateway.base_url) + connection = HTTPConnection(split.hostname, split.port, timeout=10) + try: + connection.request("GET", "/BC/mcp") + response = connection.getresponse() + start = time.monotonic() + line = b"" + while b"data:" not in line: + line = response.readline() + if not line: + break + elapsed = time.monotonic() - start + assert b"data: early" in line + # read1() flushes the event immediately; the old read() would stall until the upstream closes (~3s). + assert elapsed < 2.0 + finally: + connection.close() + gateway.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) From ba838571af4417ded6bf7ad08764fa07716e050a Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 01:21:23 +0200 Subject: [PATCH 45/52] Capture Claude tool_use + transcript via stream-json output Switch Claude Code to --output-format=stream-json --verbose so every event is emitted as JSONL: parse tool_use blocks (incl. mcp__bcmcp__* MCP calls the pre-tool-use hook misses) for tool_usage, log the session-init mcp_servers + tool count to show whether BC MCP registered in-session, and persist the full stream as a claude-transcript-.log artifact for analysis. Falls back to the hook only when the stream carried no tool calls. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/claude/agent.py | 33 +++++++--------- src/bcbench/agent/claude/metrics.py | 61 +++++++++++++++++++++++++++++ tests/test_claude_code_agent.py | 3 +- tests/test_claude_code_metrics.py | 52 +++++++++++++++++++++++- 4 files changed, 129 insertions(+), 20 deletions(-) diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index ee9ed3e7a..cbf137b10 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -1,11 +1,10 @@ -import json import shutil import subprocess from pathlib import Path import yaml -from bcbench.agent.claude.metrics import parse_metrics +from bcbench.agent.claude.metrics import parse_stream_output from bcbench.agent.shared import agent_subprocess_env, build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks, resolve_config_plugins, start_bc_mcp_gateway from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry @@ -79,7 +78,8 @@ def run_claude_code( try: cmd_args = [ claude_cmd, - "--output-format=json", + "--output-format=stream-json", # emit every event (incl. tool_use, session init) as JSONL + "--verbose", # required for stream-json in --print mode "--strict-mcp-config", # Only use MCP servers from --mcp-config, ignoring all other MCP configurations "--setting-sources=project,local", f"--model={model}", @@ -122,21 +122,18 @@ def run_claude_code( stdout: str = result.stdout.decode("utf-8", errors="replace") if result.stdout else "" logger.debug(f"Claude Code raw output: {stdout}") - metrics = None - for line in stdout.splitlines(): - striped_line: str = line.strip() - if striped_line: - try: - data = json.loads(striped_line) - if "result" in data: - logger.info(data["result"]) - metrics = parse_metrics(data) - except json.JSONDecodeError: - logger.warning(f"Skipping non-JSON line: {striped_line}") - - tool_usage: dict[str, int] | None = parse_tool_usage_from_hooks(tool_log_path) - if metrics and tool_usage: - metrics = metrics.model_copy(update={"tool_usage": tool_usage}) + # Persist the full event stream for transcript analysis (the workflow uploads *.log artifacts). + transcript_path = output_dir / f"claude-transcript-{entry.instance_id}.log" + transcript_path.write_text(stdout, encoding="utf-8") + + metrics, final_response = parse_stream_output(stdout.splitlines()) + if final_response: + logger.info(final_response) + + # The stream's tool_use events capture sub-agent and MCP tool calls; fall back to the pre-tool-use + # hook only when the stream carried none. + if metrics and not metrics.tool_usage and (hook_tool_usage := parse_tool_usage_from_hooks(tool_log_path)): + metrics = metrics.model_copy(update={"tool_usage": hook_tool_usage}) except subprocess.TimeoutExpired: logger.exception(f"Claude Code timed out after {_config.timeout.agent_execution} seconds") metrics = AgentMetrics(execution_time=_config.timeout.agent_execution) diff --git a/src/bcbench/agent/claude/metrics.py b/src/bcbench/agent/claude/metrics.py index f4ee6a3d7..bd4f1321e 100644 --- a/src/bcbench/agent/claude/metrics.py +++ b/src/bcbench/agent/claude/metrics.py @@ -1,3 +1,7 @@ +import json +from collections import Counter +from collections.abc import Sequence + from bcbench.logger import get_logger from bcbench.types import AgentMetrics @@ -41,3 +45,60 @@ def parse_metrics(data: dict) -> AgentMetrics | None: logger.warning("No metrics found in Claude Code output") return None + + +def parse_stream_output(output_lines: Sequence[str]) -> tuple[AgentMetrics | None, str | None]: + """Parse metrics + final response from `claude --output-format=stream-json --verbose` (JSONL) stdout. + + Event shapes (Claude Code): + system/init: lists the `tools` and `mcp_servers` registered for the session; logged so a run + shows whether the BC MCP tools actually connected in-session. + assistant: ``message.content`` holds ``tool_use`` blocks whose ``name`` (e.g. + ``mcp__bcmcp__bc_data_query``) captures sub-agent and MCP tool calls the pre-tool-use hook + never sees. + result: terminal event carrying duration/turns/usage and the final ``result`` text. + + Returns: + The parsed metrics (with tool usage from the stream) and the agent's final response text. + """ + tool_usage: Counter[str] = Counter() + final_response: str | None = None + metrics: AgentMetrics | None = None + + for line_number, line in enumerate(output_lines, start=1): + if not line.strip(): + continue + + try: + event = json.loads(line) + except json.JSONDecodeError as error: + logger.warning(f"Skipping invalid JSON from Claude Code output at line {line_number}: {error}") + continue + + if not isinstance(event, dict): + continue + + match event.get("type"): + case "system" if event.get("subtype") == "init": + servers = event.get("mcp_servers") + tools = event.get("tools") + tool_count = len(tools) if isinstance(tools, list) else "?" + logger.info(f"Claude session init: mcp_servers={servers}, {tool_count} tools registered") + case "assistant": + message = event.get("message") + if isinstance(message, dict): + for block in message.get("content", []): + if isinstance(block, dict) and block.get("type") == "tool_use": + name = block.get("name") + if isinstance(name, str) and name: + tool_usage[name] += 1 + case "result": + metrics = parse_metrics(event) + result_text = event.get("result") + if isinstance(result_text, str) and result_text: + final_response = result_text + + if tool_usage: + metrics = (metrics or AgentMetrics()).model_copy(update={"tool_usage": dict(tool_usage)}) + + return metrics, final_response diff --git a/tests/test_claude_code_agent.py b/tests/test_claude_code_agent.py index b62c7766c..2141c4266 100644 --- a/tests/test_claude_code_agent.py +++ b/tests/test_claude_code_agent.py @@ -43,7 +43,8 @@ def test_claude_code_excludes_user_settings_and_auto_memory(tmp_path: Path, monk assert mock_run.call_args.args[0] == [ "claude", - "--output-format=json", + "--output-format=stream-json", + "--verbose", "--strict-mcp-config", "--setting-sources=project,local", "--model=claude-test-model", diff --git a/tests/test_claude_code_metrics.py b/tests/test_claude_code_metrics.py index 8a5608a3a..654b8df2f 100644 --- a/tests/test_claude_code_metrics.py +++ b/tests/test_claude_code_metrics.py @@ -1,8 +1,10 @@ """Tests for Claude Code metrics parsing.""" +import json + import pytest -from bcbench.agent.claude.metrics import parse_metrics +from bcbench.agent.claude.metrics import parse_metrics, parse_stream_output class TestClaudeCodeMetricsParsing: @@ -115,3 +117,51 @@ def test_parse_metrics_with_model_usage(self): assert metrics.turn_count == 14 assert metrics.prompt_tokens == 41 + 22439 + 246700 assert metrics.completion_tokens == 1909 + + +class TestClaudeStreamParsing: + def _lines(self, *events: dict) -> list[str]: + return [json.dumps(event) for event in events] + + def test_counts_mcp_tool_use_across_assistant_messages(self): + lines = self._lines( + {"type": "system", "subtype": "init", "mcp_servers": [{"name": "bcmcp", "status": "connected"}], "tools": ["Bash", "mcp__bcmcp__bc_data_query"]}, + {"type": "assistant", "message": {"content": [{"type": "text", "text": "Looking"}, {"type": "tool_use", "name": "mcp__bcmcp__bc_data_find_tables", "input": {}}]}}, + {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "mcp__bcmcp__bc_data_query", "input": {}}]}}, + {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "mcp__bcmcp__bc_data_query", "input": {}}]}}, + {"type": "result", "duration_ms": 5000, "num_turns": 3, "result": "Done"}, + ) + + metrics, final_response = parse_stream_output(lines) + + assert final_response == "Done" + assert metrics is not None + assert metrics.tool_usage == {"mcp__bcmcp__bc_data_find_tables": 1, "mcp__bcmcp__bc_data_query": 2} + assert metrics.execution_time == 5.0 + assert metrics.turn_count == 3 + + def test_tool_usage_without_result_event_still_returned(self): + lines = self._lines( + {"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "Bash", "input": {}}]}}, + ) + + metrics, final_response = parse_stream_output(lines) + + assert final_response is None + assert metrics is not None + assert metrics.tool_usage == {"Bash": 1} + + def test_no_events_returns_none(self): + metrics, final_response = parse_stream_output(["", " "]) + + assert metrics is None + assert final_response is None + + def test_skips_malformed_lines(self): + lines = ["not json", json.dumps({"type": "result", "duration_ms": 1000, "result": "ok"})] + + metrics, final_response = parse_stream_output(lines) + + assert final_response == "ok" + assert metrics is not None + assert metrics.execution_time == 1.0 From 5014d2769f12b52056add479d10af27bba1ec9af Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 02:02:57 +0200 Subject: [PATCH 46/52] Raise Claude MCP timeout for BC's slow cold tools/list; log gateway requests Root cause of bcmcp 'failed' in Claude's session: BC MCP's first tools/list compiles the tool catalog (~45s cold), exceeding Claude Code's 30s default MCP_TIMEOUT, so the server is dropped and its tools never register (mslearn, being fast, connects). Reproduced locally that Claude connects fine to the gateway+mock, isolating this to BC's cold-start latency. Set MCP_TIMEOUT and MCP_TOOL_TIMEOUT to 180s for the Claude subprocess, and log each proxied request (jsonrpc method -> HTTP status, Content-Type, elapsed) so CI shows the real request pattern and timing. Adds tools/run_gateway_local.py for local repro. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/claude/agent.py | 12 ++++++++- src/bcbench/agent/shared/mcp_gateway.py | 16 ++++++++++- tools/run_gateway_local.py | 36 +++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 tools/run_gateway_local.py diff --git a/src/bcbench/agent/claude/agent.py b/src/bcbench/agent/claude/agent.py index cbf137b10..892bf4ce4 100644 --- a/src/bcbench/agent/claude/agent.py +++ b/src/bcbench/agent/claude/agent.py @@ -113,7 +113,17 @@ def run_claude_code( result = subprocess.run( cmd_args, cwd=str(repo_path), - env=agent_subprocess_env({"CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1"}), + env=agent_subprocess_env( + { + "CLAUDE_CODE_DISABLE_AUTO_MEMORY": "1", + # BC MCP's first tools/list compiles the tool catalog and can take ~45s on a cold + # container, well past Claude's 30s default MCP startup timeout -> the server is + # marked "failed" and its tools never register. Raise both the connection and tool + # execution timeouts so the slow first response is tolerated. + "MCP_TIMEOUT": "180000", + "MCP_TOOL_TIMEOUT": "180000", + } + ), timeout=_config.timeout.agent_execution, check=True, capture_output=True, diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 853e7447c..f81d330df 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -51,6 +51,16 @@ _PROBE_TIMEOUT_SECONDS = 180 +def _jsonrpc_method(body: bytes | None) -> str | None: + if not body: + return None + try: + obj = json.loads(body) + except (json.JSONDecodeError, UnicodeDecodeError): + return None + return obj.get("method") if isinstance(obj, dict) else None + + def _read_jsonrpc(response, deadline: float) -> tuple[dict, str]: # noqa: ANN001 - http.client.HTTPResponse """Parse a JSON-RPC result and return it plus a raw snippet of the response for diagnostics. @@ -211,24 +221,28 @@ def _path_allowed(self) -> bool: def _handle(self) -> None: if not self._path_allowed(): + logger.info(f"BC MCP gateway BLOCKED {self.command} {self.path} -> 403") self.send_error(403, "Forbidden") return length = self.headers.get("Content-Length") body: bytes | None = self.rfile.read(int(length)) if length else None + rpc_method = _jsonrpc_method(body) request_headers: dict[str, str] = {k: v for k, v in self.headers.items() if k.lower() not in _STRIPPED_REQUEST_HEADERS} request_headers["Host"] = f"{gateway._origin_host}:{gateway._origin_port}" request_headers.update(gateway._injected_headers) connection = HTTPConnection(gateway._origin_host, gateway._origin_port, timeout=_UPSTREAM_TIMEOUT_SECONDS) + started = time.monotonic() try: connection.request(self.command, self.path, body=body, headers=request_headers) response = connection.getresponse() gateway._note_forwarded() + logger.info(f"BC MCP gateway {self.command} {rpc_method or self.path} -> HTTP {response.status} {response.getheader('Content-Type', '')} ({time.monotonic() - started:.1f}s)") self._relay(response) except Exception: - logger.exception("BC MCP gateway failed to reach the upstream endpoint") + logger.exception(f"BC MCP gateway failed to reach upstream for {self.command} {rpc_method or self.path} after {time.monotonic() - started:.1f}s") self.send_error(502, "Bad Gateway") finally: connection.close() diff --git a/tools/run_gateway_local.py b/tools/run_gateway_local.py new file mode 100644 index 000000000..bb84ce3aa --- /dev/null +++ b/tools/run_gateway_local.py @@ -0,0 +1,36 @@ +import importlib.util +import os +import sys +import time + +_spec = importlib.util.spec_from_file_location( + "mcp_gateway", + os.path.join(os.path.dirname(__file__), "..", "src", "bcbench", "agent", "shared", "mcp_gateway.py"), +) +_mod = importlib.util.module_from_spec(_spec) +# Stub the two bcbench imports mcp_gateway needs so we don't pull the whole package. +import types # noqa: E402 + +_exc = types.ModuleType("bcbench.exceptions") +_exc.AgentError = type("AgentError", (Exception,), {}) +_log = types.ModuleType("bcbench.logger") +import logging # noqa: E402 + +logging.basicConfig(level=logging.INFO) +_log.get_logger = lambda _name: logging.getLogger("mcp_gateway") +sys.modules["bcbench.exceptions"] = _exc +sys.modules["bcbench.logger"] = _log +_spec.loader.exec_module(_mod) +start_bc_mcp_gateway = _mod.start_bc_mcp_gateway + +os.environ["BC_MCP_URL"] = sys.argv[1] # upstream mock, e.g. http://127.0.0.1:8765/BC +os.environ["BC_SERVER_USERNAME"] = "admin" +os.environ["BC_SERVER_PASSWORD"] = "secret" + +gw = start_bc_mcp_gateway(enabled=True) +print(f"GATEWAY_URL={gw.base_url}/mcp", flush=True) +try: + while True: + time.sleep(1) +except KeyboardInterrupt: + gw.stop() From 9cdeac5fd65ed419ce030628faac2998fe0c766e Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 02:41:46 +0200 Subject: [PATCH 47/52] Serve tools/list from the gateway warm-up cache BC composes the MCP tool catalog per session: the first tools/list on a fresh session takes ~45s and the server forcibly closes the connection (WinError 10054), so Claude's client (even with a raised MCP_TIMEOUT) never registers the BC tools while the warm-up probe's own session works. The catalog is identical across sessions, so cache the tools/list result during warm-up and answer the agent's tools/list from it directly, decoupling the agent from BC's cold per-session composition. Falls back to forwarding when the cache is empty. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 45 +++++++++++++++++++++---- tests/test_mcp_gateway.py | 10 ++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index f81d330df..363f8c9c7 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -51,14 +51,16 @@ _PROBE_TIMEOUT_SECONDS = 180 -def _jsonrpc_method(body: bytes | None) -> str | None: +def _jsonrpc_method_and_id(body: bytes | None) -> tuple[str | None, object]: if not body: - return None + return None, None try: obj = json.loads(body) except (json.JSONDecodeError, UnicodeDecodeError): - return None - return obj.get("method") if isinstance(obj, dict) else None + return None, None + if not isinstance(obj, dict): + return None, None + return obj.get("method"), obj.get("id") def _read_jsonrpc(response, deadline: float) -> tuple[dict, str]: # noqa: ANN001 - http.client.HTTPResponse @@ -118,6 +120,12 @@ def __init__(self, upstream_url: str, username: str, password: str, company: str self._lock = threading.Lock() self._forwarded_count = 0 self.base_url: str | None = None + # tools/list "result" object captured during warm-up. BC composes the tool catalog per MCP + # session and the first tools/list is slow (~45s) and sometimes dropped by the server, which + # blows past the agent's MCP startup timeout so the server registers zero tools. The catalog is + # identical across sessions, so once warm-up has it the gateway answers tools/list from here, + # decoupling the agent from BC's cold per-session composition. + self._cached_tools_result: dict[str, object] | None = None @property def forwarded_count(self) -> int: @@ -174,7 +182,11 @@ def _handshake_tools(self, host: str, port: int, extra_headers: dict[str, str]) self._rpc(host, port, extra_headers, "notifications/initialized", None, session_id=session_id) before_list = time.monotonic() _, listed, diag = self._rpc(host, port, extra_headers, "tools/list", {}, request_id=2, session_id=session_id) - tools = [t.get("name") for t in (listed.get("result") or {}).get("tools", []) if isinstance(t, dict)] + result = listed.get("result") + tools = [t.get("name") for t in (result or {}).get("tools", []) if isinstance(t, dict)] + if tools and isinstance(result, dict): + with self._lock: + self._cached_tools_result = result return tools, time.monotonic() - before_list, diag def probe_tools(self) -> list[str]: @@ -227,7 +239,10 @@ def _handle(self) -> None: length = self.headers.get("Content-Length") body: bytes | None = self.rfile.read(int(length)) if length else None - rpc_method = _jsonrpc_method(body) + rpc_method, rpc_id = _jsonrpc_method_and_id(body) + + if rpc_method == "tools/list" and self._serve_cached_tools(rpc_id): + return request_headers: dict[str, str] = {k: v for k, v in self.headers.items() if k.lower() not in _STRIPPED_REQUEST_HEADERS} request_headers["Host"] = f"{gateway._origin_host}:{gateway._origin_port}" @@ -247,6 +262,24 @@ def _handle(self) -> None: finally: connection.close() + def _serve_cached_tools(self, request_id: object) -> bool: + """Answer tools/list from the warm-up cache, bypassing BC's slow per-session composition.""" + with gateway._lock: + cached = gateway._cached_tools_result + if cached is None: + return False + payload = json.dumps({"jsonrpc": "2.0", "id": request_id, "result": cached}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + self.wfile.flush() + tools = cached.get("tools") if isinstance(cached, dict) else None + tool_count = len(tools) if isinstance(tools, list) else "?" + logger.info(f"BC MCP gateway tools/list -> served {tool_count} tool(s) from warm-up cache") + return True + def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse self.send_response_only(response.status) content_length: str | None = None diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index d33920bf3..6d0cc81da 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -206,6 +206,16 @@ def mcp_gateway(self, monkeypatch): def test_probe_returns_exposed_tool_names(self, mcp_gateway): assert mcp_gateway.probe_tools() == ["bc_data_find_tables", "bc_data_query"] + def test_tools_list_served_from_cache_after_warmup(self, mcp_gateway): + # start_bc_mcp_gateway already ran warm-up, populating the tools/list cache. + import json as _json + + status, _headers, body = _request(mcp_gateway.base_url, "POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":7,"method":"tools/list"}') + assert status == 200 + payload = _json.loads(body) + assert payload["id"] == 7 + assert [t["name"] for t in payload["result"]["tools"]] == ["bc_data_find_tables", "bc_data_query"] + def test_probe_never_raises_on_bad_upstream(self, monkeypatch): monkeypatch.setenv("BC_MCP_URL", "http://127.0.0.1:1/BC") # nothing listening monkeypatch.setenv("BC_SERVER_USERNAME", "admin") From 7dfeec71ccf147eae7e89e10f3cdeed8b5e4999c Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 03:23:21 +0200 Subject: [PATCH 48/52] De-stream held-open POST SSE replies in the gateway BC answers POST initialize (and tools/call) with an SSE stream it holds open after the JSON-RPC result. Mirroring that open stream stalls Claude's MCP client, which waits for the response stream to end before continuing the handshake -> it never sends tools/list and drops bcmcp as 'failed' right after initialize (reproduced locally with a held-open mock). Collapse a POST SSE reply to a single application/json response (preserving Mcp-Session-Id) and close, so the client's handshake completes; GET (the server->client channel) still streams. With this + the tools/list cache, Claude connects and registers the BC tools. Adds a held-open POST SSE regression test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 23 +++++++++++- tests/test_mcp_gateway.py | 49 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 363f8c9c7..266528d13 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -255,7 +255,14 @@ def _handle(self) -> None: response = connection.getresponse() gateway._note_forwarded() logger.info(f"BC MCP gateway {self.command} {rpc_method or self.path} -> HTTP {response.status} {response.getheader('Content-Type', '')} ({time.monotonic() - started:.1f}s)") - self._relay(response) + # A POST is request/response even when BC frames the reply as SSE and then holds the + # stream open; mirroring that open stream stalls MCP clients (e.g. Claude) that wait for + # it to end before continuing the handshake. De-stream POST SSE replies to a single + # application/json response and close. GET (the server->client channel) still streams. + if self.command == "POST" and response.status == 200 and "text/event-stream" in (response.getheader("Content-Type", "") or ""): + self._relay_post_sse(response) + else: + self._relay(response) except Exception: logger.exception(f"BC MCP gateway failed to reach upstream for {self.command} {rpc_method or self.path} after {time.monotonic() - started:.1f}s") self.send_error(502, "Bad Gateway") @@ -280,6 +287,20 @@ def _serve_cached_tools(self, request_id: object) -> bool: logger.info(f"BC MCP gateway tools/list -> served {tool_count} tool(s) from warm-up cache") return True + def _relay_post_sse(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse + """Collapse a held-open SSE reply to a single application/json response, then close.""" + session_id = response.getheader("Mcp-Session-Id") + result, _raw = _read_jsonrpc(response, deadline=time.monotonic() + _UPSTREAM_TIMEOUT_SECONDS) + payload = json.dumps(result).encode() if result else b"{}" + self.send_response(200) + self.send_header("Content-Type", "application/json") + if session_id: + self.send_header("Mcp-Session-Id", session_id) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + self.wfile.flush() + def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse self.send_response_only(response.status) content_length: str | None = None diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 6d0cc81da..7639b018d 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -245,6 +245,55 @@ def do_GET(self) -> None: time.sleep(3.0) # keep the stream open after the event, as the BC MCP endpoint does +class _HeldOpenPostSseHandler(BaseHTTPRequestHandler): + """Answers a POST with an SSE event carrying a JSON-RPC result, then holds the stream open.""" + + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log + pass + + def do_POST(self) -> None: + n = int(self.headers.get("Content-Length", 0)) + if n: + self.rfile.read(n) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Mcp-Session-Id", "sess-hold") + self.end_headers() + self.wfile.write(b'event: message\ndata: {"jsonrpc":"2.0","id":1,"result":{"ok":true}}\n\n') + self.wfile.flush() + time.sleep(30) # hold open like BC; the gateway must not wait for this + + +def test_gateway_destreams_held_open_post_sse_reply(): + server = ThreadingHTTPServer(("127.0.0.1", 0), _HeldOpenPostSseHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + gateway = BcMcpGateway(f"http://127.0.0.1:{port}/BC", "admin", "secret", None).start() + split = urlsplit(gateway.base_url) + connection = HTTPConnection(split.hostname, split.port, timeout=10) + try: + start = time.monotonic() + connection.request("POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":1,"method":"initialize"}') + response = connection.getresponse() + body = response.read() # completes because the gateway sends Content-Length and closes + elapsed = time.monotonic() - start + assert response.status == 200 + assert response.getheader("Content-Type") == "application/json" + assert response.getheader("Mcp-Session-Id") == "sess-hold" + assert json.loads(body)["result"] == {"ok": True} + # De-streamed promptly; the old behavior would hang until the upstream closed (~30s). + assert elapsed < 5.0 + finally: + connection.close() + gateway.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def test_gateway_relays_held_open_sse_event_promptly(): server = ThreadingHTTPServer(("127.0.0.1", 0), _HeldOpenSseHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) From 684a29f92d52be13296522e563ddab54639f5c60 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 04:21:17 +0200 Subject: [PATCH 49/52] Relay POST SSE faithfully and close on response; handle client disconnect Keep BC's exact SSE bytes for a POST reply (some MCP clients require the event-stream framing) but stop as soon as the JSON-RPC response event arrives, so the held-open stream doesn't stall the client. Also treat a client disconnecting mid-stream as normal instead of logging it as an upstream failure, and add de-stream timing/response-seen logging to diagnose the remaining Claude bcmcp handshake failure in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 75 +++++++++++++++++++------ tests/test_mcp_gateway.py | 11 ++-- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 266528d13..ed96903f0 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -240,6 +240,7 @@ def _handle(self) -> None: length = self.headers.get("Content-Length") body: bytes | None = self.rfile.read(int(length)) if length else None rpc_method, rpc_id = _jsonrpc_method_and_id(body) + self._response_started = False if rpc_method == "tools/list" and self._serve_cached_tools(rpc_id): return @@ -257,15 +258,25 @@ def _handle(self) -> None: logger.info(f"BC MCP gateway {self.command} {rpc_method or self.path} -> HTTP {response.status} {response.getheader('Content-Type', '')} ({time.monotonic() - started:.1f}s)") # A POST is request/response even when BC frames the reply as SSE and then holds the # stream open; mirroring that open stream stalls MCP clients (e.g. Claude) that wait for - # it to end before continuing the handshake. De-stream POST SSE replies to a single - # application/json response and close. GET (the server->client channel) still streams. + # it to end before continuing the handshake. Relay the POST SSE reply but close once the + # JSON-RPC response event arrives. GET (the server->client channel) still streams. if self.command == "POST" and response.status == 200 and "text/event-stream" in (response.getheader("Content-Type", "") or ""): - self._relay_post_sse(response) + self._relay_post_sse(rpc_method, response) else: self._relay(response) + except (ConnectionError, OSError) as error: + # The client (agent) closing its side mid-stream is normal; don't misreport it as an + # upstream failure, and don't try to send an error once the response has begun. + if self._response_started: + logger.debug(f"BC MCP gateway client disconnected during {self.command} {rpc_method or self.path}: {error}") + self.close_connection = True + else: + logger.exception(f"BC MCP gateway failed to reach upstream for {self.command} {rpc_method or self.path} after {time.monotonic() - started:.1f}s") + self.send_error(502, "Bad Gateway") except Exception: - logger.exception(f"BC MCP gateway failed to reach upstream for {self.command} {rpc_method or self.path} after {time.monotonic() - started:.1f}s") - self.send_error(502, "Bad Gateway") + logger.exception(f"BC MCP gateway error handling {self.command} {rpc_method or self.path} after {time.monotonic() - started:.1f}s") + if not self._response_started: + self.send_error(502, "Bad Gateway") finally: connection.close() @@ -276,6 +287,7 @@ def _serve_cached_tools(self, request_id: object) -> bool: if cached is None: return False payload = json.dumps({"jsonrpc": "2.0", "id": request_id, "result": cached}).encode() + self._response_started = True self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) @@ -287,21 +299,52 @@ def _serve_cached_tools(self, request_id: object) -> bool: logger.info(f"BC MCP gateway tools/list -> served {tool_count} tool(s) from warm-up cache") return True - def _relay_post_sse(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse - """Collapse a held-open SSE reply to a single application/json response, then close.""" - session_id = response.getheader("Mcp-Session-Id") - result, _raw = _read_jsonrpc(response, deadline=time.monotonic() + _UPSTREAM_TIMEOUT_SECONDS) - payload = json.dumps(result).encode() if result else b"{}" - self.send_response(200) - self.send_header("Content-Type", "application/json") - if session_id: - self.send_header("Mcp-Session-Id", session_id) - self.send_header("Content-Length", str(len(payload))) + def _relay_post_sse(self, rpc_method: str | None, response) -> None: # noqa: ANN001 - http.client.HTTPResponse + """Relay a POST SSE reply faithfully but close once the JSON-RPC response event arrives. + + BC answers a POST with an event stream it then holds open; a client waiting for the stream to + end stalls. Forward the upstream bytes unchanged as text/event-stream (so the client sees the + exact response), and stop as soon as an SSE ``data:`` line carries a JSON-RPC result/error -- + the request is answered, so the stream can close. + """ + self._response_started = True + self.send_response_only(200) + for key, value in response.getheaders(): + lowered = key.lower() + if lowered in _HOP_BY_HOP or lowered in ("content-length", "content-type"): + continue + self.send_header(key, value) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Transfer-Encoding", "chunked") self.end_headers() - self.wfile.write(payload) + + started = time.monotonic() + deadline = started + _UPSTREAM_TIMEOUT_SECONDS + saw_response = False + while time.monotonic() < deadline: + raw_line = response.readline() + if not raw_line: + break + self.wfile.write(b"%X\r\n" % len(raw_line)) + self.wfile.write(raw_line) + self.wfile.write(b"\r\n") + self.wfile.flush() + stripped = raw_line.decode("utf-8", errors="replace").strip() + if stripped.startswith("data:"): + try: + obj = json.loads(stripped[5:].strip()) + except json.JSONDecodeError: + obj = None + if isinstance(obj, dict) and ("result" in obj or "error" in obj): + saw_response = True + elif not stripped and saw_response: + break # blank line terminates the event that carried the response + self.wfile.write(b"0\r\n\r\n") self.wfile.flush() + logger.info(f"BC MCP gateway de-streamed {rpc_method} in {time.monotonic() - started:.1f}s (response_seen={saw_response})") def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse + self._response_started = True self.send_response_only(response.status) content_length: str | None = None for key, value in response.getheaders(): diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 7639b018d..dea0a30b3 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -278,13 +278,14 @@ def test_gateway_destreams_held_open_post_sse_reply(): start = time.monotonic() connection.request("POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":1,"method":"initialize"}') response = connection.getresponse() - body = response.read() # completes because the gateway sends Content-Length and closes + body = response.read() # completes because the gateway closes the stream after the response event elapsed = time.monotonic() - start assert response.status == 200 - assert response.getheader("Content-Type") == "application/json" - assert response.getheader("Mcp-Session-Id") == "sess-hold" - assert json.loads(body)["result"] == {"ok": True} - # De-streamed promptly; the old behavior would hang until the upstream closed (~30s). + # SSE framing is preserved (faithful relay), and the JSON-RPC result is carried in a data: line. + assert response.getheader("Content-Type") == "text/event-stream" + assert b'"result"' in body + assert b'"ok": true' in body or b'"ok":true' in body + # Closed promptly; the old behavior would hang until the upstream closed (~30s). assert elapsed < 5.0 finally: connection.close() From 37604de9d8c22751909e6c75ac545b2ca10f6368 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 04:53:33 +0200 Subject: [PATCH 50/52] Log BC initialize result + relayed SSE headers to diagnose Claude handshake Claude receives a valid initialize response (response_seen=True) then aborts before notifications/initialized, only against real BC (not the local mock). Log BC's initialize result content and the exact headers the gateway forwards on the relayed SSE, to pin down what real BC returns that Claude rejects. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index ed96903f0..7320409f1 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -177,7 +177,8 @@ def _rpc(self, host: str, port: int, extra_headers: dict[str, str], method: str, def _handshake_tools(self, host: str, port: int, extra_headers: dict[str, str]) -> tuple[list[str], float, str]: """initialize -> notifications/initialized -> tools/list against one target; returns (tools, tools_list_seconds, diag).""" - session_id, _, _ = self._rpc(host, port, extra_headers, "initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) + session_id, init_result, init_diag = self._rpc(host, port, extra_headers, "initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "bcbench-probe", "version": "1.0"}}, request_id=1) + logger.info(f"BC MCP initialize result: session={session_id!r} result={json.dumps(init_result)[:400]} diag={init_diag[:200]}") if session_id: self._rpc(host, port, extra_headers, "notifications/initialized", None, session_id=session_id) before_list = time.monotonic() @@ -308,11 +309,10 @@ def _relay_post_sse(self, rpc_method: str | None, response) -> None: # noqa: AN the request is answered, so the stream can close. """ self._response_started = True + forwarded_headers = [(k, v) for k, v in response.getheaders() if k.lower() not in _HOP_BY_HOP and k.lower() not in ("content-length", "content-type")] + logger.info(f"BC MCP gateway relaying {rpc_method} SSE; upstream headers forwarded: {forwarded_headers}") self.send_response_only(200) - for key, value in response.getheaders(): - lowered = key.lower() - if lowered in _HOP_BY_HOP or lowered in ("content-length", "content-type"): - continue + for key, value in forwarded_headers: self.send_header(key, value) self.send_header("Content-Type", "text/event-stream") self.send_header("Transfer-Encoding", "chunked") From d083272d8b73b01ce8da43b3551da9c8c97a93a2 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 05:35:09 +0200 Subject: [PATCH 51/52] Relay BC MCP faithfully (hold streams open); only short-circuit tools/list BC's initialize result advertises experimental x-ms-headerless and BC keeps the initialize SSE stream open as the client's event channel. Closing/collapsing that stream made Claude abort right after initialize (never sending notifications/initialized). Relay every response byte-for-byte and keep streams open exactly as BC does, so streamable-HTTP clients see the real transport. The only short-circuit is the tools/list warm-up cache (now framed as a single-event SSE to match BC), which avoids BC's slow/dropping per-session tool-catalog composition. Removes the POST SSE de-stream path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 72 ++++++------------------- tests/test_mcp_gateway.py | 28 ++++++---- 2 files changed, 33 insertions(+), 67 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index 7320409f1..c18edaba7 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -257,14 +257,12 @@ def _handle(self) -> None: response = connection.getresponse() gateway._note_forwarded() logger.info(f"BC MCP gateway {self.command} {rpc_method or self.path} -> HTTP {response.status} {response.getheader('Content-Type', '')} ({time.monotonic() - started:.1f}s)") - # A POST is request/response even when BC frames the reply as SSE and then holds the - # stream open; mirroring that open stream stalls MCP clients (e.g. Claude) that wait for - # it to end before continuing the handshake. Relay the POST SSE reply but close once the - # JSON-RPC response event arrives. GET (the server->client channel) still streams. - if self.command == "POST" and response.status == 200 and "text/event-stream" in (response.getheader("Content-Type", "") or ""): - self._relay_post_sse(rpc_method, response) - else: - self._relay(response) + # Relay faithfully, byte-for-byte, holding the stream open exactly as BC does. BC's MCP + # server keeps the initialize (and other) SSE streams open as the client's event channel; + # collapsing or closing them early breaks streamable-HTTP clients (e.g. Claude). The only + # short-circuit is the tools/list cache above, which sidesteps BC's slow/dropping + # per-session tool-catalog composition. + self._relay(response) except (ConnectionError, OSError) as error: # The client (agent) closing its side mid-stream is normal; don't misreport it as an # upstream failure, and don't try to send an error once the response has begun. @@ -282,67 +280,29 @@ def _handle(self) -> None: connection.close() def _serve_cached_tools(self, request_id: object) -> bool: - """Answer tools/list from the warm-up cache, bypassing BC's slow per-session composition.""" + """Answer tools/list from the warm-up cache, bypassing BC's slow per-session composition. + + Framed as a single-event SSE stream (then closed) to mirror how BC replies to tools/list, so + the client sees the same transport it would from the real endpoint. + """ with gateway._lock: cached = gateway._cached_tools_result if cached is None: return False - payload = json.dumps({"jsonrpc": "2.0", "id": request_id, "result": cached}).encode() + event = ("event: message\ndata: " + json.dumps({"jsonrpc": "2.0", "id": request_id, "result": cached}) + "\n\n").encode() self._response_started = True self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.send_header("Content-Length", str(len(event))) self.end_headers() - self.wfile.write(payload) + self.wfile.write(event) self.wfile.flush() tools = cached.get("tools") if isinstance(cached, dict) else None tool_count = len(tools) if isinstance(tools, list) else "?" logger.info(f"BC MCP gateway tools/list -> served {tool_count} tool(s) from warm-up cache") return True - def _relay_post_sse(self, rpc_method: str | None, response) -> None: # noqa: ANN001 - http.client.HTTPResponse - """Relay a POST SSE reply faithfully but close once the JSON-RPC response event arrives. - - BC answers a POST with an event stream it then holds open; a client waiting for the stream to - end stalls. Forward the upstream bytes unchanged as text/event-stream (so the client sees the - exact response), and stop as soon as an SSE ``data:`` line carries a JSON-RPC result/error -- - the request is answered, so the stream can close. - """ - self._response_started = True - forwarded_headers = [(k, v) for k, v in response.getheaders() if k.lower() not in _HOP_BY_HOP and k.lower() not in ("content-length", "content-type")] - logger.info(f"BC MCP gateway relaying {rpc_method} SSE; upstream headers forwarded: {forwarded_headers}") - self.send_response_only(200) - for key, value in forwarded_headers: - self.send_header(key, value) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Transfer-Encoding", "chunked") - self.end_headers() - - started = time.monotonic() - deadline = started + _UPSTREAM_TIMEOUT_SECONDS - saw_response = False - while time.monotonic() < deadline: - raw_line = response.readline() - if not raw_line: - break - self.wfile.write(b"%X\r\n" % len(raw_line)) - self.wfile.write(raw_line) - self.wfile.write(b"\r\n") - self.wfile.flush() - stripped = raw_line.decode("utf-8", errors="replace").strip() - if stripped.startswith("data:"): - try: - obj = json.loads(stripped[5:].strip()) - except json.JSONDecodeError: - obj = None - if isinstance(obj, dict) and ("result" in obj or "error" in obj): - saw_response = True - elif not stripped and saw_response: - break # blank line terminates the event that carried the response - self.wfile.write(b"0\r\n\r\n") - self.wfile.flush() - logger.info(f"BC MCP gateway de-streamed {rpc_method} in {time.monotonic() - started:.1f}s (response_seen={saw_response})") - def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse self._response_started = True self.send_response_only(response.status) diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index dea0a30b3..ef4c21992 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -210,9 +210,12 @@ def test_tools_list_served_from_cache_after_warmup(self, mcp_gateway): # start_bc_mcp_gateway already ran warm-up, populating the tools/list cache. import json as _json - status, _headers, body = _request(mcp_gateway.base_url, "POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":7,"method":"tools/list"}') + status, headers, body = _request(mcp_gateway.base_url, "POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":7,"method":"tools/list"}') assert status == 200 - payload = _json.loads(body) + # Served as a single-event SSE stream, mirroring BC's tools/list framing. + assert headers["Content-Type"] == "text/event-stream" + data_line = next(line for line in body.decode().splitlines() if line.startswith("data:")) + payload = _json.loads(data_line[len("data:") :].strip()) assert payload["id"] == 7 assert [t["name"] for t in payload["result"]["tools"]] == ["bc_data_find_tables", "bc_data_query"] @@ -263,10 +266,10 @@ def do_POST(self) -> None: self.end_headers() self.wfile.write(b'event: message\ndata: {"jsonrpc":"2.0","id":1,"result":{"ok":true}}\n\n') self.wfile.flush() - time.sleep(30) # hold open like BC; the gateway must not wait for this + time.sleep(30) # hold open like BC; the gateway relays faithfully without waiting for the end -def test_gateway_destreams_held_open_post_sse_reply(): +def test_gateway_relays_post_sse_event_promptly_without_waiting_for_close(): server = ThreadingHTTPServer(("127.0.0.1", 0), _HeldOpenPostSseHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -278,15 +281,18 @@ def test_gateway_destreams_held_open_post_sse_reply(): start = time.monotonic() connection.request("POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":1,"method":"initialize"}') response = connection.getresponse() - body = response.read() # completes because the gateway closes the stream after the response event - elapsed = time.monotonic() - start + # The gateway relays BC's SSE bytes faithfully (holding the stream open); the response event + # reaches the client promptly even though the upstream keeps the stream open. assert response.status == 200 - # SSE framing is preserved (faithful relay), and the JSON-RPC result is carried in a data: line. assert response.getheader("Content-Type") == "text/event-stream" - assert b'"result"' in body - assert b'"ok": true' in body or b'"ok":true' in body - # Closed promptly; the old behavior would hang until the upstream closed (~30s). - assert elapsed < 5.0 + line = b"" + while b"data:" not in line: + line = response.readline() + if not line: + break + elapsed = time.monotonic() - start + assert b'"result"' in line + assert elapsed < 3.0 finally: connection.close() gateway.stop() From ecedbb2f86a8cc33ed9a14d221e916eaa95423c4 Mon Sep 17 00:00:00 2001 From: Onat Buyukakkus Date: Tue, 25 Aug 2026 06:34:30 +0200 Subject: [PATCH 52/52] Strip x-ms-headerless from BC initialize so Claude's MCP client connects Bisected against a byte-exact replica of BC's initialize response: the sole cause of Claude marking bcmcp 'failed' is capabilities.experimental = {x-ms-headerless: true}. The gateway now rewrites just that first initialize result event to drop capabilities.experimental, then keeps relaying faithfully (stream held open like BC). BC still works over the standard header-based session the warm-up probe uses, so dropping the advertisement is safe. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: db34a2a0-7035-4361-b911-becb72f86e21 --- src/bcbench/agent/shared/mcp_gateway.py | 59 ++++++++++++++++++++++--- tests/test_mcp_gateway.py | 55 +++++++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/src/bcbench/agent/shared/mcp_gateway.py b/src/bcbench/agent/shared/mcp_gateway.py index c18edaba7..cdf64c8d3 100644 --- a/src/bcbench/agent/shared/mcp_gateway.py +++ b/src/bcbench/agent/shared/mcp_gateway.py @@ -257,12 +257,15 @@ def _handle(self) -> None: response = connection.getresponse() gateway._note_forwarded() logger.info(f"BC MCP gateway {self.command} {rpc_method or self.path} -> HTTP {response.status} {response.getheader('Content-Type', '')} ({time.monotonic() - started:.1f}s)") - # Relay faithfully, byte-for-byte, holding the stream open exactly as BC does. BC's MCP - # server keeps the initialize (and other) SSE streams open as the client's event channel; - # collapsing or closing them early breaks streamable-HTTP clients (e.g. Claude). The only - # short-circuit is the tools/list cache above, which sidesteps BC's slow/dropping - # per-session tool-catalog composition. - self._relay(response) + # Relay faithfully, byte-for-byte, holding streams open exactly as BC does (its MCP + # server keeps SSE streams open as the client's event channel). The one exception is the + # initialize reply: BC advertises capabilities.experimental = {"x-ms-headerless": true}, + # which makes Claude's MCP client fail the connection; strip it so the client sees a + # standard server (BC still works over the normal header-based session the probe uses). + if rpc_method == "initialize" and response.status == 200 and "text/event-stream" in (response.getheader("Content-Type", "") or ""): + self._relay_initialize(response) + else: + self._relay(response) except (ConnectionError, OSError) as error: # The client (agent) closing its side mid-stream is normal; don't misreport it as an # upstream failure, and don't try to send an error once the response has begun. @@ -303,6 +306,50 @@ def _serve_cached_tools(self, request_id: object) -> bool: logger.info(f"BC MCP gateway tools/list -> served {tool_count} tool(s) from warm-up cache") return True + def _relay_initialize(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse + """Relay the initialize SSE reply but strip capabilities.experimental from the result. + + BC advertises ``capabilities.experimental = {"x-ms-headerless": true}``; Claude's MCP client + fails the connection when it sees it (bisected against a replica of BC's exact initialize + response). Rewrite just that first result event, then keep relaying faithfully so the stream + behaves exactly like BC's for everything else. + """ + self._response_started = True + forwarded_headers = [(k, v) for k, v in response.getheaders() if k.lower() not in _HOP_BY_HOP and k.lower() not in ("content-length", "content-type")] + self.send_response_only(200) + for key, value in forwarded_headers: + self.send_header(key, value) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + + deadline = time.monotonic() + _UPSTREAM_TIMEOUT_SECONDS + rewritten = False + while time.monotonic() < deadline: + raw_line = response.readline() + if not raw_line: + break + out_line = raw_line + stripped = raw_line.decode("utf-8", errors="replace").strip() + if not rewritten and stripped.startswith("data:"): + try: + obj = json.loads(stripped[5:].strip()) + except json.JSONDecodeError: + obj = None + if isinstance(obj, dict) and isinstance(obj.get("result"), dict): + capabilities = obj["result"].get("capabilities") + removed = isinstance(capabilities, dict) and capabilities.pop("experimental", None) is not None + out_line = ("data: " + json.dumps(obj) + "\n").encode() + rewritten = True + logger.info(f"BC MCP gateway rewrote initialize result (experimental stripped={removed})") + self.wfile.write(b"%X\r\n" % len(out_line)) + self.wfile.write(out_line) + self.wfile.write(b"\r\n") + self.wfile.flush() + # Keep relaying (holding the stream open) exactly like BC until the upstream or client closes. + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + def _relay(self, response) -> None: # noqa: ANN001 - http.client.HTTPResponse self._response_started = True self.send_response_only(response.status) diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index ef4c21992..149463d84 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -301,6 +301,61 @@ def test_gateway_relays_post_sse_event_promptly_without_waiting_for_close(): thread.join(timeout=5) +class _InitializeExperimentalHandler(BaseHTTPRequestHandler): + """Answers initialize over SSE with a capabilities.experimental block, held open like BC.""" + + protocol_version = "HTTP/1.1" + + def log_message(self, format: str, *args: object) -> None: # match stdlib signature; silence access log + pass + + def do_POST(self) -> None: + import json as _json + + n = int(self.headers.get("Content-Length", 0)) + req = _json.loads(self.rfile.read(n)) if n else {} + result = {"protocolVersion": "2024-11-05", "capabilities": {"experimental": {"x-ms-headerless": True}, "tools": {}}, "serverInfo": {"name": "BC"}} + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Mcp-Session-Id", "sess-init") + self.end_headers() + self.wfile.write(("data: " + _json.dumps({"jsonrpc": "2.0", "id": req.get("id"), "result": result}) + "\n\n").encode()) + self.wfile.flush() + time.sleep(30) # hold the stream open like BC + + +def test_gateway_strips_experimental_from_initialize(): + import json as _json + + server = ThreadingHTTPServer(("127.0.0.1", 0), _InitializeExperimentalHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + gateway = BcMcpGateway(f"http://127.0.0.1:{port}/BC", "admin", "secret", None).start() + split = urlsplit(gateway.base_url) + connection = HTTPConnection(split.hostname, split.port, timeout=10) + try: + connection.request("POST", "/BC/mcp", body=b'{"jsonrpc":"2.0","id":1,"method":"initialize"}') + response = connection.getresponse() + assert response.status == 200 + line = b"" + while b"data:" not in line: + line = response.readline() + if not line: + break + payload = _json.loads(line.decode()[len("data:") :].strip()) + # The x-ms-headerless experimental capability (which breaks Claude) is stripped; the rest stays. + assert "experimental" not in payload["result"]["capabilities"] + assert "tools" in payload["result"]["capabilities"] + assert payload["result"]["protocolVersion"] == "2024-11-05" + finally: + connection.close() + gateway.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def test_gateway_relays_held_open_sse_event_promptly(): server = ThreadingHTTPServer(("127.0.0.1", 0), _HeldOpenSseHandler) thread = threading.Thread(target=server.serve_forever, daemon=True)