Add data-query category: AL query-generation benchmark

.github/workflows/bake-dataquery-gold.yml

@@ -0,0 +1,76 @@
+name: Bake Data-Query Gold Rows
+
+# Runs each data-query gold query once against a BC container and stores the resulting rows as
+# gold_rows in dataset/dataquery.jsonl, then commits them back to the branch. Evaluation then compares
+# the agent's answer to these baked rows instead of recompiling/running the gold query live per run.
+
+on:
+  workflow_dispatch:
+    inputs:
+      git-ref:
+        description: "Branch to bake and commit to"
+        required: false
+        default: ""
+
+permissions:
+  contents: write
+
+env:
+  CATEGORY: data-query
+
+jobs:
+  get-entries:
+    uses: ./.github/workflows/get-entries.yml
+    with:
+      category: data-query
+
+  bake:
+    needs: get-entries
+    runs-on: ${{ needs.get-entries.outputs.runner }}
+    if: needs.get-entries.outputs.entries != '[]'
+    timeout-minutes: 120
+    environment:
+      name: ado-read
+      deployment: false
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ inputs.git-ref || github.ref_name }}
+
+      # Provision a single BC container (all data-query entries share the same version + dataset).
+      - name: Setup BC Container
+        id: setup-env
+        timeout-minutes: 40
+        uses: ./.github/actions/setup-bc-container-repo
+        with:
+          instance-id: ${{ fromJson(needs.get-entries.outputs.entries)[0] }}
+          category: data-query
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          skip-container: "false"
+          skip-repo: "true"
+
+      - name: Setup Python with UV
+        uses: ./.github/actions/setup-python-uv
+
+      - name: Bake gold rows
+        run: uv run bcbench dataset bake-dataquery-gold --repo-path "${{ steps.setup-env.outputs.repo_path }}"
+
+      - name: Commit baked dataset
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          if (git diff --quiet -- dataset/dataquery.jsonl) {
+            Write-Output "No gold_rows changes to commit."
+          }
+          else {
+            git add dataset/dataquery.jsonl
+            git commit -m "Bake data-query gold rows (run ${{ github.run_id }})"
+            git push origin HEAD:${{ inputs.git-ref || github.ref_name }}
+          }
+        shell: pwsh

.github/workflows/claude-evaluation.yml

@@ -24,6 +24,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
           - "extensibility-request-implement"
           - "extensibility-request-triage"
       test-run:
@@ -41,6 +42,21 @@ on:
         required: false
         default: false
         type: boolean
+      bc-mcp:
+        description: "Enable the Business Central MCP server"
+        required: false
+        default: false
+        type: boolean
+      ms-learn-mcp:
+        description: "Enable the Microsoft Learn MCP server"
+        required: false
+        default: false
+        type: boolean
+      skills:
+        description: "Enable agent skills"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -151,14 +167,28 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ inputs.bc-mcp && '--bc-mcp' || '' }} `
+            ${{ inputs.ms-learn-mcp && '--ms-learn-mcp' || '' }} `
+            ${{ inputs.skills && '--skills' || '' }}
+
+      - name: Capture BC NST logs
+        if: ${{ always() && inputs.bc-mcp }}
+        shell: pwsh
+        run: |
+          # TEMPORARY (diagnostic): pull NST event log + server config + MCP Configuration row to
+          # investigate why the BC MCP endpoint exposes zero Data Query tools.
+          .\scripts\Capture-NstLogs.ps1 -ContainerName "$env:BC_CONTAINER_NAME" -OutputDir "${{ env.EVALUATION_RESULTS_DIR }}"
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6
         if: always()
         with:
           name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
-          path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
+          # TEMPORARY (remove before merge): include agent *.log to diagnose BC MCP tool use.
+          path: |
+            ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
+            ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log
           retention-days: ${{ inputs.test-run && 1 || 30 }}
 
   summarize-results:
@@ -188,4 +218,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "bc-mcp": "${{ inputs.bc-mcp }}", "ms-learn-mcp": "${{ inputs.ms-learn-mcp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

.github/workflows/copilot-evaluation.yml

@@ -29,6 +29,7 @@ on:
           - "bug-fix"
           - "test-generation"
           - "code-review"
+          - "data-query"
           - "extensibility-request-implement"
           - "extensibility-request-triage"
       test-run:
@@ -46,6 +47,21 @@ on:
         required: false
         default: false
         type: boolean
+      bc-mcp:
+        description: "Enable the Business Central MCP server"
+        required: false
+        default: false
+        type: boolean
+      ms-learn-mcp:
+        description: "Enable the Microsoft Learn MCP server"
+        required: false
+        default: false
+        type: boolean
+      skills:
+        description: "Enable agent skills"
+        required: false
+        default: false
+        type: boolean
       repeat:
         description: "Number of times to run sequentially (ignored for test runs)"
         required: false
@@ -143,7 +159,12 @@ jobs:
         timeout-minutes: 120
         shell: pwsh
         env:
-          COPILOT_GITHUB_TOKEN: ${{ github.token }}
+          # Copilot CLI must authenticate MCP with a USER token: the Actions github.token (a ghs_
+          # installation token) gets 403 from GET /copilot/mcp_registry and fails closed, blocking ALL
+          # custom MCP servers (github/copilot-cli#4346). The org policy is already allow_all, so a
+          # Copilot-licensed user PAT lets the registry fetch succeed and the BC/MS-Learn MCP load.
+          # Falls back to github.token when the secret is absent (MCP off, but completions still work).
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN || github.token }}
           GH_TOKEN: ${{ github.token }}
         run: |
           Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN"
@@ -154,14 +175,20 @@ jobs:
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
             ${{ inputs.al-mcp && '--al-mcp' || '' }} `
-            ${{ inputs.al-lsp && '--al-lsp' || '' }}
+            ${{ inputs.al-lsp && '--al-lsp' || '' }} `
+            ${{ inputs.bc-mcp && '--bc-mcp' || '' }} `
+            ${{ inputs.ms-learn-mcp && '--ms-learn-mcp' || '' }} `
+            ${{ inputs.skills && '--skills' || '' }}
 
       - name: Upload evaluation results
         uses: actions/upload-artifact@v6
         if: always()
         with:
           name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
-          path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
+          # TEMPORARY (remove before merge): include agent *.log to diagnose BC MCP tool use.
+          path: |
+            ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
+            ${{ env.EVALUATION_RESULTS_DIR }}/**/*.log
           retention-days: ${{ inputs.test-run && 1 || 30 }}
 
   summarize-results:
@@ -191,4 +218,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "bc-mcp": "${{ inputs.bc-mcp }}", "ms-learn-mcp": "${{ inputs.ms-learn-mcp }}", "skills": "${{ inputs.skills }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

.gitignore

@@ -283,3 +283,6 @@ docs/Gemfile
 docs/Gemfile.lock
 
 *.orig
+
+# Local diagnostic artifact download folders (temp, leading underscore)
+_*/

dataset/dataquery.jsonl

@@ -0,0 +1,11 @@
+{"instance_id": "dataquery__outstanding-sales-value-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order line, return the customer's number, the customer's name, and their total outstanding amount. Use open sales order lines (Sales Line records whose document type is Order) and sum the line 'Outstanding Amount' field (which is net of VAT). Customers with no open sales order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingSalesByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesLine; \"Sales Line\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-sold-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "inventory"}, "nl_prompt": "Across all posted sales invoice lines whose type is Item, return each item's number together with the total quantity sold (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 SoldQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__avg-invoice-amount-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "finance"}, "nl_prompt": "For each country/region, return the country/region code and the average posted sales invoice line amount. Average the 'Amount' field (net of VAT) over all posted sales invoice lines, grouping the lines by their bill-to customer's Country/Region Code.", "ordered": false, "gold_query": "query 50100 AvgInvoiceAmountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(AvgAmount; Amount) { Method = Average; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchase-amount-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "purchasing"}, "nl_prompt": "For each vendor that has at least one posted purchase invoice line, return the vendor's number, the vendor's name, and their total posted purchase amount. Sum the 'Amount' field (net of VAT) from posted purchase invoice lines. Vendors with no posted purchase invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PurchaseAmountByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchInvLine; \"Purch. Inv. Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                column(TotalAmount; Amount) { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__open-sales-order-count-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-13", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one open sales order, return the customer's number, the customer's name, and the number of open sales orders they have. An open sales order is a sales document whose document type is Order; count the order documents (headers), not the order lines. Customers with no open sales orders must not appear.", "ordered": false, "gold_query": "query 50100 OpenSalesOrdersByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesHeader; \"Sales Header\")\n            {\n                DataItemLink = \"Sell-to Customer No.\" = Customer.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(OrderCount) { Method = Count; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__opportunity-count-by-status-1", "environment_setup_version": "29.0", "created_at": "2026-07-10", "metadata": {"area": "crm"}, "nl_prompt": "Return the number of CRM opportunities in each status. Group the opportunities by their Status field and, for each status value that occurs, output the status and the count of opportunities with that status.", "ordered": false, "gold_query": "query 50100 OpportunityCountByStatus\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Opportunity; Opportunity)\n        {\n            column(Status; Status) { }\n            column(OpportunityCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__customer-count-by-country-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "Group all customers by their Country/Region Code and return, for each distinct code, the Country/Region Code and the number of customers that have it. Include customers whose Country/Region Code is blank as their own group. Count the Customer records (one row per distinct Country/Region Code).", "ordered": false, "gold_query": "query 50100 CustomerCountByCountry\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(CountryRegionCode; \"Country/Region Code\") { }\n            column(CustomerCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__outstanding-purchase-value-by-vendor-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "For each vendor that has at least one open purchase order line, return the vendor's number, the vendor's name, and their total outstanding amount. Use open purchase order lines (Purchase Line records whose Document Type is Order) and sum the line 'Outstanding Amount' field (net of VAT). Vendors with no open purchase order lines must not appear.", "ordered": false, "gold_query": "query 50100 OutstandingPurchaseByVendor\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Vendor; Vendor)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(PurchaseLine; \"Purchase Line\")\n            {\n                DataItemLink = \"Buy-from Vendor No.\" = Vendor.\"No.\";\n                DataItemTableFilter = \"Document Type\" = const(Order);\n                column(TotalOutstanding; \"Outstanding Amount\") { Method = Sum; }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-posted-sales-amount-by-customer-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each customer that has at least one posted sales invoice line, return the customer's number, the customer's name, and their total posted sales amount. Join posted sales invoice headers to their lines via Document No., group by the header's Bill-to Customer No., and sum the line 'Amount' field (net of VAT). Customers with no posted sales invoice lines must not appear.", "ordered": false, "gold_query": "query 50100 PostedSalesAmountByCustomer\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(Customer; Customer)\n        {\n            column(No; \"No.\") { }\n            column(Name; Name) { }\n            dataitem(SalesInvoiceHeader; \"Sales Invoice Header\")\n            {\n                DataItemLink = \"Bill-to Customer No.\" = Customer.\"No.\";\n                dataitem(SalesInvoiceLine; \"Sales Invoice Line\")\n                {\n                    DataItemLink = \"Document No.\" = SalesInvoiceHeader.\"No.\";\n                    column(TotalAmount; Amount) { Method = Sum; }\n                }\n            }\n        }\n    }\n}"}
+{"instance_id": "dataquery__line-count-per-open-sales-order-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "sales"}, "nl_prompt": "For each open sales order, return the order's document number and the number of lines it has. Use Sales Line records whose Document Type is Order, group them by Document No., and count the lines per order (one row per order document number).", "ordered": false, "gold_query": "query 50100 LineCountPerOpenSalesOrder\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(SalesLine; \"Sales Line\")\n        {\n            DataItemTableFilter = \"Document Type\" = const(Order);\n            column(DocumentNo; \"Document No.\") { }\n            column(LineCount) { Method = Count; }\n        }\n    }\n}"}
+{"instance_id": "dataquery__total-purchased-quantity-by-item-1", "environment_setup_version": "29.0", "created_at": "2026-07-28", "metadata": {"area": "purchase"}, "nl_prompt": "Across all posted purchase invoice lines whose Type is Item, return each item's number together with the total purchased quantity (the sum of the line Quantity). Produce one row per item number.", "ordered": false, "gold_query": "query 50100 PurchasedQuantityByItem\n{\n    QueryType = Normal;\n\n    elements\n    {\n        dataitem(PurchInvLine; \"Purch. Inv. Line\")\n        {\n            DataItemTableFilter = Type = const(Item);\n            column(ItemNo; \"No.\") { }\n            column(TotalQuantity; Quantity) { Method = Sum; }\n        }\n    }\n}"}