From 34358de3696b7e58749a5bee6484f41204663309 Mon Sep 17 00:00:00 2001 From: Miguel Escobar Date: Mon, 7 Sep 2026 20:06:42 -0500 Subject: [PATCH 1/2] feat: add Dataflow Gen2 data visuals skill for Claude and ChatGPT Adds knowledge for generating Dataflow Gen2 visualization documents in Power Query M. Visual is the canonical artifact name; dashboard, report, and chart are discovery synonyms. Skills: add datafactory-data-visuals.md (Claude), knowledge-data-visuals.md (ChatGPT), and docs/datafactory/data-visuals.md (repo agents); route visual requests from both client indexes; add visual symptom triage. Evals: add EVAL-INT-M-021..023 covering the five-column contract, chart-in-card column mappings, and unsupported PartTypes; add a provider-neutral parity checklist. Eval runner fixes: read files as UTF-8 so the skills arrow separator parses on Windows; correct SKILLS_DIR, which pointed at evals/claude-skills and meant no skill was ever loaded; map datafactory-destinations to the destinations/ subfiles that actually exist. --- .github/copilot-instructions.md | 1 + .../skills/datafactory.architecture/SKILL.md | 2 + .gitignore | 2 + README.md | 20 +++++ chatgpt-skills/README.md | 18 ++++ chatgpt-skills/gpt-instructions.md | 9 ++ chatgpt-skills/knowledge-data-visuals.md | 56 ++++++++++++ claude-skills/SKILL.md | 7 +- claude-skills/datafactory-data-visuals.md | 66 ++++++++++++++ docs/datafactory/data-visuals.md | 88 +++++++++++++++++++ evals/README.md | 13 ++- evals/dataflow-visuals-parity.md | 81 +++++++++++++++++ evals/integration/m-code-quality.eval.md | 70 +++++++++++++++ evals/integration/run_integration_evals.py | 39 +++++--- evals/run_evals.py | 8 +- 15 files changed, 461 insertions(+), 19 deletions(-) create mode 100644 chatgpt-skills/knowledge-data-visuals.md create mode 100644 claude-skills/datafactory-data-visuals.md create mode 100644 docs/datafactory/data-visuals.md create mode 100644 evals/dataflow-visuals-parity.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8f95c8d5..2938e154 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -28,6 +28,7 @@ When these topics come up, reference the corresponding file in `docs/datafactory | Query timeout, slow performance, chunking | `performance.md` | | Output destinations, new tables, staging | `destinations.md` | | Fast Copy, Action.Sequence, Modern Evaluator | `advanced.md` | +| Data visuals, dashboards, reports, charts, KPI cards | `data-visuals.md` | For connection management (creating/listing connections, supported types, UI form), reference `docs/connection-management.md`. diff --git a/.github/skills/datafactory.architecture/SKILL.md b/.github/skills/datafactory.architecture/SKILL.md index 7ed63e18..8bb34928 100644 --- a/.github/skills/datafactory.architecture/SKILL.md +++ b/.github/skills/datafactory.architecture/SKILL.md @@ -26,6 +26,7 @@ DataFactory.MCP.Http/ — HTTP transport layer (AspNetCore) DataFactory.MCP.Tests/ — Test suite (xUnit) DataFactory.WindowsMCP/ — Windows-specific MCP implementation claude-skills/ — Claude skill definitions (RAG pattern) +chatgpt-skills/ — ChatGPT instructions and knowledge files docs/ — Feature documentation evals/ — Evaluation test scenarios ``` @@ -89,3 +90,4 @@ For Data Factory / M language / Dataflow patterns, see `claude-skills/`: - `datafactory-destinations.md` — Output destinations, DataDestination patterns - `datafactory-performance.md` — Query tuning, chunking, query folding - `datafactory-advanced.md` — Fast Copy, Action.Sequence, Modern Evaluator +- `datafactory-data-visuals.md` — Dataflow Gen2 Visual generation and validation diff --git a/.gitignore b/.gitignore index 753fffbb..a4869851 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ tmpclaude-* **/McpApps/node_modules/ **/McpApps/dist/ **/build-output.txt +**/__pycache__/ +*.pyc diff --git a/README.md b/README.md index 8870a0f0..c7ecd8ec 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ Upload the skill files from the [`claude-skills/`](claude-skills/) folder to you - `datafactory-performance.md` - Query optimization, timeouts, chunking - `datafactory-destinations.md` - Output configuration, programmatic setup - `datafactory-advanced.md` - Fast Copy, Action.Sequence, Modern Evaluator + - `datafactory-data-visuals.md` - Dataflow Gen2 Visual generation ### What's Covered @@ -159,6 +160,7 @@ Upload the skill files from the [`claude-skills/`](claude-skills/) folder to you | **Performance** | Query timeouts, chunking strategies, filter optimization, connector selection | | **Destinations** | Lakehouse architecture, schema settings, programmatic destination configuration | | **Advanced** | `Action.Sequence` for writes, Fast Copy, Modern Evaluator | +| **Data visuals** | Visualization documents, visual selection, validation, and limitations | ### Usage @@ -180,9 +182,27 @@ Create a Custom GPT or use ChatGPT Projects with pre-built Data Factory knowledg - `knowledge-performance.md` - Query optimization, timeouts - `knowledge-destinations.md` - Output configuration - `knowledge-advanced.md` - Fast Copy, Action.Sequence + - `knowledge-data-visuals.md` - Dataflow Gen2 Visual contract and workflow See [`chatgpt-skills/README.md`](chatgpt-skills/README.md) for detailed setup options. +## Dataflow Gen2 Visual Builder (Optional) + +Generate Dataflow Gen2 data visuals in Power Query M. **Visual** is the canonical +artifact name; dashboard, report, and chart are discovery synonyms. + +| Client | Knowledge file | +|--------|----------------| +| Claude | [`claude-skills/datafactory-data-visuals.md`](claude-skills/datafactory-data-visuals.md) | +| ChatGPT | [`chatgpt-skills/knowledge-data-visuals.md`](chatgpt-skills/knowledge-data-visuals.md) | +| Repo agents | [`docs/datafactory/data-visuals.md`](docs/datafactory/data-visuals.md) | + +To confirm both clients produce equivalent results, run the checklist in +[`evals/dataflow-visuals-parity.md`](evals/dataflow-visuals-parity.md). + +These are client-side companion files. They are not included in the MCP NuGet +package, because installing an MCP server does not register skills with a client. + ## Documentation For complete documentation, see our **[Documentation Index](https://github.com/microsoft/DataFactory.MCP/blob/main/docs/index.md)**. diff --git a/chatgpt-skills/README.md b/chatgpt-skills/README.md index d6e1487b..bd34b7ca 100644 --- a/chatgpt-skills/README.md +++ b/chatgpt-skills/README.md @@ -16,6 +16,7 @@ Operational tips and best practices for working with Microsoft Fabric Data Facto - `knowledge-performance.md` - `knowledge-destinations.md` - `knowledge-advanced.md` + - `knowledge-data-visuals.md` 4. Under **Capabilities**, enable: - ✅ Code Interpreter (for M code analysis) 5. Click **Create** → **Save** @@ -41,6 +42,7 @@ Operational tips and best practices for working with Microsoft Fabric Data Facto | `knowledge-performance.md` | Query optimization, timeouts, chunking | | `knowledge-destinations.md` | Output configuration, programmatic setup | | `knowledge-advanced.md` | Fast Copy, Action.Sequence, Modern Evaluator | +| `knowledge-data-visuals.md` | Dataflow Gen2 Visual contract, workflow, and limitations | ## What's Covered @@ -66,6 +68,11 @@ Operational tips and best practices for working with Microsoft Fabric Data Facto - Fast Copy (limited transforms, fast ingestion) - Modern Evaluator (complex transforms, limited connectors) +### Data visuals +- Visualization document contract (flat five-column table) +- Closed PartType set and required properties +- Visual selection, validation, and Preview limitations + ## Usage Examples Once your GPT is created, you can ask: @@ -75,6 +82,7 @@ Once your GPT is created, you can ask: - "What's the difference between Fast Copy and Modern Evaluator?" - "Help me write an M query to aggregate sales by month" - "Explain Action.Sequence and when to use it" +- "Build a dashboard from my SalesData query" ## Comparison with Claude Skills @@ -83,3 +91,13 @@ Once your GPT is created, you can ask: | Format | Multiple .md files with YAML frontmatter | Instructions + Knowledge files | | Loading | On-demand via RAG triggers | All knowledge available | | Best for | Claude Projects | Custom GPTs or ChatGPT Projects | + +## Dataflow Gen2 Visual builder + +Visual requests are routed by `gpt-instructions.md`, which loads +`knowledge-data-visuals.md`. Visual is the canonical artifact name; dashboard, +report, and chart remain discovery synonyms. + +Use the shared cases in `../evals/dataflow-visuals-parity.md` to run the same +prompts in ChatGPT and Claude and compare both outputs with one checklist. No +provider API keys are required. diff --git a/chatgpt-skills/gpt-instructions.md b/chatgpt-skills/gpt-instructions.md index e3b0595c..78bbb40f 100644 --- a/chatgpt-skills/gpt-instructions.md +++ b/chatgpt-skills/gpt-instructions.md @@ -50,6 +50,14 @@ You are an expert assistant for Microsoft Fabric Data Factory, specializing in M 4. Offer to open the interactive form: resource `ui://datafactory/create-connection` 5. Use `list_connections` / `get_connection` to inspect existing connections +### When users ask for a visual, dashboard, report, chart, or KPI: +1. Load `knowledge-data-visuals.md` +2. Call the artifact a Visual or visualization document +3. Clarify business intent and ambiguous metric definitions +4. Generate the documented flat five-column Power Query M table +5. Validate hierarchy, property types, and exact chart-column mappings +6. Ask before persistent changes and validate the complete mashup before saving + ## Response Style - Be concise and practical @@ -78,3 +86,4 @@ in - Fast Copy only supports: combine files, select columns, change types, rename/remove columns - For connection creation, always call `list_supported_connection_types` first to get correct parameters - `gatewayId` is required for `OnPremisesGateway` and `VirtualNetworkGateway` connectivity types +- Data visuals are a Preview feature; structural validation does not prove successful Fabric rendering diff --git a/chatgpt-skills/knowledge-data-visuals.md b/chatgpt-skills/knowledge-data-visuals.md new file mode 100644 index 00000000..a9f4f071 --- /dev/null +++ b/chatgpt-skills/knowledge-data-visuals.md @@ -0,0 +1,56 @@ +# Dataflow Gen2 Data Visuals + +Use this knowledge for Dataflow Gen2 visual, visualization, dashboard, report, +chart, KPI, and visual-summary requests. **Visual** and **visualization document** +are canonical; the other terms are request synonyms. + +Data visuals are in Preview. Return one flat Power Query M table whose exact +columns are `Name` (`nullable text`), `Parent` (`nullable text`), `PartType` +(`nullable text`), `Properties` (`nullable record`), and `Data` (`any`). + +The closed PartType set is `Container`, `Card`, `Header`, `KpiCard`, `Table`, +`LineChart`, `AreaChart`, `BarChart`, `StackedBarChart`, `DonutChart`, and +`PieChart`. Never invent another value or use a nested record-tree contract. + +## Contract + +| PartType | Children | Required values | +| --- | --- | --- | +| `Container` | One or more | Optional `Direction` is `"row"` or `"column"` | +| `Card` | Exactly one | Text `Title` | +| `Header` | None | Text `Header`; optional text `FarText` | +| `KpiCard` | None | Text `Value`, text `Label`; optional text `Sub` | +| `Table` | None | Table in `Data` | +| `LineChart`, `AreaChart` | None | `XAxis`, numeric `YAxis`, table in `Data` | +| `BarChart`, `DonutChart`, `PieChart` | None | `Category`, numeric `Value`, table in `Data` | +| `StackedBarChart` | None | `Category`, numeric `Value`, `Series`, table in `Data` | + +Exactly one row has `Parent = null`. Use unique non-null names, resolvable +parents, and acyclic relationships. Charts go inside cards. Chart properties +name exact columns in `Data`; a mismatch can silently render an `undefined` +bucket. Format KPI values as text. + +Duplicate names don't raise an error, but any row parented to an ambiguous name +fails to render, so keep names unique. + +## Workflow + +1. Clarify objective, audience, measures, dimensions, time grain, filters, + targets, and exclusions. Confirm ambiguous definitions. +2. Inspect query definitions or metadata without sampling business values. + Prefer `get_dataflow_definition` when MCP tools are available. Never use + `Expression.Evaluate` or dynamically evaluate sibling query values. +3. Recommend a lightweight Visual and state assumptions. +4. Select required columns, filter invalid values, aggregate to visual grain, + sort explicitly, and cap categories before constructing the flat `#table`. +5. Begin with at most three data visuals, ten chart categories, and 50 detail + rows. Add complexity only after rendering succeeds. +6. Test with `execute_query` when possible. Ask before persistent changes. Use + `save_dataflow_definition` with `validateOnly = true` before saving because + a save replaces the complete mashup. After saving, restore required bindings + with `add_connection_to_dataflow` and validate them. + +Visuals are static, appear only in the authoring canvas, do not become refresh +output, and can slow authoring when numerous or backed by large tables. Line and +area charts do not fill missing dates. Structural validation does not prove a +successful Fabric render. \ No newline at end of file diff --git a/claude-skills/SKILL.md b/claude-skills/SKILL.md index 50420f17..75604f1b 100644 --- a/claude-skills/SKILL.md +++ b/claude-skills/SKILL.md @@ -1,6 +1,6 @@ --- name: datafactory-tips -description: Operational tips and best practices for working with Microsoft Fabric Data Factory MCP tools. Use when executing queries, troubleshooting timeouts, creating dataflows end-to-end, or working with large datasets via the DataFactory.MCP tools. +description: Operational guidance for Microsoft Fabric Data Factory MCP tools. Use when executing queries, troubleshooting timeouts, creating dataflows, or building a Dataflow Gen2 visual, dashboard, report, chart, or KPI. --- # Data Factory Tips @@ -39,6 +39,10 @@ Operational knowledge for working with Microsoft Fabric Data Factory. | Multi-source instant fail via API | Dirty dataflow or separate Lakehouse.Contents calls | `sources/multi-source.md` | | `IsNewTarget = false` fails | Direct navigation on API-created dataflow | `destinations/dest-new-table.md` | | Stale connections after revert | save_dataflow_definition doesn't remove connections | `datafactory-connections.md` (Troubleshooting) | +| Visual renders as a plain table | A required column is missing or renamed | `datafactory-data-visuals.md` | +| `Visual not recognized: ""` | PartType outside the closed set | `datafactory-data-visuals.md` | +| Chart shows one `undefined` bucket | Property names a column missing from `Data` | `datafactory-data-visuals.md` | +| `must contain exactly one root row` | Zero or multiple rows with `Parent = null` | `datafactory-data-visuals.md` | ## Knowledge Files @@ -49,6 +53,7 @@ Operational knowledge for working with Microsoft Fabric Data Factory. | `datafactory-performance.md` | Query timeouts, chunking, query folding, connector selection | | `datafactory-advanced.md` | Fast Copy limits, Action.Sequence, Modern Evaluator | | `datafactory-pipelines.md` | Pipeline creation, Dataflow activities, chaining, scheduling | +| `datafactory-data-visuals.md` | Dataflow Gen2 Visual, dashboard, report, chart, and KPI generation | ### Destination Files (read only the one you need) diff --git a/claude-skills/datafactory-data-visuals.md b/claude-skills/datafactory-data-visuals.md new file mode 100644 index 00000000..b3cd5df3 --- /dev/null +++ b/claude-skills/datafactory-data-visuals.md @@ -0,0 +1,66 @@ +# Dataflow Gen2 Data Visuals + +Use this knowledge when a user asks for a Dataflow Gen2 visual, visualization, +dashboard, report, chart, KPI, or visual summary. Call the generated artifact a +**Visual** or **visualization document**; the other terms are request synonyms. + +Data visuals are a Preview feature. A Visual is a flat Power Query M table with +these exact columns and types: + +```powerquery-m +type table [ + Name = nullable text, + Parent = nullable text, + PartType = nullable text, + Properties = nullable record, + Data = any +] +``` + +Only these PartTypes exist: `Container`, `Card`, `Header`, `KpiCard`, `Table`, +`LineChart`, `AreaChart`, `BarChart`, `StackedBarChart`, `DonutChart`, and +`PieChart`. + +## Contract + +| PartType | Children | Required values | +| --- | --- | --- | +| `Container` | One or more | Optional `Direction` is `"row"` or `"column"` | +| `Card` | Exactly one | Text `Title` | +| `Header` | None | Text `Header`; optional text `FarText` | +| `KpiCard` | None | Text `Value`, text `Label`; optional text `Sub` | +| `Table` | None | Table in `Data` | +| `LineChart`, `AreaChart` | None | `XAxis`, numeric `YAxis`, table in `Data` | +| `BarChart`, `DonutChart`, `PieChart` | None | `Category`, numeric `Value`, table in `Data` | +| `StackedBarChart` | None | `Category`, numeric `Value`, `Series`, table in `Data` | + +Exactly one row has `Parent = null`. Every `Name` is unique and non-null. Every +other `Parent` resolves to a `Name`, and relationships are acyclic. Charts go +inside cards. Chart properties contain exact column names from their `Data` +table; a bad name can silently produce one `undefined` bucket. KPI values are +formatted as text. + +Duplicate names don't raise an error, but any row parented to an ambiguous name +fails to render, so keep names unique. + +## Workflow + +1. Clarify objective, audience, measures, dimensions, time grain, filters, + targets, and exclusions. Confirm ambiguous definitions. +2. Inspect query definitions or metadata without sampling business values. + Prefer `get_dataflow_definition` when MCP tools are available. Never use + `Expression.Evaluate` or dynamically evaluate sibling query values. +3. Recommend a lightweight Visual and state assumptions. +4. Select required columns, filter invalid values, aggregate to visual grain, + sort explicitly, and cap categories before building the flat `#table`. +5. Start with at most three data visuals, ten chart categories, and 50 detail + rows. Add complexity only after rendering succeeds. +6. Test nonpersistently with `execute_query` when possible. Ask before changing + a dataflow. Use `save_dataflow_definition` with `validateOnly = true` before + saving because a save replaces the complete mashup. After saving, restore + required bindings with `add_connection_to_dataflow` and validate them. + +Visuals are static, render only in the authoring canvas, do not become refresh +output, and can slow authoring when numerous or backed by large tables. Line and +area charts do not fill missing dates. Structural validation cannot prove that +Fabric rendered the Visual successfully. \ No newline at end of file diff --git a/docs/datafactory/data-visuals.md b/docs/datafactory/data-visuals.md new file mode 100644 index 00000000..5c228591 --- /dev/null +++ b/docs/datafactory/data-visuals.md @@ -0,0 +1,88 @@ +# Data Visuals (Preview) + +Dataflow Gen2 can render a query as visuals instead of a table. The artifact is a +**Visual** (visualization document). Dashboard, report, and chart are request +synonyms only. + +## Contract + +A Visual is one flat M table. These exact column names trigger rendering: + +```m +type table [ + Name = nullable text, + Parent = nullable text, + PartType = nullable text, + Properties = nullable record, + Data = any +] +``` + +Missing or renamed columns fall back to a normal table preview. Extra columns are +ignored. Wrong value types fail evaluation. + +## Closed PartType set + +`Container`, `Card`, `Header`, `KpiCard`, `Table`, `LineChart`, `AreaChart`, +`BarChart`, `StackedBarChart`, `DonutChart`, `PieChart`. + +| PartType | Children | Required properties | Optional | +|----------|----------|---------------------|----------| +| `Container` | One or more | none | `Direction`: `"row"` or `"column"` | +| `Card` | Exactly one | `Title` | none | +| `Header` | None | `Header` | `FarText` | +| `KpiCard` | None | `Value`, `Label` (text) | `Sub` | +| `Table` | None | table in `Data` | none | +| `LineChart`, `AreaChart` | None | `XAxis`, `YAxis`, `Data` | none | +| `BarChart`, `DonutChart`, `PieChart` | None | `Category`, `Value`, `Data` | none | +| `StackedBarChart` | None | `Category`, `Value`, `Series`, `Data` | none | + +Chart properties hold **column names**, not data. `Value` and `YAxis` must name +numeric columns. Charts have no title, so nest each one in a `Card`. + +## Rules + +- Exactly one row has `Parent = null`. +- Names are unique and non-null; duplicates don't error but break descendants. +- Every other `Parent` matches an existing `Name`; unresolved parents render nothing. +- `Container` needs at least one child; `Card` needs exactly one. Everything else is a leaf. +- Format KPI numbers as text: `"$" & Number.ToText(Number.Round(x, 0))`. + +## Minimal example + +```m +let + SalesData = #table( + type table [Month = text, Revenue = number], + {{"2026-01", 12000}, {"2026-02", 15500}} + ), + VisualDocumentType = type table [ + Name = nullable text, Parent = nullable text, PartType = nullable text, + Properties = nullable record, Data = any + ] +in + #table(VisualDocumentType, { + {"sales-card", null, "Card", [Title = "Monthly sales"], null}, + {"sales-trend", "sales-card", "LineChart", [XAxis = "Month", YAxis = "Revenue"], SalesData} + }) +``` + +## Triage + +| Symptom | Cause | +|---------|-------| +| Renders as a plain table | Required column missing or renamed | +| `Visual not recognized: ""` | PartType outside the closed set | +| One `undefined` bucket | Property names a column missing from `Data` | +| `must contain exactly one root row` | Zero or multiple `Parent = null` rows | +| `Unexpected number of cells` | `Card` has other than one child | + +## Limitations + +- Preview; subject to change. +- Static only — no slicers, date pickers, or cross-filtering. +- Renders in the authoring canvas only; never part of refresh output or destinations. +- Many visuals or large `Data` tables slow authoring. +- Line and area axes don't fill missing dates. + +Full reference: [Create data visuals in Dataflow Gen2](https://learn.microsoft.com/fabric/data-factory/dataflow-gen2-data-visuals). diff --git a/evals/README.md b/evals/README.md index c37fd17e..5d88674e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -26,6 +26,10 @@ Cross-tool workflows live in `multi-step.eval.md`. | [pipelines.eval.md](pipelines.eval.md) | `ListPipelinesAsync`, `CreatePipelineAsync`, `GetPipelineAsync`, `GetPipelineDefinitionAsync`, `UpdatePipelineAsync`, `UpdatePipelineDefinitionAsync` | | [multi-step.eval.md](multi-step.eval.md) | Cross-tool orchestration scenarios | +Provider-neutral skill comparison cases are documented in +[`dataflow-visuals-parity.md`](dataflow-visuals-parity.md). They are run manually +against Claude and ChatGPT because the repository CI doesn't host either client. + ## Scenario Format Each scenario follows this template: @@ -116,9 +120,9 @@ OPENAI_API_KEY=sk-... python evals/integration/run_integration_evals.py | Connections | 5 | 5 | 5 | 15 | | Gateways | 3 | 3 | 3 | 9 | | Dataflows | 9 | 7 | 6 | 22 | -| Pipelines | 6 | 5 | 4 | 15 | +| Pipelines | 7 | 6 | 5 | 18 | | Multi-step | — | — | — | 10 | -| **Total** | **33** | **26** | **25** | **94** | +| **Total** | **34** | **27** | **26** | **97** | ### Integration Evals (M Code Quality) @@ -131,6 +135,7 @@ OPENAI_API_KEY=sk-... python evals/integration/run_integration_evals.py | Pipeline JSON | 2 | | Workflow | 3 | | Lifecycle | 2 | -| **Total** | **20** | +| Data Visuals | 3 | +| **Total** | **23** | -**Grand total: 114 evals** (94 tool-selection + 20 integration) +**Grand total: 120 evals** (97 tool-selection + 23 integration) diff --git a/evals/dataflow-visuals-parity.md b/evals/dataflow-visuals-parity.md new file mode 100644 index 00000000..b8eb3568 --- /dev/null +++ b/evals/dataflow-visuals-parity.md @@ -0,0 +1,81 @@ +# Dataflow Gen2 Visual parity checks + +Use these cases to compare the Claude and ChatGPT skill variants. They do not +require provider API keys and are not executed by the MCP server. + +## Setup + +- Claude: load `claude-skills/SKILL.md` and + `claude-skills/datafactory-data-visuals.md`. +- ChatGPT: use `chatgpt-skills/gpt-instructions.md` + with `chatgpt-skills/knowledge-data-visuals.md`. +- Start a new conversation in each client. +- Submit the same prompt without adding client-specific guidance. + +## Shared acceptance criteria + +For each case, both clients should: + +- Call the artifact a Visual or visualization document. +- Return a flat Power Query M table with `Name`, `Parent`, `PartType`, + `Properties`, and `Data`. +- Use only documented PartTypes. +- Produce exactly one root and valid parent relationships. +- Put every chart inside a `Card`. +- Map chart properties to exact source column names. +- Format KPI values as text. +- Avoid source mutation and dynamic sibling-query evaluation. +- State that data visuals are a Preview feature when limitations matter. + +Equivalent output does not require identical row names, local variable names, +formatting, prose, or visual ordering. Compare the selected PartTypes, data +aggregations, chart mappings, hierarchy validity, and safety behavior. + +## Case 1: KPI synonym routing + +> Create a minimal Dataflow Gen2 dashboard that shows a KPI with the text value +> 42 and label Active customers. Return only runnable Power Query M. + +Expected in both outputs: one valid `KpiCard` root with text `Value` and `Label`. + +## Case 2: Report synonym routing + +> Create a Dataflow Gen2 report from an existing SalesData query with Month text +> and Revenue number columns. Build one line chart titled Monthly revenue. Keep +> it lightweight and return only runnable Power Query M. + +Expected in both outputs: one `Card` containing one `LineChart`, with `XAxis = +"Month"`, `YAxis = "Revenue"`, and an aggregated, sorted data table. + +## Case 3: Multi-visual summary + +> Create a Dataflow Gen2 visual from an existing SalesData query with Category +> text and Revenue number columns. Build a header, a revenue KPI, a bar chart +> titled Revenue by category, and a detail table limited to 50 rows. Return only +> runnable Power Query M. + +Expected in both outputs: `Container`, `Header`, `KpiCard`, `Card`, `BarChart`, +and `Table`; valid hierarchy; bounded detail data; numeric category aggregation. + +## Case 4: Persistence safety + +> Add this visual query to my existing dataflow and save it. + +Expected in both responses: inspect the existing definition, preserve the full +mashup, validate before saving, obtain confirmation before the persistent save, +and restore required connection bindings afterward. + +## Recording results + +For each case record `Pass`, `Different but equivalent`, or `Fail` for: + +| Check | Claude | ChatGPT | Notes | +| --- | --- | --- | --- | +| Request routed to Visual workflow | | | | +| Required PartTypes present | | | | +| Valid hierarchy and cardinality | | | | +| Equivalent aggregation and mappings | | | | +| Preview and safety guidance aligned | | | | + +A release is ready when neither client has a `Fail`. Review any +`Different but equivalent` result to ensure the difference is presentation-only. \ No newline at end of file diff --git a/evals/integration/m-code-quality.eval.md b/evals/integration/m-code-quality.eval.md index 47af3949..6fa2966f 100644 --- a/evals/integration/m-code-quality.eval.md +++ b/evals/integration/m-code-quality.eval.md @@ -10,6 +10,7 @@ Each scenario is run **twice**: once without skills (baseline) and once with ski - `datafactory-destinations.md` — DataDestination patterns, AllowCombine - `datafactory-performance.md` — Query folding, chunking, connector choice - `datafactory-advanced.md` — Fast Copy limits, Action.Sequence +- `datafactory-data-visuals.md` — Visual document contract, closed PartType set --- @@ -434,3 +435,72 @@ Each scenario is run **twice**: once without skills (baseline) and once with ski - [ ] Re-adds connections after save - [ ] Uses `ApplyChangesIfNeeded` on first refresh - [ ] Mentions that multi-source cannot revert to single-source + +--- + +## Data Visuals + +### EVAL-INT-M-021: Visual document contract + +**Category:** Data Visuals +**Difficulty:** Medium +**Skills:** none → datafactory-data-visuals + +**User prompt:** +> Write a Power Query M query that renders a Dataflow Gen2 dashboard with a single KPI card showing the text value 42 and the label "Active customers". + +**Validation rules:** +- [ ] Contains `"KpiCard"` as the PartType value +- [ ] Contains `nullable record` in the declared table type +- [ ] Contains `#table` to construct the document +- [ ] Contains `PartType` as a declared column +- [ ] Does NOT contain `PartKind` + +**Common failure without skills:** +- Emitting a nested record tree instead of a flat five-column table +- Assigning a number to `KpiCard.Value` instead of text + +--- + +### EVAL-INT-M-022: Chart nested in a card with exact column mappings + +**Category:** Data Visuals +**Difficulty:** Hard +**Skills:** none → datafactory-data-visuals + +**User prompt:** +> I have a SalesData query with Month (text) and Revenue (number) columns. Write M that renders a Dataflow Gen2 report containing a line chart titled "Monthly revenue". + +**Validation rules:** +- [ ] Contains `"LineChart"` as the PartType value +- [ ] Contains `"Card"` as the chart's parent visual +- [ ] Contains `Title` for the card +- [ ] Contains `XAxis` mapped to the Month column name +- [ ] Contains `YAxis` mapped to the Revenue column name +- [ ] Contains `Table.Group` to aggregate to the visual grain +- [ ] Contains `Table.Sort` for deterministic ordering +- [ ] Does NOT contain `Table.FirstN` + +**Common failure without skills:** +- Parenting the chart directly to a container, so it renders without a title +- Putting column values in `XAxis`/`YAxis` instead of column names + +--- + +### EVAL-INT-M-023: Unsupported visual type + +**Category:** Data Visuals +**Difficulty:** Medium +**Skills:** none → datafactory-data-visuals + +**User prompt:** +> Add a scatter plot and a gauge to my Dataflow Gen2 visual. + +**Validation rules:** +- [ ] Does NOT contain `ScatterPlot` +- [ ] Contains `BarChart` among the supported alternatives +- [ ] Contains `PieChart` among the supported alternatives +- [ ] Contains `KpiCard` among the supported alternatives + +**Common failure without skills:** +- Inventing a plausible PartType that renders `Visual not recognized` diff --git a/evals/integration/run_integration_evals.py b/evals/integration/run_integration_evals.py index 4b2bd141..540d5d1c 100644 --- a/evals/integration/run_integration_evals.py +++ b/evals/integration/run_integration_evals.py @@ -35,14 +35,20 @@ # Skill loader # --------------------------------------------------------------------------- -SKILLS_DIR = Path(__file__).parent.parent / "claude-skills" +# evals/integration/ -> repo root +SKILLS_DIR = Path(__file__).resolve().parent.parent.parent / "claude-skills" SKILL_FILES = { - "datafactory-core": "datafactory-core.md", - "datafactory-destinations": "datafactory-destinations.md", - "datafactory-performance": "datafactory-performance.md", - "datafactory-advanced": "datafactory-advanced.md", - "datafactory-pipelines": "datafactory-pipelines.md", + "datafactory-core": ["datafactory-core.md"], + "datafactory-destinations": [ + "destinations/dest-new-table.md", + "destinations/dest-existing-table.md", + "destinations/dest-troubleshooting.md", + ], + "datafactory-performance": ["datafactory-performance.md"], + "datafactory-advanced": ["datafactory-advanced.md"], + "datafactory-pipelines": ["datafactory-pipelines.md"], + "datafactory-data-visuals": ["datafactory-data-visuals.md"], } # Always-loaded skill tip file @@ -50,15 +56,20 @@ def load_skill(name: str) -> str: - path = SKILLS_DIR / SKILL_FILES.get(name, "") - if path.exists(): - return path.read_text() - return "" + filenames = SKILL_FILES.get(name) + if not filenames: + return "" + parts = [ + (SKILLS_DIR / filename).read_text(encoding="utf-8") + for filename in filenames + if (SKILLS_DIR / filename).exists() + ] + return "\n\n---\n\n".join(parts) def load_tips() -> str: path = SKILLS_DIR / SKILL_TIPS_FILE - return path.read_text() if path.exists() else "" + return path.read_text(encoding="utf-8") if path.exists() else "" def build_system_prompt(skill_names: list[str]) -> str: @@ -173,7 +184,7 @@ def _extract_code_block(text: str, lang: str) -> Optional[str]: # --------------------------------------------------------------------------- def parse_integration_eval_file(filepath: Path) -> list[IntegrationScenario]: - text = filepath.read_text() + text = filepath.read_text(encoding="utf-8") scenarios = [] blocks = re.split(r"(?=^### EVAL-)", text, flags=re.MULTILINE) @@ -537,6 +548,10 @@ def save_results(scenarios: list[IntegrationScenario], output_path: Path): # --------------------------------------------------------------------------- def main(): + # Eval files are UTF-8; avoid UnicodeEncodeError on legacy Windows consoles. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + parser = argparse.ArgumentParser(description="Run integration evals for M code quality") parser.add_argument("--eval", help="Run a single eval by ID") parser.add_argument("--category", help="Filter by category") diff --git a/evals/run_evals.py b/evals/run_evals.py index a475d573..6fb3aa1b 100644 --- a/evals/run_evals.py +++ b/evals/run_evals.py @@ -63,7 +63,7 @@ class EvalScenario: def parse_eval_file(filepath: Path) -> list[EvalScenario]: """Parse a single .eval.md file into structured scenarios.""" - text = filepath.read_text() + text = filepath.read_text(encoding="utf-8") scenarios = [] # Split on ### EVAL- headers @@ -406,6 +406,10 @@ def save_results(scenarios: list[EvalScenario], output_path: Path): # --------------------------------------------------------------------------- def main(): + # Eval files are UTF-8; avoid UnicodeEncodeError on legacy Windows consoles. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + parser = argparse.ArgumentParser(description="Run AI evals for Data Factory MCP tools") parser.add_argument("--file", help="Run evals from a specific file (e.g., 'authentication')") parser.add_argument("--eval", help="Run a single eval by ID (e.g., 'EVAL-AUTH-001')") @@ -424,7 +428,7 @@ def main(): schema_path = evals_dir / "tools_schema.json" # Load tool schemas - tools = json.loads(schema_path.read_text())["tools"] + tools = json.loads(schema_path.read_text(encoding="utf-8"))["tools"] print(f"Loaded {len(tools)} tool definitions from {schema_path.name}") # Parse eval files From e626a4f941301274ff28310314e0de943b6078b1 Mon Sep 17 00:00:00 2001 From: Miguel Escobar Date: Tue, 8 Sep 2026 19:41:34 -0500 Subject: [PATCH 2/2] fix: handle fork PR validation results safely --- .github/workflows/pr-validation.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index da2e8d2b..450ec0db 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -40,7 +40,6 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: "10.0.x" - include-prerelease: true - name: 📦 Restore Dependencies run: dotnet restore --configfile ../nuget.config @@ -119,6 +118,12 @@ jobs: [View full results](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`; + await core.summary.addRaw(body).write(); + + if (context.payload.pull_request.head.repo.full_name !== context.payload.pull_request.base.repo.full_name) { + return; + } + // Find existing comment const comments = await github.rest.issues.listComments({ owner: context.repo.owner,