From 9bab00ad9389f5d063d550939af1722ac96b60b3 Mon Sep 17 00:00:00 2001 From: Reid-Agent <269567208+reidbaker-agent@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:25:28 -0400 Subject: [PATCH 01/14] Add run-evals skill and evals consistency tests --- .../.agents/skills/.gitignore | 1 + .../definition-of-done/evals/evals.json | 3 + .../.agents/skills/run-evals/SKILL.md | 20 +++ .../.agents/skills/run-evals/evals/evals.json | 26 ++++ tool/dart_skills_lint/dart_skills_lint.yaml | 3 + tool/dart_skills_lint/evals/README.md | 114 +++++++++++++++ .../evals/code_quality_rubric.json | 41 ++++++ .../dart-skills-lint-setup/evals/evals.json | 48 ++++++ .../test/skills_evals_test.dart | 137 ++++++++++++++++++ 9 files changed, 393 insertions(+) create mode 100644 tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md create mode 100644 tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json create mode 100644 tool/dart_skills_lint/evals/README.md create mode 100644 tool/dart_skills_lint/evals/code_quality_rubric.json create mode 100644 tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json create mode 100644 tool/dart_skills_lint/test/skills_evals_test.dart diff --git a/tool/dart_skills_lint/.agents/skills/.gitignore b/tool/dart_skills_lint/.agents/skills/.gitignore index 0072a21..d662165 100644 --- a/tool/dart_skills_lint/.agents/skills/.gitignore +++ b/tool/dart_skills_lint/.agents/skills/.gitignore @@ -7,6 +7,7 @@ !check-downstream-consumers/ !dart-skills-lint-integration/ !definition-of-done/ +!run-evals/ # Keep essential configuration and docs !.gitignore diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 7a12e81..91912a6 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -1,5 +1,8 @@ { "skill_name": "definition-of-done", + "repo-criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md new file mode 100644 index 0000000..8ee4045 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -0,0 +1,20 @@ +--- +name: run-evals +description: Run evaluations for one, multiple, or all skills using the agent orchestration framework. Make sure to use this skill whenever the user asks to run evals, test a skill's performance, run benchmarks, or compare baseline versus with-skill execution. +metadata: + internal: true +--- + +# Run Skill Evals + +Use this skill to run skill evaluations. + +1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for the exact subagent and grader prompts. +2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. +3. **Orchestrate**: For each eval prompt, spawn two `bare-agent` subagents using `Workspace: branch`: + - **Baseline**: Provide only the task prompt. + - **With-Skill**: Provide the task prompt + instructions to strictly follow the target skill file. + Instruct both to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. + **CRITICAL**: You must explicitly warn both subagents to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. +4. **Grade**: Parse the combined rubric (resolving `repo-criteria` + `evals.json` expectations). +5. **Artifact**: Grade the outputs and generate a Markdown artifact (e.g., `_eval_results.md`) containing the metadata, pass/fail rationale, and raw diffs/stdout. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json new file mode 100644 index 0000000..8b000f1 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -0,0 +1,26 @@ +{ + "skill_name": "run-evals", + "repo-criteria": [ + "evals/code_quality_rubric.json" + ], + "evals": [ + { + "id": 1, + "prompt": "Run the evals for the definition-of-done skill.", + "expected_output": "The agent orchestrates two subagents (baseline and with-skill) in branched workspaces, grades their git diffs and stdout, and outputs a markdown artifact.", + "expectations": [ + "The agent successfully triggered subagents to run the test cases.", + "A markdown artifact containing the evaluation results was created.", + "The execution run leaves the parent working tree clean, with no rogue unstaged modifications to unrelated project files caused by subagents." + ] + }, + { + "id": 2, + "prompt": "Can you test the dart-add-unit-test skill and see if it passes its rubric?", + "expected_output": "The agent identifies the target skill, spawns baseline and with-skill subagents, compares results, and writes an evaluation artifact.", + "expectations": [ + "The execution run leaves the parent working tree clean, with no rogue unstaged modifications to unrelated project files caused by subagents." + ] + } + ] +} diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index cd6926c..c116896 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -35,4 +35,7 @@ dart_skills_lint: - path: ".agents/skills/definition-of-done" rules: prevent-skills-sh-publishing: error + - path: ".agents/skills/run-evals" + rules: + prevent-skills-sh-publishing: error diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md new file mode 100644 index 0000000..241caf0 --- /dev/null +++ b/tool/dart_skills_lint/evals/README.md @@ -0,0 +1,114 @@ +# Skill Evaluations (Evals) Framework + +This directory defines the architecture, rubrics, and instructions for evaluating AI agent skills authored and maintained in this repository using an LLM Agent as Judge. + +## Core Principles & Architecture + +Evaluations in this repository use a **Two-Tiered Architecture** to separate domain-specific skill requirements from universal skill quality standards. The LLM Agent Judge evaluates execution runs against two components: + +1. **Domain Evals (`/evals/evals.json`)**: + Skill-specific task prompts, macro narrative expected outputs, and micro expectation assertions unique to that skill. + +2. **Universal Skill Quality Rubrics (`evals/*_rubric.json`)**: + Modular, shared quality rubrics injected into the LLM Agent Judge system prompt representing universal quality classes required across skills. + +--- + +### 1. Skill-Specific Evals (`/evals/evals.json`) +Each skill maintains an `evals/evals.json` file containing target task prompts and expectations: +- **`prompt`**: Realistic user prompt testing primary or edge-case workflows. +- **`expected_output`**: High-level narrative summary of success for human reviewers and judge context. +- **`expectations`**: Array of discrete, testable assertions used by the judge to compute quantitative pass rates. +- **`repo-criteria`**: Array of relative file paths to shared universal quality rubrics (e.g., `["evals/code_quality_rubric.json"]`). + +### 2. Universal Skill Quality Rubrics Framework (`evals/*_rubric.json`) +Universal skill quality expectations are structured into modular rubric classes that apply broadly across skills. + +--- + +## Key Rules for Writing Evals + +When authoring or updating `evals.json`: + +1. Maintain Schema Consistency across all `evals.json` files. + +--- + +## 🚀 Running & Validating Evals Locally + +### 1. Validate Evals Structural Consistency +Run the unit test that checks all `evals.json` files for structural consistency across the repository: + +```bash +dart test test/published_skills_evals_test.dart +``` + +### 2. Running Evals via Agent Orchestration + +Within an agentic IDE, you can run evaluations interactively by instructing a parent orchestrator agent to spawn isolated subagents for execution, and then acting as the Agent Judge itself. + +#### Subagent Execution Prompts + +For each test case in `evals.json`, spawn execution subagents using a branched workspace (`Workspace: branch`) to ensure isolation. Use these standard prompt structures: + +**With-Skill Execution Prompt:** +```text +Execute this task: +- Skill path: (Please read and STRICTLY FOLLOW the instructions in this skill file before finishing) +- Task: +- Input files: + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). +``` + +**Baseline Execution Prompt (Without Skill):** +```text +Execute this task: +- Task: +- Input files: + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). +``` + +#### Resolving `repo-criteria` for Grading + +When preparing to grade execution outputs, the orchestrator or grader agent must resolve the `repo-criteria` section from `evals.json`: + +1. **Parse `repo-criteria`**: Read the array of file paths defined under `"repo-criteria"` in `evals.json` (e.g., `["evals/code_quality_rubric.json"]`). +2. **Load Referenced Rubrics**: Open each referenced JSON file relative to the repository root and extract its `expectations` list under `evaluations`. +3. **Combine Expectations**: Concatenate the universal expectations from `repo-criteria` with the skill-specific `expectations` from `evals.json` to build the full grading payload. + +#### Agent Judge Grading + +Once execution subagents return their `git diff` and command outputs, grade the results using this prompt template: + +**Agent Judge Prompt:** +```text +You are an expert evaluator. Review the following execution outputs from an AI agent against the provided combined rubric. + +Execution Outputs: +- Git Diff: +- Command Stdout: + +Combined Rubric Expectations: + + +For each expectation, explicitly state whether it PASSED or FAILED and provide a 1-sentence justification. Finally, provide an overall PASS/FAIL grade. +``` + +#### Generating the Evaluation Artifact + +After grading the outputs, the orchestrator agent must compile the results into a markdown artifact so they can be easily reviewed. + +**Artifact Generation Prompt:** +```text +Please compile the complete evaluation results into a Markdown artifact (e.g., `_eval_results.md`). The document should include: +1. Metadata at the top (Date, Skill Name, Model Evaluated, Prompt). +2. A section for the "With-Skill Agent" containing the PASS/FAIL grade, rationale, and the raw outputs (`git diff` and command stdout). +3. A section for the "Baseline Agent" containing the PASS/FAIL grade, rationale, and its raw outputs. +``` +This artifact serves as the permanent, human-readable record of the execution run. diff --git a/tool/dart_skills_lint/evals/code_quality_rubric.json b/tool/dart_skills_lint/evals/code_quality_rubric.json new file mode 100644 index 0000000..81a4a31 --- /dev/null +++ b/tool/dart_skills_lint/evals/code_quality_rubric.json @@ -0,0 +1,41 @@ +{ + "rubric_name": "universal-code-quality-rubric", + "description": "Universal code quality class within the universal skill quality framework, injected into the LLM Agent Judge system prompt for code-generating skill execution runs.", + "evaluations": [ + { + "category": "compilation_and_health", + "description": "Does the generated or modified code compile cleanly without syntax errors, pass static analysis ('dart analyze --fatal-infos'), and run successfully?", + "expectations": [ + "The generated code compiles with zero syntax errors.", + "Static analysis passes cleanly ('dart analyze --fatal-infos' returns zero errors or warnings).", + "Existing and newly added unit, widget, or integration tests execute and pass successfully." + ] + }, + { + "category": "effective_dart_and_idioms", + "description": "Does the code adhere to Effective Dart guidelines and modern Dart 3+ language features?", + "expectations": [ + "Follows Effective Dart naming conventions, formatting, and structural guidelines.", + "Leverages Dart 3+ idioms where applicable (e.g. pattern matching, records, primary constructors, switch expressions).", + "Avoids anti-patterns such as unnecessary dynamic types or unhandled async calls." + ] + }, + { + "category": "cross_platform_compatibility", + "description": "Does the solution function correctly across targeted host platforms (macOS, Linux, Windows)?", + "expectations": [ + "File paths and CLI path parameters use platform-agnostic formatters (package:path or forward slashes '/').", + "CLI commands and shell invocation scripts use cross-platform compatible syntax.", + "Avoids hardcoded platform-specific environment paths (such as C:\\ or /usr/local/bin directly)." + ] + }, + { + "category": "directory_and_placement_hygiene", + "description": "Are all created or modified files placed in canonical repository directories?", + "expectations": [ + "Configuration files, source code, and tests are placed in their proper standard directory locations.", + "No temporary scratch files or orphaned directories are left behind after execution." + ] + } + ] +} diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json new file mode 100644 index 0000000..f03d366 --- /dev/null +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -0,0 +1,48 @@ +{ + "skill_name": "dart-skills-lint-setup", + "repo-criteria": [ + "evals/code_quality_rubric.json" + ], + "evals": [ + { + "id": 1, + "prompt": "Set up dart_skills_lint validation for the first time in our Dart project to lint skills located in .agents/skills.", + "expected_output": "The project's pubspec.yaml is updated with dart_skills_lint under dev_dependencies using a git dependency pointing to tool/dart_skills_lint, a root dart_skills_lint.yaml file is created specifying .agents/skills, and recipes for CI integration are provided.", + "expectations": [ + "dart_skills_lint is added under dev_dependencies in pubspec.yaml as a git dependency targeting https://github.com/flutter/agent-plugins.git with path tool/dart_skills_lint.", + "The dart_skills_lint.yaml file is created at the repository root so both CLI and Dart test invocations share the configuration.", + "dart_skills_lint.yaml specifies .agents/skills under directories.", + "Configures check-relative-paths: error under rules in dart_skills_lint.yaml to enforce portable relative links across skill files.", + "Guidance or recipe for CI integration or pre-commit hook is provided.", + "NOT added under runtime dependencies in pubspec.yaml.", + "NOT placed inside .agents/skills/ or a sub-folder where root CLI commands will fail to locate the config." + ] + }, + { + "id": 2, + "prompt": "A repository has pre-existing skill files in .agents/skills with lint violations. Set up dart_skills_lint without failing CI or modifying legacy skill files by generating a baseline ignore file.", + "expected_output": "dart_skills_lint is added to pubspec.yaml dev_dependencies, a root dart_skills_lint.yaml configuration is created, and dart run dart_skills_lint:cli is executed with --generate-baseline to write violations to an ignore file without mutating legacy skill content.", + "expectations": [ + "dart_skills_lint is configured under dev_dependencies in pubspec.yaml and dart_skills_lint.yaml is created at the repository root.", + "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", + "The CLI command dart run dart_skills_lint:cli is run with --skills-directory=.agents/skills --generate-baseline to write legacy failures to an ignore file.", + "An explanation is provided that pre-existing violations exit clean while violations introduced after baseline generation will still fail validation.", + "Does NOT modify, delete, or reformat any pre-existing SKILL.md files in .agents/skills during baseline generation.", + "Does NOT pass --fix during baseline generation, keeping legacy skill files intact." + ] + }, + { + "id": 3, + "prompt": "Set up dart_skills_lint to validate agent skills in .agents/skills, while isolating the package dependency inside tool/pubspec.yaml instead of adding it to the root pubspec.yaml.", + "expected_output": "The tool/ package's pubspec.yaml contains dart_skills_lint under dev_dependencies, while dart_skills_lint.yaml is placed at the repository root, ensuring the root closure is unpolluted and configuration remains centrally accessible.", + "expectations": [ + "dart_skills_lint is added under dev_dependencies in tool/pubspec.yaml rather than the root pubspec.yaml.", + "dart_skills_lint.yaml is created at the repository root directory rather than inside tool/ so both CLI and test invocations share the configuration.", + "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", + "Guidance is provided on ensuring dependency references and commit hashes remain synchronized if added to multiple pubspecs.", + "NOT placed inside tool/, which would prevent root-level CLI invocations from discovering the configuration file.", + "NOT added to the root pubspec.yaml dependencies or dev_dependencies." + ] + } + ] +} diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart new file mode 100644 index 0000000..d9760ac --- /dev/null +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -0,0 +1,137 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + group('Evals structure consistency', () { + // Ensures all evals.json files dynamically share the exact same JSON schema. + // Keys are not hardcoded to ensure enforcement remains schema-agnostic and flexible. + test('all evals.json files across skills share consistent structure and keys', () { + final List evalsFiles = [ + ..._findEvalsFiles(Directory(p.join(Directory.current.path, 'skills'))), + ..._findEvalsFiles(Directory(p.join(Directory.current.path, '.agents', 'skills'))), + ]; + + expect( + evalsFiles, + isNotEmpty, + reason: 'Should find at least one evals.json file in skills or .agents/skills.', + ); + + Set? expectedRootKeys; + String? expectedRootKeysFilePath; + Set? expectedEvalItemKeys; + String? expectedEvalItemFilePath; + + for (final evalsFile in evalsFiles) { + final Directory skillDir = evalsFile.parent.parent; + final String skillName = p.basename(skillDir.path); + + final Object? decoded = jsonDecode(evalsFile.readAsStringSync()); + expect( + decoded, + isA>(), + reason: '${evalsFile.path} must be a JSON map.', + ); + + if (decoded is! Map) { + fail('${evalsFile.path} must be a JSON map.'); + } + + expect( + decoded['skill_name'], + equals(skillName), + reason: 'skill_name in ${evalsFile.path} must match skill directory name "$skillName".', + ); + + final Set rootKeys = decoded.keys.toSet(); + if (expectedRootKeys == null) { + expectedRootKeys = rootKeys; + expectedRootKeysFilePath = evalsFile.path; + } else { + expect( + rootKeys, + equals(expectedRootKeys), + reason: + '${evalsFile.path} root keys do not match consistency pattern. ' + 'Expected keys to match the first processed file ($expectedRootKeysFilePath). ' + 'All evals.json files must share the exact same root keys.', + ); + } + + final Object? evalsRaw = decoded['evals']; + expect( + evalsRaw, + isA>(), + reason: 'evals key in ${evalsFile.path} must be a List.', + ); + + if (evalsRaw is! List) { + fail('evals key in ${evalsFile.path} must be a List.'); + } + + for (final Object? evalItem in evalsRaw) { + expect( + evalItem, + isA>(), + reason: 'Item in evals list in ${evalsFile.path} must be a JSON map.', + ); + if (evalItem is! Map) { + fail('Item in evals list in ${evalsFile.path} must be a JSON map.'); + } + + final Set itemKeys = evalItem.keys.toSet(); + if (expectedEvalItemKeys == null) { + expectedEvalItemKeys = itemKeys; + expectedEvalItemFilePath = evalsFile.path; + } else { + expect( + itemKeys, + equals(expectedEvalItemKeys), + reason: + 'Eval item in ${evalsFile.path} keys do not match consistency pattern. ' + 'Expected eval item keys to match the first processed file ($expectedEvalItemFilePath). ' + 'All eval items must share the exact same keys.', + ); + } + } + } + }); + + test('all published skills have an evals.json file', () { + final skillsDir = Directory(p.join(Directory.current.path, 'skills')); + if (!skillsDir.existsSync()) { + return; + } + + final List skillDirs = skillsDir.listSync().whereType().toList(); + + for (final skillDir in skillDirs) { + final evalsFile = File(p.join(skillDir.path, 'evals', 'evals.json')); + expect( + evalsFile.existsSync(), + isTrue, + reason: + 'Published skill "${p.basename(skillDir.path)}" is missing an evals.json file at ${evalsFile.path}', + ); + } + }); + }); +} + +List _findEvalsFiles(Directory baseDir) { + if (!baseDir.existsSync()) { + return []; + } + return baseDir + .listSync(recursive: true) + .whereType() + .where((File f) => p.basename(f.path) == 'evals.json') + .toList(); +} From 2d36209d8837c9e5d3c5434c762caffc4f72d570 Mon Sep 17 00:00:00 2001 From: Reid-Agent <269567208+reidbaker-agent@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:18:19 -0400 Subject: [PATCH 02/14] Address review feedback and stub missing evals --- .../.agents/skills/run-evals/SKILL.md | 2 -- .../.agents/skills/run-evals/evals/evals.json | 9 ++++++--- tool/dart_skills_lint/evals/README.md | 6 +++--- .../dart-skills-lint-validation/evals/evals.json | 16 ++++++++++++++++ .../dart_skills_lint/test/skills_evals_test.dart | 13 +++---------- 5 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md index 8ee4045..fc37573 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -7,8 +7,6 @@ metadata: # Run Skill Evals -Use this skill to run skill evaluations. - 1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for the exact subagent and grader prompts. 2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. 3. **Orchestrate**: For each eval prompt, spawn two `bare-agent` subagents using `Workspace: branch`: diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index 8b000f1..5840091 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -10,8 +10,9 @@ "expected_output": "The agent orchestrates two subagents (baseline and with-skill) in branched workspaces, grades their git diffs and stdout, and outputs a markdown artifact.", "expectations": [ "The agent successfully triggered subagents to run the test cases.", - "A markdown artifact containing the evaluation results was created.", - "The execution run leaves the parent working tree clean, with no rogue unstaged modifications to unrelated project files caused by subagents." + "There is an artifact with the results.", + "No files are modified in the skill folder after the eval is done running.", + "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." ] }, { @@ -19,7 +20,9 @@ "prompt": "Can you test the dart-add-unit-test skill and see if it passes its rubric?", "expected_output": "The agent identifies the target skill, spawns baseline and with-skill subagents, compares results, and writes an evaluation artifact.", "expectations": [ - "The execution run leaves the parent working tree clean, with no rogue unstaged modifications to unrelated project files caused by subagents." + "There is an artifact with the results.", + "No files are modified in the skill folder after the eval is done running.", + "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." ] } ] diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md index 241caf0..7984c41 100644 --- a/tool/dart_skills_lint/evals/README.md +++ b/tool/dart_skills_lint/evals/README.md @@ -1,6 +1,6 @@ # Skill Evaluations (Evals) Framework -This directory defines the architecture, rubrics, and instructions for evaluating AI agent skills authored and maintained in this repository using an LLM Agent as Judge. +Architecture, rubrics, and instructions for evaluating AI agent skills authored and maintained in this repository using an LLM Agent as Judge. ## Core Principles & Architecture @@ -40,7 +40,7 @@ When authoring or updating `evals.json`: Run the unit test that checks all `evals.json` files for structural consistency across the repository: ```bash -dart test test/published_skills_evals_test.dart +dart test test/skills_evals_test.dart ``` ### 2. Running Evals via Agent Orchestration @@ -79,7 +79,7 @@ Once you are done, do not commit. Just send me a message with the `git diff` of When preparing to grade execution outputs, the orchestrator or grader agent must resolve the `repo-criteria` section from `evals.json`: 1. **Parse `repo-criteria`**: Read the array of file paths defined under `"repo-criteria"` in `evals.json` (e.g., `["evals/code_quality_rubric.json"]`). -2. **Load Referenced Rubrics**: Open each referenced JSON file relative to the repository root and extract its `expectations` list under `evaluations`. +2. **Load Referenced Rubrics**: Open each referenced JSON file relative to the `tool/dart_skills_lint/` directory and extract its `expectations` list under `evaluations`. 3. **Combine Expectations**: Concatenate the universal expectations from `repo-criteria` with the skill-specific `expectations` from `evals.json` to build the full grading payload. #### Agent Judge Grading diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json new file mode 100644 index 0000000..8d3b103 --- /dev/null +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "dart-skills-lint-validation", + "repo-criteria": [ + "evals/code_quality_rubric.json" + ], + "evals": [ + { + "id": 1, + "prompt": "TODO: Add a prompt to test dart-skills-lint-validation.", + "expected_output": "TODO: Define the expected high-level outcome.", + "expectations": [ + "TODO: Add discrete expectations for the agent judge." + ] + } + ] +} diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index d9760ac..29111a5 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -16,7 +16,7 @@ void main() { final List evalsFiles = [ ..._findEvalsFiles(Directory(p.join(Directory.current.path, 'skills'))), ..._findEvalsFiles(Directory(p.join(Directory.current.path, '.agents', 'skills'))), - ]; + ]..sort((a, b) => a.path.compareTo(b.path)); expect( evalsFiles, @@ -30,9 +30,6 @@ void main() { String? expectedEvalItemFilePath; for (final evalsFile in evalsFiles) { - final Directory skillDir = evalsFile.parent.parent; - final String skillName = p.basename(skillDir.path); - final Object? decoded = jsonDecode(evalsFile.readAsStringSync()); expect( decoded, @@ -44,12 +41,6 @@ void main() { fail('${evalsFile.path} must be a JSON map.'); } - expect( - decoded['skill_name'], - equals(skillName), - reason: 'skill_name in ${evalsFile.path} must match skill directory name "$skillName".', - ); - final Set rootKeys = decoded.keys.toSet(); if (expectedRootKeys == null) { expectedRootKeys = rootKeys; @@ -104,6 +95,8 @@ void main() { } }); + // Note: We intentionally only require an evals.json file for published skills. + // Contributor skills in .agents/skills/ are not currently required to have one. test('all published skills have an evals.json file', () { final skillsDir = Directory(p.join(Directory.current.path, 'skills')); if (!skillsDir.existsSync()) { From 989e5715e55db415b4897029f1d420ba092d1bfa Mon Sep 17 00:00:00 2001 From: Reid-Agent <269567208+reidbaker-agent@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:53:47 -0400 Subject: [PATCH 03/14] chore(evals): add unary integration mode and baseline comparison examples --- .../.agents/skills/run-evals/SKILL.md | 10 ++++---- .../.agents/skills/run-evals/evals/evals.json | 15 ++++++++++-- tool/dart_skills_lint/evals/README.md | 24 ++++++++++++++++--- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md index fc37573..91fcb8d 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -9,10 +9,10 @@ metadata: 1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for the exact subagent and grader prompts. 2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. -3. **Orchestrate**: For each eval prompt, spawn two `bare-agent` subagents using `Workspace: branch`: - - **Baseline**: Provide only the task prompt. - - **With-Skill**: Provide the task prompt + instructions to strictly follow the target skill file. - Instruct both to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. - **CRITICAL**: You must explicitly warn both subagents to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. +3. **Orchestrate**: By default, run an Integration Test by spawning a single **With-Skill** `bare-agent` subagent using `Workspace: branch`. + - Provide the task prompt + instructions to strictly follow the target skill file. + - **Only if the user explicitly requests a comparison or benchmark**, also spawn a **Baseline** subagent (provided *only* the task prompt without skill instructions). + Instruct the subagent(s) to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. + **CRITICAL**: You must explicitly warn the subagent(s) to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. 4. **Grade**: Parse the combined rubric (resolving `repo-criteria` + `evals.json` expectations). 5. **Artifact**: Grade the outputs and generate a Markdown artifact (e.g., `_eval_results.md`) containing the metadata, pass/fail rationale, and raw diffs/stdout. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index 5840091..3063e92 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -7,7 +7,7 @@ { "id": 1, "prompt": "Run the evals for the definition-of-done skill.", - "expected_output": "The agent orchestrates two subagents (baseline and with-skill) in branched workspaces, grades their git diffs and stdout, and outputs a markdown artifact.", + "expected_output": "The agent orchestrates a with-skill subagent in a branched workspace, grades its git diffs and stdout, and outputs a markdown artifact.", "expectations": [ "The agent successfully triggered subagents to run the test cases.", "There is an artifact with the results.", @@ -18,12 +18,23 @@ { "id": 2, "prompt": "Can you test the dart-add-unit-test skill and see if it passes its rubric?", - "expected_output": "The agent identifies the target skill, spawns baseline and with-skill subagents, compares results, and writes an evaluation artifact.", + "expected_output": "The agent identifies the target skill, spawns a with-skill subagent, grades the result, and writes an evaluation artifact.", "expectations": [ "There is an artifact with the results.", "No files are modified in the skill folder after the eval is done running.", "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." ] + }, + { + "id": 3, + "prompt": "Run an A/B benchmark evaluation for the dart-add-unit-test skill, comparing it explicitly against a baseline without the skill.", + "expected_output": "The agent identifies the target skill, spawns both baseline and with-skill subagents, compares results, and writes a comprehensive evaluation artifact.", + "expectations": [ + "The agent successfully triggered both baseline and with-skill subagents.", + "There is an artifact with the results.", + "The evaluation artifact contains explicit sections for both the 'With-Skill Agent' and the 'Baseline Agent'.", + "No files are modified in the skill folder after the eval is done running." + ] } ] } diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md index 7984c41..c3e4d79 100644 --- a/tool/dart_skills_lint/evals/README.md +++ b/tool/dart_skills_lint/evals/README.md @@ -47,9 +47,11 @@ dart test test/skills_evals_test.dart Within an agentic IDE, you can run evaluations interactively by instructing a parent orchestrator agent to spawn isolated subagents for execution, and then acting as the Agent Judge itself. +By default, evaluations run in **Unary Integration Mode**—meaning only a With-Skill subagent is spawned and evaluated. A Baseline subagent is only spawned if the user explicitly requests a benchmark or A/B comparison. + #### Subagent Execution Prompts -For each test case in `evals.json`, spawn execution subagents using a branched workspace (`Workspace: branch`) to ensure isolation. Use these standard prompt structures: +For each test case in `evals.json`, spawn execution subagent(s) using a branched workspace (`Workspace: branch`) to ensure isolation. Use these standard prompt structures: **With-Skill Execution Prompt:** ```text @@ -63,7 +65,8 @@ WARNING: You are executing in an isolated branch workspace. Confine all file mod Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). ``` -**Baseline Execution Prompt (Without Skill):** +**Baseline Execution Prompt (Without Skill Comparison):** +Use this prompt if you are explicitly evaluating the value of a skill by comparing it against an agent that has zero specialized instructions. ```text Execute this task: - Task: @@ -74,6 +77,21 @@ WARNING: You are executing in an isolated branch workspace. Confine all file mod Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). ``` +**Baseline Execution Prompt (Main Branch Comparison):** +Use this prompt if you are evaluating a PR and want to compare the feature branch's skill performance against the stable version of the skill on `main`. +```text +Execute this task: +- Skill path: (Please read and STRICTLY FOLLOW the instructions in this skill file before finishing) +- Task: +- Input files: + +CRITICAL INSTRUCTION: Before beginning this task, you MUST run `git checkout main` to ensure you are executing the version of the skill that is currently on the main branch. + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). +``` + #### Resolving `repo-criteria` for Grading When preparing to grade execution outputs, the orchestrator or grader agent must resolve the `repo-criteria` section from `evals.json`: @@ -109,6 +127,6 @@ After grading the outputs, the orchestrator agent must compile the results into Please compile the complete evaluation results into a Markdown artifact (e.g., `_eval_results.md`). The document should include: 1. Metadata at the top (Date, Skill Name, Model Evaluated, Prompt). 2. A section for the "With-Skill Agent" containing the PASS/FAIL grade, rationale, and the raw outputs (`git diff` and command stdout). -3. A section for the "Baseline Agent" containing the PASS/FAIL grade, rationale, and its raw outputs. +3. If a comparison was explicitly requested, a section for the "Baseline Agent" containing the PASS/FAIL grade, rationale, and its raw outputs. ``` This artifact serves as the permanent, human-readable record of the execution run. From 69464d94d99cc81dd866b3414d6dbd7b9a276c0f Mon Sep 17 00:00:00 2001 From: Reid-Agent <269567208+reidbaker-agent@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:40:21 -0400 Subject: [PATCH 04/14] test: cleanly separate automated vs reasoning based validation evals utilizing absolute paths --- .../evals/evals.json | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json index 8d3b103..edd93e9 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -6,10 +6,37 @@ "evals": [ { "id": 1, - "prompt": "TODO: Add a prompt to test dart-skills-lint-validation.", - "expected_output": "TODO: Define the expected high-level outcome.", + "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with the `--check-absolute-paths` flag. The fixture points to `/tmp/this/does/not/exist.md`, which cannot be auto-fixed since it does not exist. First, create a dummy file at an absolute path (e.g., `/tmp/dummy.md`), update the link in `example/skills/invalid/SKILL.md` to point to it, and then run `--fix --dry-run` to preview the auto-fix. Finally, run `--fix` to apply the fix and verify that the link was automatically converted to a relative path.", + "expected_output": "The agent explicitly enables the absolute paths rule, sets up a real absolute file target, previews the fix via dry-run, and automatically converts the link to a relative path via the CLI.", "expectations": [ - "TODO: Add discrete expectations for the agent judge." + "The agent updated the SKILL.md link to point to an absolute path that actually exists on disk.", + "The agent ran the CLI with `--check-absolute-paths` and explicitly verified that a fixable path violation was found.", + "The agent used `--fix --dry-run` to preview the fix.", + "The agent successfully applied the fix using `--fix`, converting the absolute link into a relative one." + ] + }, + { + "id": 2, + "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with `--disallowed-field` and `--check-absolute-paths` flags to surface all violations. Interpret the failures and manually fix all three violations (skill name mismatch, disallowed field, absolute path) without using the `--fix` flag. Ensure the final run is clean.", + "expected_output": "The agent explicitly enables the extra rules, identifies all violations, applies manual corrections to each without auto-fixing, and ensures a clean run.", + "expectations": [ + "The agent executes the linter with the required flags to surface all violations.", + "The agent manually edits the skill files to fix the invalid skill name violation.", + "The agent manually edits the skill files to fix the disallowed field violation.", + "The agent manually edits the skill files to fix the absolute path violation.", + "The agent validates the directory again to confirm a clean run." + ] + }, + { + "id": 3, + "prompt": "Author a custom dart_skills_lint rule named `RequireInternalMetadataRule` that verifies if a skill's metadata contains `internal: true`. Wire it up into a new test file `test/custom_rule_test.dart`.", + "expected_output": "The agent writes a custom SkillRule and integrates it into a new unit test.", + "expectations": [ + "The agent created a class extending `SkillRule`.", + "The agent implemented the `name` and `severity` overrides.", + "The `validate` method correctly parses the `parsedYaml` to check for `internal: true`.", + "The agent created a Dart test file that imports the custom rule and passes it to `validateSkills`.", + "The newly created test compiles and passes static analysis." ] } ] From df8caf08c4b89529670f7fa803f73accc549db7c Mon Sep 17 00:00:00 2001 From: Reid-Agent <269567208+reidbaker-agent@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:11:12 -0400 Subject: [PATCH 05/14] test: add structural consistency testing for universal evals rubrics --- .../test/skills_evals_test.dart | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index 29111a5..56825a8 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -115,6 +115,88 @@ void main() { ); } }); + + test('all rubric JSON files in evals/ share consistent structure and keys', () { + final Directory rubricsDir = Directory(p.join(Directory.current.path, 'evals')); + if (!rubricsDir.existsSync()) return; + + final List rubricFiles = rubricsDir + .listSync(recursive: false) + .whereType() + .where((File f) => f.path.endsWith('.json')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + if (rubricFiles.isEmpty) return; + + Set? expectedRootKeys; + String? expectedRootKeysFilePath; + Set? expectedEvalItemKeys; + String? expectedEvalItemFilePath; + + for (final rubricFile in rubricFiles) { + final Object? decoded = jsonDecode(rubricFile.readAsStringSync()); + expect( + decoded, + isA>(), + reason: '${rubricFile.path} must be a JSON map.', + ); + + if (decoded is! Map) { + fail('${rubricFile.path} must be a JSON map.'); + } + + final Set rootKeys = decoded.keys.toSet(); + if (expectedRootKeys == null) { + expectedRootKeys = rootKeys; + expectedRootKeysFilePath = rubricFile.path; + } else { + expect( + rootKeys, + equals(expectedRootKeys), + reason: + '${rubricFile.path} root keys do not match consistency pattern. ' + 'Expected keys to match the first processed file ($expectedRootKeysFilePath).', + ); + } + + final Object? evalsRaw = decoded['evaluations']; + expect( + evalsRaw, + isA>(), + reason: 'evaluations key in ${rubricFile.path} must be a List.', + ); + + if (evalsRaw is! List) { + fail('evaluations key in ${rubricFile.path} must be a List.'); + } + + for (final Object? evalItem in evalsRaw) { + expect( + evalItem, + isA>(), + reason: 'Item in evaluations list in ${rubricFile.path} must be a JSON map.', + ); + if (evalItem is! Map) { + fail('Item in evaluations list in ${rubricFile.path} must be a JSON map.'); + } + + final Set itemKeys = evalItem.keys.toSet(); + if (expectedEvalItemKeys == null) { + expectedEvalItemKeys = itemKeys; + expectedEvalItemFilePath = rubricFile.path; + } else { + expect( + itemKeys, + equals(expectedEvalItemKeys), + reason: + 'Evaluation item in ${rubricFile.path} keys do not match consistency pattern. ' + 'Expected eval item keys to match the first processed file ($expectedEvalItemFilePath).', + ); + } + } + } + }); }); } From 2ef691f6d2f5a05214ba4a23cd51994be809f4e0 Mon Sep 17 00:00:00 2001 From: Reid-Agent <269567208+reidbaker-agent@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:16:14 -0400 Subject: [PATCH 06/14] test: deduplicate evals structural consistency checks and fix lints --- .../test/skills_evals_test.dart | 217 ++++++------------ 1 file changed, 75 insertions(+), 142 deletions(-) diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index 56825a8..0dd66d0 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -24,75 +24,7 @@ void main() { reason: 'Should find at least one evals.json file in skills or .agents/skills.', ); - Set? expectedRootKeys; - String? expectedRootKeysFilePath; - Set? expectedEvalItemKeys; - String? expectedEvalItemFilePath; - - for (final evalsFile in evalsFiles) { - final Object? decoded = jsonDecode(evalsFile.readAsStringSync()); - expect( - decoded, - isA>(), - reason: '${evalsFile.path} must be a JSON map.', - ); - - if (decoded is! Map) { - fail('${evalsFile.path} must be a JSON map.'); - } - - final Set rootKeys = decoded.keys.toSet(); - if (expectedRootKeys == null) { - expectedRootKeys = rootKeys; - expectedRootKeysFilePath = evalsFile.path; - } else { - expect( - rootKeys, - equals(expectedRootKeys), - reason: - '${evalsFile.path} root keys do not match consistency pattern. ' - 'Expected keys to match the first processed file ($expectedRootKeysFilePath). ' - 'All evals.json files must share the exact same root keys.', - ); - } - - final Object? evalsRaw = decoded['evals']; - expect( - evalsRaw, - isA>(), - reason: 'evals key in ${evalsFile.path} must be a List.', - ); - - if (evalsRaw is! List) { - fail('evals key in ${evalsFile.path} must be a List.'); - } - - for (final Object? evalItem in evalsRaw) { - expect( - evalItem, - isA>(), - reason: 'Item in evals list in ${evalsFile.path} must be a JSON map.', - ); - if (evalItem is! Map) { - fail('Item in evals list in ${evalsFile.path} must be a JSON map.'); - } - - final Set itemKeys = evalItem.keys.toSet(); - if (expectedEvalItemKeys == null) { - expectedEvalItemKeys = itemKeys; - expectedEvalItemFilePath = evalsFile.path; - } else { - expect( - itemKeys, - equals(expectedEvalItemKeys), - reason: - 'Eval item in ${evalsFile.path} keys do not match consistency pattern. ' - 'Expected eval item keys to match the first processed file ($expectedEvalItemFilePath). ' - 'All eval items must share the exact same keys.', - ); - } - } - } + _verifyStructuralConsistency(evalsFiles, 'evals'); }); // Note: We intentionally only require an evals.json file for published skills. @@ -117,87 +49,88 @@ void main() { }); test('all rubric JSON files in evals/ share consistent structure and keys', () { - final Directory rubricsDir = Directory(p.join(Directory.current.path, 'evals')); - if (!rubricsDir.existsSync()) return; + final rubricsDir = Directory(p.join(Directory.current.path, 'evals')); + if (!rubricsDir.existsSync()) { + return; + } + + final List rubricFiles = + rubricsDir + .listSync() + .whereType() + .where((File f) => f.path.endsWith('.json')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + if (rubricFiles.isEmpty) { + return; + } + + _verifyStructuralConsistency(rubricFiles, 'evaluations'); + }); + }); +} - final List rubricFiles = rubricsDir - .listSync(recursive: false) - .whereType() - .where((File f) => f.path.endsWith('.json')) - .toList() - ..sort((a, b) => a.path.compareTo(b.path)); +void _verifyStructuralConsistency(List files, String itemsKey) { + Set? expectedRootKeys; + String? expectedRootKeysFilePath; + Set? expectedItemKeys; + String? expectedItemFilePath; + + for (final file in files) { + final Object? decoded = jsonDecode(file.readAsStringSync()); + expect(decoded, isA>(), reason: '${file.path} must be a JSON map.'); + + if (decoded is! Map) { + fail('${file.path} must be a JSON map.'); + } + + final Set rootKeys = decoded.keys.toSet(); + if (expectedRootKeys == null) { + expectedRootKeys = rootKeys; + expectedRootKeysFilePath = file.path; + } else { + expect( + rootKeys, + equals(expectedRootKeys), + reason: + '${file.path} root keys do not match consistency pattern. ' + 'Expected keys to match the first processed file ($expectedRootKeysFilePath).', + ); + } - if (rubricFiles.isEmpty) return; + final Object? itemsRaw = decoded[itemsKey]; + expect(itemsRaw, isA>(), reason: '$itemsKey key in ${file.path} must be a List.'); - Set? expectedRootKeys; - String? expectedRootKeysFilePath; - Set? expectedEvalItemKeys; - String? expectedEvalItemFilePath; + if (itemsRaw is! List) { + fail('$itemsKey key in ${file.path} must be a List.'); + } - for (final rubricFile in rubricFiles) { - final Object? decoded = jsonDecode(rubricFile.readAsStringSync()); - expect( - decoded, - isA>(), - reason: '${rubricFile.path} must be a JSON map.', - ); + for (final Object? item in itemsRaw) { + expect( + item, + isA>(), + reason: 'Item in $itemsKey list in ${file.path} must be a JSON map.', + ); + if (item is! Map) { + fail('Item in $itemsKey list in ${file.path} must be a JSON map.'); + } - if (decoded is! Map) { - fail('${rubricFile.path} must be a JSON map.'); - } - - final Set rootKeys = decoded.keys.toSet(); - if (expectedRootKeys == null) { - expectedRootKeys = rootKeys; - expectedRootKeysFilePath = rubricFile.path; - } else { - expect( - rootKeys, - equals(expectedRootKeys), - reason: - '${rubricFile.path} root keys do not match consistency pattern. ' - 'Expected keys to match the first processed file ($expectedRootKeysFilePath).', - ); - } - - final Object? evalsRaw = decoded['evaluations']; + final Set itemKeys = item.keys.toSet(); + if (expectedItemKeys == null) { + expectedItemKeys = itemKeys; + expectedItemFilePath = file.path; + } else { expect( - evalsRaw, - isA>(), - reason: 'evaluations key in ${rubricFile.path} must be a List.', + itemKeys, + equals(expectedItemKeys), + reason: + 'Item in ${file.path} keys do not match consistency pattern. ' + 'Expected item keys to match the first processed file ($expectedItemFilePath).', ); - - if (evalsRaw is! List) { - fail('evaluations key in ${rubricFile.path} must be a List.'); - } - - for (final Object? evalItem in evalsRaw) { - expect( - evalItem, - isA>(), - reason: 'Item in evaluations list in ${rubricFile.path} must be a JSON map.', - ); - if (evalItem is! Map) { - fail('Item in evaluations list in ${rubricFile.path} must be a JSON map.'); - } - - final Set itemKeys = evalItem.keys.toSet(); - if (expectedEvalItemKeys == null) { - expectedEvalItemKeys = itemKeys; - expectedEvalItemFilePath = rubricFile.path; - } else { - expect( - itemKeys, - equals(expectedEvalItemKeys), - reason: - 'Evaluation item in ${rubricFile.path} keys do not match consistency pattern. ' - 'Expected eval item keys to match the first processed file ($expectedEvalItemFilePath).', - ); - } - } } - }); - }); + } + } } List _findEvalsFiles(Directory baseDir) { From 9378e8fe82194c9c46f5e14d71103cca3206fe2d Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Wed, 22 Jul 2026 14:46:08 -0400 Subject: [PATCH 07/14] Address PR 189 feedback: Refactor evals framework schema and prompts --- .../definition-of-done/evals/evals.json | 15 +- .../.agents/skills/run-evals/SKILL.md | 13 +- .../.agents/skills/run-evals/evals/evals.json | 33 +++-- .../run-evals/resources/agent_judge_prompt.md | 13 ++ .../resources/baseline_execution_prompt.md | 7 + .../resources/with_skill_execution_prompt.md | 8 ++ tool/dart_skills_lint/evals/README.md | 132 +++--------------- .../evals/code_quality_rubric.json | 34 ++--- .../dart-skills-lint-setup/evals/evals.json | 34 +++-- .../evals/evals.json | 29 ++-- .../test/skills_evals_test.dart | 24 ++-- 11 files changed, 141 insertions(+), 201 deletions(-) create mode 100644 tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md create mode 100644 tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md create mode 100644 tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 91912a6..0e42e68 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -1,19 +1,20 @@ { "skill_name": "definition-of-done", - "repo-criteria": [ - "evals/code_quality_rubric.json" - ], "evals": [ { "id": 1, "prompt": "Modify tool/dart_skills_lint/lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", - "expected_output": "The file tool/dart_skills_lint/lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", - "expectations": [ + "expected_chat_output": "The file tool/dart_skills_lint/lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", + "expected_repo_state": [ "The levenshtein.dart file contains a new comment explaining how edit distance is calculated.", "The new comment does not contain any relative temporal words such as 'now', 'currently', 'new', 'old', or 'existing'.", "The levenshtein.dart file is formatted cleanly according to dart format.", "The project has no dart analyze errors or warnings." - ] + ], + "agent_config": "reidbaker-agent" } + ], + "repo_criteria": [ + "evals/code_quality_rubric.json" ] -} +} \ No newline at end of file diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md index 91fcb8d..6ff8fd7 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -7,12 +7,13 @@ metadata: # Run Skill Evals -1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for the exact subagent and grader prompts. +1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for understanding the difference between per-skill evals and cross-skill evals. 2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. -3. **Orchestrate**: By default, run an Integration Test by spawning a single **With-Skill** `bare-agent` subagent using `Workspace: branch`. - - Provide the task prompt + instructions to strictly follow the target skill file. - - **Only if the user explicitly requests a comparison or benchmark**, also spawn a **Baseline** subagent (provided *only* the task prompt without skill instructions). +3. **Determine Agent Configuration**: Check the `agent_config` field in the target `evals.json` to determine the environment/harness to spawn. If `agent_config` is `"bare-agent"`, spawn a subagent with the `bare-agent` profile. If it is a specific contributor profile (e.g. `"reidbaker-agent"`), use that profile to provide the necessary contributor context. +4. **Orchestrate**: By default, run an Integration Test by spawning a single **With-Skill** subagent using `Workspace: branch` and the identified `agent_config`. + - Provide the task prompt. See `resources/with_skill_execution_prompt.md` for the template. + - **Only if the user explicitly requests a comparison or benchmark**, also spawn a **Baseline** subagent. See `resources/baseline_execution_prompt.md` for the template. Instruct the subagent(s) to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. **CRITICAL**: You must explicitly warn the subagent(s) to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. -4. **Grade**: Parse the combined rubric (resolving `repo-criteria` + `evals.json` expectations). -5. **Artifact**: Grade the outputs and generate a Markdown artifact (e.g., `_eval_results.md`) containing the metadata, pass/fail rationale, and raw diffs/stdout. +5. **Grade**: Parse the combined rubric (resolving `repo_criteria` + `evals.json` expectations). Use the grading instructions in `resources/agent_judge_prompt.md`. When an expectation fails, you MUST explicitly list both the expectation and what was actually found that caused the failure. +6. **Artifact**: Grade the outputs and generate a Markdown artifact (e.g., `_eval_results.md`) containing the metadata, pass/fail rationale, and raw diffs/stdout. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index 3063e92..d211ac0 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -1,40 +1,43 @@ { "skill_name": "run-evals", - "repo-criteria": [ - "evals/code_quality_rubric.json" - ], "evals": [ { "id": 1, "prompt": "Run the evals for the definition-of-done skill.", - "expected_output": "The agent orchestrates a with-skill subagent in a branched workspace, grades its git diffs and stdout, and outputs a markdown artifact.", - "expectations": [ + "expected_chat_output": "The agent orchestrates a with-skill subagent in a branched workspace, grades its git diffs and stdout, and outputs a markdown artifact.", + "expected_repo_state": [ "The agent successfully triggered subagents to run the test cases.", "There is an artifact with the results.", "No files are modified in the skill folder after the eval is done running.", "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." - ] + ], + "agent_config": "reidbaker-agent" }, { "id": 2, - "prompt": "Can you test the dart-add-unit-test skill and see if it passes its rubric?", - "expected_output": "The agent identifies the target skill, spawns a with-skill subagent, grades the result, and writes an evaluation artifact.", - "expectations": [ + "prompt": "Can you test the definition-of-done skill and see if it passes its rubric?", + "expected_chat_output": "The agent identifies the target skill, spawns a with-skill subagent, grades the result, and writes an evaluation artifact.", + "expected_repo_state": [ "There is an artifact with the results.", "No files are modified in the skill folder after the eval is done running.", "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." - ] + ], + "agent_config": "reidbaker-agent" }, { "id": 3, - "prompt": "Run an A/B benchmark evaluation for the dart-add-unit-test skill, comparing it explicitly against a baseline without the skill.", - "expected_output": "The agent identifies the target skill, spawns both baseline and with-skill subagents, compares results, and writes a comprehensive evaluation artifact.", - "expectations": [ + "prompt": "Run an A/B benchmark evaluation for the definition-of-done skill, comparing it explicitly against a baseline without the skill.", + "expected_chat_output": "The agent identifies the target skill, spawns both baseline and with-skill subagents, compares results, and writes a comprehensive evaluation artifact.", + "expected_repo_state": [ "The agent successfully triggered both baseline and with-skill subagents.", "There is an artifact with the results.", "The evaluation artifact contains explicit sections for both the 'With-Skill Agent' and the 'Baseline Agent'.", "No files are modified in the skill folder after the eval is done running." - ] + ], + "agent_config": "reidbaker-agent" } + ], + "repo_criteria": [ + "evals/code_quality_rubric.json" ] -} +} \ No newline at end of file diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md new file mode 100644 index 0000000..e568622 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md @@ -0,0 +1,13 @@ +You are an expert evaluator. Review the following execution outputs from an AI agent against the provided combined rubric. + +Execution Outputs: +- Git Diff: +- Command Stdout: + +Combined Rubric Expectations: + + +For every single expectation, explicitly state whether it PASSED or FAILED and provide a 1-sentence justification. +IMPORTANT: When an expectation fails, explicitly list the expectation and explain exactly what you found that caused the failure. + +Finally, provide an overall PASS/FAIL grade. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md new file mode 100644 index 0000000..d073ab7 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md @@ -0,0 +1,7 @@ +Execute this task: +- Task: +- Input files: + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md new file mode 100644 index 0000000..de672a6 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md @@ -0,0 +1,8 @@ +Execute this task: +- Skill path: (Please read and STRICTLY FOLLOW the instructions in this skill file before finishing) +- Task: +- Input files: + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md index c3e4d79..b00f969 100644 --- a/tool/dart_skills_lint/evals/README.md +++ b/tool/dart_skills_lint/evals/README.md @@ -1,38 +1,37 @@ -# Skill Evaluations (Evals) Framework +# Skill Evaluations -Architecture, rubrics, and instructions for evaluating AI agent skills authored and maintained in this repository using an LLM Agent as Judge. +Architecture, rubrics, and instructions for evaluating AI agent skills authored and maintained in this repository. +**Note:** These evaluations are essentially unit tests for the skills within the `dart_skills_lint` package and its internal ecosystem. They are *not* intended to be a generic evaluation framework for other agent client plugins or tools outside of this specific domain. -## Core Principles & Architecture +## What Should (and Shouldn't) Be Evaluated -Evaluations in this repository use a **Two-Tiered Architecture** to separate domain-specific skill requirements from universal skill quality standards. The LLM Agent Judge evaluates execution runs against two components: +**DO Evaluate:** +- Core workflows of a skill (e.g., adding a dependency, running validation checks). +- Specific edge cases that a skill claims to handle (e.g., legacy integration paths without `--fix`). +- Whether a skill correctly leaves the repository in a compilable, passing state. -1. **Domain Evals (`/evals/evals.json`)**: - Skill-specific task prompts, macro narrative expected outputs, and micro expectation assertions unique to that skill. +**DO NOT Evaluate:** +- Trivial syntax formatting that `dart format` already fixes perfectly. +- Complete system architectures that take longer than a few minutes to generate and verify. +- Skills that are outside the scope of `dart_skills_lint` (e.g. general flutter app creation). -2. **Universal Skill Quality Rubrics (`evals/*_rubric.json`)**: - Modular, shared quality rubrics injected into the LLM Agent Judge system prompt representing universal quality classes required across skills. +## Core Principles & Architecture ---- +Evaluations in this repository use a **Two-Tiered Architecture** to separate domain-specific skill requirements from universal skill quality standards. -### 1. Skill-Specific Evals (`/evals/evals.json`) +### 1. Per-Skill Evals (`/evals/evals.json`) Each skill maintains an `evals/evals.json` file containing target task prompts and expectations: - **`prompt`**: Realistic user prompt testing primary or edge-case workflows. -- **`expected_output`**: High-level narrative summary of success for human reviewers and judge context. -- **`expectations`**: Array of discrete, testable assertions used by the judge to compute quantitative pass rates. -- **`repo-criteria`**: Array of relative file paths to shared universal quality rubrics (e.g., `["evals/code_quality_rubric.json"]`). +- **`expected_chat_output`**: High-level narrative summary of what the LLM should say/give to the user. +- **`expected_repo_state`**: Array of discrete, testable assertions regarding the end state of the repository and tracked files. +- **`repo_criteria`**: Array of relative file paths to shared universal quality rubrics (e.g., `["evals/code_quality_rubric.json"]`). +- **`agent_config`**: The model configuration/harness used when executing the eval against the skill. For published skills, use `"bare-agent"`. For internal contributor skills, use the internal agent profile (e.g., `"reidbaker-agent"`). -### 2. Universal Skill Quality Rubrics Framework (`evals/*_rubric.json`) +### 2. Cross-Skill Evals (`evals/*_rubric.json`) Universal skill quality expectations are structured into modular rubric classes that apply broadly across skills. ---- - -## Key Rules for Writing Evals - -When authoring or updating `evals.json`: - -1. Maintain Schema Consistency across all `evals.json` files. - ---- +## Cross-Cutting Rules +Skills that author or modify code MUST adhere to the universal code quality expectations defined in `code_quality_rubric.json`. This ensures that generated code compiles cleanly, adheres to Effective Dart, works across platforms, and is placed in standard canonical directories. ## 🚀 Running & Validating Evals Locally @@ -44,89 +43,4 @@ dart test test/skills_evals_test.dart ``` ### 2. Running Evals via Agent Orchestration - -Within an agentic IDE, you can run evaluations interactively by instructing a parent orchestrator agent to spawn isolated subagents for execution, and then acting as the Agent Judge itself. - -By default, evaluations run in **Unary Integration Mode**—meaning only a With-Skill subagent is spawned and evaluated. A Baseline subagent is only spawned if the user explicitly requests a benchmark or A/B comparison. - -#### Subagent Execution Prompts - -For each test case in `evals.json`, spawn execution subagent(s) using a branched workspace (`Workspace: branch`) to ensure isolation. Use these standard prompt structures: - -**With-Skill Execution Prompt:** -```text -Execute this task: -- Skill path: (Please read and STRICTLY FOLLOW the instructions in this skill file before finishing) -- Task: -- Input files: - -WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. - -Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). -``` - -**Baseline Execution Prompt (Without Skill Comparison):** -Use this prompt if you are explicitly evaluating the value of a skill by comparing it against an agent that has zero specialized instructions. -```text -Execute this task: -- Task: -- Input files: - -WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. - -Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). -``` - -**Baseline Execution Prompt (Main Branch Comparison):** -Use this prompt if you are evaluating a PR and want to compare the feature branch's skill performance against the stable version of the skill on `main`. -```text -Execute this task: -- Skill path: (Please read and STRICTLY FOLLOW the instructions in this skill file before finishing) -- Task: -- Input files: - -CRITICAL INSTRUCTION: Before beginning this task, you MUST run `git checkout main` to ensure you are executing the version of the skill that is currently on the main branch. - -WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. - -Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). -``` - -#### Resolving `repo-criteria` for Grading - -When preparing to grade execution outputs, the orchestrator or grader agent must resolve the `repo-criteria` section from `evals.json`: - -1. **Parse `repo-criteria`**: Read the array of file paths defined under `"repo-criteria"` in `evals.json` (e.g., `["evals/code_quality_rubric.json"]`). -2. **Load Referenced Rubrics**: Open each referenced JSON file relative to the `tool/dart_skills_lint/` directory and extract its `expectations` list under `evaluations`. -3. **Combine Expectations**: Concatenate the universal expectations from `repo-criteria` with the skill-specific `expectations` from `evals.json` to build the full grading payload. - -#### Agent Judge Grading - -Once execution subagents return their `git diff` and command outputs, grade the results using this prompt template: - -**Agent Judge Prompt:** -```text -You are an expert evaluator. Review the following execution outputs from an AI agent against the provided combined rubric. - -Execution Outputs: -- Git Diff: -- Command Stdout: - -Combined Rubric Expectations: - - -For each expectation, explicitly state whether it PASSED or FAILED and provide a 1-sentence justification. Finally, provide an overall PASS/FAIL grade. -``` - -#### Generating the Evaluation Artifact - -After grading the outputs, the orchestrator agent must compile the results into a markdown artifact so they can be easily reviewed. - -**Artifact Generation Prompt:** -```text -Please compile the complete evaluation results into a Markdown artifact (e.g., `_eval_results.md`). The document should include: -1. Metadata at the top (Date, Skill Name, Model Evaluated, Prompt). -2. A section for the "With-Skill Agent" containing the PASS/FAIL grade, rationale, and the raw outputs (`git diff` and command stdout). -3. If a comparison was explicitly requested, a section for the "Baseline Agent" containing the PASS/FAIL grade, rationale, and its raw outputs. -``` -This artifact serves as the permanent, human-readable record of the execution run. +You should use the `/run-evals` skill to run evaluations. The bulk of the execution logic and prompts are located within the `run-evals` skill itself (`.agents/skills/run-evals/SKILL.md`). The environment, model, and harness are determined by the `agent_config` specified in the corresponding `evals.json` file. diff --git a/tool/dart_skills_lint/evals/code_quality_rubric.json b/tool/dart_skills_lint/evals/code_quality_rubric.json index 81a4a31..e451409 100644 --- a/tool/dart_skills_lint/evals/code_quality_rubric.json +++ b/tool/dart_skills_lint/evals/code_quality_rubric.json @@ -1,41 +1,35 @@ { - "rubric_name": "universal-code-quality-rubric", - "description": "Universal code quality class within the universal skill quality framework, injected into the LLM Agent Judge system prompt for code-generating skill execution runs.", - "evaluations": [ + "evals": [ { - "category": "compilation_and_health", - "description": "Does the generated or modified code compile cleanly without syntax errors, pass static analysis ('dart analyze --fatal-infos'), and run successfully?", - "expectations": [ + "id": "compilation_and_health", + "expected_repo_state": [ "The generated code compiles with zero syntax errors.", "Static analysis passes cleanly ('dart analyze --fatal-infos' returns zero errors or warnings).", "Existing and newly added unit, widget, or integration tests execute and pass successfully." ] }, { - "category": "effective_dart_and_idioms", - "description": "Does the code adhere to Effective Dart guidelines and modern Dart 3+ language features?", - "expectations": [ - "Follows Effective Dart naming conventions, formatting, and structural guidelines.", - "Leverages Dart 3+ idioms where applicable (e.g. pattern matching, records, primary constructors, switch expressions).", - "Avoids anti-patterns such as unnecessary dynamic types or unhandled async calls." + "id": "effective_dart_and_idioms", + "expected_repo_state": [ + "Follows Effective Dart style guidelines as enforced by standard 'core' and 'recommended' linter rules.", + "Leverages Dart 3 idioms such as switch expressions and exhaustive pattern matching when conditional logic branches on algebraic types.", + "Avoids anti-patterns such as unnecessary dynamic types, raw Strings instead of string interpolation, or unhandled Futures." ] }, { - "category": "cross_platform_compatibility", - "description": "Does the solution function correctly across targeted host platforms (macOS, Linux, Windows)?", - "expectations": [ + "id": "cross_platform_compatibility", + "expected_repo_state": [ "File paths and CLI path parameters use platform-agnostic formatters (package:path or forward slashes '/').", "CLI commands and shell invocation scripts use cross-platform compatible syntax.", "Avoids hardcoded platform-specific environment paths (such as C:\\ or /usr/local/bin directly)." ] }, { - "category": "directory_and_placement_hygiene", - "description": "Are all created or modified files placed in canonical repository directories?", - "expectations": [ - "Configuration files, source code, and tests are placed in their proper standard directory locations.", + "id": "directory_and_placement_hygiene", + "expected_repo_state": [ + "Source code must be placed in 'lib/', tests in 'test/', and executable scripts in 'bin/' or 'tool/'.", "No temporary scratch files or orphaned directories are left behind after execution." ] } ] -} +} \ No newline at end of file diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json index f03d366..5dfc940 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -1,48 +1,52 @@ { "skill_name": "dart-skills-lint-setup", - "repo-criteria": [ - "evals/code_quality_rubric.json" - ], "evals": [ { "id": 1, "prompt": "Set up dart_skills_lint validation for the first time in our Dart project to lint skills located in .agents/skills.", - "expected_output": "The project's pubspec.yaml is updated with dart_skills_lint under dev_dependencies using a git dependency pointing to tool/dart_skills_lint, a root dart_skills_lint.yaml file is created specifying .agents/skills, and recipes for CI integration are provided.", - "expectations": [ + "expected_chat_output": "The project's pubspec.yaml is updated with dart_skills_lint under dev_dependencies using a git dependency pointing to tool/dart_skills_lint, a root dart_skills_lint.yaml file is created specifying .agents/skills, and recipes for CI integration are provided.", + "expected_repo_state": [ "dart_skills_lint is added under dev_dependencies in pubspec.yaml as a git dependency targeting https://github.com/flutter/agent-plugins.git with path tool/dart_skills_lint.", "The dart_skills_lint.yaml file is created at the repository root so both CLI and Dart test invocations share the configuration.", "dart_skills_lint.yaml specifies .agents/skills under directories.", "Configures check-relative-paths: error under rules in dart_skills_lint.yaml to enforce portable relative links across skill files.", "Guidance or recipe for CI integration or pre-commit hook is provided.", "NOT added under runtime dependencies in pubspec.yaml.", - "NOT placed inside .agents/skills/ or a sub-folder where root CLI commands will fail to locate the config." - ] + "NOT placed inside .agents/skills/ or a sub-folder where root CLI commands will fail to locate the config.", + "No deprecated code from dart_skills_lint is used in the generated code.", + "There is a dart test created that fails when a skill defined in dart_skills_lint.yaml has an error." + ], + "agent_config": "bare-agent" }, { "id": 2, "prompt": "A repository has pre-existing skill files in .agents/skills with lint violations. Set up dart_skills_lint without failing CI or modifying legacy skill files by generating a baseline ignore file.", - "expected_output": "dart_skills_lint is added to pubspec.yaml dev_dependencies, a root dart_skills_lint.yaml configuration is created, and dart run dart_skills_lint:cli is executed with --generate-baseline to write violations to an ignore file without mutating legacy skill content.", - "expectations": [ + "expected_chat_output": "dart_skills_lint is added to pubspec.yaml dev_dependencies, a root dart_skills_lint.yaml configuration is created, and dart run dart_skills_lint:cli is executed with --generate-baseline to write violations to an ignore file without mutating legacy skill content.", + "expected_repo_state": [ "dart_skills_lint is configured under dev_dependencies in pubspec.yaml and dart_skills_lint.yaml is created at the repository root.", "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", "The CLI command dart run dart_skills_lint:cli is run with --skills-directory=.agents/skills --generate-baseline to write legacy failures to an ignore file.", "An explanation is provided that pre-existing violations exit clean while violations introduced after baseline generation will still fail validation.", "Does NOT modify, delete, or reformat any pre-existing SKILL.md files in .agents/skills during baseline generation.", "Does NOT pass --fix during baseline generation, keeping legacy skill files intact." - ] + ], + "agent_config": "bare-agent" }, { "id": 3, "prompt": "Set up dart_skills_lint to validate agent skills in .agents/skills, while isolating the package dependency inside tool/pubspec.yaml instead of adding it to the root pubspec.yaml.", - "expected_output": "The tool/ package's pubspec.yaml contains dart_skills_lint under dev_dependencies, while dart_skills_lint.yaml is placed at the repository root, ensuring the root closure is unpolluted and configuration remains centrally accessible.", - "expectations": [ + "expected_chat_output": "The tool/ package's pubspec.yaml contains dart_skills_lint under dev_dependencies, while dart_skills_lint.yaml is placed at the repository root, ensuring the root closure is unpolluted and configuration remains centrally accessible.", + "expected_repo_state": [ "dart_skills_lint is added under dev_dependencies in tool/pubspec.yaml rather than the root pubspec.yaml.", "dart_skills_lint.yaml is created at the repository root directory rather than inside tool/ so both CLI and test invocations share the configuration.", "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", - "Guidance is provided on ensuring dependency references and commit hashes remain synchronized if added to multiple pubspecs.", "NOT placed inside tool/, which would prevent root-level CLI invocations from discovering the configuration file.", "NOT added to the root pubspec.yaml dependencies or dev_dependencies." - ] + ], + "agent_config": "bare-agent" } + ], + "repo_criteria": [ + "evals/code_quality_rubric.json" ] -} +} \ No newline at end of file diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json index edd93e9..87386f5 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -1,43 +1,46 @@ { "skill_name": "dart-skills-lint-validation", - "repo-criteria": [ - "evals/code_quality_rubric.json" - ], "evals": [ { "id": 1, "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with the `--check-absolute-paths` flag. The fixture points to `/tmp/this/does/not/exist.md`, which cannot be auto-fixed since it does not exist. First, create a dummy file at an absolute path (e.g., `/tmp/dummy.md`), update the link in `example/skills/invalid/SKILL.md` to point to it, and then run `--fix --dry-run` to preview the auto-fix. Finally, run `--fix` to apply the fix and verify that the link was automatically converted to a relative path.", - "expected_output": "The agent explicitly enables the absolute paths rule, sets up a real absolute file target, previews the fix via dry-run, and automatically converts the link to a relative path via the CLI.", - "expectations": [ + "expected_chat_output": "The agent explicitly enables the absolute paths rule, sets up a real absolute file target, previews the fix via dry-run, and automatically converts the link to a relative path via the CLI.", + "expected_repo_state": [ "The agent updated the SKILL.md link to point to an absolute path that actually exists on disk.", "The agent ran the CLI with `--check-absolute-paths` and explicitly verified that a fixable path violation was found.", "The agent used `--fix --dry-run` to preview the fix.", "The agent successfully applied the fix using `--fix`, converting the absolute link into a relative one." - ] + ], + "agent_config": "bare-agent" }, { "id": 2, "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with `--disallowed-field` and `--check-absolute-paths` flags to surface all violations. Interpret the failures and manually fix all three violations (skill name mismatch, disallowed field, absolute path) without using the `--fix` flag. Ensure the final run is clean.", - "expected_output": "The agent explicitly enables the extra rules, identifies all violations, applies manual corrections to each without auto-fixing, and ensures a clean run.", - "expectations": [ + "expected_chat_output": "The agent explicitly enables the extra rules, identifies all violations, applies manual corrections to each without auto-fixing, and ensures a clean run.", + "expected_repo_state": [ "The agent executes the linter with the required flags to surface all violations.", "The agent manually edits the skill files to fix the invalid skill name violation.", "The agent manually edits the skill files to fix the disallowed field violation.", "The agent manually edits the skill files to fix the absolute path violation.", "The agent validates the directory again to confirm a clean run." - ] + ], + "agent_config": "bare-agent" }, { "id": 3, "prompt": "Author a custom dart_skills_lint rule named `RequireInternalMetadataRule` that verifies if a skill's metadata contains `internal: true`. Wire it up into a new test file `test/custom_rule_test.dart`.", - "expected_output": "The agent writes a custom SkillRule and integrates it into a new unit test.", - "expectations": [ + "expected_chat_output": "The agent writes a custom SkillRule and integrates it into a new unit test.", + "expected_repo_state": [ "The agent created a class extending `SkillRule`.", "The agent implemented the `name` and `severity` overrides.", "The `validate` method correctly parses the `parsedYaml` to check for `internal: true`.", "The agent created a Dart test file that imports the custom rule and passes it to `validateSkills`.", "The newly created test compiles and passes static analysis." - ] + ], + "agent_config": "bare-agent" } + ], + "repo_criteria": [ + "evals/code_quality_rubric.json" ] -} +} \ No newline at end of file diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index 0dd66d0..6278872 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -66,7 +66,7 @@ void main() { return; } - _verifyStructuralConsistency(rubricFiles, 'evaluations'); + _verifyStructuralConsistency(rubricFiles, 'evals'); }); }); } @@ -81,11 +81,8 @@ void _verifyStructuralConsistency(List files, String itemsKey) { final Object? decoded = jsonDecode(file.readAsStringSync()); expect(decoded, isA>(), reason: '${file.path} must be a JSON map.'); - if (decoded is! Map) { - fail('${file.path} must be a JSON map.'); - } - - final Set rootKeys = decoded.keys.toSet(); + final decodedMap = decoded! as Map; + final Set rootKeys = decodedMap.keys.toSet(); if (expectedRootKeys == null) { expectedRootKeys = rootKeys; expectedRootKeysFilePath = file.path; @@ -99,24 +96,19 @@ void _verifyStructuralConsistency(List files, String itemsKey) { ); } - final Object? itemsRaw = decoded[itemsKey]; + final Object? itemsRaw = decodedMap[itemsKey]; expect(itemsRaw, isA>(), reason: '$itemsKey key in ${file.path} must be a List.'); - if (itemsRaw is! List) { - fail('$itemsKey key in ${file.path} must be a List.'); - } - - for (final Object? item in itemsRaw) { + final itemsList = itemsRaw! as List; + for (final Object? item in itemsList) { expect( item, isA>(), reason: 'Item in $itemsKey list in ${file.path} must be a JSON map.', ); - if (item is! Map) { - fail('Item in $itemsKey list in ${file.path} must be a JSON map.'); - } - final Set itemKeys = item.keys.toSet(); + final itemMap = item! as Map; + final Set itemKeys = itemMap.keys.toSet(); if (expectedItemKeys == null) { expectedItemKeys = itemKeys; expectedItemFilePath = file.path; From 3e5d8cea94528fb4179a0b0affe61dbc138d2110 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Wed, 22 Jul 2026 16:12:27 -0400 Subject: [PATCH 08/14] Prevent evals from modifying files in main working directory --- tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md | 2 +- .../.agents/skills/run-evals/evals/evals.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md index 6ff8fd7..8fa173c 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -11,7 +11,7 @@ metadata: 2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. 3. **Determine Agent Configuration**: Check the `agent_config` field in the target `evals.json` to determine the environment/harness to spawn. If `agent_config` is `"bare-agent"`, spawn a subagent with the `bare-agent` profile. If it is a specific contributor profile (e.g. `"reidbaker-agent"`), use that profile to provide the necessary contributor context. 4. **Orchestrate**: By default, run an Integration Test by spawning a single **With-Skill** subagent using `Workspace: branch` and the identified `agent_config`. - - Provide the task prompt. See `resources/with_skill_execution_prompt.md` for the template. + - Provide the task prompt. See `resources/with_skill_execution_prompt.md` for the template. When filling in ``, you MUST use a relative path from the repository root, not an absolute path, to prevent the subagent from accidentally anchoring to and modifying the parent workspace. - **Only if the user explicitly requests a comparison or benchmark**, also spawn a **Baseline** subagent. See `resources/baseline_execution_prompt.md` for the template. Instruct the subagent(s) to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. **CRITICAL**: You must explicitly warn the subagent(s) to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index d211ac0..b7e99a1 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -8,7 +8,7 @@ "expected_repo_state": [ "The agent successfully triggered subagents to run the test cases.", "There is an artifact with the results.", - "No files are modified in the skill folder after the eval is done running.", + "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree).", "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." ], "agent_config": "reidbaker-agent" @@ -19,7 +19,7 @@ "expected_chat_output": "The agent identifies the target skill, spawns a with-skill subagent, grades the result, and writes an evaluation artifact.", "expected_repo_state": [ "There is an artifact with the results.", - "No files are modified in the skill folder after the eval is done running.", + "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree).", "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." ], "agent_config": "reidbaker-agent" @@ -32,7 +32,7 @@ "The agent successfully triggered both baseline and with-skill subagents.", "There is an artifact with the results.", "The evaluation artifact contains explicit sections for both the 'With-Skill Agent' and the 'Baseline Agent'.", - "No files are modified in the skill folder after the eval is done running." + "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree)." ], "agent_config": "reidbaker-agent" } From c05fb0577543777022232d38a199c1ecd1de3e6d Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 12:29:45 -0400 Subject: [PATCH 09/14] Update evals and SKILL.md to enforce strict DoD output format --- .../skills/definition-of-done/SKILL.md | 35 +++++++++++-------- .../definition-of-done/evals/evals.json | 5 ++- .../.agents/skills/run-evals/evals/evals.json | 18 ++++++++-- .../dart-skills-lint-setup/evals/evals.json | 17 +++++++-- .../evals/evals.json | 16 +++++++-- 5 files changed, 66 insertions(+), 25 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md b/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md index 4ed4c9f..6ee901b 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md @@ -13,21 +13,26 @@ Use this skill to ensure that all work meets the repository standards before dec Before stating that a task is complete, you MUST execute and pass the following checks: -1. **Formatting**: Run `dart format .` to format files, or `dart format --output=none --set-exit-if-changed .` to check without modifying. Ensure all files are formatted correctly. +1. **Format**: Run `dart format .` to format files, or `dart format --output=none --set-exit-if-changed .` to check without modifying. Ensure all files are formatted correctly. 2. **Analysis**: Run `dart analyze --fatal-infos` and ensure there are zero issues (including info-level issues). -3. **Metrics/Linter**: Run `dart run dart_code_linter:metrics analyze lib` and ensure there are zero issues. This checks for cyclomatic complexity and custom rules like file naming and redundant async. +3. **Metrics**: Run `dart run dart_code_linter:metrics analyze lib` and ensure there are zero issues. This checks for cyclomatic complexity and custom rules like file naming and redundant async. 4. **Tests**: Run `dart test` and ensure all tests pass successfully. -5. **Skill Validation**: If any skill files were modified, run `dart run dart_skills_lint -d .agents/skills` to ensure they are valid. +5. **Skills**: If any skill files were modified, run `dart run dart_skills_lint -d .agents/skills` to ensure they are valid. 6. **Changelog**: Ensure `CHANGELOG.md` is updated if the task includes user-facing features, bug fixes, or behavioral changes. Audit all entries against the *previously released version* (do not document changes to intermediate PR development code or new unreleased APIs as breaking changes). -7. **Temporal Words**: Ensure that code and code comments contain no relative temporal terms (e.g., 'now', 'currently', 'new', 'old', 'existing behavior'). - -## 🚦 Completion Checklist - -- [ ] Code is formatted (`dart format .` or checked with `--output=none --set-exit-if-changed .`). -- [ ] Analysis is clean (`dart analyze --fatal-infos`). -- [ ] Metrics/Linter are clean (`dart run dart_code_linter:metrics analyze lib`). -- [ ] Tests are passing (`dart test`). -- [ ] Skills validated if modified (`dart run dart_skills_lint -d .agents/skills`). -- [ ] `CHANGELOG.md` updated and audited against the previously released version (no intermediate or unreleased breaking changes listed). -- [ ] Verified that code and code comments contain no relative temporal terms (e.g., 'now', 'currently', 'new', 'old', 'existing behavior'). -- [ ] Documentation is updated. +7. **Temporal**: Ensure that code and code comments contain no relative temporal terms (e.g., 'now', 'currently', 'new', 'old', 'existing behavior'). +8. **Documentation**: Ensure that any relevant documentation is updated. + +## 🚦 Output Formatting + +You MUST include a text list of all mandatory verification steps in your final response to the user. Use the exact following format: +- Use `[x] : ` if the step was completed. +- Use `[ ] : ` if the step was skipped or not applicable. + +CRITICAL: Do not just copy the step description text. You MUST use the exact short Identifier (e.g. `Format`, `Analysis`, `Metrics`, `Tests`, `Skills`, `Changelog`, `Temporal`, `Documentation`) followed by a colon and your short explanation. + +Examples: +- `[x] Format: dart format success.` +- `[x] Analysis: Static clean (0 issues, dart analyze --fatal-infos).` +- `[ ] Skills: Skipped because dart_skills_lint is not installed.` +- `[x] Changelog: (N/A) Not necessary since we're updating internal eval fixtures.` +- `[x] Temporal: no added words.` diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 0e42e68..6eb4262 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -4,7 +4,10 @@ { "id": 1, "prompt": "Modify tool/dart_skills_lint/lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", - "expected_chat_output": "The file tool/dart_skills_lint/lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", + "expected_chat_output": [ + "The agent confirms it added an explanatory comment about how edit distance is calculated to levenshtein.dart.", + "The agent outputs a text list of all definition-of-done rules in the format '[x] Identifier: Explanation' (if done) or '[ ] Identifier: Skipped explanation' (if skipped)." + ], "expected_repo_state": [ "The levenshtein.dart file contains a new comment explaining how edit distance is calculated.", "The new comment does not contain any relative temporal words such as 'now', 'currently', 'new', 'old', or 'existing'.", diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index b7e99a1..740ed2c 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -4,7 +4,11 @@ { "id": 1, "prompt": "Run the evals for the definition-of-done skill.", - "expected_chat_output": "The agent orchestrates a with-skill subagent in a branched workspace, grades its git diffs and stdout, and outputs a markdown artifact.", + "expected_chat_output": [ + "The agent confirms it ran the evals for the definition-of-done skill by orchestrating a subagent.", + "The agent provides a summary of the grading results in the chat.", + "The agent mentions it has outputted a markdown artifact with the detailed results." + ], "expected_repo_state": [ "The agent successfully triggered subagents to run the test cases.", "There is an artifact with the results.", @@ -16,7 +20,11 @@ { "id": 2, "prompt": "Can you test the definition-of-done skill and see if it passes its rubric?", - "expected_chat_output": "The agent identifies the target skill, spawns a with-skill subagent, grades the result, and writes an evaluation artifact.", + "expected_chat_output": [ + "The agent confirms it tested the definition-of-done skill against its rubric.", + "The agent provides a summary of whether it passed or failed in the chat.", + "The agent mentions it has written the detailed evaluation to a markdown artifact." + ], "expected_repo_state": [ "There is an artifact with the results.", "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree).", @@ -27,7 +35,11 @@ { "id": 3, "prompt": "Run an A/B benchmark evaluation for the definition-of-done skill, comparing it explicitly against a baseline without the skill.", - "expected_chat_output": "The agent identifies the target skill, spawns both baseline and with-skill subagents, compares results, and writes a comprehensive evaluation artifact.", + "expected_chat_output": [ + "The agent confirms it ran an A/B benchmark evaluation for the definition-of-done skill.", + "The agent provides a high-level summary comparing the baseline and with-skill agents in the chat.", + "The agent points the user to the comprehensive evaluation artifact for details." + ], "expected_repo_state": [ "The agent successfully triggered both baseline and with-skill subagents.", "There is an artifact with the results.", diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json index 5dfc940..61c02bc 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -4,7 +4,11 @@ { "id": 1, "prompt": "Set up dart_skills_lint validation for the first time in our Dart project to lint skills located in .agents/skills.", - "expected_chat_output": "The project's pubspec.yaml is updated with dart_skills_lint under dev_dependencies using a git dependency pointing to tool/dart_skills_lint, a root dart_skills_lint.yaml file is created specifying .agents/skills, and recipes for CI integration are provided.", + "expected_chat_output": [ + "The agent confirms it has updated pubspec.yaml with dart_skills_lint as a git dependency.", + "The agent states it has created a root dart_skills_lint.yaml configuration file.", + "The agent provides guidance or recipes for integrating the validation step into CI." + ], "expected_repo_state": [ "dart_skills_lint is added under dev_dependencies in pubspec.yaml as a git dependency targeting https://github.com/flutter/agent-plugins.git with path tool/dart_skills_lint.", "The dart_skills_lint.yaml file is created at the repository root so both CLI and Dart test invocations share the configuration.", @@ -21,7 +25,11 @@ { "id": 2, "prompt": "A repository has pre-existing skill files in .agents/skills with lint violations. Set up dart_skills_lint without failing CI or modifying legacy skill files by generating a baseline ignore file.", - "expected_chat_output": "dart_skills_lint is added to pubspec.yaml dev_dependencies, a root dart_skills_lint.yaml configuration is created, and dart run dart_skills_lint:cli is executed with --generate-baseline to write violations to an ignore file without mutating legacy skill content.", + "expected_chat_output": [ + "The agent confirms adding dart_skills_lint to pubspec.yaml and creating a root configuration file.", + "The agent states it executed the CLI with --generate-baseline.", + "The agent explicitly mentions that pre-existing violations were written to an ignore file and legacy skill content was not mutated." + ], "expected_repo_state": [ "dart_skills_lint is configured under dev_dependencies in pubspec.yaml and dart_skills_lint.yaml is created at the repository root.", "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", @@ -35,7 +43,10 @@ { "id": 3, "prompt": "Set up dart_skills_lint to validate agent skills in .agents/skills, while isolating the package dependency inside tool/pubspec.yaml instead of adding it to the root pubspec.yaml.", - "expected_chat_output": "The tool/ package's pubspec.yaml contains dart_skills_lint under dev_dependencies, while dart_skills_lint.yaml is placed at the repository root, ensuring the root closure is unpolluted and configuration remains centrally accessible.", + "expected_chat_output": [ + "The agent states it added dart_skills_lint to the tool/ package's pubspec.yaml instead of the root.", + "The agent confirms it placed dart_skills_lint.yaml at the repository root." + ], "expected_repo_state": [ "dart_skills_lint is added under dev_dependencies in tool/pubspec.yaml rather than the root pubspec.yaml.", "dart_skills_lint.yaml is created at the repository root directory rather than inside tool/ so both CLI and test invocations share the configuration.", diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json index 87386f5..12c83cd 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -4,7 +4,10 @@ { "id": 1, "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with the `--check-absolute-paths` flag. The fixture points to `/tmp/this/does/not/exist.md`, which cannot be auto-fixed since it does not exist. First, create a dummy file at an absolute path (e.g., `/tmp/dummy.md`), update the link in `example/skills/invalid/SKILL.md` to point to it, and then run `--fix --dry-run` to preview the auto-fix. Finally, run `--fix` to apply the fix and verify that the link was automatically converted to a relative path.", - "expected_chat_output": "The agent explicitly enables the absolute paths rule, sets up a real absolute file target, previews the fix via dry-run, and automatically converts the link to a relative path via the CLI.", + "expected_chat_output": [ + "The agent mentions it ran the linter with --fix --dry-run to preview changes.", + "The agent confirms it ran --fix and successfully converted the absolute path to a relative one." + ], "expected_repo_state": [ "The agent updated the SKILL.md link to point to an absolute path that actually exists on disk.", "The agent ran the CLI with `--check-absolute-paths` and explicitly verified that a fixable path violation was found.", @@ -16,7 +19,11 @@ { "id": 2, "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with `--disallowed-field` and `--check-absolute-paths` flags to surface all violations. Interpret the failures and manually fix all three violations (skill name mismatch, disallowed field, absolute path) without using the `--fix` flag. Ensure the final run is clean.", - "expected_chat_output": "The agent explicitly enables the extra rules, identifies all violations, applies manual corrections to each without auto-fixing, and ensures a clean run.", + "expected_chat_output": [ + "The agent confirms it ran the linter with the extra flags and identified the three violations.", + "The agent explicitly states it manually fixed the skill name mismatch, disallowed field, and absolute path violations.", + "The agent confirms the final validation run was clean." + ], "expected_repo_state": [ "The agent executes the linter with the required flags to surface all violations.", "The agent manually edits the skill files to fix the invalid skill name violation.", @@ -29,7 +36,10 @@ { "id": 3, "prompt": "Author a custom dart_skills_lint rule named `RequireInternalMetadataRule` that verifies if a skill's metadata contains `internal: true`. Wire it up into a new test file `test/custom_rule_test.dart`.", - "expected_chat_output": "The agent writes a custom SkillRule and integrates it into a new unit test.", + "expected_chat_output": [ + "The agent states it authored the custom RequireInternalMetadataRule.", + "The agent confirms it wired the new rule up into a new test file test/custom_rule_test.dart." + ], "expected_repo_state": [ "The agent created a class extending `SkillRule`.", "The agent implemented the `name` and `severity` overrides.", From 582449ca358aa74f81c60217735e4a5218d63473 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 12:33:16 -0400 Subject: [PATCH 10/14] Simplify SKILL.md formatting instructions --- .../dart_skills_lint/.agents/skills/definition-of-done/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md b/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md index 6ee901b..ec61916 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md @@ -28,7 +28,7 @@ You MUST include a text list of all mandatory verification steps in your final r - Use `[x] : ` if the step was completed. - Use `[ ] : ` if the step was skipped or not applicable. -CRITICAL: Do not just copy the step description text. You MUST use the exact short Identifier (e.g. `Format`, `Analysis`, `Metrics`, `Tests`, `Skills`, `Changelog`, `Temporal`, `Documentation`) followed by a colon and your short explanation. +CRITICAL: Do not just copy the full step description text. You MUST use the exact bolded Identifier from the Mandatory Verification Steps list above, followed by a colon and your short explanation. Examples: - `[x] Format: dart format success.` From 4c416c8f72a6923d1b9ce5ccd14765929ede6a8c Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 12:42:40 -0400 Subject: [PATCH 11/14] Relax chat output expectations in evals to prevent arbitrary failures --- .../.agents/skills/run-evals/evals/evals.json | 12 +++--------- .../skills/dart-skills-lint-setup/evals/evals.json | 11 +++-------- .../dart-skills-lint-validation/evals/evals.json | 10 +++------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index 740ed2c..c49215b 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -5,9 +5,7 @@ "id": 1, "prompt": "Run the evals for the definition-of-done skill.", "expected_chat_output": [ - "The agent confirms it ran the evals for the definition-of-done skill by orchestrating a subagent.", - "The agent provides a summary of the grading results in the chat.", - "The agent mentions it has outputted a markdown artifact with the detailed results." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "The agent successfully triggered subagents to run the test cases.", @@ -21,9 +19,7 @@ "id": 2, "prompt": "Can you test the definition-of-done skill and see if it passes its rubric?", "expected_chat_output": [ - "The agent confirms it tested the definition-of-done skill against its rubric.", - "The agent provides a summary of whether it passed or failed in the chat.", - "The agent mentions it has written the detailed evaluation to a markdown artifact." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "There is an artifact with the results.", @@ -36,9 +32,7 @@ "id": 3, "prompt": "Run an A/B benchmark evaluation for the definition-of-done skill, comparing it explicitly against a baseline without the skill.", "expected_chat_output": [ - "The agent confirms it ran an A/B benchmark evaluation for the definition-of-done skill.", - "The agent provides a high-level summary comparing the baseline and with-skill agents in the chat.", - "The agent points the user to the comprehensive evaluation artifact for details." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "The agent successfully triggered both baseline and with-skill subagents.", diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json index 61c02bc..11ec252 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -5,9 +5,7 @@ "id": 1, "prompt": "Set up dart_skills_lint validation for the first time in our Dart project to lint skills located in .agents/skills.", "expected_chat_output": [ - "The agent confirms it has updated pubspec.yaml with dart_skills_lint as a git dependency.", - "The agent states it has created a root dart_skills_lint.yaml configuration file.", - "The agent provides guidance or recipes for integrating the validation step into CI." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "dart_skills_lint is added under dev_dependencies in pubspec.yaml as a git dependency targeting https://github.com/flutter/agent-plugins.git with path tool/dart_skills_lint.", @@ -26,9 +24,7 @@ "id": 2, "prompt": "A repository has pre-existing skill files in .agents/skills with lint violations. Set up dart_skills_lint without failing CI or modifying legacy skill files by generating a baseline ignore file.", "expected_chat_output": [ - "The agent confirms adding dart_skills_lint to pubspec.yaml and creating a root configuration file.", - "The agent states it executed the CLI with --generate-baseline.", - "The agent explicitly mentions that pre-existing violations were written to an ignore file and legacy skill content was not mutated." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "dart_skills_lint is configured under dev_dependencies in pubspec.yaml and dart_skills_lint.yaml is created at the repository root.", @@ -44,8 +40,7 @@ "id": 3, "prompt": "Set up dart_skills_lint to validate agent skills in .agents/skills, while isolating the package dependency inside tool/pubspec.yaml instead of adding it to the root pubspec.yaml.", "expected_chat_output": [ - "The agent states it added dart_skills_lint to the tool/ package's pubspec.yaml instead of the root.", - "The agent confirms it placed dart_skills_lint.yaml at the repository root." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "dart_skills_lint is added under dev_dependencies in tool/pubspec.yaml rather than the root pubspec.yaml.", diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json index 12c83cd..c0732f2 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -5,8 +5,7 @@ "id": 1, "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with the `--check-absolute-paths` flag. The fixture points to `/tmp/this/does/not/exist.md`, which cannot be auto-fixed since it does not exist. First, create a dummy file at an absolute path (e.g., `/tmp/dummy.md`), update the link in `example/skills/invalid/SKILL.md` to point to it, and then run `--fix --dry-run` to preview the auto-fix. Finally, run `--fix` to apply the fix and verify that the link was automatically converted to a relative path.", "expected_chat_output": [ - "The agent mentions it ran the linter with --fix --dry-run to preview changes.", - "The agent confirms it ran --fix and successfully converted the absolute path to a relative one." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "The agent updated the SKILL.md link to point to an absolute path that actually exists on disk.", @@ -20,9 +19,7 @@ "id": 2, "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with `--disallowed-field` and `--check-absolute-paths` flags to surface all violations. Interpret the failures and manually fix all three violations (skill name mismatch, disallowed field, absolute path) without using the `--fix` flag. Ensure the final run is clean.", "expected_chat_output": [ - "The agent confirms it ran the linter with the extra flags and identified the three violations.", - "The agent explicitly states it manually fixed the skill name mismatch, disallowed field, and absolute path violations.", - "The agent confirms the final validation run was clean." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "The agent executes the linter with the required flags to surface all violations.", @@ -37,8 +34,7 @@ "id": 3, "prompt": "Author a custom dart_skills_lint rule named `RequireInternalMetadataRule` that verifies if a skill's metadata contains `internal: true`. Wire it up into a new test file `test/custom_rule_test.dart`.", "expected_chat_output": [ - "The agent states it authored the custom RequireInternalMetadataRule.", - "The agent confirms it wired the new rule up into a new test file test/custom_rule_test.dart." + "Any natural language output summarizing the completed work is acceptable." ], "expected_repo_state": [ "The agent created a class extending `SkillRule`.", From 38ef90ec06168fb3a8347ed5ee93cb2378a291f5 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 13:23:10 -0400 Subject: [PATCH 12/14] Refactor unit tests to use Dart 3 switch expressions --- .../test/skills_evals_test.dart | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index 6278872..2f289f5 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -79,9 +79,10 @@ void _verifyStructuralConsistency(List files, String itemsKey) { for (final file in files) { final Object? decoded = jsonDecode(file.readAsStringSync()); - expect(decoded, isA>(), reason: '${file.path} must be a JSON map.'); - - final decodedMap = decoded! as Map; + final Map decodedMap = switch (decoded) { + final Map map => map, + _ => fail('${file.path} must be a JSON map.'), + }; final Set rootKeys = decodedMap.keys.toSet(); if (expectedRootKeys == null) { expectedRootKeys = rootKeys; @@ -97,17 +98,15 @@ void _verifyStructuralConsistency(List files, String itemsKey) { } final Object? itemsRaw = decodedMap[itemsKey]; - expect(itemsRaw, isA>(), reason: '$itemsKey key in ${file.path} must be a List.'); - - final itemsList = itemsRaw! as List; + final List itemsList = switch (itemsRaw) { + final List list => list, + _ => fail('$itemsKey key in ${file.path} must be a List.'), + }; for (final Object? item in itemsList) { - expect( - item, - isA>(), - reason: 'Item in $itemsKey list in ${file.path} must be a JSON map.', - ); - - final itemMap = item! as Map; + final Map itemMap = switch (item) { + final Map map => map, + _ => fail('Item in $itemsKey list in ${file.path} must be a JSON map.'), + }; final Set itemKeys = itemMap.keys.toSet(); if (expectedItemKeys == null) { expectedItemKeys = itemKeys; From 909de2e2ddfcf9975c9f5a8cb7b201d9f1f1bd86 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 14:01:13 -0400 Subject: [PATCH 13/14] Remove redundant skill_name property from all evals.json files --- .../.agents/skills/definition-of-done/evals/evals.json | 1 - tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json | 1 - .../skills/dart-skills-lint-setup/evals/evals.json | 1 - .../skills/dart-skills-lint-validation/evals/evals.json | 1 - 4 files changed, 4 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 6eb4262..5c28bda 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -1,5 +1,4 @@ { - "skill_name": "definition-of-done", "evals": [ { "id": 1, diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index c49215b..9b97ea6 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -1,5 +1,4 @@ { - "skill_name": "run-evals", "evals": [ { "id": 1, diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json index 11ec252..2d7894e 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -1,5 +1,4 @@ { - "skill_name": "dart-skills-lint-setup", "evals": [ { "id": 1, diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json index c0732f2..8aad282 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -1,5 +1,4 @@ { - "skill_name": "dart-skills-lint-validation", "evals": [ { "id": 1, From d3d8ee2cf193a63c21c28fe033b565bbf68bbacd Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 14:02:43 -0400 Subject: [PATCH 14/14] Move repo_criteria above evals array in all evals.json files for better readability --- .../.agents/skills/definition-of-done/evals/evals.json | 6 +++--- .../.agents/skills/run-evals/evals/evals.json | 6 +++--- .../skills/dart-skills-lint-setup/evals/evals.json | 6 +++--- .../skills/dart-skills-lint-validation/evals/evals.json | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 5c28bda..687ac4f 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -1,4 +1,7 @@ { + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, @@ -15,8 +18,5 @@ ], "agent_config": "reidbaker-agent" } - ], - "repo_criteria": [ - "evals/code_quality_rubric.json" ] } \ No newline at end of file diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json index 9b97ea6..e0f8094 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -1,4 +1,7 @@ { + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, @@ -41,8 +44,5 @@ ], "agent_config": "reidbaker-agent" } - ], - "repo_criteria": [ - "evals/code_quality_rubric.json" ] } \ No newline at end of file diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json index 2d7894e..ddad175 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -1,4 +1,7 @@ { + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, @@ -50,8 +53,5 @@ ], "agent_config": "bare-agent" } - ], - "repo_criteria": [ - "evals/code_quality_rubric.json" ] } \ No newline at end of file diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json index 8aad282..cc58775 100644 --- a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -1,4 +1,7 @@ { + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, @@ -44,8 +47,5 @@ ], "agent_config": "bare-agent" } - ], - "repo_criteria": [ - "evals/code_quality_rubric.json" ] } \ No newline at end of file