diff --git a/tool/dart_skills_lint/.agents/skills/.gitignore b/tool/dart_skills_lint/.agents/skills/.gitignore index 59fb7f3..2e208c8 100644 --- a/tool/dart_skills_lint/.agents/skills/.gitignore +++ b/tool/dart_skills_lint/.agents/skills/.gitignore @@ -8,6 +8,7 @@ !check-downstream-consumers/ !dart-skills-lint-integration/ !definition-of-done/ +!run-evals/ # Keep essential configuration and docs !.gitignore diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md b/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md index 4ed4c9f..ec61916 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/SKILL.md @@ -13,21 +13,26 @@ Use this skill to ensure that all work meets the repository standards before dec Before stating that a task is complete, you MUST execute and pass the following checks: -1. **Formatting**: Run `dart format .` to format files, or `dart format --output=none --set-exit-if-changed .` to check without modifying. Ensure all files are formatted correctly. +1. **Format**: Run `dart format .` to format files, or `dart format --output=none --set-exit-if-changed .` to check without modifying. Ensure all files are formatted correctly. 2. **Analysis**: Run `dart analyze --fatal-infos` and ensure there are zero issues (including info-level issues). -3. **Metrics/Linter**: Run `dart run dart_code_linter:metrics analyze lib` and ensure there are zero issues. This checks for cyclomatic complexity and custom rules like file naming and redundant async. +3. **Metrics**: Run `dart run dart_code_linter:metrics analyze lib` and ensure there are zero issues. This checks for cyclomatic complexity and custom rules like file naming and redundant async. 4. **Tests**: Run `dart test` and ensure all tests pass successfully. -5. **Skill Validation**: If any skill files were modified, run `dart run dart_skills_lint -d .agents/skills` to ensure they are valid. +5. **Skills**: If any skill files were modified, run `dart run dart_skills_lint -d .agents/skills` to ensure they are valid. 6. **Changelog**: Ensure `CHANGELOG.md` is updated if the task includes user-facing features, bug fixes, or behavioral changes. Audit all entries against the *previously released version* (do not document changes to intermediate PR development code or new unreleased APIs as breaking changes). -7. **Temporal Words**: Ensure that code and code comments contain no relative temporal terms (e.g., 'now', 'currently', 'new', 'old', 'existing behavior'). - -## 🚦 Completion Checklist - -- [ ] Code is formatted (`dart format .` or checked with `--output=none --set-exit-if-changed .`). -- [ ] Analysis is clean (`dart analyze --fatal-infos`). -- [ ] Metrics/Linter are clean (`dart run dart_code_linter:metrics analyze lib`). -- [ ] Tests are passing (`dart test`). -- [ ] Skills validated if modified (`dart run dart_skills_lint -d .agents/skills`). -- [ ] `CHANGELOG.md` updated and audited against the previously released version (no intermediate or unreleased breaking changes listed). -- [ ] Verified that code and code comments contain no relative temporal terms (e.g., 'now', 'currently', 'new', 'old', 'existing behavior'). -- [ ] Documentation is updated. +7. **Temporal**: Ensure that code and code comments contain no relative temporal terms (e.g., 'now', 'currently', 'new', 'old', 'existing behavior'). +8. **Documentation**: Ensure that any relevant documentation is updated. + +## 🚦 Output Formatting + +You MUST include a text list of all mandatory verification steps in your final response to the user. Use the exact following format: +- Use `[x] : ` if the step was completed. +- Use `[ ] : ` if the step was skipped or not applicable. + +CRITICAL: Do not just copy the full step description text. You MUST use the exact bolded Identifier from the Mandatory Verification Steps list above, followed by a colon and your short explanation. + +Examples: +- `[x] Format: dart format success.` +- `[x] Analysis: Static clean (0 issues, dart analyze --fatal-infos).` +- `[ ] Skills: Skipped because dart_skills_lint is not installed.` +- `[x] Changelog: (N/A) Not necessary since we're updating internal eval fixtures.` +- `[x] Temporal: no added words.` diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 7a12e81..687ac4f 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -1,16 +1,22 @@ { - "skill_name": "definition-of-done", + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, "prompt": "Modify tool/dart_skills_lint/lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", - "expected_output": "The file tool/dart_skills_lint/lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", - "expectations": [ + "expected_chat_output": [ + "The agent confirms it added an explanatory comment about how edit distance is calculated to levenshtein.dart.", + "The agent outputs a text list of all definition-of-done rules in the format '[x] Identifier: Explanation' (if done) or '[ ] Identifier: Skipped explanation' (if skipped)." + ], + "expected_repo_state": [ "The levenshtein.dart file contains a new comment explaining how edit distance is calculated.", "The new comment does not contain any relative temporal words such as 'now', 'currently', 'new', 'old', or 'existing'.", "The levenshtein.dart file is formatted cleanly according to dart format.", "The project has no dart analyze errors or warnings." - ] + ], + "agent_config": "reidbaker-agent" } ] -} +} \ No newline at end of file diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md new file mode 100644 index 0000000..8fa173c --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -0,0 +1,19 @@ +--- +name: run-evals +description: Run evaluations for one, multiple, or all skills using the agent orchestration framework. Make sure to use this skill whenever the user asks to run evals, test a skill's performance, run benchmarks, or compare baseline versus with-skill execution. +metadata: + internal: true +--- + +# Run Skill Evals + +1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for understanding the difference between per-skill evals and cross-skill evals. +2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. +3. **Determine Agent Configuration**: Check the `agent_config` field in the target `evals.json` to determine the environment/harness to spawn. If `agent_config` is `"bare-agent"`, spawn a subagent with the `bare-agent` profile. If it is a specific contributor profile (e.g. `"reidbaker-agent"`), use that profile to provide the necessary contributor context. +4. **Orchestrate**: By default, run an Integration Test by spawning a single **With-Skill** subagent using `Workspace: branch` and the identified `agent_config`. + - Provide the task prompt. See `resources/with_skill_execution_prompt.md` for the template. When filling in ``, you MUST use a relative path from the repository root, not an absolute path, to prevent the subagent from accidentally anchoring to and modifying the parent workspace. + - **Only if the user explicitly requests a comparison or benchmark**, also spawn a **Baseline** subagent. See `resources/baseline_execution_prompt.md` for the template. + Instruct the subagent(s) to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. + **CRITICAL**: You must explicitly warn the subagent(s) to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. +5. **Grade**: Parse the combined rubric (resolving `repo_criteria` + `evals.json` expectations). Use the grading instructions in `resources/agent_judge_prompt.md`. When an expectation fails, you MUST explicitly list both the expectation and what was actually found that caused the failure. +6. **Artifact**: Grade the outputs and generate a Markdown artifact (e.g., `_eval_results.md`) containing the metadata, pass/fail rationale, and raw diffs/stdout. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json new file mode 100644 index 0000000..e0f8094 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/evals/evals.json @@ -0,0 +1,48 @@ +{ + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], + "evals": [ + { + "id": 1, + "prompt": "Run the evals for the definition-of-done skill.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The agent successfully triggered subagents to run the test cases.", + "There is an artifact with the results.", + "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree).", + "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." + ], + "agent_config": "reidbaker-agent" + }, + { + "id": 2, + "prompt": "Can you test the definition-of-done skill and see if it passes its rubric?", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "There is an artifact with the results.", + "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree).", + "The evaluation artifact includes evals sourced from code_quality_rubric.json for a skill that produces code." + ], + "agent_config": "reidbaker-agent" + }, + { + "id": 3, + "prompt": "Run an A/B benchmark evaluation for the definition-of-done skill, comparing it explicitly against a baseline without the skill.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The agent successfully triggered both baseline and with-skill subagents.", + "There is an artifact with the results.", + "The evaluation artifact contains explicit sections for both the 'With-Skill Agent' and the 'Baseline Agent'.", + "No files are modified in the parent workspace after the eval is done running (all modifications must be confined to the isolated subagent worktree)." + ], + "agent_config": "reidbaker-agent" + } + ] +} \ No newline at end of file diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md new file mode 100644 index 0000000..e568622 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/agent_judge_prompt.md @@ -0,0 +1,13 @@ +You are an expert evaluator. Review the following execution outputs from an AI agent against the provided combined rubric. + +Execution Outputs: +- Git Diff: +- Command Stdout: + +Combined Rubric Expectations: + + +For every single expectation, explicitly state whether it PASSED or FAILED and provide a 1-sentence justification. +IMPORTANT: When an expectation fails, explicitly list the expectation and explain exactly what you found that caused the failure. + +Finally, provide an overall PASS/FAIL grade. diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md new file mode 100644 index 0000000..d073ab7 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md @@ -0,0 +1,7 @@ +Execute this task: +- Task: +- Input files: + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md new file mode 100644 index 0000000..de672a6 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md @@ -0,0 +1,8 @@ +Execute this task: +- Skill path: (Please read and STRICTLY FOLLOW the instructions in this skill file before finishing) +- Task: +- Input files: + +WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. + +Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index 9acdab2..bcee3ef 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -35,6 +35,9 @@ dart_skills_lint: - path: ".agents/skills/definition-of-done" rules: prevent-skills-sh-publishing: error + - path: ".agents/skills/run-evals" + rules: + prevent-skills-sh-publishing: error - path: ".agents/skills/contributor-pr-description" rules: prevent-skills-sh-publishing: error diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md new file mode 100644 index 0000000..b00f969 --- /dev/null +++ b/tool/dart_skills_lint/evals/README.md @@ -0,0 +1,46 @@ +# Skill Evaluations + +Architecture, rubrics, and instructions for evaluating AI agent skills authored and maintained in this repository. +**Note:** These evaluations are essentially unit tests for the skills within the `dart_skills_lint` package and its internal ecosystem. They are *not* intended to be a generic evaluation framework for other agent client plugins or tools outside of this specific domain. + +## What Should (and Shouldn't) Be Evaluated + +**DO Evaluate:** +- Core workflows of a skill (e.g., adding a dependency, running validation checks). +- Specific edge cases that a skill claims to handle (e.g., legacy integration paths without `--fix`). +- Whether a skill correctly leaves the repository in a compilable, passing state. + +**DO NOT Evaluate:** +- Trivial syntax formatting that `dart format` already fixes perfectly. +- Complete system architectures that take longer than a few minutes to generate and verify. +- Skills that are outside the scope of `dart_skills_lint` (e.g. general flutter app creation). + +## Core Principles & Architecture + +Evaluations in this repository use a **Two-Tiered Architecture** to separate domain-specific skill requirements from universal skill quality standards. + +### 1. Per-Skill Evals (`/evals/evals.json`) +Each skill maintains an `evals/evals.json` file containing target task prompts and expectations: +- **`prompt`**: Realistic user prompt testing primary or edge-case workflows. +- **`expected_chat_output`**: High-level narrative summary of what the LLM should say/give to the user. +- **`expected_repo_state`**: Array of discrete, testable assertions regarding the end state of the repository and tracked files. +- **`repo_criteria`**: Array of relative file paths to shared universal quality rubrics (e.g., `["evals/code_quality_rubric.json"]`). +- **`agent_config`**: The model configuration/harness used when executing the eval against the skill. For published skills, use `"bare-agent"`. For internal contributor skills, use the internal agent profile (e.g., `"reidbaker-agent"`). + +### 2. Cross-Skill Evals (`evals/*_rubric.json`) +Universal skill quality expectations are structured into modular rubric classes that apply broadly across skills. + +## Cross-Cutting Rules +Skills that author or modify code MUST adhere to the universal code quality expectations defined in `code_quality_rubric.json`. This ensures that generated code compiles cleanly, adheres to Effective Dart, works across platforms, and is placed in standard canonical directories. + +## 🚀 Running & Validating Evals Locally + +### 1. Validate Evals Structural Consistency +Run the unit test that checks all `evals.json` files for structural consistency across the repository: + +```bash +dart test test/skills_evals_test.dart +``` + +### 2. Running Evals via Agent Orchestration +You should use the `/run-evals` skill to run evaluations. The bulk of the execution logic and prompts are located within the `run-evals` skill itself (`.agents/skills/run-evals/SKILL.md`). The environment, model, and harness are determined by the `agent_config` specified in the corresponding `evals.json` file. diff --git a/tool/dart_skills_lint/evals/code_quality_rubric.json b/tool/dart_skills_lint/evals/code_quality_rubric.json new file mode 100644 index 0000000..e451409 --- /dev/null +++ b/tool/dart_skills_lint/evals/code_quality_rubric.json @@ -0,0 +1,35 @@ +{ + "evals": [ + { + "id": "compilation_and_health", + "expected_repo_state": [ + "The generated code compiles with zero syntax errors.", + "Static analysis passes cleanly ('dart analyze --fatal-infos' returns zero errors or warnings).", + "Existing and newly added unit, widget, or integration tests execute and pass successfully." + ] + }, + { + "id": "effective_dart_and_idioms", + "expected_repo_state": [ + "Follows Effective Dart style guidelines as enforced by standard 'core' and 'recommended' linter rules.", + "Leverages Dart 3 idioms such as switch expressions and exhaustive pattern matching when conditional logic branches on algebraic types.", + "Avoids anti-patterns such as unnecessary dynamic types, raw Strings instead of string interpolation, or unhandled Futures." + ] + }, + { + "id": "cross_platform_compatibility", + "expected_repo_state": [ + "File paths and CLI path parameters use platform-agnostic formatters (package:path or forward slashes '/').", + "CLI commands and shell invocation scripts use cross-platform compatible syntax.", + "Avoids hardcoded platform-specific environment paths (such as C:\\ or /usr/local/bin directly)." + ] + }, + { + "id": "directory_and_placement_hygiene", + "expected_repo_state": [ + "Source code must be placed in 'lib/', tests in 'test/', and executable scripts in 'bin/' or 'tool/'.", + "No temporary scratch files or orphaned directories are left behind after execution." + ] + } + ] +} \ No newline at end of file diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json new file mode 100644 index 0000000..ddad175 --- /dev/null +++ b/tool/dart_skills_lint/skills/dart-skills-lint-setup/evals/evals.json @@ -0,0 +1,57 @@ +{ + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], + "evals": [ + { + "id": 1, + "prompt": "Set up dart_skills_lint validation for the first time in our Dart project to lint skills located in .agents/skills.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "dart_skills_lint is added under dev_dependencies in pubspec.yaml as a git dependency targeting https://github.com/flutter/agent-plugins.git with path tool/dart_skills_lint.", + "The dart_skills_lint.yaml file is created at the repository root so both CLI and Dart test invocations share the configuration.", + "dart_skills_lint.yaml specifies .agents/skills under directories.", + "Configures check-relative-paths: error under rules in dart_skills_lint.yaml to enforce portable relative links across skill files.", + "Guidance or recipe for CI integration or pre-commit hook is provided.", + "NOT added under runtime dependencies in pubspec.yaml.", + "NOT placed inside .agents/skills/ or a sub-folder where root CLI commands will fail to locate the config.", + "No deprecated code from dart_skills_lint is used in the generated code.", + "There is a dart test created that fails when a skill defined in dart_skills_lint.yaml has an error." + ], + "agent_config": "bare-agent" + }, + { + "id": 2, + "prompt": "A repository has pre-existing skill files in .agents/skills with lint violations. Set up dart_skills_lint without failing CI or modifying legacy skill files by generating a baseline ignore file.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "dart_skills_lint is configured under dev_dependencies in pubspec.yaml and dart_skills_lint.yaml is created at the repository root.", + "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", + "The CLI command dart run dart_skills_lint:cli is run with --skills-directory=.agents/skills --generate-baseline to write legacy failures to an ignore file.", + "An explanation is provided that pre-existing violations exit clean while violations introduced after baseline generation will still fail validation.", + "Does NOT modify, delete, or reformat any pre-existing SKILL.md files in .agents/skills during baseline generation.", + "Does NOT pass --fix during baseline generation, keeping legacy skill files intact." + ], + "agent_config": "bare-agent" + }, + { + "id": 3, + "prompt": "Set up dart_skills_lint to validate agent skills in .agents/skills, while isolating the package dependency inside tool/pubspec.yaml instead of adding it to the root pubspec.yaml.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "dart_skills_lint is added under dev_dependencies in tool/pubspec.yaml rather than the root pubspec.yaml.", + "dart_skills_lint.yaml is created at the repository root directory rather than inside tool/ so both CLI and test invocations share the configuration.", + "Configures check-relative-paths: error in dart_skills_lint.yaml to enforce portable path references.", + "NOT placed inside tool/, which would prevent root-level CLI invocations from discovering the configuration file.", + "NOT added to the root pubspec.yaml dependencies or dev_dependencies." + ], + "agent_config": "bare-agent" + } + ] +} \ No newline at end of file diff --git a/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json new file mode 100644 index 0000000..cc58775 --- /dev/null +++ b/tool/dart_skills_lint/skills/dart-skills-lint-validation/evals/evals.json @@ -0,0 +1,51 @@ +{ + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], + "evals": [ + { + "id": 1, + "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with the `--check-absolute-paths` flag. The fixture points to `/tmp/this/does/not/exist.md`, which cannot be auto-fixed since it does not exist. First, create a dummy file at an absolute path (e.g., `/tmp/dummy.md`), update the link in `example/skills/invalid/SKILL.md` to point to it, and then run `--fix --dry-run` to preview the auto-fix. Finally, run `--fix` to apply the fix and verify that the link was automatically converted to a relative path.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The agent updated the SKILL.md link to point to an absolute path that actually exists on disk.", + "The agent ran the CLI with `--check-absolute-paths` and explicitly verified that a fixable path violation was found.", + "The agent used `--fix --dry-run` to preview the fix.", + "The agent successfully applied the fix using `--fix`, converting the absolute link into a relative one." + ], + "agent_config": "bare-agent" + }, + { + "id": 2, + "prompt": "Run dart_skills_lint validation on `example/skills/invalid` with `--disallowed-field` and `--check-absolute-paths` flags to surface all violations. Interpret the failures and manually fix all three violations (skill name mismatch, disallowed field, absolute path) without using the `--fix` flag. Ensure the final run is clean.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The agent executes the linter with the required flags to surface all violations.", + "The agent manually edits the skill files to fix the invalid skill name violation.", + "The agent manually edits the skill files to fix the disallowed field violation.", + "The agent manually edits the skill files to fix the absolute path violation.", + "The agent validates the directory again to confirm a clean run." + ], + "agent_config": "bare-agent" + }, + { + "id": 3, + "prompt": "Author a custom dart_skills_lint rule named `RequireInternalMetadataRule` that verifies if a skill's metadata contains `internal: true`. Wire it up into a new test file `test/custom_rule_test.dart`.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The agent created a class extending `SkillRule`.", + "The agent implemented the `name` and `severity` overrides.", + "The `validate` method correctly parses the `parsedYaml` to check for `internal: true`.", + "The agent created a Dart test file that imports the custom rule and passes it to `validateSkills`.", + "The newly created test compiles and passes static analysis." + ], + "agent_config": "bare-agent" + } + ] +} \ No newline at end of file diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart new file mode 100644 index 0000000..2f289f5 --- /dev/null +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -0,0 +1,136 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +void main() { + group('Evals structure consistency', () { + // Ensures all evals.json files dynamically share the exact same JSON schema. + // Keys are not hardcoded to ensure enforcement remains schema-agnostic and flexible. + test('all evals.json files across skills share consistent structure and keys', () { + final List evalsFiles = [ + ..._findEvalsFiles(Directory(p.join(Directory.current.path, 'skills'))), + ..._findEvalsFiles(Directory(p.join(Directory.current.path, '.agents', 'skills'))), + ]..sort((a, b) => a.path.compareTo(b.path)); + + expect( + evalsFiles, + isNotEmpty, + reason: 'Should find at least one evals.json file in skills or .agents/skills.', + ); + + _verifyStructuralConsistency(evalsFiles, 'evals'); + }); + + // Note: We intentionally only require an evals.json file for published skills. + // Contributor skills in .agents/skills/ are not currently required to have one. + test('all published skills have an evals.json file', () { + final skillsDir = Directory(p.join(Directory.current.path, 'skills')); + if (!skillsDir.existsSync()) { + return; + } + + final List skillDirs = skillsDir.listSync().whereType().toList(); + + for (final skillDir in skillDirs) { + final evalsFile = File(p.join(skillDir.path, 'evals', 'evals.json')); + expect( + evalsFile.existsSync(), + isTrue, + reason: + 'Published skill "${p.basename(skillDir.path)}" is missing an evals.json file at ${evalsFile.path}', + ); + } + }); + + test('all rubric JSON files in evals/ share consistent structure and keys', () { + final rubricsDir = Directory(p.join(Directory.current.path, 'evals')); + if (!rubricsDir.existsSync()) { + return; + } + + final List rubricFiles = + rubricsDir + .listSync() + .whereType() + .where((File f) => f.path.endsWith('.json')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + if (rubricFiles.isEmpty) { + return; + } + + _verifyStructuralConsistency(rubricFiles, 'evals'); + }); + }); +} + +void _verifyStructuralConsistency(List files, String itemsKey) { + Set? expectedRootKeys; + String? expectedRootKeysFilePath; + Set? expectedItemKeys; + String? expectedItemFilePath; + + for (final file in files) { + final Object? decoded = jsonDecode(file.readAsStringSync()); + final Map decodedMap = switch (decoded) { + final Map map => map, + _ => fail('${file.path} must be a JSON map.'), + }; + final Set rootKeys = decodedMap.keys.toSet(); + if (expectedRootKeys == null) { + expectedRootKeys = rootKeys; + expectedRootKeysFilePath = file.path; + } else { + expect( + rootKeys, + equals(expectedRootKeys), + reason: + '${file.path} root keys do not match consistency pattern. ' + 'Expected keys to match the first processed file ($expectedRootKeysFilePath).', + ); + } + + final Object? itemsRaw = decodedMap[itemsKey]; + final List itemsList = switch (itemsRaw) { + final List list => list, + _ => fail('$itemsKey key in ${file.path} must be a List.'), + }; + for (final Object? item in itemsList) { + final Map itemMap = switch (item) { + final Map map => map, + _ => fail('Item in $itemsKey list in ${file.path} must be a JSON map.'), + }; + final Set itemKeys = itemMap.keys.toSet(); + if (expectedItemKeys == null) { + expectedItemKeys = itemKeys; + expectedItemFilePath = file.path; + } else { + expect( + itemKeys, + equals(expectedItemKeys), + reason: + 'Item in ${file.path} keys do not match consistency pattern. ' + 'Expected item keys to match the first processed file ($expectedItemFilePath).', + ); + } + } + } +} + +List _findEvalsFiles(Directory baseDir) { + if (!baseDir.existsSync()) { + return []; + } + return baseDir + .listSync(recursive: true) + .whereType() + .where((File f) => p.basename(f.path) == 'evals.json') + .toList(); +}