From c77841e4e2b7d215696455f370bb72e210d9a38b Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 11:52:36 -0400 Subject: [PATCH 1/4] Add evals for add-dart-lint-validation-rule --- .../evals/evals.json | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json diff --git a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json new file mode 100644 index 0000000..53b4f64 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json @@ -0,0 +1,37 @@ +{ + "skill_name": "add-dart-lint-validation-rule", + "evals": [ + { + "id": 1, + "prompt": "Author a custom dart_skills_lint rule named `RequireSpecificMetadataRule` that verifies if a skill's metadata contains a `required_version` field. Wire it up into a new test file `test/require_specific_metadata_rule_test.dart` using an in-memory SkillContext.", + "expected_chat_output": "The agent identifies the skill, spawns a with-skill subagent, and successfully implements the rule.", + "expected_repo_state": [ + "The agent successfully authored `RequireSpecificMetadataRule` extending `SkillRule` in `lib/src/rules/require_specific_metadata_rule.dart`.", + "The rule was properly added to `RuleRegistry.allChecks` and the `createRule` switch statement in `lib/src/rule_registry.dart`.", + "The `validate` method correctly parses `context.parsedYaml` to check for `required_version` in the metadata.", + "The agent authored unit tests utilizing an in-memory raw string, `loadYaml`, and a mock `SkillContext` in `test/require_specific_metadata_rule_test.dart`.", + "The toggleable flag `--[no-]require-specific-metadata` was correctly documented in `README.md`.", + "A proper rule documentation entry was added in `RULES.md`, including severity, fixability, and diagnostic shape.", + "The agent proactively documented the rule constraints in `SPECIFICATION.md`." + ], + "agent_config": "reidbaker-agent" + }, + { + "id": 2, + "prompt": "Create a new rule `FileExistenceRule` that ensures a specific supplementary file `info.txt` exists alongside `SKILL.md`. Make sure this rule is disabled by default. Write tests using the temporary directory approach.", + "expected_chat_output": "The agent creates the rule and physical directory tests.", + "expected_repo_state": [ + "The `FileExistenceRule` class was successfully generated extending `SkillRule` in `lib/src/rules/file_existence_rule.dart`.", + "Included correctly in `RuleRegistry.allChecks` as `disabled` in `lib/src/rule_registry.dart`.", + "The agent successfully utilized `Directory.systemTemp.createTemp` and `File.writeAsString` for integration testing in `test/rules/file_existence_rule_test.dart`.", + "The rule flag `--[no-]file-existence` was documented in `README.md`.", + "A proper rule documentation entry was added outlining the diagnostics and behavior in `RULES.md`.", + "Temporary files were safely cleaned up via `tempDir.delete(recursive: true)` in `tearDown`." + ], + "agent_config": "reidbaker-agent" + } + ], + "repo_criteria": [ + "evals/code_quality_rubric.json" + ] +} From 80700d2bffb52bf969626959d38fa3101cd7be44 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 14:38:48 -0400 Subject: [PATCH 2/4] Update SKILL.md instructions for README.md flags formatting --- .../.agents/skills/add-dart-lint-validation-rule/SKILL.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md index 6db18ab..9275546 100644 --- a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md @@ -183,7 +183,9 @@ If the rule interacts with CLI flags or configuration files, add a test in `test When a new rule is introduced, verify that you synchronize sibling markdown files! 1. **`README.md`:** - * Add your flag under the **Usage** and **Flags** sections so users know it exists. + * Add your flag under the **Flags** section (under **Usage**) so users know it exists. + * **CRITICAL FORMATTING:** You MUST use the exact format `- \`--[no-]\`: . (Disabled by default if applicable)`. + * **CRITICAL NAMING:** Ensure the flag string matches the `ruleName` EXACTLY. For example, if the `ruleName` is `file-existence`, the flag MUST be documented as `--[no-]file-existence` (do NOT hallucinate a `check-` prefix like `--[no-]check-file-existence`). Do NOT add empty bullet points. 2. **`RULES.md`:** * Add a new entry for your rule documenting its default severity, fixability, what it checks, diagnostic shape, auto-fix behavior, and how to disable it. This is strictly required by the `rules_md_consistency_test.dart` test. 3. **`documentation/knowledge/SPECIFICATION.md`:** @@ -196,7 +198,7 @@ When a new rule is introduced, verify that you synchronize sibling markdown file - [ ] Rule class created in `lib/src/rules/`. - [ ] Rule registered in `lib/src/rule_registry.dart`. - [ ] Unit tests added in `test/` using in-memory `SkillContext`. -- [ ] Usage listed in `README.md`. +- [ ] **CRITICAL**: Usage flag correctly documented in `README.md` under Flags (ensure flag string matches `ruleName` EXACTLY and format is correct). - [ ] Rule documented in `RULES.md`. - [ ] Schema documented in `documentation/knowledge/SPECIFICATION.md` (if applicable). - [ ] Run `dart format .` to format code. From 22dbac37ad401ff878af4000f9047e251ba8cdd1 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 15:00:01 -0400 Subject: [PATCH 3/4] Apply schema changes to add-dart-lint-validation-rule evals --- .../evals/evals.json | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json index 53b4f64..8039b0a 100644 --- a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/evals/evals.json @@ -1,10 +1,14 @@ { - "skill_name": "add-dart-lint-validation-rule", + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, "prompt": "Author a custom dart_skills_lint rule named `RequireSpecificMetadataRule` that verifies if a skill's metadata contains a `required_version` field. Wire it up into a new test file `test/require_specific_metadata_rule_test.dart` using an in-memory SkillContext.", - "expected_chat_output": "The agent identifies the skill, spawns a with-skill subagent, and successfully implements the rule.", + "expected_chat_output": [ + "The agent identifies the skill, spawns a with-skill subagent, and successfully implements the rule." + ], "expected_repo_state": [ "The agent successfully authored `RequireSpecificMetadataRule` extending `SkillRule` in `lib/src/rules/require_specific_metadata_rule.dart`.", "The rule was properly added to `RuleRegistry.allChecks` and the `createRule` switch statement in `lib/src/rule_registry.dart`.", @@ -19,7 +23,9 @@ { "id": 2, "prompt": "Create a new rule `FileExistenceRule` that ensures a specific supplementary file `info.txt` exists alongside `SKILL.md`. Make sure this rule is disabled by default. Write tests using the temporary directory approach.", - "expected_chat_output": "The agent creates the rule and physical directory tests.", + "expected_chat_output": [ + "The agent creates the rule and physical directory tests." + ], "expected_repo_state": [ "The `FileExistenceRule` class was successfully generated extending `SkillRule` in `lib/src/rules/file_existence_rule.dart`.", "Included correctly in `RuleRegistry.allChecks` as `disabled` in `lib/src/rule_registry.dart`.", @@ -30,8 +36,5 @@ ], "agent_config": "reidbaker-agent" } - ], - "repo_criteria": [ - "evals/code_quality_rubric.json" ] } From 79c679d5dd7378836c1358176da3b72e51d72a9e Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 15:05:42 -0400 Subject: [PATCH 4/4] Fix trailing whitespace in add-dart-lint-validation-rule SKILL.md --- .../.agents/skills/add-dart-lint-validation-rule/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md index 9275546..208f9af 100644 --- a/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/add-dart-lint-validation-rule/SKILL.md @@ -183,7 +183,7 @@ If the rule interacts with CLI flags or configuration files, add a test in `test When a new rule is introduced, verify that you synchronize sibling markdown files! 1. **`README.md`:** - * Add your flag under the **Flags** section (under **Usage**) so users know it exists. + * Add your flag under the **Flags** section (under **Usage**) so users know it exists. * **CRITICAL FORMATTING:** You MUST use the exact format `- \`--[no-]\`: . (Disabled by default if applicable)`. * **CRITICAL NAMING:** Ensure the flag string matches the `ruleName` EXACTLY. For example, if the `ruleName` is `file-existence`, the flag MUST be documented as `--[no-]file-existence` (do NOT hallucinate a `check-` prefix like `--[no-]check-file-existence`). Do NOT add empty bullet points. 2. **`RULES.md`:**