From 3c92a1e81293fa8c8c04cd81802128cf6ebbc883 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 13:23:20 -0400 Subject: [PATCH 01/10] Add static dataset and meta-eval for code quality rubric --- .../.agents/skills/test-rubric/SKILL.md | 7 +++++++ .../skills/test-rubric/evals/evals.json | 21 +++++++++++++++++++ .../test_data/bad_code/tmp/bad_script.dart | 15 +++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md create mode 100644 tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json create mode 100644 tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md b/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md new file mode 100644 index 0000000..a18817f --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md @@ -0,0 +1,7 @@ +--- +name: test-rubric +description: A dummy skill solely for running meta-evaluations against the code quality rubric. only call this skill when instructed. +--- +# Test Rubric + +This skill does nothing and is only used as a container for evaluating the code quality rubric against bad fixtures. diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json b/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json new file mode 100644 index 0000000..bd42e68 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json @@ -0,0 +1,21 @@ +{ + "skill_name": "test-rubric", + "evals": [ + { + "id": 1, + "prompt": "Grade the code in 'evals/test_data/bad_code/tmp/bad_script.dart' using the code_quality_rubric.json.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The evaluation artifact explicitly flags a failure for directory placement hygiene.", + "The evaluation artifact explicitly flags a failure for effective Dart idioms.", + "The evaluation artifact explicitly flags a failure for cross platform compatibility." + ], + "agent_config": "reidbaker-agent" + } + ], + "repo_criteria": [ + "evals/code_quality_rubric.json" + ] +} diff --git a/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart b/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart new file mode 100644 index 0000000..25abae6 --- /dev/null +++ b/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart @@ -0,0 +1,15 @@ +// ignore_for_file: unused_local_variable, prefer_final_locals, avoid_print, use_raw_strings + +// Violates placement hygiene (in tmp/ instead of bin/ or lib/) +// Violates cross-platform compatibility (hardcoded Windows path) +// Violates effective Dart idioms (raw strings instead of interpolation) + +void main() { + var user = 'Test'; + + // Anti-pattern: Raw string concatenation instead of interpolation + print(r'Hello ' + user); + + // Anti-pattern: Hardcoded platform specific path + var path = 'C:\\my\\path\\data.txt'; +} From 5e71e4f4ef31fd6c134d36d4933238941ebd0aa8 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 13:26:11 -0400 Subject: [PATCH 02/10] Add positive test case and README docs for meta-evals --- .../.agents/skills/test-rubric/evals/evals.json | 11 +++++++++++ tool/dart_skills_lint/evals/README.md | 11 +++++++++++ .../evals/test_data/ok_code/bin/good_script.dart | 15 +++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json b/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json index bd42e68..7087423 100644 --- a/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json @@ -13,6 +13,17 @@ "The evaluation artifact explicitly flags a failure for cross platform compatibility." ], "agent_config": "reidbaker-agent" + }, + { + "id": 2, + "prompt": "Grade the code in 'evals/test_data/ok_code/bin/good_script.dart' using the code_quality_rubric.json.", + "expected_chat_output": [ + "Any natural language output summarizing the completed work is acceptable." + ], + "expected_repo_state": [ + "The evaluation artifact explicitly flags that all code quality criteria passed successfully without any failures." + ], + "agent_config": "reidbaker-agent" } ], "repo_criteria": [ diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md index b00f969..64ee59c 100644 --- a/tool/dart_skills_lint/evals/README.md +++ b/tool/dart_skills_lint/evals/README.md @@ -44,3 +44,14 @@ dart test test/skills_evals_test.dart ### 2. Running Evals via Agent Orchestration You should use the `/run-evals` skill to run evaluations. The bulk of the execution logic and prompts are located within the `run-evals` skill itself (`.agents/skills/run-evals/SKILL.md`). The environment, model, and harness are determined by the `agent_config` specified in the corresponding `evals.json` file. + +### 3. Testing Meta-Evals (Testing the Rubrics) +To ensure our universal rubrics correctly catch anti-patterns (and permit clean code), we use meta-evaluations. The `test-rubric` skill is a dummy skill that contains evals strictly intended to grade static fixtures located in `evals/test_data/`. + +To run the meta-evals and verify the rubrics: +1. Ensure the `test-rubric` skill is available in your workspace. +2. Run the `run-evals` skill and specify `test-rubric`. + ```bash + dart run bin/cli.dart -d .agents/skills test-rubric + ``` + Or invoke `/run-evals` and ask the agent to run the evaluations for the `test-rubric` skill. diff --git a/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart b/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart new file mode 100644 index 0000000..c43fb17 --- /dev/null +++ b/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart @@ -0,0 +1,15 @@ +import 'package:path/path.dart' as p; + +void main() { + const user = 'Test'; + + // Interpolation (clean Dart idiom) + const message = 'Hello $user'; + + // Cross platform path + final String path = p.join('my', 'path', 'data.txt'); + + if (message.isEmpty || path.isEmpty) { + throw Exception('Validation failed'); + } +} From 139b5a18760f76ac84445793e35c39d0b047af59 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 14:03:24 -0400 Subject: [PATCH 03/10] Update evals.json schema: move repo_criteria to top and remove skill_name --- .../.agents/skills/test-rubric/evals/evals.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json b/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json index 7087423..789c0d2 100644 --- a/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json @@ -1,5 +1,7 @@ { - "skill_name": "test-rubric", + "repo_criteria": [ + "evals/code_quality_rubric.json" + ], "evals": [ { "id": 1, @@ -25,8 +27,5 @@ ], "agent_config": "reidbaker-agent" } - ], - "repo_criteria": [ - "evals/code_quality_rubric.json" ] } From 3104642f13702da2edecee009fb8a4f1d99715ad Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 15:01:25 -0400 Subject: [PATCH 04/10] Address remaining PR 194 feedback: fix test-rubric lints --- tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md | 2 ++ tool/dart_skills_lint/dart_skills_lint.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md b/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md index a18817f..4d18ac2 100644 --- a/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md @@ -1,6 +1,8 @@ --- name: test-rubric description: A dummy skill solely for running meta-evaluations against the code quality rubric. only call this skill when instructed. +metadata: + internal: true --- # Test Rubric diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index bcee3ef..3de0c93 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -38,7 +38,7 @@ dart_skills_lint: - path: ".agents/skills/run-evals" rules: prevent-skills-sh-publishing: error - - path: ".agents/skills/contributor-pr-description" + - path: ".agents/skills/test-rubric" rules: prevent-skills-sh-publishing: error From cc0793a7de812864f89695911e253b6a28a435b7 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 15:08:15 -0400 Subject: [PATCH 05/10] Address PR review: resolve package directory dynamically in tests using Isolate.resolvePackageUri --- .../test/skills_evals_test.dart | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index 2f289f5..f2214f2 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -4,6 +4,7 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:isolate'; import 'package:path/path.dart' as p; import 'package:test/test.dart'; @@ -12,10 +13,15 @@ void main() { group('Evals structure consistency', () { // Ensures all evals.json files dynamically share the exact same JSON schema. // Keys are not hardcoded to ensure enforcement remains schema-agnostic and flexible. - test('all evals.json files across skills share consistent structure and keys', () { + test('all evals.json files across skills share consistent structure and keys', () async { + final Uri? packageUri = await Isolate.resolvePackageUri( + Uri.parse('package:dart_skills_lint/'), + ); + final String packageRoot = packageUri!.resolve('..').toFilePath(); + final List evalsFiles = [ - ..._findEvalsFiles(Directory(p.join(Directory.current.path, 'skills'))), - ..._findEvalsFiles(Directory(p.join(Directory.current.path, '.agents', 'skills'))), + ..._findEvalsFiles(Directory(p.join(packageRoot, 'skills'))), + ..._findEvalsFiles(Directory(p.join(packageRoot, '.agents', 'skills'))), ]..sort((a, b) => a.path.compareTo(b.path)); expect( @@ -29,8 +35,13 @@ void main() { // Note: We intentionally only require an evals.json file for published skills. // Contributor skills in .agents/skills/ are not currently required to have one. - test('all published skills have an evals.json file', () { - final skillsDir = Directory(p.join(Directory.current.path, 'skills')); + test('all published skills have an evals.json file', () async { + final Uri? packageUri = await Isolate.resolvePackageUri( + Uri.parse('package:dart_skills_lint/'), + ); + final String packageRoot = packageUri!.resolve('..').toFilePath(); + + final skillsDir = Directory(p.join(packageRoot, 'skills')); if (!skillsDir.existsSync()) { return; } @@ -48,8 +59,13 @@ void main() { } }); - test('all rubric JSON files in evals/ share consistent structure and keys', () { - final rubricsDir = Directory(p.join(Directory.current.path, 'evals')); + test('all rubric JSON files in evals/ share consistent structure and keys', () async { + final Uri? packageUri = await Isolate.resolvePackageUri( + Uri.parse('package:dart_skills_lint/'), + ); + final String packageRoot = packageUri!.resolve('..').toFilePath(); + + final rubricsDir = Directory(p.join(packageRoot, 'evals')); if (!rubricsDir.existsSync()) { return; } From aafba2e41f4a3b57d9b979bfe00dd467a4f98085 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 15:17:32 -0400 Subject: [PATCH 06/10] Add contributor-pr-description to prevent-skills-sh-publishing config --- tool/dart_skills_lint/dart_skills_lint.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index 3de0c93..a3ef298 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -41,4 +41,7 @@ dart_skills_lint: - path: ".agents/skills/test-rubric" rules: prevent-skills-sh-publishing: error + - path: ".agents/skills/contributor-pr-description" + rules: + prevent-skills-sh-publishing: error From a3616a85bfe194f0c6879bcbd9d11b0ce937cb22 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 17:52:13 -0400 Subject: [PATCH 07/10] Refactor test-rubric dummy skill into standalone code_quality_rubric_evals.json --- .../.agents/skills/run-evals/SKILL.md | 6 +++--- .../.agents/skills/test-rubric/SKILL.md | 9 --------- tool/dart_skills_lint/dart_skills_lint.yaml | 3 --- tool/dart_skills_lint/evals/README.md | 12 +++--------- .../code_quality_rubric_evals.json} | 0 tool/dart_skills_lint/test/skills_evals_test.dart | 12 ++++++------ 6 files changed, 12 insertions(+), 30 deletions(-) delete mode 100644 tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md rename tool/dart_skills_lint/{.agents/skills/test-rubric/evals/evals.json => evals/code_quality_rubric_evals.json} (100%) diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md index 8fa173c..cba32a8 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/SKILL.md @@ -8,10 +8,10 @@ metadata: # Run Skill Evals 1. **Read Framework**: Read `tool/dart_skills_lint/evals/README.md` for understanding the difference between per-skill evals and cross-skill evals. -2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. -3. **Determine Agent Configuration**: Check the `agent_config` field in the target `evals.json` to determine the environment/harness to spawn. If `agent_config` is `"bare-agent"`, spawn a subagent with the `bare-agent` profile. If it is a specific contributor profile (e.g. `"reidbaker-agent"`), use that profile to provide the necessary contributor context. +2. **Locate Targets**: Find target `evals/evals.json` files inside `.agents/skills/` and/or `skills/`. For cross-skill evaluations, look for `*_evals.json` files directly in `tool/dart_skills_lint/evals/`. +3. **Determine Agent Configuration**: Check the `agent_config` field in the target target JSON file to determine the environment/harness to spawn. If `agent_config` is `"bare-agent"`, spawn a subagent with the `bare-agent` profile. If it is a specific contributor profile (e.g. `"reidbaker-agent"`), use that profile to provide the necessary contributor context. 4. **Orchestrate**: By default, run an Integration Test by spawning a single **With-Skill** subagent using `Workspace: branch` and the identified `agent_config`. - - Provide the task prompt. See `resources/with_skill_execution_prompt.md` for the template. When filling in ``, you MUST use a relative path from the repository root, not an absolute path, to prevent the subagent from accidentally anchoring to and modifying the parent workspace. + - Provide the task prompt. See `resources/with_skill_execution_prompt.md` for the template. When filling in ``, you MUST use a relative path from the repository root, not an absolute path. If you are running a cross-skill evaluation, fill in `` with `"none (cross-skill meta-eval)"`. - **Only if the user explicitly requests a comparison or benchmark**, also spawn a **Baseline** subagent. See `resources/baseline_execution_prompt.md` for the template. Instruct the subagent(s) to return their `git diff` and verification outputs (`dart format`, `dart analyze`, `dart test`) without committing. **CRITICAL**: You must explicitly warn the subagent(s) to confine all file edits strictly to their current working directory and avoid using absolute paths to modify the parent workspace. diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md b/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md deleted file mode 100644 index 4d18ac2..0000000 --- a/tool/dart_skills_lint/.agents/skills/test-rubric/SKILL.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: test-rubric -description: A dummy skill solely for running meta-evaluations against the code quality rubric. only call this skill when instructed. -metadata: - internal: true ---- -# Test Rubric - -This skill does nothing and is only used as a container for evaluating the code quality rubric against bad fixtures. diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index a3ef298..bcee3ef 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -38,9 +38,6 @@ dart_skills_lint: - path: ".agents/skills/run-evals" rules: prevent-skills-sh-publishing: error - - path: ".agents/skills/test-rubric" - rules: - prevent-skills-sh-publishing: error - path: ".agents/skills/contributor-pr-description" rules: prevent-skills-sh-publishing: error diff --git a/tool/dart_skills_lint/evals/README.md b/tool/dart_skills_lint/evals/README.md index 64ee59c..d55dc47 100644 --- a/tool/dart_skills_lint/evals/README.md +++ b/tool/dart_skills_lint/evals/README.md @@ -46,12 +46,6 @@ dart test test/skills_evals_test.dart You should use the `/run-evals` skill to run evaluations. The bulk of the execution logic and prompts are located within the `run-evals` skill itself (`.agents/skills/run-evals/SKILL.md`). The environment, model, and harness are determined by the `agent_config` specified in the corresponding `evals.json` file. ### 3. Testing Meta-Evals (Testing the Rubrics) -To ensure our universal rubrics correctly catch anti-patterns (and permit clean code), we use meta-evaluations. The `test-rubric` skill is a dummy skill that contains evals strictly intended to grade static fixtures located in `evals/test_data/`. - -To run the meta-evals and verify the rubrics: -1. Ensure the `test-rubric` skill is available in your workspace. -2. Run the `run-evals` skill and specify `test-rubric`. - ```bash - dart run bin/cli.dart -d .agents/skills test-rubric - ``` - Or invoke `/run-evals` and ask the agent to run the evaluations for the `test-rubric` skill. +To ensure our universal rubrics correctly catch anti-patterns (and permit clean code), we use meta-evaluations. Standalone cross-skill evaluations are defined as `evals/*_evals.json` files (e.g., `evals/code_quality_rubric_evals.json`). These files contain evals strictly intended to grade static fixtures located in `evals/test_data/`. + +To run the meta-evals and verify the rubrics, invoke the `/run-evals` skill and ask the agent to run the standalone `code_quality_rubric_evals.json` file. diff --git a/tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json b/tool/dart_skills_lint/evals/code_quality_rubric_evals.json similarity index 100% rename from tool/dart_skills_lint/.agents/skills/test-rubric/evals/evals.json rename to tool/dart_skills_lint/evals/code_quality_rubric_evals.json diff --git a/tool/dart_skills_lint/test/skills_evals_test.dart b/tool/dart_skills_lint/test/skills_evals_test.dart index f2214f2..5c02ebe 100644 --- a/tool/dart_skills_lint/test/skills_evals_test.dart +++ b/tool/dart_skills_lint/test/skills_evals_test.dart @@ -22,6 +22,7 @@ void main() { final List evalsFiles = [ ..._findEvalsFiles(Directory(p.join(packageRoot, 'skills'))), ..._findEvalsFiles(Directory(p.join(packageRoot, '.agents', 'skills'))), + ..._findEvalsFiles(Directory(p.join(packageRoot, 'evals'))), ]..sort((a, b) => a.path.compareTo(b.path)); expect( @@ -74,7 +75,7 @@ void main() { rubricsDir .listSync() .whereType() - .where((File f) => f.path.endsWith('.json')) + .where((File f) => f.path.endsWith('.json') && !f.path.endsWith('_evals.json')) .toList() ..sort((a, b) => a.path.compareTo(b.path)); @@ -144,9 +145,8 @@ List _findEvalsFiles(Directory baseDir) { if (!baseDir.existsSync()) { return []; } - return baseDir - .listSync(recursive: true) - .whereType() - .where((File f) => p.basename(f.path) == 'evals.json') - .toList(); + return baseDir.listSync(recursive: true).whereType().where((File f) { + final String name = p.basename(f.path); + return name == 'evals.json' || name.endsWith('_evals.json'); + }).toList(); } From 8cb2f8f9b2709057d76e6f35a396a60420d22cc7 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 18:19:51 -0400 Subject: [PATCH 08/10] Update baseline execution prompt to avoid fixing code on grading tasks --- .../skills/run-evals/resources/baseline_execution_prompt.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md index d073ab7..a9a4a98 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/baseline_execution_prompt.md @@ -5,3 +5,4 @@ Execute this task: WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). +NOTE: If your task is strictly to grade, review, or evaluate code, do NOT fix the issues you find. Leave the code exactly as it is, even if verification commands fail. Your job is only to report the evaluation results. From bc82b1f1b3f861817255e7764ac2fdb19011d7f3 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 23:21:54 -0400 Subject: [PATCH 09/10] Add copyright headers to test fixtures --- .../evals/test_data/bad_code/tmp/bad_script.dart | 4 ++++ .../evals/test_data/ok_code/bin/good_script.dart | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart b/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart index 25abae6..ffc0a5a 100644 --- a/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart +++ b/tool/dart_skills_lint/evals/test_data/bad_code/tmp/bad_script.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + // ignore_for_file: unused_local_variable, prefer_final_locals, avoid_print, use_raw_strings // Violates placement hygiene (in tmp/ instead of bin/ or lib/) diff --git a/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart b/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart index c43fb17..41eccf4 100644 --- a/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart +++ b/tool/dart_skills_lint/evals/test_data/ok_code/bin/good_script.dart @@ -1,3 +1,7 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + import 'package:path/path.dart' as p; void main() { From 95b7efc4adb48086f1773707b822d27c82a2a79f Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Thu, 23 Jul 2026 23:23:42 -0400 Subject: [PATCH 10/10] Update with_skill execution prompt to avoid fixing code on grading tasks --- .../skills/run-evals/resources/with_skill_execution_prompt.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md b/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md index de672a6..d9341be 100644 --- a/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md +++ b/tool/dart_skills_lint/.agents/skills/run-evals/resources/with_skill_execution_prompt.md @@ -6,3 +6,4 @@ Execute this task: WARNING: You are executing in an isolated branch workspace. Confine all file modifications strictly to your current working directory. Do NOT use absolute paths to modify files in the parent workspace. Once you are done, do not commit. Just send me a message with the `git diff` of your changes, and the output of running verification commands (e.g., `dart format`, `dart analyze`, `dart test`). +NOTE: If your task is strictly to grade, review, or evaluate code, do NOT fix the issues you find. Leave the code exactly as it is, even if verification commands fail. Your job is only to report the evaluation results.