From 22ad90f54800641cf0e96d330b37b0d007367778 Mon Sep 17 00:00:00 2001 From: CodeSigils Date: Fri, 21 Aug 2026 18:46:03 +0300 Subject: [PATCH] feat: polish, advisory baseline tests, and v0.1.2 release - Add 12 tests for advisory baseline load/save/diff (test_common.py) - Add summary line to cron-health.py output - Add pyproject.toml URLs and classifiers - Improve docstrings in _common.py - Implement P9 advisory baseline (_common.py + cron-health.py) - Bump version to 0.1.2 (pyproject.toml + CITATION.cff) --- CITATION.cff | 4 +- CONTRIBUTING.md | 2 + README.md | 3 +- advisory-baseline.json | 4 ++ proposals/ROADMAP.md | 33 +++++++++--- pyproject.toml | 19 ++++++- scripts/_common.py | 55 +++++++++++++++++++- scripts/check-readme-tree.py | 2 +- scripts/cron-health.py | 73 ++++++++++++++++++++++---- scripts/test_common.py | 99 ++++++++++++++++++++++++++++++++++++ uv.lock | 2 +- 11 files changed, 270 insertions(+), 26 deletions(-) create mode 100644 advisory-baseline.json create mode 100644 scripts/test_common.py diff --git a/CITATION.cff b/CITATION.cff index 683dc6c..18cc94f 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,8 +5,8 @@ type: software authors: - family-names: CodeSigils license: MIT -version: "0.1.1" -date-released: "2026-07-25" +version: "0.1.2" +date-released: "2026-08-21" repository-code: "https://github.com/CodeSigils/skill-discovery" keywords: - agent diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index beb5e98..c393530 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,8 +50,10 @@ Run the relevant checks before opening a pull request: ```bash uv sync --locked --only-dev uv run python scripts/validate-ci.py +uv run python scripts/validate-ci.py --self-test uv run python scripts/check-version-consistency.py uv run python scripts/check-readme-tree.py +uv run python scripts/cron-health.py uv run ruff check .github/scripts/ scripts/ uv run python .github/scripts/test_validators.py uv run python -m pytest .github/scripts/test_integration.py -v diff --git a/README.md b/README.md index b4e9913..9ec08f6 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ skill-discovery/ │ ├── check-readme-tree.py # README layout vs disk check │ ├── check-version-consistency.py # CITATION.cff ↔ pyproject.toml version │ ├── cron-health.py # weekly link rot, reference integrity, budget +│ ├── test_common.py # tests for _common.py utilities │ ├── test_validate_skill.py # tests for validate-skill │ ├── validate-ci.py # CI workflow structural validator │ ├── validate-evaluation-fixtures.py # offline discovery report-contract check @@ -240,7 +241,7 @@ independently; use the verified local installation paths above. | Payload, documentation, evaluation fixtures, and dependency validation | Every push and pull request | CI reports failures that must be fixed before merge. | | External contract reachability and URL drift | Weekly schedule or manual dispatch | CI refreshes the evidence manifest through bounded checks and opens a PR when changes need review. | | Research expiry and reference accuracy | Weekly schedule or manual review | A maintainer reviews expiring research and updates dated references or the affected guidance. | -| Internal link rot, reference integrity, SKILL.md budget | Weekly schedule or manual dispatch | Detect-only checks warn when markdown links break, reference files go missing, or the skill payload exceeds budget. | +| Internal link rot, reference integrity, SKILL.md budget | Weekly schedule or manual dispatch | Detect-only checks warn when markdown links break, reference files go missing, or the skill payload exceeds budget. Known warnings are suppressed by the advisory baseline. | ## Security diff --git a/advisory-baseline.json b/advisory-baseline.json new file mode 100644 index 0000000..422f0e7 --- /dev/null +++ b/advisory-baseline.json @@ -0,0 +1,4 @@ +{ + "reviewed": "2026-08-21", + "warnings": [] +} diff --git a/proposals/ROADMAP.md b/proposals/ROADMAP.md index 2deb3bb..52c32c3 100644 --- a/proposals/ROADMAP.md +++ b/proposals/ROADMAP.md @@ -165,15 +165,9 @@ from known noise. - Format validation — `validate-ci.py` covers structure -## Recommendation: Advisory Baseline +## Advisory Baseline ✅ IMPLEMENTED (P9) -**Priority:** Medium. Not blocking any user-facing feature. -**Effort:** ~2-3 hours. -**Value:** Reduces weekly CI noise by filtering known warnings. -**Files affected:** `scripts/cron-health.py`, new `advisory-baseline.json`. - -Add as a future improvement when the weekly cron starts producing noise. -Not urgent — current weekly cron is functional and all warnings are valid. +Implemented 2026-08-21. See P9 section for details. ## Improvement Ideas ✅ COMPLETE @@ -553,6 +547,29 @@ Code review identified 5 cleanup items. All fixed in PR #28. Net: -14 lines. All 35 tests pass, ruff clean, LSP clean. +### P9 — Advisory baseline (2026-08-21) ✅ COMPLETE + +Weekly health checks (`cron-health.py`) now diff current warnings against a +baseline snapshot. Only NEW warnings are reported; known warnings are +suppressed. Resolved warnings (in baseline but no longer produced) are +flagged for baseline update. + +**Pattern source:** [awesome-agent-trust](https://github.com/CodeSigils/awesome-agent-trust) +advisory-baseline.json pattern, adapted for flat warning strings. + +**Files affected:** +- `scripts/_common.py` — `load_advisory_baseline()`, `save_advisory_baseline()`, `diff_advisories()` +- `scripts/cron-health.py` — baseline diff in `main()`, `--update-baseline` flag +- `advisory-baseline.json` — new file, snapshots current warnings + +**Usage:** +```bash +cron-health.py # run all checks, report vs baseline +cron-health.py --update-baseline # snapshot current warnings as new baseline +``` + +**Status:** Implemented 2026-08-21. + ### What we're NOT doing (Phase 3) - CODEOWNERS — solo project, no reviewers to assign diff --git a/pyproject.toml b/pyproject.toml index a15109e..0e9387e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "skill-discovery" -version = "0.1.1" +version = "0.1.2" description = "A methodology for finding and evaluating agent skills" readme = "README.md" requires-python = ">=3.10" @@ -8,6 +8,23 @@ license = "MIT" authors = [ { name = "CodeSigils" }, ] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries", +] + +[project.urls] +Homepage = "https://github.com/CodeSigils/skill-discovery" +Repository = "https://github.com/CodeSigils/skill-discovery" +Issues = "https://github.com/CodeSigils/skill-discovery/issues" [dependency-groups] dev = [ diff --git a/scripts/_common.py b/scripts/_common.py index a6284e6..1451638 100644 --- a/scripts/_common.py +++ b/scripts/_common.py @@ -50,7 +50,7 @@ def parse_expiry_date(value: Any) -> date | None: def check_fences(content: str, label: str) -> list[str]: - """Require matched fences and a language tag on opening fences.""" + """Require matched code fences and a language tag on opening fences.""" errors: list[str] = [] opening: tuple[int, str] | None = None for line_number, line in enumerate(content.splitlines(), start=1): @@ -131,6 +131,57 @@ def fail(message: str, hint: str | None = None) -> None: print(f" HINT: {hint}", file=sys.stderr) +# ── advisory baseline ───────────────────────────────────────────────────── + +ADVISORY_BASELINE_PATH = ROOT / "advisory-baseline.json" + + +def load_advisory_baseline(path: Path | None = None) -> list[str]: + """Load the advisory baseline file and return known warning strings. + + Returns an empty list if the file doesn't exist or is empty. + """ + import json + + path = path or ADVISORY_BASELINE_PATH + if not path.exists(): + return [] + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return [] + return data.get("warnings", []) if isinstance(data, dict) else [] + + +def save_advisory_baseline(warnings: list[str], path: Path | None = None) -> None: + """Save current warnings as the new baseline.""" + import json + from datetime import date + + path = path or ADVISORY_BASELINE_PATH + data = { + "reviewed": date.today().isoformat(), + "warnings": sorted(warnings), + } + path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def diff_advisories( + current: list[str], baseline: list[str] +) -> tuple[list[str], list[str]]: + """Diff current warnings against baseline. + + Returns (new, resolved) where: + - new: warnings in current but not in baseline (NEW issues) + - resolved: warnings in baseline but not in current (RESOLVED issues) + """ + current_set = set(current) + baseline_set = set(baseline) + new = sorted(current_set - baseline_set) + resolved = sorted(baseline_set - current_set) + return new, resolved + + # ── unsafe probe detection ──────────────────────────────────────────────── UNSAFE_PROBE_PATTERNS: list[tuple[str, str]] = [ @@ -161,7 +212,7 @@ def check_unsafe_probes(content: str, label: str) -> list[str]: def check_reference_sizes(skill_md: Path, root: Path) -> list[str]: - """Verify referenced files are within size budgets.""" + """Verify referenced files are within size budgets (50B–50KB).""" errors: list[str] = [] label = str(skill_md.relative_to(root)) content = skill_md.read_text(encoding="utf-8") diff --git a/scripts/check-readme-tree.py b/scripts/check-readme-tree.py index 7010fcd..12eaaef 100644 --- a/scripts/check-readme-tree.py +++ b/scripts/check-readme-tree.py @@ -17,7 +17,7 @@ # Directories to exclude from reverse drift check EXCLUDE_DIRS = {".git", "node_modules", ".omo", "__pycache__", ".ruff_cache"} -EXCLUDE_FILES = {".gitignore", "uv.lock", "CITATION.cff"} +EXCLUDE_FILES = {".gitignore", "uv.lock", "CITATION.cff", "advisory-baseline.json"} def git_tracked_files() -> set[str]: diff --git a/scripts/cron-health.py b/scripts/cron-health.py index ebdb163..b5d006c 100644 --- a/scripts/cron-health.py +++ b/scripts/cron-health.py @@ -7,13 +7,25 @@ 3. skill-budget – SKILL.md line count against 350-line warning threshold Exit code 0 = all checks passed, 1 = at least one issue found. + +Usage: + cron-health.py Run all checks, report vs baseline + cron-health.py --check internal-link-rot Run single check + cron-health.py --update-baseline Snapshot current warnings as new baseline """ from __future__ import annotations import re import sys -from _common import ROOT, SKILL_REF_RE, find_markdown_files +from _common import ( + ROOT, + SKILL_REF_RE, + diff_advisories, + find_markdown_files, + load_advisory_baseline, + save_advisory_baseline, +) RELATIVE_LINK_RE = re.compile(r"\[.*?\]\(((?!https?://|mailto:|#)[^)]+)\)") @@ -77,17 +89,53 @@ def check_skill_budget() -> list[str]: } -def main(check: str | None = None) -> int: +def main(check: str | None = None, update_baseline: bool = False) -> int: checks_to_run = {check: CHECKS[check]} if check else CHECKS - all_ok = True + all_warnings: list[str] = [] for name, fn in checks_to_run.items(): issues = fn() - if issues: - all_ok = False - for issue in issues: - print(f"⚠️ {name}: {issue}", file=sys.stderr) - else: - print(f"✅ {name}: OK") + for issue in issues: + all_warnings.append(f"{name}: {issue}") + + if update_baseline: + save_advisory_baseline(all_warnings) + print(f"Baseline updated: {len(all_warnings)} warnings saved") + return 0 + + baseline = load_advisory_baseline() + new, resolved = diff_advisories(all_warnings, baseline) + + if baseline: + known = len(all_warnings) - len(new) + if known > 0: + print(f"ℹ️ {known} known warning(s) suppressed by baseline") + for w in resolved: + print(f"✅ RESOLVED (baseline): {w}") + else: + new = all_warnings + + all_ok = True + for w in new: + all_ok = False + print(f"⚠️ NEW: {w}", file=sys.stderr) + for w in all_warnings: + if w not in new: + print(f"✅ {w}") + if not all_warnings: + print("✅ All checks passed") + return 0 + + n_new = len(new) + n_known = len(all_warnings) - n_new + n_resolved = len(resolved) + parts = [] + if n_new: + parts.append(f"{n_new} new") + if n_known: + parts.append(f"{n_known} known") + if n_resolved: + parts.append(f"{n_resolved} resolved") + print(f"Summary: {', '.join(parts)} warning(s)") return 0 if all_ok else 1 @@ -101,5 +149,10 @@ def main(check: str | None = None) -> int: default=None, help="Run a single check (default: all)", ) + parser.add_argument( + "--update-baseline", + action="store_true", + help="Save current warnings as the new advisory baseline", + ) args = parser.parse_args() - sys.exit(main(check=args.check)) + sys.exit(main(check=args.check, update_baseline=args.update_baseline)) diff --git a/scripts/test_common.py b/scripts/test_common.py new file mode 100644 index 0000000..552933e --- /dev/null +++ b/scripts/test_common.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Tests for shared utilities in _common.py.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from _common import ( + diff_advisories, + load_advisory_baseline, + save_advisory_baseline, +) + + +class AdvisoryBaselineTests(unittest.TestCase): + """Tests for advisory baseline load/save/diff functions.""" + + def test_load_missing_file(self): + with tempfile.TemporaryDirectory() as tmp: + result = load_advisory_baseline(Path(tmp) / "nonexistent.json") + self.assertEqual(result, []) + + def test_load_empty_warnings(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "baseline.json" + path.write_text(json.dumps({"reviewed": "2026-01-01", "warnings": []})) + result = load_advisory_baseline(path) + self.assertEqual(result, []) + + def test_load_with_warnings(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "baseline.json" + path.write_text( + json.dumps({"reviewed": "2026-01-01", "warnings": ["w1", "w2"]}) + ) + result = load_advisory_baseline(path) + self.assertEqual(result, ["w1", "w2"]) + + def test_load_invalid_json(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "baseline.json" + path.write_text("not json {{{") + result = load_advisory_baseline(path) + self.assertEqual(result, []) + + def test_load_non_dict_shape(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "baseline.json" + path.write_text(json.dumps("just a string")) + result = load_advisory_baseline(path) + self.assertEqual(result, []) + + def test_save_creates_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "baseline.json" + save_advisory_baseline(["w2", "w1"], path=path) + data = json.loads(path.read_text()) + self.assertEqual(data["warnings"], ["w1", "w2"]) # sorted + self.assertIn("reviewed", data) + + def test_save_overwrites(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "baseline.json" + save_advisory_baseline(["old"], path=path) + save_advisory_baseline(["new"], path=path) + data = json.loads(path.read_text()) + self.assertEqual(data["warnings"], ["new"]) + + def test_diff_new_warnings(self): + new, resolved = diff_advisories(["a", "b", "c"], ["a"]) + self.assertEqual(new, ["b", "c"]) + self.assertEqual(resolved, []) + + def test_diff_resolved_warnings(self): + new, resolved = diff_advisories(["a"], ["a", "b", "c"]) + self.assertEqual(new, []) + self.assertEqual(resolved, ["b", "c"]) + + def test_diff_mixed(self): + new, resolved = diff_advisories(["a", "d"], ["a", "b"]) + self.assertEqual(new, ["d"]) + self.assertEqual(resolved, ["b"]) + + def test_diff_empty_both(self): + new, resolved = diff_advisories([], []) + self.assertEqual(new, []) + self.assertEqual(resolved, []) + + def test_diff_duplicates_dont_affect(self): + new, resolved = diff_advisories(["a", "a"], ["a"]) + self.assertEqual(new, []) + self.assertEqual(resolved, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 2a4a196..c55c78b 100644 --- a/uv.lock +++ b/uv.lock @@ -168,7 +168,7 @@ wheels = [ [[package]] name = "skill-discovery" -version = "0.1.1" +version = "0.1.2" source = { virtual = "." } [package.dev-dependencies]