From e022930070bd71666fb48ff4776d09f50aeb0107 Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Sat, 18 Jul 2026 05:53:11 -0500 Subject: [PATCH 1/3] ci: add npm + PyPI publish workflows for eval Adapted from extract's publish-npm.yml/publish-pypi.yml templates: - publish-pypi.yml: release-triggered, PyPI Trusted Publisher (environment: pypi), no prompts-copy step (eval has no bundled-resource packaging need extract has). Working directory is repo root, matching eval's flat pyproject.toml layout (extract uses packages/python/). - publish-npm.yml: release-triggered + workflow_dispatch, npm Trusted Publishing (--provenance, no token). working-directory: ts, matching eval's ts/ subdirectory layout. Uses `npm run typecheck` in place of extract's `npm run build` -- eval's tsconfig.json has noEmit: true and package.json has no build script; the package ships raw TS source (exports -> ./src/mod.ts) by design, so typecheck is the correct pre-publish safety gate, not a missing step. Both verified locally end-to-end short of the actual registry upload: python -m build + twine check both PASSED; npm ci + npm run typecheck clean; npm publish --dry-run confirms correct package name/scope (@synapt/eval) and tarball contents (38 files, 15.2kB). Note: package.json/jsr.json/README all establish @synapt/eval as the existing npm scope (not @synapt-dev/eval) -- following the codebase's established convention rather than extract's scope. Does not trigger a real publish. PyPI + npm Trusted Publisher registration (external, registry-side) and cutting an actual GitHub release both remain outside this PR's scope. Ref #2 (does not Close -- that closes once a real publish succeeds). --- .github/workflows/publish-npm.yml | 57 ++++++++++++++++++++++++++++++ .github/workflows/publish-pypi.yml | 28 +++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 .github/workflows/publish-npm.yml create mode 100644 .github/workflows/publish-pypi.yml diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml new file mode 100644 index 0000000..3a84768 --- /dev/null +++ b/.github/workflows/publish-npm.yml @@ -0,0 +1,57 @@ +name: Publish @synapt/eval to npm + +on: + release: + types: [published] + workflow_dispatch: + # Manual re-trigger — needed because this workflow has no version-mismatch retry path: + # a `release` event fires it exactly once, and if npm's version hadn't moved yet at that + # point (e.g. a Python-only release, or this repo's own version-sync-deferred slices), + # there is no automatic second attempt once the npm version is later bumped. + +permissions: + contents: write + id-token: write + +jobs: + publish: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ts + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + + - run: npm ci + + - run: npm run typecheck + + - name: Check if this version is already published + id: check + run: | + VERSION=$(node -p "require('./package.json').version") + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + if npm view "@synapt/eval@$VERSION" version >/dev/null 2>&1; then + echo "already_published=true" >> "$GITHUB_OUTPUT" + echo "@synapt/eval@$VERSION is already published to npm -- skipping publish (this is expected on every release where only the PyPI side bumped, not an error)." + else + echo "already_published=false" >> "$GITHUB_OUTPUT" + fi + + - run: npm publish --provenance --access public + if: steps.check.outputs.already_published != 'true' + + - name: Generate SBOM + if: steps.check.outputs.already_published != 'true' + run: npm sbom --omit=dev --sbom-format cyclonedx > sbom.cdx.json + + - name: Upload SBOM to release + if: steps.check.outputs.already_published != 'true' && github.event_name == 'release' + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload ${{ github.event.release.tag_name }} sbom.cdx.json --clobber diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 0000000..fcd69d7 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,28 @@ +name: Publish synapt-eval to PyPI + +on: + release: + types: [published] + +permissions: + contents: read + id-token: write + +jobs: + publish: + runs-on: ubuntu-latest + environment: pypi + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - run: pip install build + + - run: python -m build + + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + with: + packages-dir: dist/ From d0fa962c495e19e6f5f71708554ed314d0b4780e Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Sat, 18 Jul 2026 05:58:22 -0500 Subject: [PATCH 2/3] fix: rename npm package to @synapt-dev/eval @synapt/eval was stale internal naming from before the npm scope was decided. Per Layne's 2026-07-17 docket answer and shipped precedent (extract already publishes as @synapt-dev/extract), the correct scope is @synapt-dev. Renamed package.json, all README/CHANGELOG/docs references, the two migration example comments, and publish-npm.yml's name + already-published check to match. package-lock.json regenerated via npm install, not hand-edited. Verified after rename: npm run typecheck clean, npm publish --dry-run confirms name: @synapt-dev/eval with the same 38-file/15.2kB tarball as before (no content change, name only). Deliberately NOT renamed: ts/jsr.json. JSR scope registration is a separate, independent decision from npm's -- owning @synapt-dev on npmjs.com says nothing about JSR scope ownership, and JSR publishing is out of scope for this task entirely (no workflow, not requested). Flagging rather than guessing. This matters now specifically because npm Trusted Publisher registration for the @synapt-dev org is happening this morning -- a scope mismatch would have failed the publish at the registry. --- .github/workflows/publish-npm.yml | 6 +++--- CHANGELOG.md | 2 +- README.md | 2 +- docs/ts-migration.md | 14 +++++++------- ts/examples/migration/after.ts | 2 +- ts/examples/migration/before.ts | 2 +- ts/package-lock.json | 4 ++-- ts/package.json | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 3a84768..675b575 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -1,4 +1,4 @@ -name: Publish @synapt/eval to npm +name: Publish @synapt-dev/eval to npm on: release: @@ -36,9 +36,9 @@ jobs: run: | VERSION=$(node -p "require('./package.json').version") echo "version=$VERSION" >> "$GITHUB_OUTPUT" - if npm view "@synapt/eval@$VERSION" version >/dev/null 2>&1; then + if npm view "@synapt-dev/eval@$VERSION" version >/dev/null 2>&1; then echo "already_published=true" >> "$GITHUB_OUTPUT" - echo "@synapt/eval@$VERSION is already published to npm -- skipping publish (this is expected on every release where only the PyPI side bumped, not an error)." + echo "@synapt-dev/eval@$VERSION is already published to npm -- skipping publish (this is expected on every release where only the PyPI side bumped, not an error)." else echo "already_published=false" >> "$GITHUB_OUTPUT" fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 674de0b..b5ab4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v0.1.0 (2026-05-07) -Initial release of @synapt/eval. +Initial release of @synapt-dev/eval. ### Components diff --git a/README.md b/README.md index e8b68cd..23cb259 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# @synapt/eval +# @synapt-dev/eval [![PyPI](https://img.shields.io/pypi/v/synapt-eval)](https://pypi.org/project/synapt-eval/) [![Python](https://img.shields.io/pypi/pyversions/synapt-eval)](https://pypi.org/project/synapt-eval/) diff --git a/docs/ts-migration.md b/docs/ts-migration.md index 06a647d..378608b 100644 --- a/docs/ts-migration.md +++ b/docs/ts-migration.md @@ -1,6 +1,6 @@ # TypeScript Migration Guide -Migrate from a monolithic eval script to composable `@synapt/eval` adapters. +Migrate from a monolithic eval script to composable `@synapt-dev/eval` adapters. ## Before vs After @@ -15,7 +15,7 @@ See the working examples in `ts/examples/migration/`: Extract your retrieval/generation call into an adapter class: ```typescript -import type { RetrievalAdapter, RetrievalCandidate } from "@synapt/eval"; +import type { RetrievalAdapter, RetrievalCandidate } from "@synapt-dev/eval"; class MyRetrievalAdapter implements RetrievalAdapter { async retrieve(query: string, k = 10): Promise { @@ -30,7 +30,7 @@ class MyRetrievalAdapter implements RetrievalAdapter { Replace hand-rolled precision/recall with standard primitives: ```typescript -import { precisionAtK, recallAtK } from "@synapt/eval"; +import { precisionAtK, recallAtK } from "@synapt-dev/eval"; const p5 = precisionAtK(retrievedIds, expectedIds, 5); const r10 = recallAtK(retrievedIds, expectedIds, 10); @@ -41,7 +41,7 @@ const r10 = recallAtK(retrievedIds, expectedIds, 10); Replace `if (p5 < 0.5) failures.push(...)` with: ```typescript -import { SuggestionEngine } from "@synapt/eval"; +import { SuggestionEngine } from "@synapt-dev/eval"; const engine = SuggestionEngine.withDefaults(); const suggestions = engine.evaluateAll(results); @@ -53,7 +53,7 @@ const suggestions = engine.evaluateAll(results); Replace `console.log` with structured output: ```typescript -import { composeReportCard, generateMarkdown } from "@synapt/eval"; +import { composeReportCard, generateMarkdown } from "@synapt-dev/eval"; const card = composeReportCard({ results, suggestions }); console.log(generateMarkdown(card)); @@ -75,7 +75,7 @@ Write results to JSON and use the GitHub Action: With adapters, adding generation eval is one new class: ```typescript -import type { GenerationAdapter, GenerationOutput } from "@synapt/eval"; +import type { GenerationAdapter, GenerationOutput } from "@synapt-dev/eval"; class MyGenerationAdapter implements GenerationAdapter { async generate(query: string, context?: unknown[]): Promise { @@ -90,7 +90,7 @@ class MyGenerationAdapter implements GenerationAdapter { Extend the suggestion engine with domain-specific rules: ```typescript -import { SuggestionEngine, suggestionRule, SEVERITY_WARNING } from "@synapt/eval"; +import { SuggestionEngine, suggestionRule, SEVERITY_WARNING } from "@synapt-dev/eval"; const latencyRule = suggestionRule({ name: "high_latency" })((result) => { // Custom rule for your domain diff --git a/ts/examples/migration/after.ts b/ts/examples/migration/after.ts index 01e9584..a936f08 100644 --- a/ts/examples/migration/after.ts +++ b/ts/examples/migration/after.ts @@ -1,5 +1,5 @@ /** - * AFTER: Composable eval with @synapt/eval. + * AFTER: Composable eval with @synapt-dev/eval. * * Benefits: * - Adapter pattern decouples eval from backend implementation diff --git a/ts/examples/migration/before.ts b/ts/examples/migration/before.ts index 7a0d8f1..7742b00 100644 --- a/ts/examples/migration/before.ts +++ b/ts/examples/migration/before.ts @@ -1,5 +1,5 @@ /** - * BEFORE: Monolithic eval pattern (typical pre-@synapt/eval approach). + * BEFORE: Monolithic eval pattern (typical pre-@synapt-dev/eval approach). * * Problems with this pattern: * - Scoring, assertion, and reporting are interleaved diff --git a/ts/package-lock.json b/ts/package-lock.json index 9529865..a187347 100644 --- a/ts/package-lock.json +++ b/ts/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@synapt/eval", + "name": "@synapt-dev/eval", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@synapt/eval", + "name": "@synapt-dev/eval", "version": "0.1.0", "license": "MIT", "devDependencies": { diff --git a/ts/package.json b/ts/package.json index 8ced107..470b1c0 100644 --- a/ts/package.json +++ b/ts/package.json @@ -1,5 +1,5 @@ { - "name": "@synapt/eval", + "name": "@synapt-dev/eval", "version": "0.1.0", "description": "Domain-agnostic eval framework for AI applications", "type": "module", From feba8f23a538cf2c635ae72dcf59c7f564d3506a Mon Sep 17 00:00:00 2001 From: Layne Penney Date: Sat, 18 Jul 2026 10:59:19 -0500 Subject: [PATCH 3/3] fix: address Atlas reviewer-2 findings on publish workflows Three blockers from Atlas's round-2 review at d0fa962, each fixed and mutation-verified against his exact named mutant: 1. npm OIDC trusted publishing requires package.json's repository.url to exactly match the GitHub repo (npm trusted-publisher docs). Added the repository field (git/synapt-dev/eval.git, directory: ts) -- the dry-run couldn't exercise OIDC so this passed review-round-1 despite being a real publish-time failure. Mutation-verified: removing the field fails a repository-metadata check. 2. The PyPI workflow ran build machinery (checkout, pip install, python -m build) inside the same job holding id-token: write, exposing the OIDC-token-granting permission to project/build/ transitive-dependency code -- exactly what pypa's trusted-publishing guidance says not to do. Split into an unprivileged build job (contents: read only, uploads dist/ as an artifact) and a minimal publish job (id-token: write only, downloads the artifact, runs only the pypa publish action). Pinned actions/upload-artifact@v4.6.2 and actions/download-artifact@v4.3.0 to their real tag SHAs (queried via git ls-remote against the actual actions repos, not guessed). Mutation-verified: re-adding a build command to the publish job fails a job-structure check; workflow-level permissions no longer grant id-token to any job by default. 3. The npm workflow's SBOM generation and upload steps shared the same already_published guard as npm publish itself. If publish succeeds but SBOM generation/upload then fails, a retry sees already_published=true and skips both SBOM steps too -- job goes green, release ships without its SBOM. Decoupled: SBOM generation now runs unconditionally (computing it doesn't depend on who published), upload still gates on the release event but not on already_published (--clobber already makes the upload retry-safe). Mutation-verified: reapplying the already_published condition to the SBOM generation step fails a retry-fixture check. Also adopted Opus's round-1 non-blocking suggestion (skip-existing: true on the PyPI publish step) for symmetry with npm's already- published handling -- Atlas's round-2 review confirmed it mechanically valid and left it as an author's-call addition. Re-verified end-to-end after all changes: both YAML files parse, both JSON files parse, python -m build + twine check PASSED, npm ci + typecheck clean, npm publish --dry-run confirms unchanged tarball shape (38 files, @synapt-dev/eval@0.1.0) with the new repository field present. --- .github/workflows/publish-npm.yml | 3 +-- .github/workflows/publish-pypi.yml | 22 +++++++++++++++++++--- ts/package.json | 5 +++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 675b575..d28bdf4 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -47,11 +47,10 @@ jobs: if: steps.check.outputs.already_published != 'true' - name: Generate SBOM - if: steps.check.outputs.already_published != 'true' run: npm sbom --omit=dev --sbom-format cyclonedx > sbom.cdx.json - name: Upload SBOM to release - if: steps.check.outputs.already_published != 'true' && github.event_name == 'release' + if: github.event_name == 'release' env: GH_TOKEN: ${{ github.token }} run: gh release upload ${{ github.event.release.tag_name }} sbom.cdx.json --clobber diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index fcd69d7..04d1397 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -6,12 +6,10 @@ on: permissions: contents: read - id-token: write jobs: - publish: + build: runs-on: ubuntu-latest - environment: pypi steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -23,6 +21,24 @@ jobs: - run: python -m build + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: pypi-dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: pypi-dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: packages-dir: dist/ + skip-existing: true diff --git a/ts/package.json b/ts/package.json index 470b1c0..dd065ed 100644 --- a/ts/package.json +++ b/ts/package.json @@ -2,6 +2,11 @@ "name": "@synapt-dev/eval", "version": "0.1.0", "description": "Domain-agnostic eval framework for AI applications", + "repository": { + "type": "git", + "url": "https://github.com/synapt-dev/eval.git", + "directory": "ts" + }, "type": "module", "exports": { ".": "./src/mod.ts"