diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27ea6bfb..44332db3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - name: Validate GitHub Actions workflows uses: docker://rhysd/actionlint:1.7.11 with: - args: .github/workflows/ci.yml .github/workflows/release.yml + args: .github/workflows/ci.yml .github/workflows/release.yml .github/workflows/turbo.yml - uses: pnpm/action-setup@v6 - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/turbo.yml b/.github/workflows/turbo.yml new file mode 100644 index 00000000..5d051e08 --- /dev/null +++ b/.github/workflows/turbo.yml @@ -0,0 +1,94 @@ +name: Turbo + +# Additive Turborepo lane. The required gates stay in ci.yml; this workflow +# proves the turbo.json task graph end to end on every change: +# - pull requests run the AFFECTED graph only (merge-base diff) +# - pushes to main run the FULL graph +# Remote caching activates as soon as the two secrets below exist; without +# them turbo falls back to the local cache and every step still works. + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: turbo-${{ github.ref }} + cancel-in-progress: true + +env: + # Vercel Remote Cache. Absent secrets degrade to local-only caching. + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + # Artifact signature verification (remoteCache.signature in turbo.json). + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} + +jobs: + graph: + name: turbo ${{ github.event_name == 'pull_request' && '--affected' || '(full)' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + # --affected diffs against the merge base; depth 1 hides it. + fetch-depth: 0 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v4 + with: + node-version: 22.19.0 + cache: pnpm + - run: pnpm install --frozen-lockfile --ignore-scripts + # Same jj setup as ci.yml: the jj and host-contract suites exercise a + # real jj binary and refuse to skip silently. + - name: Install jj + uses: taiki-e/install-action@v2 + with: + tool: jj-cli@0.39.0 + - name: Initialize colocated jj repository + run: jj git init --colocate + # --affected diffs against TURBO_SCM_BASE when set. On pull requests, pin + # it to the base SHA so the affected graph is exactly this PR's changes + # no matter what the checkout's merge history looks like. Pushes to main + # run the full graph, so no base is needed there. + - name: Pin the affected base to the PR base + if: github.event_name == 'pull_request' + run: echo "TURBO_SCM_BASE=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_ENV" + # The graph itself is a deliverable: what turbo THINKS must run for this + # diff, as JSON, before anything executes. + - name: Record the affected graph + if: github.event_name == 'pull_request' + run: pnpm exec turbo run build check lint circular test --affected --dry=json > affected-graph.json + - name: Build + run: pnpm exec turbo run build ${{ github.event_name == 'pull_request' && '--affected' || '' }} --summarize + - name: Check + lint + circular + run: pnpm exec turbo run check lint circular ${{ github.event_name == 'pull_request' && '--affected' || '' }} --summarize + # The heavy vitest suites carry finite 30s per-test budgets that host + # parallelism starves (see the smthrs-shadow job in ci.yml), so the test + # fan-out stays at concurrency 2. --continue runs every affected suite + # even when a sibling fails, so one red package never hides another. + - name: Test + run: pnpm exec turbo run test ${{ github.event_name == 'pull_request' && '--affected' || '' }} --concurrency=2 --continue --summarize + # Boundaries is experimental and currently reports 210 pre-existing + # findings on this repository (config-file imports that leave the + # package, undeclared test dependencies, one circular package pair). + # Advisory until those are triaged; the engine-tier rule it encodes is + # enforced below by the query-based assertion, which is green. + - name: Boundaries (advisory) + continue-on-error: true + run: pnpm exec turbo boundaries + - name: Engine boundary assertion (turbo query) + run: node scripts/turbo-engine-boundary.mjs + # Root gates are not affected-filterable (//# tasks read the whole + # workspace), so they always run. + - name: Root gates + run: pnpm exec turbo run '//#browser' '//#test:scripts' '//#docs' --summarize + - name: Upload run summaries + if: always() + uses: actions/upload-artifact@v4 + with: + name: turbo-run-summaries + path: | + .turbo/runs/ + affected-graph.json + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index e058eb21..5f588a3b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ dist/ +.turbo/ +/out/ /flows .smithers/ .artifacts/ diff --git a/apps/server/turbo.json b/apps/server/turbo.json new file mode 100644 index 00000000..bdf3a5ab --- /dev/null +++ b/apps/server/turbo.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://turborepo.dev/schema.json", + "extends": ["//"], + "tasks": { + "deploy": { + "dependsOn": ["smithers-ui#build"], + "inputs": ["src/**", "scripts/**", "wrangler.jsonc", "tsconfig.json", "package.json"], + "outputs": ["deploy-receipts/**"], + "cache": false, + "interactive": true, + "passThroughEnv": ["CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID"] + }, + "deploy:dry": { + "dependsOn": ["smithers-ui#build"], + "inputs": ["src/**", "scripts/**", "wrangler.jsonc", "tsconfig.json", "package.json"], + "outputs": ["deploy-receipts/**"], + "cache": false + }, + "deploy:dry-run": { + "dependsOn": ["smithers-ui#build"], + "inputs": ["src/**", "scripts/**", "wrangler.jsonc", "tsconfig.json", "package.json"], + "outputs": [], + "cache": false + }, + "test": { + "inputs": ["src/**", "scripts/**", "tsconfig.json", "package.json"], + "outputs": [] + }, + "dev": { + "dependsOn": ["smithers-ui#build"], + "cache": false, + "persistent": true, + "interactive": true + }, + "serve:local": { + "dependsOn": ["smithers-ui#build"], + "cache": false, + "persistent": true, + "interactive": true + }, + "seed:allowlist": { + "cache": false, + "outputs": [], + "passThroughEnv": ["CLOUDFLARE_API_TOKEN", "CLOUDFLARE_ACCOUNT_ID"] + } + } +} diff --git a/apps/shared/turbo.json b/apps/shared/turbo.json new file mode 100644 index 00000000..8dae7b96 --- /dev/null +++ b/apps/shared/turbo.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://turborepo.dev/schema.json", + "extends": ["//"], + "tasks": { + "test": { + "inputs": ["src/**", "tsconfig.json", "package.json"], + "outputs": [] + } + } +} diff --git a/apps/tui/turbo.json b/apps/tui/turbo.json new file mode 100644 index 00000000..86722a64 --- /dev/null +++ b/apps/tui/turbo.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://turborepo.dev/schema.json", + "extends": ["//"], + "tasks": { + "test": { + "inputs": ["src/**", "scripts/**", "tsconfig.json", "package.json"], + "outputs": [] + }, + "smoke": { + "cache": false, + "interactive": true, + "outputs": [] + } + } +} diff --git a/apps/ui/turbo.json b/apps/ui/turbo.json new file mode 100644 index 00000000..b935b9f7 --- /dev/null +++ b/apps/ui/turbo.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://turborepo.dev/schema.json", + "extends": ["//"], + "tasks": { + "build": { + "inputs": [ + "src/**", + "vite.config.ts", + "postcss.config.js", + "tailwind.config.js", + "electrobun.config.ts", + "tsconfig.json", + "package.json" + ], + "outputs": ["dist/**"], + "env": ["SMITHERS_CHAT_URL", "SMITHERS_CHAT_ORIGIN"] + }, + "test": { + "inputs": ["src/**", "scripts/**", "tsconfig.json", "package.json"], + "outputs": [] + }, + "dev": { + "cache": false, + "persistent": true, + "interactive": true, + "passThroughEnv": ["SMITHERS_DEV_UPSTREAM", "SMITHERS_CHAT_URL", "SMITHERS_CHAT_ORIGIN", "SMITHERS_APP_URL"] + }, + "dev:hmr": { + "cache": false, + "persistent": true, + "interactive": true, + "passThroughEnv": ["SMITHERS_DEV_UPSTREAM", "SMITHERS_CHAT_URL", "SMITHERS_CHAT_ORIGIN"] + }, + "hmr": { + "cache": false, + "persistent": true, + "interactive": true, + "passThroughEnv": ["SMITHERS_DEV_UPSTREAM", "SMITHERS_CHAT_URL", "SMITHERS_CHAT_ORIGIN"] + }, + "web": { + "cache": false, + "persistent": true, + "interactive": true, + "passThroughEnv": ["SMITHERS_DEV_UPSTREAM", "SMITHERS_CHAT_URL", "SMITHERS_CHAT_ORIGIN"] + }, + "start": { + "dependsOn": ["build"], + "cache": false, + "persistent": true, + "interactive": true + }, + "start:canary": { + "cache": false, + "persistent": true, + "interactive": true, + "passThroughEnv": ["SMITHERS_APP_URL"] + }, + "build:canary": { + "dependsOn": ["build"], + "outputs": [], + "cache": false, + "passThroughEnv": ["SMITHERS_APP_URL"] + }, + "serve:local": { + "cache": false, + "persistent": true, + "interactive": true + }, + "test:e2e:web": { + "cache": false, + "outputs": [] + }, + "test:e2e:web:context": { + "cache": false, + "outputs": [] + }, + "test:e2e:web:shell": { + "cache": false, + "outputs": [] + }, + "test:e2e:worker": { + "cache": false, + "outputs": [] + }, + "checklist": { + "cache": false, + "outputs": [] + } + } +} diff --git a/docs/build-systems/turborepo.md b/docs/build-systems/turborepo.md new file mode 100644 index 00000000..a0006d7b --- /dev/null +++ b/docs/build-systems/turborepo.md @@ -0,0 +1,341 @@ +# Turborepo for flows + +What it looks like to automate this repository with Turborepo 2.10.11, and how +that compares to the in-repo build system (`BUILD.ts`, `@smthrs/targets`, +`@smthrs/build`, `@smthrs/build-cli`). Every claim below was executed in this +worktree; the numbers are from a 16-core aarch64 macOS host with 29 GB free +memory. + +## 1. What was added, file by file + +| File | Responsibility | +| --- | --- | +| `turbo.json` | The whole task graph. Tasks `build`, `check`, `lint`, `format`, `circular`, `test`, `coverage`, `typecheck`, `dev`; root gates as `//#browser`, `//#test:scripts`, `//#docs`, `//#cargo:fmt`, `//#cargo:clippy`, `//#cargo:test`; `envMode: "strict"`; `globalEnv` / `globalDependencies`; `remoteCache` with `signature: true`; the `boundaries` tag rules; `ui: "tui"`. | +| `packages/*/turbo.json` (45) | Package Configurations. Each `extends: ["//"]` and carries one Boundaries tag (`engine` / `agent` / `tooling`) mapped from the manifest's `smthrs.group`. Three lines each; no task overrides. | +| `packages/jj/turbo.json` | Adds `build:wasm`: inputs reach out of the package to `../../crates/flows-jj/**`, `../../Cargo.lock`, `../../rust-toolchain.toml`, `../../vendor/jj/**`; output is `wasm/flows_jj.wasm`; `cache: false` because the artifact is a reproducibility contract (below). | +| `packages/chain/turbo.json` | Adds `prompts` (regenerates the checked-in `src/internal/prompts.ts` from `prompts/*.mdx`; a mutation task, so `cache: false`). | +| `packages/evals/turbo.json` | Adds `score-gate`. | +| `packages/build/infra/turbo.json` | `check`/`test` with this package's real input set (`worker/**`, `alchemy.run.ts`), and `deploy` / `deploy:dev` as uncached interactive tasks whose Cloudflare and cache-token secrets ride `passThroughEnv`. | +| `apps/ui/turbo.json` | `build` outputs `dist/**` (vite root is `src/mainview`); `dev`, `hmr`, `web`, `start`, `serve:local` are `persistent: true`, `interactive: true`, uncached; e2e and checklist scripts are uncached one-shots. | +| `apps/server/turbo.json` | `deploy`, `deploy:dry`, `deploy:dry-run`, `dev`, `serve:local` all `dependsOn: ["smithers-ui#build"]` — the wrangler assets directory is `../ui/dist`, so the SPA build is a real edge in the graph. Deploys are uncached; `CLOUDFLARE_API_TOKEN` / `CLOUDFLARE_ACCOUNT_ID` are `passThroughEnv`. | +| `apps/shared/turbo.json`, `apps/tui/turbo.json`, `examples/turbo.json` | Narrow `test`/`check` inputs to what those suites read. | +| `package.json` | Adds the `turbo@2.10.11` devDependency and five root scripts the `//#` tasks invoke: `docs`, `test:scripts`, `cargo:fmt`, `cargo:clippy`, `cargo:test`. No existing script changed. | +| `pnpm-lock.yaml` | Regenerated by `pnpm install` with the one new dependency. | +| `.gitignore` | Adds `.turbo/` and `/out/`. | +| `.github/workflows/turbo.yml` | The CI lane: `--affected` on pull requests, full graph on main, `--summarize` + run-summary artifacts, advisory `turbo boundaries`, the query-based engine-boundary assertion, and the root gates. | +| `.github/workflows/ci.yml` | One line: `turbo.yml` added to the actionlint args. | +| `scripts/turbo-engine-boundary.mjs` | The release-train rule (engine never depends on agent/tooling) asserted through `turbo query`'s GraphQL package graph. | +| `packages/flows/test/vitestCoverageIsolation.test.ts` | The issue-#166 pin of the root script roster, widened for the five new scripts — the review path the pin's own comment prescribes. | +| `docs/build-systems/turborepo.md` | This document. | + +## 2. How Turborepo models what `BUILD.ts` models + +### Target definition + +`BUILD.ts`: a target is a typed TypeScript value with declared inputs, +outputs, toolchain, and secrets. A label addresses it (`//packages/engine:lib`). + +Turborepo: a target is a `(package, script)` pair. The task definition in +`turbo.json` carries inputs, outputs, env, and dependency edges; the command +itself lives in the package's `package.json` scripts and is opaque to Turbo — +it schedules `pnpm run