From 8c337631324fff1c261dd13755d0416bb43a9a18 Mon Sep 17 00:00:00 2001 From: psmyrdek Date: Thu, 13 Aug 2026 15:32:12 +0200 Subject: [PATCH 1/3] feat: add bench-kit command (init + update stub) '10x bench-kit init [dir]' creates a benchmark instance from the przeprogramowani/10x-bench-kit template: preflight (git required, container engine advisory), shallow clone at a tag, materialization without git history, fresh git init + initial commit, and an instance manifest in .bench-kit/instance.json. Re-running init on an existing instance repairs missing template files without touching company content. 'bench-kit update' is a not_implemented stub. CAC has no nested command groups, so the command follows the auth precedent: one 'bench-kit ' command dispatching internally. Side effects are injectable (BenchKitDeps) so tests exercise the full flow against fixture trees without network or real git. Co-Authored-By: Claude Fable 5 --- src/commands/bench-kit.ts | 342 ++++++++++++++++++++++++++++++++ src/index.ts | 2 + tests/bench-kit-command.test.ts | 274 +++++++++++++++++++++++++ 3 files changed, 618 insertions(+) create mode 100644 src/commands/bench-kit.ts create mode 100644 tests/bench-kit-command.test.ts diff --git a/src/commands/bench-kit.ts b/src/commands/bench-kit.ts new file mode 100644 index 0000000..5ff9768 --- /dev/null +++ b/src/commands/bench-kit.ts @@ -0,0 +1,342 @@ +/** + * 10x bench-kit — installer/updater for benchmark instances. + * + * `bench-kit init` is deliberately a *thin, deterministic* installer: it + * knows nothing about the template's internal structure beyond the + * `.bench-kit/` marker directory. Everything judgment-based (rubrics, + * tasks, stack-specific images) happens later, via agent skills inside + * the instance — never here. + * + * `bench-kit update` (zone-aware template upgrade) lands in a later phase. + */ + +import { spawn } from "node:child_process"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import type { CAC } from "cac"; +import { + ExitCodes, + type GlobalFlags, + type OutputContext, + exitNotImplemented, + output, + outputError, + resolveContext, + verbose, +} from "../lib/output"; + +export const TEMPLATE_REPO_URL = "https://github.com/przeprogramowani/10x-bench-kit"; + +/** Instance manifest written next to the template's VERSION file. */ +export interface InstanceManifest { + templateVersion: string; + templateRef: string; + templateSource: string; + initializedAt: string; +} + +interface BenchKitFlags extends GlobalFlags { + templateVersion?: string; + yes?: boolean; +} + +/** + * Side-effectful collaborators, injectable for tests (DI over module + * mocking, per repo convention). The default implementation shells out + * to git; tests substitute a fake that materializes a fixture tree. + */ +export interface BenchKitDeps { + /** Resolves true when `cmd` can be spawned (used for preflight). */ + toolAvailable(cmd: string): Promise; + /** Clones the template at `ref` (null = default branch) into `destDir`. */ + cloneTemplate(ref: string | null, destDir: string): Promise<{ ok: boolean; error: string }>; + /** Runs git with `args` inside `cwd` (fresh `git init` + first commit). */ + runGit(args: string[], cwd: string): Promise<{ ok: boolean; error: string }>; + now(): Date; +} + +// CAC has no nested command groups (a name with a space never matches), so +// bench-kit follows the `auth` precedent: one command dispatching on an +// action argument. +export function registerBenchKitCommand(cli: CAC): void { + cli + .command("bench-kit [dir]", "Manage a benchmark instance (actions: init, update)") + .option("--template-version ", "Template tag to install (default: latest)") + .option("--yes", "Run non-interactively, accepting defaults") + .example("10x bench-kit init my-benchmark") + .example("10x bench-kit init my-benchmark --template-version v0.1.0") + .action(async (action: string, dir: string | undefined, options: BenchKitFlags) => { + const ctx = resolveContext(options); + if (action === "init") { + await runBenchKitInit(ctx, dir, options); + return; + } + if (action === "update") { + exitNotImplemented("bench-kit update", "a later bench-kit phase", options); + } + outputError( + ctx, + "unknown_action", + `'${action}' is not a bench-kit action.`, + ExitCodes.USAGE, + "Run '10x bench-kit init [dir]' or '10x bench-kit update'.", + ); + }); +} + +export async function runBenchKitInit( + ctx: OutputContext, + dirArg: string | undefined, + options: BenchKitFlags, + deps: BenchKitDeps = defaultDeps, +): Promise { + const targetDir = resolve(dirArg ?? "."); + const requestedRef = normalizeRef(ctx, options.templateVersion); + + await preflight(ctx, deps); + + const existingVersion = readInstanceVersion(targetDir); + const repair = existingVersion !== null; + + if (repair && requestedRef !== null) { + outputError( + ctx, + "version_conflict", + `This directory already holds a benchmark instance on template version ${existingVersion}.`, + ExitCodes.USAGE, + "Run '10x bench-kit update' to change the template version of an existing instance.", + ); + } + if (!repair && existsSync(targetDir) && readdirSync(targetDir).length > 0) { + outputError( + ctx, + "target_not_empty", + `Directory '${targetDir}' is not empty and is not a benchmark instance.`, + ExitCodes.ERROR, + "Run '10x bench-kit init ' with an empty or new directory.", + ); + } + + // Materialize the template into a scratch clone first, so a failed + // download can never leave a half-written instance behind. + const scratch = mkdtempSync(join(tmpdir(), "bench-kit-")); + try { + verbose(ctx, `cloning ${TEMPLATE_REPO_URL} (${requestedRef ?? "latest"}) into ${scratch}`); + const clone = await deps.cloneTemplate(requestedRef, scratch); + if (!clone.ok) { + outputError( + ctx, + "clone_failed", + `Could not download the template from ${TEMPLATE_REPO_URL}.`, + ExitCodes.ERROR, + clone.error + ? `Git said: ${clone.error.trim()}` + : "Check your internet connection and run '10x bench-kit init' again.", + ); + } + + const templateVersion = readTemplateVersion(ctx, scratch); + mkdirSync(targetDir, { recursive: true }); + const copied = materialize(scratch, targetDir, { skipExisting: repair }); + + const manifest: InstanceManifest = { + templateVersion, + templateRef: requestedRef ?? "latest", + templateSource: TEMPLATE_REPO_URL, + initializedAt: deps.now().toISOString(), + }; + writeFileSync( + join(targetDir, ".bench-kit", "instance.json"), + `${JSON.stringify(manifest, null, 2)}\n`, + ); + + let committed = false; + if (!repair) { + committed = await freshGitInit(ctx, deps, targetDir, templateVersion); + } + + const humanLines = repair + ? [ + `Repaired the benchmark instance in '${targetDir}' (template ${templateVersion}).`, + `Restored ${copied} missing file${copied === 1 ? "" : "s"}; your tasks, evaluation pool and config were not touched.`, + ] + : [ + `Created a benchmark instance in '${targetDir}' from template ${templateVersion}.`, + committed + ? "Initialized a fresh git repository with an initial commit." + : "Initialized a fresh git repository (initial commit skipped — commit the files yourself).", + "Next: wire up secrets, then run 'bench validate' before the first run.", + ]; + output(ctx, humanLines.join("\n"), { + dir: targetDir, + mode: repair ? "repair" : "init", + templateVersion, + templateRef: manifest.templateRef, + filesCopied: copied, + gitInitialized: !repair, + committed, + }); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +} + +function normalizeRef(ctx: OutputContext, raw: string | undefined): string | null { + if (raw === undefined) return null; + const ref = raw.trim(); + if (ref === "" || ref === "latest") return null; + if (!/^[A-Za-z0-9._/-]+$/.test(ref)) { + outputError( + ctx, + "invalid_template_version", + `'${raw}' is not a valid template tag.`, + ExitCodes.USAGE, + "Pass a tag name, for example '10x bench-kit init --template-version v0.1.0'.", + ); + } + return ref; +} + +async function preflight(ctx: OutputContext, deps: BenchKitDeps): Promise { + if (!(await deps.toolAvailable("git"))) { + outputError( + ctx, + "preflight_failed", + "Git is required to download the template and initialize the instance.", + ExitCodes.ERROR, + "Install git (https://git-scm.com) and run '10x bench-kit init' again.", + ); + } + // The trial runtime needs a container engine, but the skeleton does not — + // missing Docker/Podman is a warning, not a blocker. + const hasEngine = (await deps.toolAvailable("docker")) || (await deps.toolAvailable("podman")); + if (!hasEngine) { + verbose(ctx, "neither docker nor podman found — benchmark runs will need one later"); + } +} + +/** Returns the instance's template version, or null when `dir` is not an instance. */ +function readInstanceVersion(dir: string): string | null { + const versionFile = join(dir, ".bench-kit", "VERSION"); + if (!existsSync(versionFile)) return null; + return readFileSync(versionFile, "utf8").trim(); +} + +function readTemplateVersion(ctx: OutputContext, cloneDir: string): string { + const versionFile = join(cloneDir, ".bench-kit", "VERSION"); + if (!existsSync(versionFile)) { + outputError( + ctx, + "invalid_template", + "The downloaded template has no .bench-kit/VERSION file.", + ExitCodes.ERROR, + "Pass a valid tag via '10x bench-kit init --template-version '.", + ); + } + return readFileSync(versionFile, "utf8").trim(); +} + +/** + * Copies the clone into the target without git history. In repair mode + * existing files are never overwritten — company content is untouchable. + * Returns the number of files copied. + */ +function materialize( + srcDir: string, + destDir: string, + opts: { skipExisting: boolean }, +): number { + let copied = 0; + const walk = (rel: string): void => { + for (const entry of readdirSync(join(srcDir, rel), { withFileTypes: true })) { + if (rel === "" && entry.name === ".git") continue; + const relPath = join(rel, entry.name); + const from = join(srcDir, relPath); + const to = join(destDir, relPath); + if (entry.isDirectory()) { + mkdirSync(to, { recursive: true }); + walk(relPath); + continue; + } + if (opts.skipExisting && existsSync(to)) continue; + cpSync(from, to); + copied++; + } + }; + walk(""); + return copied; +} + +/** Fresh `git init` + first commit. A failed commit degrades to a warning. */ +async function freshGitInit( + ctx: OutputContext, + deps: BenchKitDeps, + dir: string, + templateVersion: string, +): Promise { + const init = await deps.runGit(["init"], dir); + if (!init.ok) { + outputError( + ctx, + "git_init_failed", + "Could not initialize a git repository in the instance directory.", + ExitCodes.ERROR, + init.error ? `Git said: ${init.error.trim()}` : undefined, + ); + } + const add = await deps.runGit(["add", "-A"], dir); + const commit = add.ok + ? await deps.runGit( + ["commit", "-m", `chore: bench-kit init (template ${templateVersion})`], + dir, + ) + : add; + if (!commit.ok) { + verbose(ctx, `initial commit failed (${commit.error.trim()}) — files are staged, commit manually`); + return false; + } + return true; +} + +// --------------------------------------------------------------------------- +// Default (real) side effects +// --------------------------------------------------------------------------- + +function run(cmd: string, args: string[], cwd?: string): Promise<{ ok: boolean; error: string }> { + return new Promise((resolvePromise) => { + const child = spawn(cmd, args, { cwd, stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.on("error", (err) => resolvePromise({ ok: false, error: err.message })); + child.on("close", (code) => resolvePromise({ ok: code === 0, error: stderr })); + }); +} + +const defaultDeps: BenchKitDeps = { + async toolAvailable(cmd) { + const result = await run(cmd, ["--version"]); + return result.ok; + }, + cloneTemplate(ref, destDir) { + const args = ["clone", "--depth", "1"]; + if (ref !== null) args.push("--branch", ref); + args.push(TEMPLATE_REPO_URL, destDir); + return run("git", args); + }, + runGit(args, cwd) { + return run("git", args, cwd); + }, + now: () => new Date(), +}; diff --git a/src/index.ts b/src/index.ts index 1ba27f2..e4d2426 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import cac from "cac"; import packageJson from "../package.json" with { type: "json" }; import { registerAuthCommand } from "./commands/auth"; import { registerBenchCommand } from "./commands/bench"; +import { registerBenchKitCommand } from "./commands/bench-kit"; import { registerDoctorCommand } from "./commands/doctor"; import { registerGetCommand } from "./commands/get"; import { registerListCommand } from "./commands/list"; @@ -19,6 +20,7 @@ registerListCommand(cli); registerSyncCommand(cli); registerDoctorCommand(cli); registerBenchCommand(cli); +registerBenchKitCommand(cli); cli.help(); cli.version(packageJson.version); diff --git a/tests/bench-kit-command.test.ts b/tests/bench-kit-command.test.ts new file mode 100644 index 0000000..1b505e3 --- /dev/null +++ b/tests/bench-kit-command.test.ts @@ -0,0 +1,274 @@ +/** + * 10x bench-kit — command-level behavior. + * + * Uses dependency injection (BenchKitDeps) instead of module mocking: + * cloneTemplate materializes a fixture template tree, runGit records calls. + * All filesystem work happens in per-test temp directories. + */ + +import { afterEach, describe, expect, it } from "bun:test"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import cac from "cac"; +import { + type BenchKitDeps, + registerBenchKitCommand, + runBenchKitInit, +} from "../src/commands/bench-kit"; +import type { OutputContext } from "../src/lib/output"; + +interface CaptureResult { + stdout: string; + stderr: string; + exitCode?: number; +} + +function captureStreams(fn: () => Promise): Promise { + return new Promise((resolve) => { + const realExit = process.exit; + const realStdoutWrite = process.stdout.write.bind(process.stdout); + const realStderrWrite = process.stderr.write.bind(process.stderr); + let stdout = ""; + let stderr = ""; + process.stdout.write = ((chunk: string | Uint8Array) => { + stdout += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString(); + return true; + }) as typeof process.stdout.write; + process.stderr.write = ((chunk: string | Uint8Array) => { + stderr += typeof chunk === "string" ? chunk : Buffer.from(chunk).toString(); + return true; + }) as typeof process.stderr.write; + process.exit = ((code?: number) => { + throw Object.assign(new Error("__exit__"), { __exitCode: code }); + }) as typeof process.exit; + + fn() + .then(() => resolve({ stdout, stderr })) + .catch((err: unknown) => { + if (err && typeof err === "object" && "__exitCode" in err) { + resolve({ + stdout, + stderr, + exitCode: (err as { __exitCode: number }).__exitCode, + }); + } else { + resolve({ + stdout, + stderr: `${stderr}\n[uncaught: ${err instanceof Error ? err.message : String(err)}]`, + }); + } + }) + .finally(() => { + process.stdout.write = realStdoutWrite; + process.stderr.write = realStderrWrite; + process.exit = realExit; + }); + }); +} + +const JSON_CTX: OutputContext = { json: true, verbose: false }; + +const tempDirs: string[] = []; + +function tempDir(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + while (tempDirs.length > 0) { + rmSync(tempDirs.pop()!, { recursive: true, force: true }); + } +}); + +/** Builds a minimal template fixture (with a fake .git to prove it's stripped). */ +function buildTemplateFixture(version = "0.1.0"): string { + const dir = tempDir("bench-kit-template-"); + mkdirSync(join(dir, ".git"), { recursive: true }); + writeFileSync(join(dir, ".git", "HEAD"), "ref: refs/heads/main\n"); + mkdirSync(join(dir, ".bench-kit"), { recursive: true }); + writeFileSync(join(dir, ".bench-kit", "VERSION"), `${version}\n`); + mkdirSync(join(dir, "tasks", "demo"), { recursive: true }); + writeFileSync(join(dir, "tasks", "demo", "prompt.md"), "demo prompt\n"); + writeFileSync(join(dir, "bench.config.yaml"), "base_repos: []\n"); + return dir; +} + +interface FakeDepsResult { + deps: BenchKitDeps; + gitCalls: string[][]; +} + +function fakeDeps(templateDir: string, overrides: Partial = {}): FakeDepsResult { + const gitCalls: string[][] = []; + const deps: BenchKitDeps = { + toolAvailable: () => Promise.resolve(true), + cloneTemplate: (_ref, destDir) => { + cpSync(templateDir, destDir, { recursive: true }); + return Promise.resolve({ ok: true, error: "" }); + }, + runGit: (args, _cwd) => { + gitCalls.push(args); + return Promise.resolve({ ok: true, error: "" }); + }, + now: () => new Date("2026-08-13T12:00:00.000Z"), + ...overrides, + }; + return { deps, gitCalls }; +} + +function parseEnvelope(stdout: string): { status: string; data?: any; error?: any } { + return JSON.parse(stdout.trim()); +} + +describe("10x bench-kit init", () => { + it("materializes the template without git history and inits a fresh repo", async () => { + const template = buildTemplateFixture(); + const target = join(tempDir("bench-kit-target-"), "instance"); + const { deps, gitCalls } = fakeDeps(template); + + const result = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, {}, deps), + ); + + expect(result.exitCode).toBeUndefined(); + expect(existsSync(join(target, ".bench-kit", "VERSION"))).toBe(true); + expect(existsSync(join(target, "tasks", "demo", "prompt.md"))).toBe(true); + expect(existsSync(join(target, ".git", "HEAD"))).toBe(false); + + const manifest = JSON.parse(readFileSync(join(target, ".bench-kit", "instance.json"), "utf8")); + expect(manifest.templateVersion).toBe("0.1.0"); + expect(manifest.templateRef).toBe("latest"); + expect(manifest.initializedAt).toBe("2026-08-13T12:00:00.000Z"); + + expect(gitCalls[0]).toEqual(["init"]); + expect(gitCalls[1]).toEqual(["add", "-A"]); + expect(gitCalls[2]?.[0]).toBe("commit"); + + const envelope = parseEnvelope(result.stdout); + expect(envelope.status).toBe("ok"); + expect(envelope.data.mode).toBe("init"); + expect(envelope.data.committed).toBe(true); + }); + + it("refuses a non-empty directory that is not an instance", async () => { + const template = buildTemplateFixture(); + const target = tempDir("bench-kit-target-"); + writeFileSync(join(target, "unrelated.txt"), "not an instance\n"); + const { deps, gitCalls } = fakeDeps(template); + + const result = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, {}, deps), + ); + + expect(result.exitCode).toBe(1); + const envelope = parseEnvelope(result.stdout); + expect(envelope.status).toBe("error"); + expect(envelope.error.code).toBe("target_not_empty"); + expect(gitCalls.length).toBe(0); + }); + + it("repairs an existing instance without touching company content", async () => { + const template = buildTemplateFixture(); + const target = tempDir("bench-kit-target-"); + // Existing instance: VERSION present, company file edited, template file missing. + mkdirSync(join(target, ".bench-kit"), { recursive: true }); + writeFileSync(join(target, ".bench-kit", "VERSION"), "0.1.0\n"); + writeFileSync(join(target, "bench.config.yaml"), "base_repos: [edited by company]\n"); + const { deps, gitCalls } = fakeDeps(template); + + const result = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, {}, deps), + ); + + expect(result.exitCode).toBeUndefined(); + // Missing template file restored… + expect(existsSync(join(target, "tasks", "demo", "prompt.md"))).toBe(true); + // …company content untouched… + expect(readFileSync(join(target, "bench.config.yaml"), "utf8")).toContain("edited by company"); + // …and no fresh git init in repair mode. + expect(gitCalls.length).toBe(0); + + const envelope = parseEnvelope(result.stdout); + expect(envelope.data.mode).toBe("repair"); + }); + + it("rejects --template-version on an existing instance, pointing to update", async () => { + const template = buildTemplateFixture(); + const target = tempDir("bench-kit-target-"); + mkdirSync(join(target, ".bench-kit"), { recursive: true }); + writeFileSync(join(target, ".bench-kit", "VERSION"), "0.1.0\n"); + const { deps } = fakeDeps(template); + + const result = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, { templateVersion: "v0.2.0" }, deps), + ); + + expect(result.exitCode).toBe(2); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("version_conflict"); + expect(envelope.error.hint).toContain("10x bench-kit update"); + }); + + it("fails preflight when git is missing", async () => { + const template = buildTemplateFixture(); + const target = join(tempDir("bench-kit-target-"), "instance"); + const { deps } = fakeDeps(template, { + toolAvailable: (cmd) => Promise.resolve(cmd !== "git"), + }); + + const result = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, {}, deps), + ); + + expect(result.exitCode).toBe(1); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("preflight_failed"); + }); + + it("surfaces clone failures without leaving a half-written instance", async () => { + const template = buildTemplateFixture(); + const target = join(tempDir("bench-kit-target-"), "instance"); + const { deps } = fakeDeps(template, { + cloneTemplate: () => Promise.resolve({ ok: false, error: "fatal: repository not found" }), + }); + + const result = await captureStreams(() => + runBenchKitInit(JSON_CTX, target, {}, deps), + ); + + expect(result.exitCode).toBe(1); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("clone_failed"); + expect(existsSync(target)).toBe(false); + }); +}); + +describe("10x bench-kit dispatch", () => { + async function runCli(argv: string[]): Promise { + return captureStreams(async () => { + const cli = cac("10x"); + cli.option("--json", "Output as JSON (auto-detected when piped)"); + cli.option("--verbose", "Show detailed output on stderr"); + registerBenchKitCommand(cli); + cli.parse(["bun", "10x", ...argv], { run: false }); + await cli.runMatchedCommand(); + }); + } + + it("routes 'update' to the not_implemented stub", async () => { + const result = await runCli(["bench-kit", "update", "--json"]); + expect(result.exitCode).toBe(1); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("not_implemented"); + }); + + it("rejects an unknown action with usage exit code", async () => { + const result = await runCli(["bench-kit", "frobnicate", "--json"]); + expect(result.exitCode).toBe(2); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("unknown_action"); + }); +}); From f0587e98359b9e4e2531cab9de6fd4083b5f4105 Mon Sep 17 00:00:00 2001 From: psmyrdek Date: Thu, 13 Aug 2026 20:16:04 +0200 Subject: [PATCH 2/3] feat: gate experimental commands behind TENX_CLI_EXPERIMENTAL New src/lib/experimental.ts exposes experimentalEnabled() and requireExperimental(), which exits with a stable 'experimental_locked' envelope (exit 4 FORBIDDEN) unless TENX_CLI_EXPERIMENTAL=1 (or 'true') is set. bench-kit stays registered so help remains discoverable and the command never dies silently, but its action is locked before any side effects. This lets the branch merge to master ahead of the feature being student-ready. Co-Authored-By: Claude Fable 5 --- src/commands/bench-kit.ts | 7 +++- src/lib/experimental.ts | 39 ++++++++++++++++++++ tests/bench-kit-command.test.ts | 65 +++++++++++++++++++++++++++------ 3 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 src/lib/experimental.ts diff --git a/src/commands/bench-kit.ts b/src/commands/bench-kit.ts index 5ff9768..b4a7d1e 100644 --- a/src/commands/bench-kit.ts +++ b/src/commands/bench-kit.ts @@ -24,6 +24,7 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import type { CAC } from "cac"; +import { requireExperimental } from "../lib/experimental"; import { ExitCodes, type GlobalFlags, @@ -70,12 +71,16 @@ export interface BenchKitDeps { // action argument. export function registerBenchKitCommand(cli: CAC): void { cli - .command("bench-kit [dir]", "Manage a benchmark instance (actions: init, update)") + .command( + "bench-kit [dir]", + "Manage a benchmark instance (actions: init, update; experimental)", + ) .option("--template-version ", "Template tag to install (default: latest)") .option("--yes", "Run non-interactively, accepting defaults") .example("10x bench-kit init my-benchmark") .example("10x bench-kit init my-benchmark --template-version v0.1.0") .action(async (action: string, dir: string | undefined, options: BenchKitFlags) => { + requireExperimental(`bench-kit ${action}`, options); const ctx = resolveContext(options); if (action === "init") { await runBenchKitInit(ctx, dir, options); diff --git a/src/lib/experimental.ts b/src/lib/experimental.ts new file mode 100644 index 0000000..4aea303 --- /dev/null +++ b/src/lib/experimental.ts @@ -0,0 +1,39 @@ +/** + * Gating for experimental commands. + * + * Commands that should ship to master before they are ready for students + * register normally (so `10x ` never dies silently and help stays + * discoverable) but call `requireExperimental` first: without the opt-in + * env var the action exits with a stable, parseable error envelope. + * + * Opt-in: TENX_CLI_EXPERIMENTAL=1 (or "true"). + */ + +import { ExitCodes, type GlobalFlags, outputError, resolveContext } from "./output"; + +export const EXPERIMENTAL_ENV = "TENX_CLI_EXPERIMENTAL"; + +export function experimentalEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env[EXPERIMENTAL_ENV]; + return value === "1" || value === "true"; +} + +/** + * Exits with `experimental_locked` unless the experimental opt-in is set. + * Call at the top of an action callback, before any side effects. + */ +export function requireExperimental( + command: string, + flags: GlobalFlags, + env: NodeJS.ProcessEnv = process.env, +): void { + if (experimentalEnabled(env)) return; + const ctx = resolveContext(flags); + outputError( + ctx, + "experimental_locked", + `'10x ${command}' is experimental and currently locked.`, + ExitCodes.FORBIDDEN, + `Set ${EXPERIMENTAL_ENV}=1 and run '10x ${command}' again.`, + ); +} diff --git a/tests/bench-kit-command.test.ts b/tests/bench-kit-command.test.ts index 1b505e3..75e48e2 100644 --- a/tests/bench-kit-command.test.ts +++ b/tests/bench-kit-command.test.ts @@ -6,7 +6,7 @@ * All filesystem work happens in per-test temp directories. */ -import { afterEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -16,6 +16,7 @@ import { registerBenchKitCommand, runBenchKitInit, } from "../src/commands/bench-kit"; +import { EXPERIMENTAL_ENV, experimentalEnabled } from "../src/lib/experimental"; import type { OutputContext } from "../src/lib/output"; interface CaptureResult { @@ -246,17 +247,59 @@ describe("10x bench-kit init", () => { }); }); +async function runCli(argv: string[]): Promise { + return captureStreams(async () => { + const cli = cac("10x"); + cli.option("--json", "Output as JSON (auto-detected when piped)"); + cli.option("--verbose", "Show detailed output on stderr"); + registerBenchKitCommand(cli); + cli.parse(["bun", "10x", ...argv], { run: false }); + await cli.runMatchedCommand(); + }); +} + +describe("experimental gate", () => { + const savedEnv = process.env[EXPERIMENTAL_ENV]; + + afterEach(() => { + if (savedEnv === undefined) { + delete process.env[EXPERIMENTAL_ENV]; + } else { + process.env[EXPERIMENTAL_ENV] = savedEnv; + } + }); + + it("is off by default and accepts 1 / true", () => { + expect(experimentalEnabled({})).toBe(false); + expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "0" })).toBe(false); + expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "1" })).toBe(true); + expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "true" })).toBe(true); + }); + + it("locks bench-kit actions before any side effects", async () => { + delete process.env[EXPERIMENTAL_ENV]; + const result = await runCli(["bench-kit", "init", "some-dir", "--json"]); + expect(result.exitCode).toBe(4); + const envelope = parseEnvelope(result.stdout); + expect(envelope.error.code).toBe("experimental_locked"); + expect(envelope.error.hint).toContain(EXPERIMENTAL_ENV); + }); +}); + describe("10x bench-kit dispatch", () => { - async function runCli(argv: string[]): Promise { - return captureStreams(async () => { - const cli = cac("10x"); - cli.option("--json", "Output as JSON (auto-detected when piped)"); - cli.option("--verbose", "Show detailed output on stderr"); - registerBenchKitCommand(cli); - cli.parse(["bun", "10x", ...argv], { run: false }); - await cli.runMatchedCommand(); - }); - } + const savedEnv = process.env[EXPERIMENTAL_ENV]; + + beforeEach(() => { + process.env[EXPERIMENTAL_ENV] = "1"; + }); + + afterEach(() => { + if (savedEnv === undefined) { + delete process.env[EXPERIMENTAL_ENV]; + } else { + process.env[EXPERIMENTAL_ENV] = savedEnv; + } + }); it("routes 'update' to the not_implemented stub", async () => { const result = await runCli(["bench-kit", "update", "--json"]); From f6b1517583a21d4753d4f44071951e2a8d098e2f Mon Sep 17 00:00:00 2001 From: psmyrdek Date: Thu, 13 Aug 2026 20:17:31 +0200 Subject: [PATCH 3/3] feat: hide experimental commands entirely without the opt-in All-or-nothing per review: registerBenchKitCommand returns early when TENX_CLI_EXPERIMENTAL is unset, so bench-kit is absent from --help and behaves like any unknown command instead of appearing half-locked. The requireExperimental gate stays in the action as defense in depth. Co-Authored-By: Claude Fable 5 --- src/commands/bench-kit.ts | 6 +++++- tests/bench-kit-command.test.ts | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/commands/bench-kit.ts b/src/commands/bench-kit.ts index b4a7d1e..377bac3 100644 --- a/src/commands/bench-kit.ts +++ b/src/commands/bench-kit.ts @@ -24,7 +24,7 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import type { CAC } from "cac"; -import { requireExperimental } from "../lib/experimental"; +import { experimentalEnabled, requireExperimental } from "../lib/experimental"; import { ExitCodes, type GlobalFlags, @@ -70,6 +70,10 @@ export interface BenchKitDeps { // bench-kit follows the `auth` precedent: one command dispatching on an // action argument. export function registerBenchKitCommand(cli: CAC): void { + // Experimental commands are all-or-nothing: without the opt-in the + // command is not registered at all — absent from help and behaving like + // any unknown command — instead of showing up half-locked. + if (!experimentalEnabled()) return; cli .command( "bench-kit [dir]", diff --git a/tests/bench-kit-command.test.ts b/tests/bench-kit-command.test.ts index 75e48e2..48103c0 100644 --- a/tests/bench-kit-command.test.ts +++ b/tests/bench-kit-command.test.ts @@ -276,13 +276,23 @@ describe("experimental gate", () => { expect(experimentalEnabled({ [EXPERIMENTAL_ENV]: "true" })).toBe(true); }); - it("locks bench-kit actions before any side effects", async () => { + it("keeps bench-kit fully hidden without the opt-in", async () => { delete process.env[EXPERIMENTAL_ENV]; + const cli = cac("10x"); + registerBenchKitCommand(cli); + expect(cli.commands.map((c) => c.name)).not.toContain("bench-kit"); + + // Invoking it behaves like any unknown command: no output, no error. const result = await runCli(["bench-kit", "init", "some-dir", "--json"]); - expect(result.exitCode).toBe(4); - const envelope = parseEnvelope(result.stdout); - expect(envelope.error.code).toBe("experimental_locked"); - expect(envelope.error.hint).toContain(EXPERIMENTAL_ENV); + expect(result.exitCode).toBeUndefined(); + expect(result.stdout).toBe(""); + }); + + it("registers bench-kit when the opt-in is set", () => { + process.env[EXPERIMENTAL_ENV] = "1"; + const cli = cac("10x"); + registerBenchKitCommand(cli); + expect(cli.commands.map((c) => c.name)).toContain("bench-kit"); }); });