From 769ae8073f56a0ee9de1fcabbd5a23e58e7469fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20R=C3=B8ed?= Date: Sat, 29 Aug 2026 14:13:05 +0200 Subject: [PATCH] bench:compare: interleave the whole-process samples of the two builds The comparison ran control's whole benchmark and then experiment's, so a runner speeding up or slowing down over those minutes showed as a delta on the last cases. The whole-process cases now live in test/bench-process.mjs and the driver samples them itself, alternating control and experiment; the cached cases re-seed before each sample because a cache entry is one slot per source file, whichever build wrote it last. --- README.md | 2 +- scripts/bench-compare.mjs | 32 +++++++++++++-- test/bench-process.mjs | 86 +++++++++++++++++++++++++++++++++++++++ test/validate.bench.mjs | 81 ++++++++---------------------------- 4 files changed, 132 insertions(+), 69 deletions(-) create mode 100644 test/bench-process.mjs diff --git a/README.md b/README.md index 6d5d2ce..3efe086 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ Add the Ember/Glimmer language IDs to your project's `.vscode/settings.json`: ## Benchmarks -`pnpm bench` runs the mitata benchmarks in `test/validate.bench.mjs`: `extractAttrTypeMap` per fixture, and whole `dist/run.js` runs over `examples/` (cold, warm, one cached file, `--no-glint`). `pnpm bench:compare --base main` runs them against a base branch side by side; CI posts that comparison on pull requests labelled `run-bench`. Deltas under 5 % are run-to-run noise and 5–10 % may be; the benchmarks exist to catch order-of-magnitude regressions such as a backend start-up or an uncached per-file cost. +`pnpm bench` runs the mitata benchmarks in `test/validate.bench.mjs`: `extractAttrTypeMap` per fixture, and whole `dist/run.js` runs over `examples/` (cold, warm, one cached file, `--no-glint`; `test/bench-process.mjs`). `pnpm bench:compare --base main` runs them against a base branch: the in-process benchmarks one side after the other in separate processes, the whole-process cases with the two builds' samples interleaved so runner drift lands on both sides alike. CI posts that comparison on pull requests labelled `run-bench`. Deltas under 5 % are run-to-run noise and 5–10 % may be; the benchmarks exist to catch order-of-magnitude regressions such as a backend start-up or an uncached per-file cost. ## Glint integration diff --git a/scripts/bench-compare.mjs b/scripts/bench-compare.mjs index bd2e471..8617310 100644 --- a/scripts/bench-compare.mjs +++ b/scripts/bench-compare.mjs @@ -14,6 +14,8 @@ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { CACHED_CASES, ms, PROCESS_CASES, PROCESS_SAMPLES, sample, seedCache, SUBSET, trial } from '../test/bench-process.mjs'; + const args = process.argv.slice(2); const baseIdx = args.indexOf('--base'); const BASE_BRANCH = baseIdx !== -1 ? args[baseIdx + 1] : 'main'; @@ -71,8 +73,9 @@ try { // does not pick `@types/*` up on its own. run('pnpm exec tsc --types node', { cwd: CONTROL_DIR, stdio: ['inherit', 'pipe', 'inherit'] }); - // One process per side, so neither copy's code layout or optimisation - // state affects the other. Control first, then experiment. + // In-process benchmarks: one process per side, so neither copy's code + // layout or optimisation state affects the other. Control first, then + // experiment. const benchScript = join(ROOT, 'test/validate.bench.mjs'); const hasTaskset = process.platform === 'linux' && spawnSync('which', ['taskset'], { stdio: 'pipe' }).status === 0; @@ -85,7 +88,7 @@ try { for (const [side, dist] of sides) { console.error(`\nRunning benchmarks: ${side} (${dist})\n`); const jsonFile = join(CONTROL_DIR, `${side}.json`); - const nodeArgs = ['--expose-gc', '--max-old-space-size=4096', benchScript, '--dist', dist, '--json', jsonFile]; + const nodeArgs = ['--expose-gc', '--max-old-space-size=4096', benchScript, '--dist', dist, '--json', jsonFile, '--skip-process']; const result = spawnSync(hasTaskset ? 'taskset' : 'node', hasTaskset ? ['-c', '0', 'node', ...nodeArgs] : nodeArgs, { stdio: 'inherit', cwd: ROOT, @@ -98,6 +101,29 @@ try { results[side] = JSON.parse(readFileSync(jsonFile, 'utf8')); } + // Whole-process cases: the samples of the two builds are interleaved + // (control, experiment, control, …) so that the runner slowing down or + // speeding up over the minutes this takes lands on both sides alike — + // run one side after the other and a drift shows up as a regression. + process.env['HVE_NO_CACHE'] = '1'; + process.env['HVE_TS_BACKEND'] ??= 'tsgo'; + console.error(`\nWhole process (min / p50 of ${PROCESS_SAMPLES} interleaved runs, ${SUBSET.length} files)\n`); + for (const [, dist] of sides) seedCache(dist); + for (const [name, runCase] of Object.entries(PROCESS_CASES)) { + const samples = Object.fromEntries(sides.map(([side]) => [side, []])); + for (let i = 0; i < PROCESS_SAMPLES; i++) { + for (const [side, dist] of sides) { + if (CACHED_CASES.has(name)) seedCache(dist); + samples[side].push(sample(runCase, dist)); + } + } + for (const [side] of sides) { + const t = trial(name, samples[side], side); + console.error(` ${name.padEnd(28)} ${side.padEnd(10)} ${ms(t.runs[0].stats.min).padStart(9)} / ${ms(t.runs[0].stats.p50).padStart(9)}`); + results[side].benchmarks.push({ alias: name, runs: t.runs.map((r) => ({ ...r, name })) }); + } + } + // Merge into one file with `(control)` / `(experiment)` runs, the shape // the formatters read. if (process.env.BENCH_JSON_OUTPUT) { diff --git a/test/bench-process.mjs b/test/bench-process.mjs new file mode 100644 index 0000000..e0d19f9 --- /dev/null +++ b/test/bench-process.mjs @@ -0,0 +1,86 @@ +/** + * The whole-process benchmark cases: `dist/run.js` over a fixed subset of + * `examples/` — cold (cache off), warm (all cached), one cached file, and + * `--no-glint`. They catch backend start-up and per-run costs that no single + * in-process call can see, and take seconds each, so they are timed with a + * few samples (min and median) rather than by mitata. + * + * Shared by `test/validate.bench.mjs` (one build) and + * `scripts/bench-compare.mjs`, which runs the samples of the two builds + * interleaved so that runner drift lands on both sides alike. + */ + +import { spawnSync } from 'node:child_process'; +import { readdirSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const ROOT = fileURLToPath(new URL('..', import.meta.url)); +export const PROCESS_SAMPLES = 3; + +// A fixed subset keeps each run short; the costs these cases guard against +// (backend start-up, uncached per-file work) show at any size. +export const SUBSET = [ + 'test/bench/large.gts', + ...readdirSync(resolve(ROOT, 'examples')) + .filter((f) => f.endsWith('.gts')) + .sort() + .slice(0, 20) + .map((f) => `examples/${f}`), +]; +// A small file: this case measures start-up, not template work. +const ONE = [SUBSET[1]]; +const CACHED = { HVE_NO_CACHE: '' }; + +/** One `dist/run.js` run of the build at `dist`. `HVE_NO_CACHE` is inherited unless `env` sets it. */ +export function validate(dist, cliArgs, env = {}) { + const result = spawnSync(process.execPath, [resolve(dist, 'dist/run.js'), ...cliArgs], { + cwd: ROOT, + env: { ...process.env, ...env }, + stdio: ['ignore', 'ignore', 'pipe'], + }); + // Exit 1 means findings (the examples have some); anything else is a crash. + if (result.status !== 0 && result.status !== 1) { + throw new Error(`run.js exited ${result.status}:\n${result.stderr}`); + } +} + +/** Populates the disk cache for the warm cases (entries are keyed by plugin source, so builds do not share them). */ +export function seedCache(dist) { + validate(dist, ['--glint', ...SUBSET], CACHED); +} + +/** + * Cases that read the disk cache. An entry is one slot per source file + * (`lib/cache.ts` `entryPath`), whichever build wrote it last, so when two + * builds alternate on the same fixtures each must re-seed before it samples. + */ +export const CACHED_CASES = new Set(['warm run (all cached)', 'one cached file']); + +export const PROCESS_CASES = { + 'cold run (cache off)': (dist) => validate(dist, ['--glint', ...SUBSET]), + 'warm run (all cached)': (dist) => validate(dist, ['--glint', ...SUBSET], CACHED), + 'one cached file': (dist) => validate(dist, ['--glint', ...ONE], CACHED), + 'no glint': (dist) => validate(dist, ['--no-glint', ...SUBSET]), +}; + +/** Nanoseconds one run of `runCase` took. */ +export function sample(runCase, dist) { + const t = process.hrtime.bigint(); + runCase(dist); + return Number(process.hrtime.bigint() - t); +} + +/** Same shape as a mitata trial's stats, from a few samples. */ +export function stats(samples) { + const sorted = [...samples].sort((a, b) => a - b); + const p50 = sorted[Math.floor(sorted.length / 2)]; + return { avg: samples.reduce((a, b) => a + b, 0) / samples.length, min: sorted[0], max: sorted.at(-1), p50, p75: p50, p99: sorted.at(-1), samples: sorted }; +} + +export const ms = (ns) => `${(ns / 1e6).toFixed(0)} ms`; + +/** A trial in the mitata JSON shape, so the formatters read both kinds alike. */ +export function trial(name, samples, side) { + return { alias: name, runs: [{ name: side ? `${name} (${side})` : name, args: {}, stats: stats(samples) }] }; +} diff --git a/test/validate.bench.mjs b/test/validate.bench.mjs index 0da74b3..831e942 100644 --- a/test/validate.bench.mjs +++ b/test/validate.bench.mjs @@ -9,32 +9,31 @@ * * - In-process: `extractAttrTypeMap` per fixture (the extraction and * resolver work, with the disk cache off). - * - Whole process: `dist/run.js` over a fixed subset of `examples/` — cold - * (cache off), warm (all cached), one cached file, and `--no-glint`. These - * catch backend start-up and per-run costs that no single call can see; - * they are timed with a few samples each rather than by mitata. + * - Whole process (`test/bench-process.mjs`): `dist/run.js` over a fixed + * subset of `examples/`. `--skip-process` leaves them out: the comparison + * driver runs them itself, interleaving the two builds. * * Both sides need `dist/` built. The harness is adapted from ember-estree. */ -import { spawnSync } from 'node:child_process'; -import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { readFileSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; import { bench, do_not_optimize, run } from 'mitata'; +import { ms, PROCESS_CASES, PROCESS_SAMPLES, ROOT, sample, seedCache, SUBSET, trial } from './bench-process.mjs'; + const args = process.argv.slice(2); const flag = (name) => { const i = args.indexOf(name); return i !== -1 ? resolve(args[i + 1]) : null; }; -const ROOT = fileURLToPath(new URL('..', import.meta.url)); // The build under test. `scripts/bench-compare.mjs` runs this script once // per side, in separate processes: two copies in one V8 heap skew the // numbers by 10-15 % on identical code. const DIST = flag('--dist') ?? ROOT; const JSON_OUT = flag('--json') ?? process.env['BENCH_JSON_OUTPUT']; +const SKIP_PROCESS = args.includes('--skip-process'); // The in-process benchmarks measure extraction, not the disk cache. process.env['HVE_NO_CACHE'] = '1'; @@ -60,43 +59,7 @@ for (const { filename, contents } of Object.values(FIXTURES)) { } globalThis.gc?.(); -// A whole `dist/run.js` run. `HVE_NO_CACHE` is inherited from this process -// unless the case sets it. -function validate(cliArgs, env = {}) { - const result = spawnSync(process.execPath, [resolve(DIST, 'dist/run.js'), ...cliArgs], { - cwd: ROOT, - env: { ...process.env, ...env }, - stdio: ['ignore', 'ignore', 'pipe'], - }); - // Exit 1 means findings (the examples have some); anything else is a crash. - if (result.status !== 0 && result.status !== 1) { - throw new Error(`run.js exited ${result.status}:\n${result.stderr}`); - } -} - -// A fixed subset keeps each run short; the costs these cases guard against -// (backend start-up, uncached per-file work) show at any size. -const SUBSET = [ - 'test/bench/large.gts', - ...readdirSync(resolve(ROOT, 'examples')) - .filter((f) => f.endsWith('.gts')) - .sort() - .slice(0, 20) - .map((f) => `examples/${f}`), -]; -// A small file: this case measures start-up, not template work. -const ONE = [SUBSET[1]]; -const CACHED = { HVE_NO_CACHE: '' }; -const PROCESS_CASES = { - 'cold run (cache off)': () => validate(['--glint', ...SUBSET]), - 'warm run (all cached)': () => validate(['--glint', ...SUBSET], CACHED), - 'one cached file': () => validate(['--glint', ...ONE], CACHED), - 'no glint': () => validate(['--no-glint', ...SUBSET]), -}; - -// Populate the disk cache for the warm cases (entries are keyed by plugin -// source, so the two sides of a comparison do not share them). -validate(['--glint', ...SUBSET], CACHED); +if (!SKIP_PROCESS) seedCache(DIST); for (const [name, { filename, contents }] of Object.entries(FIXTURES)) { globalThis.gc?.(); @@ -105,28 +68,16 @@ for (const [name, { filename, contents }] of Object.entries(FIXTURES)) { const result = await run({ colors: false, throw: true }); -// Whole-process cases take seconds each, so they are timed here with a -// few samples (min and median) instead of mitata's twelve-sample minimum, -// and reported in the same shape as the mitata trials. -const PROCESS_SAMPLES = 3; -function stats(samples) { - const sorted = [...samples].sort((a, b) => a - b); - const p50 = sorted[Math.floor(sorted.length / 2)]; - return { avg: samples.reduce((a, b) => a + b, 0) / samples.length, min: sorted[0], max: sorted.at(-1), p50, p75: p50, p99: sorted.at(-1), samples: sorted }; -} -const ms = (ns) => `${(ns / 1e6).toFixed(0)} ms`; const processTrials = []; -console.log('\nwhole process (min / p50 of %d runs, %d files)', PROCESS_SAMPLES, SUBSET.length); -for (const [name, runCase] of Object.entries(PROCESS_CASES)) { - const samples = []; - for (let i = 0; i < PROCESS_SAMPLES; i++) { - const t = process.hrtime.bigint(); - runCase(); - samples.push(Number(process.hrtime.bigint() - t)); +if (!SKIP_PROCESS) { + console.log('\nwhole process (min / p50 of %d runs, %d files)', PROCESS_SAMPLES, SUBSET.length); + for (const [name, runCase] of Object.entries(PROCESS_CASES)) { + const samples = []; + for (let i = 0; i < PROCESS_SAMPLES; i++) samples.push(sample(runCase, DIST)); + const t = trial(name, samples); + console.log(` ${name.padEnd(28)} ${ms(t.runs[0].stats.min).padStart(9)} / ${ms(t.runs[0].stats.p50).padStart(9)}`); + processTrials.push(t); } - const st = stats(samples); - console.log(` ${name.padEnd(28)} ${ms(st.min).padStart(9)} / ${ms(st.p50).padStart(9)}`); - processTrials.push({ alias: name, runs: [{ name, args: {}, stats: st }] }); } if (JSON_OUT) {