From bd6ea412c6732b3b946a2827fcaac3a1c8f2e863 Mon Sep 17 00:00:00 2001 From: "Sebastian \"Sebbie\" Silbermann" Date: Mon, 24 Aug 2026 14:10:02 +0200 Subject: [PATCH] [ci] Shard build workers by measured bundle build times (#37353) `build_and_lint` assigns its `[bundle, bundleType]` pairs to 25 workers per channel by round-robin, which leaves the slowest worker with 56-62 seconds of rollup time while the mean worker has 40-42 seconds (measured from the timestamped `BUILDING`/`COMPLETE` lines in recent `main` run logs). This change shards by measured build time instead, and the measurement maintains itself: `yarn build` writes the timing results into `build/__shard_timings__/`, which rides along inside the existing per-worker artifacts. `process_artifacts_combined`, which already downloads all 50 artifacts and is off the critical path, combines them into `build-weights.json` and saves it to the actions cache under a per-run key. Readers restore the most recent entry via a `restore-keys` prefix. Only pushes can save to cache keys that PRs can read, so pull request runs benefit from the weights but cannot poison them. The new measurement is always written verbatim rather than merged with previous weights, so removed bundles drop out instead of accumulating. A per-bundle diff against the previous weights is logged so that we can monitor whether single-run variance is too high, in which case shards should be determined from timings across the last N runs instead. Even on this PR a perfect prediction would've only gained us ?s for the slowest shard. Co-authored-by: Claude Code (kimi-k3[1m]) --- .github/workflows/runtime_build_and_test.yml | 36 +++++++ scripts/ci/merge-build-weights.js | 96 +++++++++++++++++ scripts/rollup/build-all-release-channels.js | 5 +- scripts/rollup/build.js | 21 +++- scripts/rollup/sharding.js | 104 +++++++++++++++++++ 5 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 scripts/ci/merge-build-weights.js create mode 100644 scripts/rollup/sharding.js diff --git a/.github/workflows/runtime_build_and_test.yml b/.github/workflows/runtime_build_and_test.yml index 8789cd75e8f..307f4b62cbf 100644 --- a/.github/workflows/runtime_build_and_test.yml +++ b/.github/workflows/runtime_build_and_test.yml @@ -302,11 +302,21 @@ jobs: if: steps.node_modules.outputs.cache-hit != 'true' - run: yarn --cwd compiler install --frozen-lockfile if: steps.node_modules.outputs.cache-hit != 'true' + - name: Restore build shard weights + uses: actions/cache/restore@v4 + with: + # Written by process_artifacts_combined on every push. The + # restore-keys prefix picks up the most recent entry. On a miss the + # build falls back to round-robin sharding. + path: build-weights.json + key: build-weights-v1-${{ github.run_id }} + restore-keys: build-weights-v1- - run: yarn build --index=${{ matrix.worker_id }} --total=25 --r=${{ matrix.release_channel }} --ci env: CI: github RELEASE_CHANNEL: ${{ matrix.release_channel }} NODE_INDEX: ${{ matrix.worker_id }} + BUILD_SHARD_WEIGHTS: build-weights.json - name: Lint build run: yarn lint-build - name: Display structure of build @@ -471,6 +481,32 @@ jobs: pattern: _build_* path: build merge-multiple: true + # Only used to log weight variance; the new measurement is what gets + # saved, so removed bundles drop out instead of accumulating. + - name: Restore previous build shard weights + uses: actions/cache/restore@v4 + with: + # Must match the save step's path exactly: the cache version is a + # hash of the path, so a different path never matches the key. + path: build-weights.json + key: build-weights-v1-${{ github.run_id }} + restore-keys: build-weights-v1- + - name: Update build shard weights + run: node scripts/ci/merge-build-weights.js + - name: Save build shard weights + # Pull request saves land in the pull request's own merge-ref cache + # scope, which only that pull request can read; fork pull requests + # have a read-only cache token, so their save degrades to a warning. + # A re-run of this same workflow run reuses the cache key, and cache + # entries are immutable. Weights are an optimization, so a failed + # save must not break artifact processing. + continue-on-error: true + uses: actions/cache/save@v4 + with: + path: build-weights.json + key: build-weights-v1-${{ github.run_id }} + # Keep the shard timings out of the released tarball. + - run: rm -rf build/__shard_timings__ - name: Display structure of build run: ls -R build - run: echo ${{ github.event.pull_request.head.sha || github.sha }} >> build/COMMIT_SHA diff --git a/scripts/ci/merge-build-weights.js b/scripts/ci/merge-build-weights.js new file mode 100644 index 00000000000..e0c3b20bc2f --- /dev/null +++ b/scripts/ci/merge-build-weights.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +'use strict'; + +// Combines the per-worker shard timings of the current run +// (build/__shard_timings__/*.json, written by scripts/rollup/build.js) into +// build-weights.json, which the workflow then saves back to the actions +// cache. The previous weights (restored to build-weights.json by the +// workflow) are only used to log a diff for variance monitoring; the new +// measurement is always written verbatim so that removed bundles drop out +// instead of accumulating. If the logged variance turns out to be too high +// for stable shards, weights should be aggregated across the last N runs +// instead. Weights feed scripts/rollup/sharding.js. This script never fails: +// the weights are an optimization, so a broken update must not break +// artifact processing. + +const fs = require('fs'); + +const TIMINGS_DIR = 'build/__shard_timings__'; +// The workflow restores the previous weights to OUT_PATH itself, so this +// script reads them from there before overwriting with the new measurement. +const OUT_PATH = 'build-weights.json'; + +function logDiff(fresh, previous) { + const deltas = []; + Object.keys(fresh).forEach(key => { + if (previous[key] !== undefined) { + deltas.push({key, delta: fresh[key] - previous[key]}); + } + }); + if (deltas.length === 0) { + return; + } + deltas.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta)); + const absDeltas = deltas + .map(entry => Math.abs(entry.delta)) + .sort((a, b) => a - b); + const mean = + absDeltas.reduce((sum, delta) => sum + delta, 0) / absDeltas.length; + const p95 = absDeltas[Math.floor(absDeltas.length * 0.95)]; + console.log( + `Weight changes vs the previous run: mean |delta| = ${mean.toFixed(2)}s, ` + + `p95 = ${p95.toFixed(1)}s across ${deltas.length} bundles. ` + + 'High variance here means shards should be determined from multiple runs.' + ); + console.log('Largest changes:'); + deltas.slice(0, 10).forEach(entry => { + console.log( + ` ${entry.delta >= 0 ? '+' : ''}${entry.delta.toFixed(1)}s ${entry.key}` + ); + }); +} + +function main() { + const fresh = {}; + const files = fs + .readdirSync(TIMINGS_DIR) + .filter(name => name.endsWith('.json')); + files.forEach(name => { + const timings = JSON.parse( + fs.readFileSync(TIMINGS_DIR + '/' + name, 'utf8') + ); + Object.keys(timings).forEach(key => { + fresh[key] = timings[key]; + }); + }); + const freshKeys = Object.keys(fresh); + let previous = {}; + try { + previous = JSON.parse(fs.readFileSync(OUT_PATH, 'utf8')).weights; + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + // Expected before the first weights have ever been saved. + console.log('No previous weights found, skipping the diff.'); + } + logDiff(fresh, previous); + fs.writeFileSync( + OUT_PATH, + JSON.stringify({version: 1, weights: fresh}, null, 2) + '\n' + ); + console.log(`Wrote ${freshKeys.length} weights to ${OUT_PATH}.`); +} + +try { + main(); +} catch (error) { + console.log( + 'Could not update build shard weights, keeping the previous ones.', + error + ); + // The restored previous weights may still sit at OUT_PATH; delete them so + // the save step cannot republish data this run did not produce. + fs.rmSync(OUT_PATH, {force: true}); +} diff --git a/scripts/rollup/build-all-release-channels.js b/scripts/rollup/build-all-release-channels.js index e4ffbadfa6e..4a5bbe14eb4 100644 --- a/scripts/rollup/build-all-release-channels.js +++ b/scripts/rollup/build-all-release-channels.js @@ -388,7 +388,10 @@ function processExperimental(buildDir, version) { if ( pathName !== 'oss-experimental' && pathName !== 'facebook-www' && - pathName !== 'sizes-experimental' + pathName !== 'sizes-experimental' && + // Not a duplicate: this worker's shard timings, merged into the build + // weights cache by process_artifacts_combined. + pathName !== '__shard_timings__' ) { fs.rmSync(path.join(buildDir, pathName), { recursive: true, diff --git a/scripts/rollup/build.js b/scripts/rollup/build.js index cde75fa0250..5eb251d35b7 100644 --- a/scripts/rollup/build.js +++ b/scripts/rollup/build.js @@ -12,6 +12,7 @@ const stripBanner = require('rollup-plugin-strip-banner'); const chalk = require('chalk'); const resolve = require('@rollup/plugin-node-resolve').nodeResolve; const fs = require('fs'); +const {performance} = require('perf_hooks'); const childProcess = require('child_process'); const argv = require('minimist')(process.argv.slice(2)); const Modules = require('./modules'); @@ -23,6 +24,7 @@ const useForks = require('./plugins/use-forks-plugin'); const dynamicImports = require('./plugins/dynamic-imports'); const externalRuntime = require('./plugins/external-runtime-plugin'); const Packaging = require('./packaging'); +const {selectShard, writeShardTimings} = require('./sharding'); const {asyncRimRaf} = require('./utils'); const codeFrame = require('@babel/code-frame').default; const Wrappers = require('./wrappers'); @@ -866,19 +868,36 @@ async function buildEverything() { return !shouldSkipBundle(bundle, bundleType); }); + // Prefixed with the channel because feature-flag forks change the cost of + // some heavy bundles. + const shardKeyOf = ([bundle, bundleType]) => + process.env.RELEASE_CHANNEL + + '/' + + getFilename(bundle, bundleType) + + ' (' + + bundleType.toLowerCase() + + ')'; + if (process.env.CI_TOTAL && process.env.CI_INDEX) { const nodeTotal = parseInt(process.env.CI_TOTAL, 10); const nodeIndex = parseInt(process.env.CI_INDEX, 10); - bundles = bundles.filter((_, i) => i % nodeTotal === nodeIndex); + bundles = selectShard(bundles, shardKeyOf, nodeTotal, nodeIndex); } + const shardTimings = []; // eslint-disable-next-line no-for-of-loops/no-for-of-loops for (const [bundle, bundleType] of bundles) { if (bundle.prebuild) { runShellCommand(bundle.prebuild); } + const start = performance.now(); await createBundle(bundle, bundleType); + shardTimings.push({ + key: shardKeyOf([bundle, bundleType]), + seconds: (performance.now() - start) / 1000, + }); } + writeShardTimings(shardTimings); await Packaging.copyAllShims(); await Packaging.prepareNpmPackages(); diff --git a/scripts/rollup/sharding.js b/scripts/rollup/sharding.js new file mode 100644 index 00000000000..a5ea6da94fe --- /dev/null +++ b/scripts/rollup/sharding.js @@ -0,0 +1,104 @@ +'use strict'; + +const fs = require('fs'); + +function readWeights() { + const weightsPath = process.env.BUILD_SHARD_WEIGHTS; + if (!weightsPath) { + return null; + } + let weights; + try { + weights = JSON.parse(fs.readFileSync(weightsPath, 'utf8')).weights; + } catch (error) { + if (error.code === 'ENOENT') { + // Expected before the first weights have ever been saved. + console.log('No build shard weights found, using round-robin sharding.'); + return null; + } + throw error; + } + if (weights === null || typeof weights !== 'object') { + return null; + } + return weights; +} + +// Persists the durations this worker measured so that +// process_artifacts_combined can merge them into the shared weights cache. +// No-op outside sharded CI builds. +function writeShardTimings(timings) { + const nodeIndex = process.env.CI_INDEX; + if (!process.env.CI_TOTAL || !nodeIndex) { + return; + } + const dir = 'build/__shard_timings__'; + fs.mkdirSync(dir, {recursive: true}); + const result = {}; + timings.forEach(timing => { + result[timing.key] = Math.round(timing.seconds * 10) / 10; + }); + fs.writeFileSync( + dir + '/' + nodeIndex + '-' + process.env.RELEASE_CHANNEL + '.json', + JSON.stringify(result) + ); +} + +// Assigns work items to CI workers. With measured per-item durations (see +// scripts/ci/merge-build-weights.js), items are assigned longest-first to +// the currently least-loaded worker so that workers finish around the same +// time. Every worker computes the full assignment and then picks its own +// bin, so the ordering below must stay deterministic. Without weights we +// fall back to round-robin. +function selectShard(items, keyFn, nodeTotal, nodeIndex) { + const weights = readWeights(); + const weightedKeys = weights === null ? [] : Object.keys(weights); + if (weightedKeys.length === 0) { + return items.filter((_, i) => i % nodeTotal === nodeIndex); + } + const keys = items.map(keyFn); + const sortedWeights = weightedKeys + .map(key => weights[key]) + .sort((a, b) => a - b); + const defaultWeight = sortedWeights[Math.floor(sortedWeights.length / 2)]; + const weightOf = index => { + const weight = weights[keys[index]]; + return weight === undefined ? defaultWeight : weight; + }; + const order = items + .map((_, i) => i) + .sort((a, b) => { + const delta = weightOf(b) - weightOf(a); + if (delta !== 0) { + return delta; + } + if (keys[a] !== keys[b]) { + return keys[a] < keys[b] ? -1 : 1; + } + return a - b; + }); + const bins = []; + for (let i = 0; i < nodeTotal; i++) { + bins.push({load: 0, indices: []}); + } + order.forEach(i => { + // The first bin wins ties so that the assignment stays deterministic. + let target = bins[0]; + for (let j = 1; j < bins.length; j++) { + if (bins[j].load < target.load) { + target = bins[j]; + } + } + target.load += weightOf(i); + target.indices.push(i); + }); + const shard = bins[nodeIndex].indices.sort((a, b) => a - b); + console.log( + `Sharding by measured build time: worker ${nodeIndex + 1}/${nodeTotal} ` + + `builds ${shard.length} of ${items.length} bundles ` + + `(~${Math.round(bins[nodeIndex].load)}s of rollup time).` + ); + return shard.map(i => items[i]); +} + +module.exports = {selectShard, writeShardTimings};