diff --git a/.github/workflows/runtime_build_and_test.yml b/.github/workflows/runtime_build_and_test.yml index 8789cd75e8f..307f4b62cbf 100644 --- a/.github/workflows/runtime_build_and_test.yml +++ b/.github/workflows/runtime_build_and_test.yml @@ -302,11 +302,21 @@ jobs: if: steps.node_modules.outputs.cache-hit != 'true' - run: yarn --cwd compiler install --frozen-lockfile if: steps.node_modules.outputs.cache-hit != 'true' + - name: Restore build shard weights + uses: actions/cache/restore@v4 + with: + # Written by process_artifacts_combined on every push. The + # restore-keys prefix picks up the most recent entry. On a miss the + # build falls back to round-robin sharding. + path: build-weights.json + key: build-weights-v1-${{ github.run_id }} + restore-keys: build-weights-v1- - run: yarn build --index=${{ matrix.worker_id }} --total=25 --r=${{ matrix.release_channel }} --ci env: CI: github RELEASE_CHANNEL: ${{ matrix.release_channel }} NODE_INDEX: ${{ matrix.worker_id }} + BUILD_SHARD_WEIGHTS: build-weights.json - name: Lint build run: yarn lint-build - name: Display structure of build @@ -471,6 +481,32 @@ jobs: pattern: _build_* path: build merge-multiple: true + # Only used to log weight variance; the new measurement is what gets + # saved, so removed bundles drop out instead of accumulating. + - name: Restore previous build shard weights + uses: actions/cache/restore@v4 + with: + # Must match the save step's path exactly: the cache version is a + # hash of the path, so a different path never matches the key. + path: build-weights.json + key: build-weights-v1-${{ github.run_id }} + restore-keys: build-weights-v1- + - name: Update build shard weights + run: node scripts/ci/merge-build-weights.js + - name: Save build shard weights + # Pull request saves land in the pull request's own merge-ref cache + # scope, which only that pull request can read; fork pull requests + # have a read-only cache token, so their save degrades to a warning. + # A re-run of this same workflow run reuses the cache key, and cache + # entries are immutable. Weights are an optimization, so a failed + # save must not break artifact processing. + continue-on-error: true + uses: actions/cache/save@v4 + with: + path: build-weights.json + key: build-weights-v1-${{ github.run_id }} + # Keep the shard timings out of the released tarball. + - run: rm -rf build/__shard_timings__ - name: Display structure of build run: ls -R build - run: echo ${{ github.event.pull_request.head.sha || github.sha }} >> build/COMMIT_SHA diff --git a/scripts/ci/merge-build-weights.js b/scripts/ci/merge-build-weights.js new file mode 100644 index 00000000000..e0c3b20bc2f --- /dev/null +++ b/scripts/ci/merge-build-weights.js @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +'use strict'; + +// Combines the per-worker shard timings of the current run +// (build/__shard_timings__/*.json, written by scripts/rollup/build.js) into +// build-weights.json, which the workflow then saves back to the actions +// cache. The previous weights (restored to build-weights.json by the +// workflow) are only used to log a diff for variance monitoring; the new +// measurement is always written verbatim so that removed bundles drop out +// instead of accumulating. If the logged variance turns out to be too high +// for stable shards, weights should be aggregated across the last N runs +// instead. Weights feed scripts/rollup/sharding.js. This script never fails: +// the weights are an optimization, so a broken update must not break +// artifact processing. + +const fs = require('fs'); + +const TIMINGS_DIR = 'build/__shard_timings__'; +// The workflow restores the previous weights to OUT_PATH itself, so this +// script reads them from there before overwriting with the new measurement. +const OUT_PATH = 'build-weights.json'; + +function logDiff(fresh, previous) { + const deltas = []; + Object.keys(fresh).forEach(key => { + if (previous[key] !== undefined) { + deltas.push({key, delta: fresh[key] - previous[key]}); + } + }); + if (deltas.length === 0) { + return; + } + deltas.sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta)); + const absDeltas = deltas + .map(entry => Math.abs(entry.delta)) + .sort((a, b) => a - b); + const mean = + absDeltas.reduce((sum, delta) => sum + delta, 0) / absDeltas.length; + const p95 = absDeltas[Math.floor(absDeltas.length * 0.95)]; + console.log( + `Weight changes vs the previous run: mean |delta| = ${mean.toFixed(2)}s, ` + + `p95 = ${p95.toFixed(1)}s across ${deltas.length} bundles. ` + + 'High variance here means shards should be determined from multiple runs.' + ); + console.log('Largest changes:'); + deltas.slice(0, 10).forEach(entry => { + console.log( + ` ${entry.delta >= 0 ? '+' : ''}${entry.delta.toFixed(1)}s ${entry.key}` + ); + }); +} + +function main() { + const fresh = {}; + const files = fs + .readdirSync(TIMINGS_DIR) + .filter(name => name.endsWith('.json')); + files.forEach(name => { + const timings = JSON.parse( + fs.readFileSync(TIMINGS_DIR + '/' + name, 'utf8') + ); + Object.keys(timings).forEach(key => { + fresh[key] = timings[key]; + }); + }); + const freshKeys = Object.keys(fresh); + let previous = {}; + try { + previous = JSON.parse(fs.readFileSync(OUT_PATH, 'utf8')).weights; + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + // Expected before the first weights have ever been saved. + console.log('No previous weights found, skipping the diff.'); + } + logDiff(fresh, previous); + fs.writeFileSync( + OUT_PATH, + JSON.stringify({version: 1, weights: fresh}, null, 2) + '\n' + ); + console.log(`Wrote ${freshKeys.length} weights to ${OUT_PATH}.`); +} + +try { + main(); +} catch (error) { + console.log( + 'Could not update build shard weights, keeping the previous ones.', + error + ); + // The restored previous weights may still sit at OUT_PATH; delete them so + // the save step cannot republish data this run did not produce. + fs.rmSync(OUT_PATH, {force: true}); +} diff --git a/scripts/rollup/build-all-release-channels.js b/scripts/rollup/build-all-release-channels.js index e4ffbadfa6e..4a5bbe14eb4 100644 --- a/scripts/rollup/build-all-release-channels.js +++ b/scripts/rollup/build-all-release-channels.js @@ -388,7 +388,10 @@ function processExperimental(buildDir, version) { if ( pathName !== 'oss-experimental' && pathName !== 'facebook-www' && - pathName !== 'sizes-experimental' + pathName !== 'sizes-experimental' && + // Not a duplicate: this worker's shard timings, merged into the build + // weights cache by process_artifacts_combined. + pathName !== '__shard_timings__' ) { fs.rmSync(path.join(buildDir, pathName), { recursive: true, diff --git a/scripts/rollup/build.js b/scripts/rollup/build.js index cde75fa0250..5eb251d35b7 100644 --- a/scripts/rollup/build.js +++ b/scripts/rollup/build.js @@ -12,6 +12,7 @@ const stripBanner = require('rollup-plugin-strip-banner'); const chalk = require('chalk'); const resolve = require('@rollup/plugin-node-resolve').nodeResolve; const fs = require('fs'); +const {performance} = require('perf_hooks'); const childProcess = require('child_process'); const argv = require('minimist')(process.argv.slice(2)); const Modules = require('./modules'); @@ -23,6 +24,7 @@ const useForks = require('./plugins/use-forks-plugin'); const dynamicImports = require('./plugins/dynamic-imports'); const externalRuntime = require('./plugins/external-runtime-plugin'); const Packaging = require('./packaging'); +const {selectShard, writeShardTimings} = require('./sharding'); const {asyncRimRaf} = require('./utils'); const codeFrame = require('@babel/code-frame').default; const Wrappers = require('./wrappers'); @@ -866,19 +868,36 @@ async function buildEverything() { return !shouldSkipBundle(bundle, bundleType); }); + // Prefixed with the channel because feature-flag forks change the cost of + // some heavy bundles. + const shardKeyOf = ([bundle, bundleType]) => + process.env.RELEASE_CHANNEL + + '/' + + getFilename(bundle, bundleType) + + ' (' + + bundleType.toLowerCase() + + ')'; + if (process.env.CI_TOTAL && process.env.CI_INDEX) { const nodeTotal = parseInt(process.env.CI_TOTAL, 10); const nodeIndex = parseInt(process.env.CI_INDEX, 10); - bundles = bundles.filter((_, i) => i % nodeTotal === nodeIndex); + bundles = selectShard(bundles, shardKeyOf, nodeTotal, nodeIndex); } + const shardTimings = []; // eslint-disable-next-line no-for-of-loops/no-for-of-loops for (const [bundle, bundleType] of bundles) { if (bundle.prebuild) { runShellCommand(bundle.prebuild); } + const start = performance.now(); await createBundle(bundle, bundleType); + shardTimings.push({ + key: shardKeyOf([bundle, bundleType]), + seconds: (performance.now() - start) / 1000, + }); } + writeShardTimings(shardTimings); await Packaging.copyAllShims(); await Packaging.prepareNpmPackages(); diff --git a/scripts/rollup/sharding.js b/scripts/rollup/sharding.js new file mode 100644 index 00000000000..a5ea6da94fe --- /dev/null +++ b/scripts/rollup/sharding.js @@ -0,0 +1,104 @@ +'use strict'; + +const fs = require('fs'); + +function readWeights() { + const weightsPath = process.env.BUILD_SHARD_WEIGHTS; + if (!weightsPath) { + return null; + } + let weights; + try { + weights = JSON.parse(fs.readFileSync(weightsPath, 'utf8')).weights; + } catch (error) { + if (error.code === 'ENOENT') { + // Expected before the first weights have ever been saved. + console.log('No build shard weights found, using round-robin sharding.'); + return null; + } + throw error; + } + if (weights === null || typeof weights !== 'object') { + return null; + } + return weights; +} + +// Persists the durations this worker measured so that +// process_artifacts_combined can merge them into the shared weights cache. +// No-op outside sharded CI builds. +function writeShardTimings(timings) { + const nodeIndex = process.env.CI_INDEX; + if (!process.env.CI_TOTAL || !nodeIndex) { + return; + } + const dir = 'build/__shard_timings__'; + fs.mkdirSync(dir, {recursive: true}); + const result = {}; + timings.forEach(timing => { + result[timing.key] = Math.round(timing.seconds * 10) / 10; + }); + fs.writeFileSync( + dir + '/' + nodeIndex + '-' + process.env.RELEASE_CHANNEL + '.json', + JSON.stringify(result) + ); +} + +// Assigns work items to CI workers. With measured per-item durations (see +// scripts/ci/merge-build-weights.js), items are assigned longest-first to +// the currently least-loaded worker so that workers finish around the same +// time. Every worker computes the full assignment and then picks its own +// bin, so the ordering below must stay deterministic. Without weights we +// fall back to round-robin. +function selectShard(items, keyFn, nodeTotal, nodeIndex) { + const weights = readWeights(); + const weightedKeys = weights === null ? [] : Object.keys(weights); + if (weightedKeys.length === 0) { + return items.filter((_, i) => i % nodeTotal === nodeIndex); + } + const keys = items.map(keyFn); + const sortedWeights = weightedKeys + .map(key => weights[key]) + .sort((a, b) => a - b); + const defaultWeight = sortedWeights[Math.floor(sortedWeights.length / 2)]; + const weightOf = index => { + const weight = weights[keys[index]]; + return weight === undefined ? defaultWeight : weight; + }; + const order = items + .map((_, i) => i) + .sort((a, b) => { + const delta = weightOf(b) - weightOf(a); + if (delta !== 0) { + return delta; + } + if (keys[a] !== keys[b]) { + return keys[a] < keys[b] ? -1 : 1; + } + return a - b; + }); + const bins = []; + for (let i = 0; i < nodeTotal; i++) { + bins.push({load: 0, indices: []}); + } + order.forEach(i => { + // The first bin wins ties so that the assignment stays deterministic. + let target = bins[0]; + for (let j = 1; j < bins.length; j++) { + if (bins[j].load < target.load) { + target = bins[j]; + } + } + target.load += weightOf(i); + target.indices.push(i); + }); + const shard = bins[nodeIndex].indices.sort((a, b) => a - b); + console.log( + `Sharding by measured build time: worker ${nodeIndex + 1}/${nodeTotal} ` + + `builds ${shard.length} of ${items.length} bundles ` + + `(~${Math.round(bins[nodeIndex].load)}s of rollup time).` + ); + return shard.map(i => items[i]); +} + +module.exports = {selectShard, writeShardTimings};