From 45b779f907ee286ae9ee24f89810605e89d65d0c Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 17 Aug 2026 23:04:16 -0400 Subject: [PATCH 1/2] Disable pnpm's implicit install-on-run to fix the CI Synapse startup flake pnpm 11 defaults verify-deps-before-run to `install`, so every `pnpm run` / `pnpm exec` first checks node_modules freshness and silently runs a workspace-wide install when the check fails. In CI the check never passes, even immediately after the init action's `pnpm install --frozen-lockfile` (and even after one of these implicit installs completes), so every pnpm invocation in a job pays a redundant 4-16s install. The fatal case is the service boot: `run-p` launches ~9 pnpm scripts at once, each kicking off its own concurrent install into the same node_modules. They stomp each other's bin links (the "Failed to create bin ... ENOENT" warnings visible in green shards) and occasionally deadlock; when start:matrix's install is the one that wedges, the Synapse container is never created and the shard fails with "Failed to reach Synapse ... after 60 attempts (~300s)". This is the mechanism behind the recurring "Synapse startup race" flake, which hit 12 host shards across three PRs today alone. Setting verifyDepsBeforeRun: false restores pnpm 10 behavior: scripts run against node_modules as-is, and installs happen only when explicitly requested. Co-Authored-By: Claude Fable 5 --- pnpm-workspace.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 96e487c5f35..f7953459284 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -257,6 +257,15 @@ catalog: yaml: ^2.5.1 yargs: ^17.5.1 +# pnpm 11 defaults this to `install`, making every `pnpm run` first check +# node_modules freshness and silently run a full install when the check +# fails. The check never passes in CI, so each of the ~9 services `run-p` +# boots kicks off its own concurrent workspace-wide install into the same +# node_modules; when they collide badly enough, start:matrix never reaches +# `docker run` and the shard dies with "Failed to reach Synapse". Disabling +# restores pnpm 10 behavior: scripts run against node_modules as-is. +verifyDepsBeforeRun: false + minimumReleaseAge: 1440 minimumReleaseAgeExclude: - "@glint/ember-tsc" From 3691ab49c6fc815e3adc584db2bf7fb5f166bfe4 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 17 Aug 2026 23:07:13 -0400 Subject: [PATCH 2/2] Trigger CI when pnpm-workspace.yaml changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pnpm 11 reads workspace settings (catalog, overrides, patchedDependencies, allowBuilds, verifyDepsBeforeRun) from pnpm-workspace.yaml, but no workflow path filter included it — a settings-only change that doesn't touch the lockfile ran zero suites. Add it alongside pnpm-lock.yaml in every filter, including ci-host's index-cache invalidation case, since overrides and patches can change indexing behavior the same way a lockfile change can. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci-host.yaml | 3 ++- .github/workflows/ci-software-factory.yaml | 1 + .github/workflows/ci.yaml | 1 + .github/workflows/pr-boxel-ui.yml | 1 + .github/workflows/preview-host.yml | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-host.yaml b/.github/workflows/ci-host.yaml index 1346ea281fc..60bcecc3471 100644 --- a/.github/workflows/ci-host.yaml +++ b/.github/workflows/ci-host.yaml @@ -16,6 +16,7 @@ on: - ".github/workflows/ci-host.yaml" - "package.json" - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" # CI boot path: see comment on the equivalent block in ci.yaml. - ".mise.toml" - ".github/actions/**" @@ -149,7 +150,7 @@ jobs: use_cache=true while IFS= read -r file; do case "$file" in - packages/realm-server/*|packages/runtime-common/*|packages/postgres/*|packages/boxel-ui/*|package.json|pnpm-lock.yaml) + packages/realm-server/*|packages/runtime-common/*|packages/postgres/*|packages/boxel-ui/*|package.json|pnpm-lock.yaml|pnpm-workspace.yaml) use_cache=false echo "Index-behavior change detected: $file" break diff --git a/.github/workflows/ci-software-factory.yaml b/.github/workflows/ci-software-factory.yaml index cf2a6bbab57..0d8b8a2f24c 100644 --- a/.github/workflows/ci-software-factory.yaml +++ b/.github/workflows/ci-software-factory.yaml @@ -20,6 +20,7 @@ on: - "packages/software-factory/**" - "package.json" - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" # CI boot path: see comment on the equivalent block in ci.yaml. - ".mise.toml" - ".github/actions/**" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae6ff4b60d0..9cc60358a02 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -54,6 +54,7 @@ jobs: - 'packages/runtime-common/**' - 'package.json' - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' # CI boot path. Each gated suite runs `./.github/actions/init` # (which calls `jdx/mise-action`, sourcing `.mise.toml` for tool # versions) and then `mise run test-services:*` (which sources diff --git a/.github/workflows/pr-boxel-ui.yml b/.github/workflows/pr-boxel-ui.yml index 35878b92eb1..464af24625b 100644 --- a/.github/workflows/pr-boxel-ui.yml +++ b/.github/workflows/pr-boxel-ui.yml @@ -8,6 +8,7 @@ on: - ".github/workflows/pr-boxel-ui.yml" - "package.json" - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" permissions: contents: read diff --git a/.github/workflows/preview-host.yml b/.github/workflows/preview-host.yml index 591826f3dad..2ec9cc40d2d 100644 --- a/.github/workflows/preview-host.yml +++ b/.github/workflows/preview-host.yml @@ -10,6 +10,7 @@ on: - ".github/workflows/preview-host.yml" - "package.json" - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" concurrency: group: preview-host-${{ github.head_ref || github.run_id }}