From 129aaa2713ede1da30c3b184c40cbf0aa56ce240 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Sat, 5 Sep 2026 02:19:10 +0100 Subject: [PATCH 1/3] ci(smart-ci): add the nightly coordinator decision module CI-10 slice 1 (#2334): a pure decision over the last deep-qualified receipt, the main head and tree SHAs, the diff since that receipt, the clock and the weekly UTC slot. Verdicts are no-change, affected, weekly-full and full-sweep; every fail-closed path selects the complete sweep with a stable reason id, and a quiet night is an explicit receipt rather than a skipped workflow. Path-to-group mapping reuses matchGroups() from lib/plan.mjs. The group-to-suite table is a module constant that CI10-2 moves into ci/policy.v1.json. No workflow behaviour changes in this slice. --- scripts/ci/smart-ci/nightly-coordinator.mjs | 687 ++++++++++++++++++++ 1 file changed, 687 insertions(+) create mode 100644 scripts/ci/smart-ci/nightly-coordinator.mjs diff --git a/scripts/ci/smart-ci/nightly-coordinator.mjs b/scripts/ci/smart-ci/nightly-coordinator.mjs new file mode 100644 index 000000000..e6b4a231a --- /dev/null +++ b/scripts/ci/smart-ci/nightly-coordinator.mjs @@ -0,0 +1,687 @@ +#!/usr/bin/env node +// Smart CI nightly coordinator — decision module (ADR-0066 §Decision 10, CI-10 #2334, tracker CI-00 #2324). +// +// One owner answers "what changed on `main` since the last complete deep qualification, and which +// deep suites would produce new evidence tonight". This slice (CI10-1) is the pure decision only: +// it changes no workflow behaviour, and nothing here schedules, skips or gates a job. Wiring the +// verdict into `ci-nightly.yml` / `nightly-quality.yml` is CI10-2, the weekly sweep is CI10-3, and +// the release collapse is CI10-4. +// +// FAIL-CLOSED, matching `docs/ci/SMART_CI.md` invariant 2 (unknown change = full escalation): +// a missing, unreadable or incomplete last receipt, an unreachable diff, an unmapped path, a +// changed control path, a policy path group with no suite mapping, an invalid policy, an +// unparseable clock, an invalid weekly slot or any internal error selects the complete sweep. +// A quiet night is an explicit `no-change` RECEIPT, never a skipped workflow (invariant 1 forbids +// a skip reporting as success to branch protection). +// +// Verdicts +// no-change the current tree SHA equals the last qualified tree SHA, or the diff maps only to +// path groups with no deep suite. Selected: none. This is the honest green receipt. +// affected the union of the deep suites of the matched path groups. +// weekly-full the configured weekly UTC slot forces the complete sweep regardless of the diff. +// full-sweep fail-closed escalation, or an explicit `--force-full` dispatch. +// Precedence: full-sweep > weekly-full > no-change (identical tree) > affected. +// +// Receipt shape (JSON written by the CLI). No `ci/schemas` entry in this slice: CI10-2 adds one +// alongside the policy move described under GROUP_DEEP_SUITES, so the shape is specified here. +// { +// schemaVersion: 1, +// kind: "nightly-plan", +// generatedAtUtc: string, // echoed from the `nowUtc` input, never Date.now() +// policyId: string|null, +// policyDigest: string|null, +// verdict: "no-change"|"affected"|"weekly-full"|"full-sweep", +// reasons: string[], // sorted, stable ids; never empty +// current: { headSha: string|null, treeSha: string|null }, +// lastQualified: { headSha: string, treeSha: string, completedAtUtc: string, complete: boolean }|null, +// lastQualifiedUnavailableReason: string|null, +// duplicateQualification: boolean, +// duplicateQualificationReason: string, +// weeklySlot: { utcDay: number|null, nowUtcDay: number|null, matched: boolean }, +// forceFull: boolean, +// changedFilesAvailable: boolean, +// changedFileCount: number|null, +// matchedGroups: string[], +// unmappedPaths: string[], +// controlPathsChanged: string[], +// selectedSuites: string[], // canonical NIGHTLY_DEEP_SUITES order +// skippedSuites: [{ suite: string, reason: string }] +// } +// +// CLI usage (every input explicit; no network and no git calls anywhere in this module): +// node scripts/ci/smart-ci/nightly-coordinator.mjs --policy ci/policy.v1.json \ +// --head-sha --tree-sha [--last-receipt ] [--changed-files ] \ +// [--now ] [--weekly-slot <0-6>] [--force-full] \ +// [--out artifacts/nightly-plan.json] [--out-md artifacts/nightly-plan.md] \ +// [--summary "$GITHUB_STEP_SUMMARY"] +// Omitting `--changed-files`, or pointing it at a missing file, means the diff is UNAVAILABLE and +// escalates; an existing empty file means an empty diff. Omitting `--last-receipt` means no last +// receipt and escalates. The CLI always exits 0: the verdict is the output, not the exit status. + +import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { matchGroups, policyDigest, validatePolicy } from './lib/plan.mjs'; + +export const NIGHTLY_COORDINATOR_SCHEMA_VERSION = 1; +export const NIGHTLY_COORDINATOR_KIND = 'nightly-plan'; + +/** + * The deep suites that exist today, as the job ids of `.github/workflows/ci-nightly.yml` + * (the first nine) and `.github/workflows/nightly-quality.yml` (the last three). Declaration + * order is the canonical render order, so a receipt is byte-stable for identical input. + */ +export const NIGHTLY_DEEP_SUITES = Object.freeze([ + 'openapi-guardrail', + 'developer-portal', + 'backend-solution', + 'e2e-smoke', + 'load-concurrency-harness', + 'performance-regression-gate', + 'e2e-cross-browser', + 'container-images', + 'sast-scanning', + 'backend-coverage', + 'frontend-coverage', + 'dependency-security-signals', +]); + +const BACKEND_SUITES = Object.freeze([ + 'backend-solution', + 'backend-coverage', + 'load-concurrency-harness', + 'performance-regression-gate', + 'container-images', +]); +const FRONTEND_SUITES = Object.freeze([ + 'frontend-coverage', + 'e2e-smoke', + 'e2e-cross-browser', +]); +const DEPENDENCY_SUITES = Object.freeze([ + 'dependency-security-signals', + 'sast-scanning', + 'container-images', +]); +const API_CONTRACT_SUITES = Object.freeze(['openapi-guardrail', 'developer-portal']); + +/** + * Path group id (from `ci/policy.v1.json` `pathGroups`) to the deep suites a change in that group + * can produce new evidence for. Deliberately conservative: a group selects a superset rather than + * the minimum, and only a genuinely evidence-free group (docs, repo metadata, agent tooling) + * selects nothing. Dependency manifests and lockfiles are `controlPaths` in the policy, so they + * escalate to the full sweep before this table is consulted; `backend-project-files` carries the + * dependency suites for the manifests that are not control paths. + * + * CI10-2 moves this table into `ci/policy.v1.json` next to `pathGroups` so the policy digest covers + * it; it lives here in slice 1 so the decision function can be proven before the policy schema + * changes. Until then `nightlyCoordinatorMappingErrors()` and its test enumerate every policy group + * id against this table, so a new group cannot silently select nothing: a policy group with no + * entry here is a fail-closed full sweep, not an empty selection. + */ +export const GROUP_DEEP_SUITES = Object.freeze({ + 'docs': Object.freeze([]), + 'repo-metadata': Object.freeze([]), + 'agent-tooling': Object.freeze([]), + 'mcp-config': Object.freeze(['backend-solution', 'e2e-smoke']), + 'worktree-helpers': Object.freeze([]), + 'governance-scripts': Object.freeze([]), + 'scripts-other': Object.freeze([]), + 'backend-domain': BACKEND_SUITES, + 'backend-application': BACKEND_SUITES, + 'backend-infrastructure': BACKEND_SUITES, + 'persistence-migrations': BACKEND_SUITES, + 'backend-api': Object.freeze([...BACKEND_SUITES, ...API_CONTRACT_SUITES, 'e2e-smoke']), + 'mcp-process': Object.freeze([...BACKEND_SUITES, 'e2e-smoke']), + 'auth-security': Object.freeze([...BACKEND_SUITES, 'e2e-smoke', 'sast-scanning']), + 'capture-proposal-executor': Object.freeze([...BACKEND_SUITES, 'e2e-smoke']), + 'backend-cli': BACKEND_SUITES, + 'backend-tests': BACKEND_SUITES, + 'backend-project-files': Object.freeze([...BACKEND_SUITES, ...DEPENDENCY_SUITES]), + 'frontend-src': FRONTEND_SUITES, + 'frontend-e2e': FRONTEND_SUITES, + 'launchers-windows': Object.freeze([...BACKEND_SUITES, ...FRONTEND_SUITES]), + 'containers-deploy': DEPENDENCY_SUITES, + 'load-and-evals': Object.freeze(['load-concurrency-harness', 'performance-regression-gate']), +}); + +/** + * Stable reason ids. Every verdict carries at least one. `unmapped-path` and `control-path-change` + * deliberately repeat the planner's escalation vocabulary in `lib/plan.mjs`. + */ +export const REASONS = Object.freeze({ + affectedGroups: 'affected-groups', + controlPathChanged: 'control-path-change', + coordinatorError: 'coordinator-error', + diffUnavailable: 'diff-unavailable', + forceFull: 'force-full-requested', + groupNotMapped: 'group-not-in-suite-map', + headShaInvalid: 'current-head-sha-invalid', + identicalTree: 'identical-tree-sha', + lastReceiptIncomplete: 'last-receipt-incomplete', + lastReceiptMissing: 'last-receipt-missing', + lastReceiptUnreadable: 'last-receipt-unreadable', + noDeepSuiteGroups: 'no-deep-suite-groups', + nowUnparseable: 'now-unparseable', + policyInvalid: 'policy-invalid', + treeShaInvalid: 'current-tree-sha-invalid', + unmappedPath: 'unmapped-path', + weeklySlot: 'weekly-slot', + weeklySlotInvalid: 'weekly-slot-invalid', +}); + +/** Why a deep suite is not selected. */ +export const SKIP_REASONS = Object.freeze({ + noChange: 'no-change', + notAffected: 'not-selected-by-changed-groups', +}); + +function isObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isSha(value) { + return /^[0-9a-f]{40}$/i.test(String(value ?? '')); +} + +function normaliseSha(value) { + return isSha(value) ? String(value).toLowerCase() : null; +} + +function isIsoTimestamp(value) { + return typeof value === 'string' && value.length > 0 && !Number.isNaN(Date.parse(value)); +} + +/** Codepoint sort, matching `lib/plan.mjs`; locale-sensitive ordering would not be byte-stable. */ +function uniqueSorted(values) { + return [...new Set(values.map(String))].sort(); +} + +/** Order a suite set by the canonical NIGHTLY_DEEP_SUITES declaration order. */ +function orderSuites(suites) { + const wanted = new Set(suites.map(String)); + return NIGHTLY_DEEP_SUITES.filter((suite) => wanted.has(suite)); +} + +/** + * Consistency errors in GROUP_DEEP_SUITES itself, given a policy document. Empty means the table + * covers every policy path group, names no unknown deep suite, and names no group the policy has + * dropped. Sorted so the message set is stable. + * @param {object} policy parsed `ci/policy.v1.json` + * @returns {string[]} + */ +export function nightlyCoordinatorMappingErrors(policy) { + const errors = []; + const known = new Set(NIGHTLY_DEEP_SUITES); + for (const [groupId, suites] of Object.entries(GROUP_DEEP_SUITES)) { + for (const suite of suites) { + if (!known.has(suite)) errors.push(`group ${groupId} names unknown deep suite ${suite}`); + } + } + const groups = isObject(policy) && Array.isArray(policy.pathGroups) ? policy.pathGroups : []; + for (const group of groups) { + const id = isObject(group) && typeof group.id === 'string' ? group.id : null; + if (id === null) { + errors.push('policy pathGroups contains an entry without a string id'); + continue; + } + if (!Object.hasOwn(GROUP_DEEP_SUITES, id)) errors.push(`policy path group ${id} has no deep-suite mapping`); + } + const policyIds = new Set(groups.map((group) => (isObject(group) && typeof group.id === 'string' ? group.id : ''))); + for (const groupId of Object.keys(GROUP_DEEP_SUITES)) { + if (!policyIds.has(groupId)) errors.push(`deep-suite mapping names unknown policy path group ${groupId}`); + } + return errors.sort(); +} + +/** + * Duplicate qualification: the current tree SHA was already deep-qualified by the last receipt. + * CI-12 (#2336) consumes this as the nightly half of its duplicate-qualification flag. + * + * A partially failed nightly never qualified anything (issue edge case: some deep suites green, + * some red), so an incomplete receipt reports `last-receipt-incomplete` rather than a duplicate. + * + * @param {string|null} treeSha current `main` tree SHA + * @param {object|null} lastQualified last deep-qualified receipt, or null + * @returns {{ duplicate: boolean, reason: string, treeSha: string|null, lastQualifiedTreeSha: string|null }} + */ +export function detectDuplicateQualification(treeSha, lastQualified) { + const current = normaliseSha(treeSha); + if (current === null) { + return { duplicate: false, reason: 'current-tree-sha-invalid', treeSha: null, lastQualifiedTreeSha: null }; + } + if (!isObject(lastQualified)) { + return { duplicate: false, reason: 'no-last-receipt', treeSha: current, lastQualifiedTreeSha: null }; + } + const previous = normaliseSha(lastQualified.treeSha); + if (previous === null) { + return { duplicate: false, reason: 'last-receipt-tree-sha-invalid', treeSha: current, lastQualifiedTreeSha: null }; + } + if (previous !== current) { + return { duplicate: false, reason: 'tree-sha-differs', treeSha: current, lastQualifiedTreeSha: previous }; + } + if (lastQualified.complete !== true) { + return { duplicate: false, reason: REASONS.lastReceiptIncomplete, treeSha: current, lastQualifiedTreeSha: previous }; + } + return { duplicate: true, reason: 'tree-sha-already-qualified', treeSha: current, lastQualifiedTreeSha: previous }; +} + +/** + * Validate the last deep-qualified receipt. + * @param {object|null} lastQualified + * @param {string|null} [explicitReason] reason supplied by the caller when the receipt is null + * because it could not be read rather than because it does not exist + * @returns {{ receipt: object|null, reason: string|null }} `reason === null` means usable + */ +export function normaliseLastQualified(lastQualified, explicitReason = null) { + if (lastQualified === null || lastQualified === undefined) { + return { receipt: null, reason: explicitReason ?? REASONS.lastReceiptMissing }; + } + if (!isObject(lastQualified)) return { receipt: null, reason: REASONS.lastReceiptUnreadable }; + const headSha = normaliseSha(lastQualified.headSha); + const treeSha = normaliseSha(lastQualified.treeSha); + if (headSha === null || treeSha === null || !isIsoTimestamp(lastQualified.completedAtUtc)) { + return { receipt: null, reason: REASONS.lastReceiptUnreadable }; + } + const completedAtUtc = new Date(lastQualified.completedAtUtc).toISOString(); + // The "last deep-qualified SHA" only advances on a complete success. + if (lastQualified.complete !== true) { + return { + receipt: { headSha, treeSha, completedAtUtc, complete: false }, + reason: REASONS.lastReceiptIncomplete, + }; + } + return { receipt: { headSha, treeSha, completedAtUtc, complete: true }, reason: null }; +} + +function buildReceipt(fields) { + const selected = orderSuites(fields.selectedSuites ?? []); + const selectedSet = new Set(selected); + const skipReason = fields.skipReason ?? SKIP_REASONS.notAffected; + return { + schemaVersion: NIGHTLY_COORDINATOR_SCHEMA_VERSION, + kind: NIGHTLY_COORDINATOR_KIND, + generatedAtUtc: fields.generatedAtUtc, + policyId: fields.policyId, + policyDigest: fields.policyDigest, + verdict: fields.verdict, + reasons: uniqueSorted(fields.reasons), + current: { headSha: fields.headSha, treeSha: fields.treeSha }, + lastQualified: fields.lastQualified, + lastQualifiedUnavailableReason: fields.lastQualifiedUnavailableReason, + duplicateQualification: fields.duplicate.duplicate, + duplicateQualificationReason: fields.duplicate.reason, + weeklySlot: fields.weeklySlot, + forceFull: fields.forceFull, + changedFilesAvailable: fields.changedFilesAvailable, + changedFileCount: fields.changedFileCount, + matchedGroups: fields.matchedGroups ?? [], + unmappedPaths: fields.unmappedPaths ?? [], + controlPathsChanged: fields.controlPathsChanged ?? [], + selectedSuites: selected, + skippedSuites: NIGHTLY_DEEP_SUITES + .filter((suite) => !selectedSet.has(suite)) + .map((suite) => ({ suite, reason: skipReason })), + }; +} + +/** + * Decide tonight's nightly plan. Pure: identical input always yields an identical receipt, and + * nothing here reads the network, the clock or git. + * + * @param {object} input + * @param {object} input.policy parsed `ci/policy.v1.json` + * @param {string|null} [input.policyDigest] policyDigest() of the policy file bytes + * @param {object|null} input.lastQualified last deep-qualified receipt + * `{ headSha, treeSha, completedAtUtc, complete }`, or null when missing or unreadable + * @param {string|null} [input.lastQualifiedReason] escalation reason when `lastQualified` is null + * because it could not be read rather than because it does not exist + * @param {string} input.headSha current `main` head SHA + * @param {string} input.treeSha current `main` tree SHA + * @param {string[]|null} input.changedFiles paths changed between the last qualified SHA and the + * current head, or null when that diff is unavailable (force-moved history, expired receipt) + * @param {string} input.nowUtc ISO-8601 timestamp; also the receipt's generatedAtUtc + * @param {number|null} [input.weeklySlotUtcDay] UTC weekday 0..6 that forces the full sweep + * @param {boolean} [input.forceFull] workflow_dispatch escalation + * @returns {object} the receipt described in this file's header + */ +export function decideNightlyPlan(input) { + const source = isObject(input) ? input : {}; + const policyId = isObject(source.policy) && typeof source.policy.policyId === 'string' + ? source.policy.policyId + : null; + const policyDigestValue = typeof source.policyDigest === 'string' ? source.policyDigest : null; + const headSha = normaliseSha(source.headSha); + const treeSha = normaliseSha(source.treeSha); + const forceFull = source.forceFull === true; + const nowUtc = isIsoTimestamp(source.nowUtc) ? new Date(source.nowUtc).toISOString() : null; + const weeklySlotRequested = source.weeklySlotUtcDay ?? null; + const weeklySlotValid = weeklySlotRequested === null + || (Number.isInteger(weeklySlotRequested) && weeklySlotRequested >= 0 && weeklySlotRequested <= 6); + const weeklySlotUtcDay = weeklySlotValid ? weeklySlotRequested : null; + const nowUtcDay = nowUtc === null ? null : new Date(nowUtc).getUTCDay(); + const weeklySlot = { + utcDay: weeklySlotUtcDay, + nowUtcDay, + matched: weeklySlotUtcDay !== null && nowUtcDay !== null && weeklySlotUtcDay === nowUtcDay, + }; + const lastQualifiedRaw = source.lastQualified ?? null; + const duplicate = detectDuplicateQualification(treeSha, lastQualifiedRaw); + // generatedAtUtc must never fall back to Date.now(): the receipt has to be reproducible. + const generatedAtUtc = nowUtc ?? '1970-01-01T00:00:00.000Z'; + + const base = { + generatedAtUtc, + policyId, + policyDigest: policyDigestValue, + headSha, + treeSha, + duplicate, + weeklySlot, + forceFull, + }; + + try { + const { receipt: lastQualified, reason: lastQualifiedReason } = normaliseLastQualified( + lastQualifiedRaw, + typeof source.lastQualifiedReason === 'string' ? source.lastQualifiedReason : null, + ); + const changedFiles = Array.isArray(source.changedFiles) + ? source.changedFiles.map(String).filter((path) => path.length > 0) + : null; + const changedFilesAvailable = changedFiles !== null; + const common = { + ...base, + lastQualified, + lastQualifiedUnavailableReason: lastQualifiedReason, + changedFilesAvailable, + changedFileCount: changedFilesAvailable ? new Set(changedFiles).size : null, + }; + + // 1. Fail-closed escalations, and the explicit dispatch override. + const escalations = []; + if (forceFull) escalations.push(REASONS.forceFull); + if (lastQualifiedReason !== null) escalations.push(lastQualifiedReason); + if (headSha === null) escalations.push(REASONS.headShaInvalid); + if (treeSha === null) escalations.push(REASONS.treeShaInvalid); + if (nowUtc === null) escalations.push(REASONS.nowUnparseable); + if (!weeklySlotValid) escalations.push(REASONS.weeklySlotInvalid); + const policyErrors = isObject(source.policy) ? validatePolicy(source.policy) : ['policy is not an object']; + if (policyErrors.length > 0) escalations.push(REASONS.policyInvalid); + if (escalations.length > 0) { + return buildReceipt({ + ...common, verdict: 'full-sweep', reasons: escalations, selectedSuites: NIGHTLY_DEEP_SUITES, + }); + } + + // 2. The weekly entropy sweep runs regardless of the diff. + if (weeklySlot.matched) { + return buildReceipt({ + ...common, verdict: 'weekly-full', reasons: [REASONS.weeklySlot], selectedSuites: NIGHTLY_DEEP_SUITES, + }); + } + + // 3. Identical content is identical evidence, whether or not the diff is reachable. + if (duplicate.duplicate) { + return buildReceipt({ + ...common, + verdict: 'no-change', + reasons: [REASONS.identicalTree], + selectedSuites: [], + skipReason: SKIP_REASONS.noChange, + }); + } + + // 4. Without a diff there is no basis for selection. + if (!changedFilesAvailable) { + return buildReceipt({ + ...common, verdict: 'full-sweep', reasons: [REASONS.diffUnavailable], selectedSuites: NIGHTLY_DEEP_SUITES, + }); + } + + // 5. Map paths to policy groups with the planner's matcher, never a second glob implementation. + const { groups, unmapped, controlPathsChanged } = matchGroups(uniqueSorted(changedFiles), source.policy); + const matchedGroups = [...groups.keys()].sort(); + const withPaths = { + ...common, + matchedGroups, + unmappedPaths: uniqueSorted(unmapped), + controlPathsChanged: uniqueSorted(controlPathsChanged), + }; + const pathEscalations = []; + if (unmapped.length > 0) pathEscalations.push(REASONS.unmappedPath); + if (controlPathsChanged.length > 0) pathEscalations.push(REASONS.controlPathChanged); + if (matchedGroups.some((groupId) => !Object.hasOwn(GROUP_DEEP_SUITES, groupId))) { + pathEscalations.push(REASONS.groupNotMapped); + } + if (pathEscalations.length > 0) { + return buildReceipt({ + ...withPaths, verdict: 'full-sweep', reasons: pathEscalations, selectedSuites: NIGHTLY_DEEP_SUITES, + }); + } + + // 6. The union of the matched groups' deep suites. + const selected = new Set(); + for (const groupId of matchedGroups) for (const suite of GROUP_DEEP_SUITES[groupId]) selected.add(suite); + if (selected.size === 0) { + return buildReceipt({ + ...withPaths, + verdict: 'no-change', + reasons: [REASONS.noDeepSuiteGroups], + selectedSuites: [], + skipReason: SKIP_REASONS.noChange, + }); + } + return buildReceipt({ + ...withPaths, verdict: 'affected', reasons: [REASONS.affectedGroups], selectedSuites: [...selected], + }); + } catch { + // Invariant 2: a coordinator defect escalates rather than producing a selective plan. + return buildReceipt({ + ...base, + lastQualified: null, + lastQualifiedUnavailableReason: REASONS.lastReceiptUnreadable, + changedFilesAvailable: false, + changedFileCount: null, + verdict: 'full-sweep', + reasons: [REASONS.coordinatorError], + selectedSuites: NIGHTLY_DEEP_SUITES, + }); + } +} + +function markdownCell(value) { + return String(value ?? '').replace(/\|/g, '\\|').replace(/[\r\n]+/g, ' '); +} + +function shortSha(value) { + return value === null || value === undefined ? 'unknown' : String(value).slice(0, 12); +} + +/** + * Markdown for `$GITHUB_STEP_SUMMARY`. Derived only from the receipt, so it is deterministic too. + * @param {object} receipt a receipt from decideNightlyPlan() + * @returns {string} + */ +export function renderNightlySummary(receipt) { + const selectedSet = new Set(receipt.selectedSuites); + const lastQualified = receipt.lastQualified + ? `head \`${shortSha(receipt.lastQualified.headSha)}\`, tree \`${shortSha(receipt.lastQualified.treeSha)}\`` + + ` at \`${markdownCell(receipt.lastQualified.completedAtUtc)}\`` + + ` (complete: ${receipt.lastQualified.complete ? 'yes' : 'no'})` + : `none (\`${markdownCell(receipt.lastQualifiedUnavailableReason ?? 'unknown')}\`)`; + const weekly = receipt.weeklySlot.utcDay === null ? 'not configured' : `UTC day ${receipt.weeklySlot.utcDay}`; + const today = receipt.weeklySlot.nowUtcDay === null ? 'unknown' : String(receipt.weeklySlot.nowUtcDay); + const lines = [ + '# Smart CI nightly coordinator', + '', + `- Verdict: **${markdownCell(receipt.verdict)}**`, + `- Reasons: ${receipt.reasons.map((reason) => `\`${markdownCell(reason)}\``).join(', ')}`, + `- Current: head \`${shortSha(receipt.current.headSha)}\`, tree \`${shortSha(receipt.current.treeSha)}\``, + `- Last qualified: ${lastQualified}`, + `- Duplicate qualification: **${receipt.duplicateQualification ? 'yes' : 'no'}**` + + ` (\`${markdownCell(receipt.duplicateQualificationReason)}\`)`, + `- Changed files: ${receipt.changedFilesAvailable ? String(receipt.changedFileCount) : 'unavailable'}`, + `- Weekly slot: ${weekly} (today ${today}, matched ${receipt.weeklySlot.matched ? 'yes' : 'no'})`, + `- Matched path groups: ${receipt.matchedGroups.length > 0 + ? receipt.matchedGroups.map((group) => `\`${markdownCell(group)}\``).join(', ') + : 'none'}`, + '', + '## Deep suites', + '', + '| Suite | Decision | Reason |', + '| --- | --- | --- |', + ]; + const skipReasons = new Map(receipt.skippedSuites.map((entry) => [entry.suite, entry.reason])); + for (const suite of NIGHTLY_DEEP_SUITES) { + const selected = selectedSet.has(suite); + const reason = selected ? receipt.verdict : skipReasons.get(suite) ?? 'not-selected'; + lines.push(`| \`${markdownCell(suite)}\` | ${selected ? 'run' : 'skip'} | ${markdownCell(reason)} |`); + } + if (receipt.unmappedPaths.length > 0) { + lines.push('', `Unmapped paths: ${receipt.unmappedPaths.map((path) => `\`${markdownCell(path)}\``).join(', ')}`); + } + if (receipt.controlPathsChanged.length > 0) { + lines.push('', `Changed control paths: ${receipt.controlPathsChanged.map((path) => `\`${markdownCell(path)}\``).join(', ')}`); + } + return `${lines.join('\n')}\n`; +} + +/** + * One path per line, or TSV `statuspathprevious_path`, matching plan.mjs's list format. + * @param {string} text + * @returns {string[]} + */ +export function parseChangedFileList(text) { + const paths = []; + for (const rawLine of String(text).split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + const parts = line.split('\t'); + if (parts.length >= 2) { + if (parts[1]) paths.push(parts[1]); + if (parts[2]) paths.push(parts[2]); + } else { + paths.push(parts[0]); + } + } + return paths; +} + +export const USAGE = 'usage: nightly-coordinator.mjs --policy --head-sha --tree-sha ' + + ' [--last-receipt ] [--changed-files ] [--now ] [--weekly-slot <0-6>]' + + ' [--force-full] [--out ] [--out-md ] [--summary ]'; + +/** + * Parse the CLI arguments. Throws on an unknown flag so a workflow typo is loud. + * @param {string[]} argv `process.argv.slice(2)` + */ +export function parseArgs(argv) { + const args = { + policy: 'ci/policy.v1.json', + lastReceipt: null, + headSha: null, + treeSha: null, + changedFiles: null, + now: null, + weeklySlotUtcDay: null, + forceFull: false, + out: null, + outMarkdown: null, + summary: null, + help: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = () => argv[++index]; + switch (arg) { + case '--policy': args.policy = next(); break; + case '--last-receipt': args.lastReceipt = next(); break; + case '--head-sha': args.headSha = next(); break; + case '--tree-sha': args.treeSha = next(); break; + case '--changed-files': args.changedFiles = next(); break; + case '--now': args.now = next(); break; + case '--weekly-slot': args.weeklySlotUtcDay = Number(next()); break; + case '--force-full': args.forceFull = true; break; + case '--out': args.out = next(); break; + case '--out-md': args.outMarkdown = next(); break; + case '--summary': args.summary = next(); break; + case '--help': args.help = true; break; + default: throw new Error(`Unknown argument: ${arg}`); + } + } + return args; +} + +function writeOutput(path, contents) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, contents); +} + +/** + * Assemble the pure input from files and run the decision. Exported for the CLI test; it reads + * the named files and performs no network, git or clock access. + * @param {ReturnType} args + * @returns {object} the receipt + */ +export function runCoordinator(args) { + let policy = null; + let digest = null; + try { + const policyText = readFileSync(args.policy, 'utf8'); + digest = policyDigest(policyText); + policy = JSON.parse(policyText); + } catch { + policy = null; + digest = null; + } + let lastQualified = null; + let lastQualifiedReason = REASONS.lastReceiptMissing; + if (args.lastReceipt) { + if (!existsSync(args.lastReceipt)) { + lastQualifiedReason = REASONS.lastReceiptMissing; + } else { + try { + lastQualified = JSON.parse(readFileSync(args.lastReceipt, 'utf8')); + lastQualifiedReason = null; + } catch { + lastQualified = null; + lastQualifiedReason = REASONS.lastReceiptUnreadable; + } + } + } + let changedFiles = null; + if (args.changedFiles && existsSync(args.changedFiles)) { + changedFiles = parseChangedFileList(readFileSync(args.changedFiles, 'utf8')); + } + return decideNightlyPlan({ + policy, + policyDigest: digest, + lastQualified, + lastQualifiedReason, + headSha: args.headSha, + treeSha: args.treeSha, + changedFiles, + nowUtc: args.now, + // Passed through unchanged: `--weekly-slot friday` arrives as NaN and must escalate rather + // than silently disabling the weekly sweep. + weeklySlotUtcDay: args.weeklySlotUtcDay, + forceFull: args.forceFull === true, + }); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(USAGE); + return; + } + const receipt = runCoordinator(args); + const json = `${JSON.stringify(receipt, null, 2)}\n`; + const markdown = renderNightlySummary(receipt); + if (args.out) writeOutput(args.out, json); + if (args.outMarkdown) writeOutput(args.outMarkdown, markdown); + if (args.summary) appendFileSync(args.summary, markdown); + process.stdout.write(markdown); +} + +if (process.argv[1] && /nightly-coordinator\.mjs$/.test(process.argv[1])) main(); From f6a3ff65a3bc0d1947721e6ad0c8b079d11a929d Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Sat, 5 Sep 2026 02:19:16 +0100 Subject: [PATCH 2/3] test(smart-ci): cover the nightly coordinator decision plan The issue's head-start test plan (#2334): no-change with both SHAs, backend-only selection without the browser matrix, the weekly slot overriding an empty diff, missing / unreadable / incomplete last receipt, an unreachable diff, duplicate qualification, an unmapped path, a changed control path, determinism, and CLI argv parsing with file output through a temp directory. Adds the enumeration test that asserts every ci/policy.v1.json path group has a deep-suite mapping in both directions, so a new policy group cannot silently select nothing. --- .../ci/smart-ci/nightly-coordinator.test.mjs | 514 ++++++++++++++++++ 1 file changed, 514 insertions(+) create mode 100644 scripts/ci/smart-ci/nightly-coordinator.test.mjs diff --git a/scripts/ci/smart-ci/nightly-coordinator.test.mjs b/scripts/ci/smart-ci/nightly-coordinator.test.mjs new file mode 100644 index 000000000..05efd4dcb --- /dev/null +++ b/scripts/ci/smart-ci/nightly-coordinator.test.mjs @@ -0,0 +1,514 @@ +// CI-10 slice 1 (#2334): the nightly coordinator decision module. Covers the issue's head-start +// test plan in docs/analysis/2026-08-30-acceleration-bundle/issues/2334-*.md plus the policy-group +// enumeration that keeps GROUP_DEEP_SUITES honest as `ci/policy.v1.json` grows. + +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { policyDigest } from './lib/plan.mjs'; +import { + GROUP_DEEP_SUITES, + NIGHTLY_COORDINATOR_KIND, + NIGHTLY_COORDINATOR_SCHEMA_VERSION, + NIGHTLY_DEEP_SUITES, + REASONS, + SKIP_REASONS, + decideNightlyPlan, + detectDuplicateQualification, + nightlyCoordinatorMappingErrors, + normaliseLastQualified, + parseArgs, + parseChangedFileList, + renderNightlySummary, +} from './nightly-coordinator.mjs'; + +const policyPath = fileURLToPath(new URL('../../../ci/policy.v1.json', import.meta.url)); +const policyText = readFileSync(policyPath, 'utf8'); +const policy = JSON.parse(policyText); +const digest = policyDigest(policyText); + +const HEAD = 'a'.repeat(40); +const TREE = 'b'.repeat(40); +const LAST_HEAD = 'c'.repeat(40); +const LAST_TREE = 'd'.repeat(40); +// 2026-09-03 is a Thursday (UTC day 4); 2026-09-05 is a Saturday (UTC day 6). +const THURSDAY = '2026-09-03T03:25:00.000Z'; +const SATURDAY = '2026-09-05T03:25:00.000Z'; + +function lastReceipt(overrides = {}) { + return { + headSha: LAST_HEAD, + treeSha: LAST_TREE, + completedAtUtc: '2026-09-02T03:59:00.000Z', + complete: true, + ...overrides, + }; +} + +function coordinatorInput(overrides = {}) { + return { + policy, + policyDigest: digest, + lastQualified: lastReceipt(), + headSha: HEAD, + treeSha: TREE, + changedFiles: [], + nowUtc: THURSDAY, + weeklySlotUtcDay: 6, + forceFull: false, + ...overrides, + }; +} + +test('the checked-in policy sanity: the fixture SHAs and the weekly slot days are what the tests assume', () => { + assert.equal(new Date(THURSDAY).getUTCDay(), 4); + assert.equal(new Date(SATURDAY).getUTCDay(), 6); + assert.equal(NIGHTLY_DEEP_SUITES.length, 12); + assert.equal(new Set(NIGHTLY_DEEP_SUITES).size, 12); +}); + +test('no relevant change gives no-change and a receipt naming both SHAs', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + treeSha: LAST_TREE, + changedFiles: [], + })); + + assert.equal(receipt.schemaVersion, NIGHTLY_COORDINATOR_SCHEMA_VERSION); + assert.equal(receipt.kind, NIGHTLY_COORDINATOR_KIND); + assert.equal(receipt.verdict, 'no-change'); + assert.deepEqual(receipt.reasons, [REASONS.identicalTree]); + assert.equal(receipt.current.headSha, HEAD); + assert.equal(receipt.current.treeSha, LAST_TREE); + assert.equal(receipt.lastQualified.headSha, LAST_HEAD); + assert.equal(receipt.lastQualified.treeSha, LAST_TREE); + assert.equal(receipt.lastQualifiedUnavailableReason, null); + assert.deepEqual(receipt.selectedSuites, []); + // Invariant 1: an explicit receipt for every suite, not a skipped workflow. + assert.equal(receipt.skippedSuites.length, NIGHTLY_DEEP_SUITES.length); + for (const entry of receipt.skippedSuites) assert.equal(entry.reason, SKIP_REASONS.noChange); + assert.equal(receipt.generatedAtUtc, THURSDAY); + assert.equal(receipt.policyId, policy.policyId); + assert.equal(receipt.policyDigest, digest); +}); + +test('a docs-only change maps to a group with no deep suite and is also no-change', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + changedFiles: ['docs/STATUS.md', 'autodoc/AGENT_INDEX.md'], + })); + + assert.equal(receipt.verdict, 'no-change'); + assert.deepEqual(receipt.reasons, [REASONS.noDeepSuiteGroups]); + assert.deepEqual(receipt.matchedGroups, ['docs']); + assert.deepEqual(receipt.selectedSuites, []); + assert.equal(receipt.changedFileCount, 2); +}); + +test('a backend-only change selects the backend deep suites and not the browser matrix', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + changedFiles: [ + 'backend/src/Taskdeck.Domain/Boards/Board.cs', + 'backend/src/Taskdeck.Application/Boards/CreateBoardHandler.cs', + ], + })); + + assert.equal(receipt.verdict, 'affected'); + assert.deepEqual(receipt.reasons, [REASONS.affectedGroups]); + assert.deepEqual(receipt.matchedGroups, ['backend-application', 'backend-domain']); + assert.deepEqual(receipt.selectedSuites, [ + 'backend-solution', + 'load-concurrency-harness', + 'performance-regression-gate', + 'container-images', + 'backend-coverage', + ]); + assert.ok(!receipt.selectedSuites.includes('e2e-cross-browser')); + assert.ok(!receipt.selectedSuites.includes('e2e-smoke')); + assert.ok(!receipt.selectedSuites.includes('frontend-coverage')); + const skipped = new Map(receipt.skippedSuites.map((entry) => [entry.suite, entry.reason])); + assert.equal(skipped.get('e2e-cross-browser'), SKIP_REASONS.notAffected); +}); + +test('a frontend-only change selects the browser suites and no backend solution run', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + changedFiles: ['frontend/taskdeck-web/src/views/BoardView.vue'], + })); + + assert.equal(receipt.verdict, 'affected'); + assert.deepEqual(receipt.matchedGroups, ['frontend-src']); + assert.deepEqual(receipt.selectedSuites, ['e2e-smoke', 'e2e-cross-browser', 'frontend-coverage']); + assert.ok(!receipt.selectedSuites.includes('backend-solution')); +}); + +test('the weekly slot forces weekly-full even when the diff is empty', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + nowUtc: SATURDAY, + weeklySlotUtcDay: 6, + changedFiles: [], + })); + + assert.equal(receipt.verdict, 'weekly-full'); + assert.deepEqual(receipt.reasons, [REASONS.weeklySlot]); + assert.deepEqual(receipt.weeklySlot, { utcDay: 6, nowUtcDay: 6, matched: true }); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); + assert.deepEqual(receipt.skippedSuites, []); +}); + +test('the weekly slot forces weekly-full even when the tree SHA is unchanged', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + nowUtc: SATURDAY, + weeklySlotUtcDay: 6, + treeSha: LAST_TREE, + changedFiles: [], + })); + + assert.equal(receipt.verdict, 'weekly-full'); + assert.equal(receipt.duplicateQualification, true); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); +}); + +test('an out-of-range or unparseable weekly slot fails closed rather than disabling the sweep', () => { + for (const slot of [7, -1, 1.5, Number.NaN, 'saturday']) { + const receipt = decideNightlyPlan(coordinatorInput({ weeklySlotUtcDay: slot })); + assert.equal(receipt.verdict, 'full-sweep', `slot ${String(slot)}`); + assert.ok(receipt.reasons.includes(REASONS.weeklySlotInvalid), `slot ${String(slot)}`); + } + const unconfigured = decideNightlyPlan(coordinatorInput({ weeklySlotUtcDay: null })); + assert.equal(unconfigured.verdict, 'no-change'); + assert.deepEqual(unconfigured.weeklySlot, { utcDay: null, nowUtcDay: 4, matched: false }); +}); + +test('a missing last receipt gives full-sweep, never no-change', () => { + const receipt = decideNightlyPlan(coordinatorInput({ lastQualified: null, treeSha: LAST_TREE })); + + assert.equal(receipt.verdict, 'full-sweep'); + assert.deepEqual(receipt.reasons, [REASONS.lastReceiptMissing]); + assert.equal(receipt.lastQualified, null); + assert.equal(receipt.lastQualifiedUnavailableReason, REASONS.lastReceiptMissing); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); +}); + +test('an unreadable last receipt gives full-sweep', () => { + const unreadable = [ + 'not-an-object', + lastReceipt({ headSha: 'nope' }), + lastReceipt({ treeSha: null }), + lastReceipt({ completedAtUtc: 'never' }), + ]; + for (const value of unreadable) { + const receipt = decideNightlyPlan(coordinatorInput({ lastQualified: value })); + assert.equal(receipt.verdict, 'full-sweep'); + assert.deepEqual(receipt.reasons, [REASONS.lastReceiptUnreadable]); + assert.equal(receipt.lastQualified, null); + } + + const explicit = decideNightlyPlan(coordinatorInput({ + lastQualified: null, + lastQualifiedReason: REASONS.lastReceiptUnreadable, + })); + assert.equal(explicit.verdict, 'full-sweep'); + assert.deepEqual(explicit.reasons, [REASONS.lastReceiptUnreadable]); +}); + +test('an incomplete last receipt gives full-sweep and never advances the marker', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + lastQualified: lastReceipt({ complete: false }), + treeSha: LAST_TREE, + changedFiles: [], + })); + + assert.equal(receipt.verdict, 'full-sweep'); + assert.deepEqual(receipt.reasons, [REASONS.lastReceiptIncomplete]); + assert.equal(receipt.lastQualified.complete, false); + assert.equal(receipt.duplicateQualification, false); + assert.equal(receipt.duplicateQualificationReason, REASONS.lastReceiptIncomplete); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); + + const normalised = normaliseLastQualified(lastReceipt({ complete: false })); + assert.equal(normalised.reason, REASONS.lastReceiptIncomplete); + assert.equal(normalised.receipt.complete, false); +}); + +test('an unreachable diff gives full-sweep with the reason in the receipt', () => { + const receipt = decideNightlyPlan(coordinatorInput({ changedFiles: null })); + + assert.equal(receipt.verdict, 'full-sweep'); + assert.deepEqual(receipt.reasons, [REASONS.diffUnavailable]); + assert.equal(receipt.changedFilesAvailable, false); + assert.equal(receipt.changedFileCount, null); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); +}); + +test('duplicate qualification is detected from the tree SHA', () => { + assert.deepEqual(detectDuplicateQualification(LAST_TREE, lastReceipt()), { + duplicate: true, + reason: 'tree-sha-already-qualified', + treeSha: LAST_TREE, + lastQualifiedTreeSha: LAST_TREE, + }); + assert.equal(detectDuplicateQualification(TREE, lastReceipt()).duplicate, false); + assert.equal(detectDuplicateQualification(TREE, lastReceipt()).reason, 'tree-sha-differs'); + assert.equal(detectDuplicateQualification(TREE, null).reason, 'no-last-receipt'); + assert.equal(detectDuplicateQualification('short', lastReceipt()).reason, 'current-tree-sha-invalid'); + assert.equal(detectDuplicateQualification(TREE, lastReceipt({ treeSha: 'x' })).reason, 'last-receipt-tree-sha-invalid'); + // Case-insensitive: the same tree in upper case is still the same tree. + assert.equal(detectDuplicateQualification(LAST_TREE.toUpperCase(), lastReceipt()).duplicate, true); + + const receipt = decideNightlyPlan(coordinatorInput({ treeSha: LAST_TREE })); + assert.equal(receipt.duplicateQualification, true); + assert.equal(receipt.duplicateQualificationReason, 'tree-sha-already-qualified'); +}); + +test('an unmapped path gives full-sweep', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + changedFiles: ['some-new-top-level-thing.bin', 'backend/src/Taskdeck.Domain/Boards/Board.cs'], + })); + + assert.equal(receipt.verdict, 'full-sweep'); + assert.deepEqual(receipt.reasons, [REASONS.unmappedPath]); + assert.deepEqual(receipt.unmappedPaths, ['some-new-top-level-thing.bin']); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); +}); + +test('a control-path change gives full-sweep', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + changedFiles: ['.github/workflows/ci-nightly.yml', 'docs/ci/SMART_CI.md'], + })); + + assert.equal(receipt.verdict, 'full-sweep'); + assert.ok(receipt.reasons.includes(REASONS.controlPathChanged)); + assert.deepEqual(receipt.controlPathsChanged, ['.github/workflows/ci-nightly.yml']); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); +}); + +test('an invalid policy, an invalid SHA, an unparseable clock and force-full each fail closed', () => { + const invalidPolicy = decideNightlyPlan(coordinatorInput({ policy: { policyId: 'broken' } })); + assert.equal(invalidPolicy.verdict, 'full-sweep'); + assert.ok(invalidPolicy.reasons.includes(REASONS.policyInvalid)); + + const noPolicy = decideNightlyPlan(coordinatorInput({ policy: null })); + assert.equal(noPolicy.verdict, 'full-sweep'); + assert.ok(noPolicy.reasons.includes(REASONS.policyInvalid)); + + const badHead = decideNightlyPlan(coordinatorInput({ headSha: 'HEAD' })); + assert.equal(badHead.verdict, 'full-sweep'); + assert.ok(badHead.reasons.includes(REASONS.headShaInvalid)); + assert.equal(badHead.current.headSha, null); + + const badTree = decideNightlyPlan(coordinatorInput({ treeSha: '' })); + assert.equal(badTree.verdict, 'full-sweep'); + assert.ok(badTree.reasons.includes(REASONS.treeShaInvalid)); + + const badClock = decideNightlyPlan(coordinatorInput({ nowUtc: 'tonight' })); + assert.equal(badClock.verdict, 'full-sweep'); + assert.ok(badClock.reasons.includes(REASONS.nowUnparseable)); + assert.equal(badClock.generatedAtUtc, '1970-01-01T00:00:00.000Z'); + + const forced = decideNightlyPlan(coordinatorInput({ forceFull: true, treeSha: LAST_TREE })); + assert.equal(forced.verdict, 'full-sweep'); + assert.ok(forced.reasons.includes(REASONS.forceFull)); + assert.equal(forced.forceFull, true); + + const nothing = decideNightlyPlan(undefined); + assert.equal(nothing.verdict, 'full-sweep'); + assert.deepEqual(nothing.selectedSuites, [...NIGHTLY_DEEP_SUITES]); +}); + +test('every policy path group has a deep-suite mapping and every mapped suite exists', () => { + assert.deepEqual(nightlyCoordinatorMappingErrors(policy), []); + + const policyIds = policy.pathGroups.map((group) => group.id).sort(); + assert.deepEqual(Object.keys(GROUP_DEEP_SUITES).sort(), policyIds); + + const known = new Set(NIGHTLY_DEEP_SUITES); + for (const [groupId, suites] of Object.entries(GROUP_DEEP_SUITES)) { + for (const suite of suites) assert.ok(known.has(suite), `${groupId} names unknown suite ${suite}`); + } + + // A new policy group with no mapping is reported, and at decision time it fails closed. + const grown = { ...policy, pathGroups: [...policy.pathGroups, { id: 'brand-new-surface', riskFloor: 'R2', patterns: ['brand-new/**'], lanes: [] }] }; + const errors = nightlyCoordinatorMappingErrors(grown); + assert.deepEqual(errors, ['policy path group brand-new-surface has no deep-suite mapping']); + + const receipt = decideNightlyPlan(coordinatorInput({ policy: grown, changedFiles: ['brand-new/thing.txt'] })); + assert.equal(receipt.verdict, 'full-sweep'); + assert.ok(receipt.reasons.includes(REASONS.groupNotMapped)); + assert.deepEqual(receipt.selectedSuites, [...NIGHTLY_DEEP_SUITES]); + + // A dropped policy group is reported too, so the table cannot rot in the other direction. + const shrunk = { ...policy, pathGroups: policy.pathGroups.filter((group) => group.id !== 'load-and-evals') }; + assert.deepEqual(nightlyCoordinatorMappingErrors(shrunk), [ + 'deep-suite mapping names unknown policy path group load-and-evals', + ]); +}); + +test('two calls with the same input produce byte-identical JSON and markdown', () => { + const input = () => coordinatorInput({ + changedFiles: [ + 'frontend/taskdeck-web/src/views/BoardView.vue', + 'backend/src/Taskdeck.Api/Endpoints/BoardEndpoints.cs', + 'backend/src/Taskdeck.Api/Endpoints/BoardEndpoints.cs', + ], + }); + const first = decideNightlyPlan(input()); + const second = decideNightlyPlan(input()); + + assert.equal(JSON.stringify(first, null, 2), JSON.stringify(second, null, 2)); + assert.equal(renderNightlySummary(first), renderNightlySummary(second)); + assert.equal(first.verdict, 'affected'); + assert.equal(first.changedFileCount, 2, 'duplicate paths are counted once'); + + // Input order must not move a byte either. + const reordered = decideNightlyPlan(coordinatorInput({ + changedFiles: [ + 'backend/src/Taskdeck.Api/Endpoints/BoardEndpoints.cs', + 'frontend/taskdeck-web/src/views/BoardView.vue', + ], + })); + const forward = decideNightlyPlan(coordinatorInput({ + changedFiles: [ + 'frontend/taskdeck-web/src/views/BoardView.vue', + 'backend/src/Taskdeck.Api/Endpoints/BoardEndpoints.cs', + ], + })); + assert.equal(JSON.stringify(reordered), JSON.stringify(forward)); +}); + +test('the markdown summary renders every deep suite and the verdict', () => { + const receipt = decideNightlyPlan(coordinatorInput({ + changedFiles: ['backend/src/Taskdeck.Domain/Boards/Board.cs'], + })); + const markdown = renderNightlySummary(receipt); + + assert.match(markdown, /^# Smart CI nightly coordinator$/m); + assert.match(markdown, /- Verdict: \*\*affected\*\*/); + for (const suite of NIGHTLY_DEEP_SUITES) assert.ok(markdown.includes(`\`${suite}\``), suite); + assert.match(markdown, /\| `backend-solution` \| run \| affected \|/); + assert.match(markdown, /\| `e2e-cross-browser` \| skip \| not-selected-by-changed-groups \|/); +}); + +test('parseChangedFileList reads plain lines and TSV rename rows', () => { + assert.deepEqual(parseChangedFileList('docs/a.md\n\nbackend/b.cs\n'), ['docs/a.md', 'backend/b.cs']); + assert.deepEqual(parseChangedFileList('modified\tdocs/a.md\t\nrenamed\tdocs/new.md\tdocs/old.md\n'), [ + 'docs/a.md', + 'docs/new.md', + 'docs/old.md', + ]); +}); + +test('parseArgs reads every documented flag and rejects an unknown one', () => { + const args = parseArgs([ + '--policy', 'ci/policy.v1.json', + '--last-receipt', 'last.json', + '--head-sha', HEAD, + '--tree-sha', TREE, + '--changed-files', 'changed.txt', + '--now', THURSDAY, + '--weekly-slot', '6', + '--force-full', + '--out', 'out.json', + '--out-md', 'out.md', + '--summary', 'summary.md', + ]); + + assert.deepEqual(args, { + policy: 'ci/policy.v1.json', + lastReceipt: 'last.json', + headSha: HEAD, + treeSha: TREE, + changedFiles: 'changed.txt', + now: THURSDAY, + weeklySlotUtcDay: 6, + forceFull: true, + out: 'out.json', + outMarkdown: 'out.md', + summary: 'summary.md', + help: false, + }); + assert.equal(parseArgs([]).policy, 'ci/policy.v1.json'); + assert.equal(parseArgs(['--help']).help, true); + assert.throws(() => parseArgs(['--nope']), /Unknown argument: --nope/); +}); + +test('the CLI writes the receipt JSON and the markdown summary to files', () => { + const root = mkdtempSync(join(tmpdir(), 'taskdeck-nightly-coordinator-')); + const receiptPath = join(root, 'last.json'); + const changedPath = join(root, 'changed.tsv'); + const outPath = join(root, 'nested', 'nightly-plan.json'); + const outMarkdownPath = join(root, 'nested', 'nightly-plan.md'); + const summaryPath = join(root, 'step-summary.md'); + + try { + writeFileSync(receiptPath, `${JSON.stringify(lastReceipt())}\n`); + writeFileSync(changedPath, 'modified\tbackend/src/Taskdeck.Domain/Boards/Board.cs\t\n'); + writeFileSync(summaryPath, ''); + + const stdout = execFileSync(process.execPath, [ + fileURLToPath(new URL('./nightly-coordinator.mjs', import.meta.url)), + '--policy', policyPath, + '--last-receipt', receiptPath, + '--head-sha', HEAD, + '--tree-sha', TREE, + '--changed-files', changedPath, + '--now', THURSDAY, + '--weekly-slot', '6', + '--out', outPath, + '--out-md', outMarkdownPath, + '--summary', summaryPath, + ], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + + const receipt = JSON.parse(readFileSync(outPath, 'utf8')); + assert.equal(receipt.kind, NIGHTLY_COORDINATOR_KIND); + assert.equal(receipt.verdict, 'affected'); + assert.deepEqual(receipt.matchedGroups, ['backend-domain']); + assert.equal(receipt.current.headSha, HEAD); + assert.equal(receipt.lastQualified.treeSha, LAST_TREE); + assert.equal(receipt.policyDigest, digest); + assert.equal(receipt.generatedAtUtc, THURSDAY); + assert.ok(!receipt.selectedSuites.includes('e2e-cross-browser')); + + const markdown = readFileSync(outMarkdownPath, 'utf8'); + assert.equal(markdown, renderNightlySummary(receipt)); + assert.equal(readFileSync(summaryPath, 'utf8'), markdown); + assert.equal(stdout, markdown); + + // A missing changed-file list is an unavailable diff, not an empty one. + const missingDiffOut = join(root, 'missing-diff.json'); + execFileSync(process.execPath, [ + fileURLToPath(new URL('./nightly-coordinator.mjs', import.meta.url)), + '--policy', policyPath, + '--last-receipt', receiptPath, + '--head-sha', HEAD, + '--tree-sha', TREE, + '--changed-files', join(root, 'does-not-exist.tsv'), + '--now', THURSDAY, + '--out', missingDiffOut, + ], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + const missingDiff = JSON.parse(readFileSync(missingDiffOut, 'utf8')); + assert.equal(missingDiff.verdict, 'full-sweep'); + assert.deepEqual(missingDiff.reasons, [REASONS.diffUnavailable]); + + // An unreadable last receipt escalates rather than throwing. + const badReceiptPath = join(root, 'corrupt.json'); + writeFileSync(badReceiptPath, '{ not json'); + const badReceiptOut = join(root, 'corrupt-plan.json'); + execFileSync(process.execPath, [ + fileURLToPath(new URL('./nightly-coordinator.mjs', import.meta.url)), + '--policy', policyPath, + '--last-receipt', badReceiptPath, + '--head-sha', HEAD, + '--tree-sha', TREE, + '--changed-files', changedPath, + '--now', THURSDAY, + '--out', badReceiptOut, + ], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + const badReceipt = JSON.parse(readFileSync(badReceiptOut, 'utf8')); + assert.equal(badReceipt.verdict, 'full-sweep'); + assert.deepEqual(badReceipt.reasons, [REASONS.lastReceiptUnreadable]); + } finally { + rmSync(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); + } +}); From 929331247dbfbc0ced442fbe95da776ccc933516 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Sat, 5 Sep 2026 02:33:25 +0100 Subject: [PATCH 3/3] fix(smart-ci): close nightly suite selection under the workflow needs edges Five ci-nightly.yml jobs declare needs: backend-solution, so a receipt that selected e2e-smoke, e2e-cross-browser, container-images or the k6 suites without backend-solution promised evidence GitHub Actions would skip. Declare the needs edges as SUITE_PREREQUISITES, close every receipt selection under them, and enumerate the workflow edges in the tests so a new edge cannot drift away from the table. --- scripts/ci/smart-ci/nightly-coordinator.mjs | 77 ++++++++++- .../ci/smart-ci/nightly-coordinator.test.mjs | 124 +++++++++++++++++- 2 files changed, 195 insertions(+), 6 deletions(-) diff --git a/scripts/ci/smart-ci/nightly-coordinator.mjs b/scripts/ci/smart-ci/nightly-coordinator.mjs index e6b4a231a..fdf8db23e 100644 --- a/scripts/ci/smart-ci/nightly-coordinator.mjs +++ b/scripts/ci/smart-ci/nightly-coordinator.mjs @@ -17,7 +17,8 @@ // Verdicts // no-change the current tree SHA equals the last qualified tree SHA, or the diff maps only to // path groups with no deep suite. Selected: none. This is the honest green receipt. -// affected the union of the deep suites of the matched path groups. +// affected the union of the deep suites of the matched path groups, closed under the +// `needs:` edges of the nightly workflows (SUITE_PREREQUISITES). // weekly-full the configured weekly UTC slot forces the complete sweep regardless of the diff. // full-sweep fail-closed escalation, or an explicit `--force-full` dispatch. // Precedence: full-sweep > weekly-full > no-change (identical tree) > affected. @@ -44,7 +45,7 @@ // matchedGroups: string[], // unmappedPaths: string[], // controlPathsChanged: string[], -// selectedSuites: string[], // canonical NIGHTLY_DEEP_SUITES order +// selectedSuites: string[], // canonical order, closed under SUITE_PREREQUISITES // skippedSuites: [{ suite: string, reason: string }] // } // @@ -85,6 +86,27 @@ export const NIGHTLY_DEEP_SUITES = Object.freeze([ 'dependency-security-signals', ]); +/** + * The `needs:` edges of `.github/workflows/ci-nightly.yml`: five deep suites cannot start until + * `backend-solution` succeeds, because they consume the solution build it publishes. + * `nightly-quality.yml` declares no `needs:` at all, so its three suites appear here with none. + * + * A selected suite must drag its prerequisites in. GitHub Actions skips a job whose `needs:` + * dependency was skipped (unless its `if:` uses `always()`/`!cancelled()`, which these jobs do + * not), so selecting `e2e-smoke` without `backend-solution` would promise evidence that never + * runs — SMART_CI invariant 1, a skip must never read as evidence. The closure lives in this + * module rather than in each GROUP_DEEP_SUITES entry so a new group cannot reintroduce the gap. + * CI10-2, which wires the receipt into the workflows, must keep this table and the workflow + * graph in step; `nightlyCoordinatorSuiteGraphErrors()` and its test enforce the shape. + */ +export const SUITE_PREREQUISITES = Object.freeze({ + 'e2e-smoke': Object.freeze(['backend-solution']), + 'load-concurrency-harness': Object.freeze(['backend-solution']), + 'performance-regression-gate': Object.freeze(['backend-solution']), + 'e2e-cross-browser': Object.freeze(['backend-solution']), + 'container-images': Object.freeze(['backend-solution']), +}); + const BACKEND_SUITES = Object.freeze([ 'backend-solution', 'backend-coverage', @@ -112,6 +134,10 @@ const API_CONTRACT_SUITES = Object.freeze(['openapi-guardrail', 'developer-porta * escalate to the full sweep before this table is consulted; `backend-project-files` carries the * dependency suites for the manifests that are not control paths. * + * An entry names the suites whose EVIDENCE the group needs; it does not have to name the suites + * those depend on. Every selection is closed under SUITE_PREREQUISITES before it reaches a receipt, + * so a frontend entry that names `e2e-smoke` still gets `backend-solution` in `selectedSuites`. + * * CI10-2 moves this table into `ci/policy.v1.json` next to `pathGroups` so the policy digest covers * it; it lives here in slice 1 so the decision function can be proven before the policy schema * changes. Until then `nightlyCoordinatorMappingErrors()` and its test enumerate every policy group @@ -202,6 +228,51 @@ function orderSuites(suites) { return NIGHTLY_DEEP_SUITES.filter((suite) => wanted.has(suite)); } +/** + * Close a suite selection under SUITE_PREREQUISITES, transitively. Applied to every receipt, so + * `selectedSuites` is always a set the workflow can actually run: no entry depends on a suite the + * same receipt lists as skipped. Adding only, never removing, so it can never shrink a full sweep. + * @param {Iterable} suites + * @returns {string[]} the closed set, unordered (buildReceipt applies the canonical order) + */ +export function closeUnderPrerequisites(suites) { + const closed = new Set([...suites].map(String)); + const pending = [...closed]; + while (pending.length > 0) { + const suite = pending.pop(); + const needs = Object.hasOwn(SUITE_PREREQUISITES, suite) ? SUITE_PREREQUISITES[suite] : []; + for (const need of needs) { + if (!closed.has(need)) { + closed.add(need); + pending.push(need); + } + } + } + return [...closed]; +} + +/** + * Consistency errors in SUITE_PREREQUISITES itself: an edge may only name deep suites this module + * knows, and the graph must be acyclic so the closure terminates on a fixed point. Sorted so the + * message set is stable. The test additionally compares this table against the `needs:` edges + * parsed out of the nightly workflows, which is what catches drift when a workflow gains an edge. + * @returns {string[]} + */ +export function nightlyCoordinatorSuiteGraphErrors() { + const errors = []; + const known = new Set(NIGHTLY_DEEP_SUITES); + for (const [suite, needs] of Object.entries(SUITE_PREREQUISITES)) { + if (!known.has(suite)) errors.push(`prerequisite table names unknown deep suite ${suite}`); + for (const need of needs) { + if (!known.has(need)) errors.push(`deep suite ${suite} needs unknown deep suite ${need}`); + } + if (closeUnderPrerequisites(needs).includes(suite)) { + errors.push(`deep suite ${suite} depends on itself`); + } + } + return errors.sort(); +} + /** * Consistency errors in GROUP_DEEP_SUITES itself, given a policy document. Empty means the table * covers every policy path group, names no unknown deep suite, and names no group the policy has @@ -294,7 +365,7 @@ export function normaliseLastQualified(lastQualified, explicitReason = null) { } function buildReceipt(fields) { - const selected = orderSuites(fields.selectedSuites ?? []); + const selected = orderSuites(closeUnderPrerequisites(fields.selectedSuites ?? [])); const selectedSet = new Set(selected); const skipReason = fields.skipReason ?? SKIP_REASONS.notAffected; return { diff --git a/scripts/ci/smart-ci/nightly-coordinator.test.mjs b/scripts/ci/smart-ci/nightly-coordinator.test.mjs index 05efd4dcb..d8c5ed43b 100644 --- a/scripts/ci/smart-ci/nightly-coordinator.test.mjs +++ b/scripts/ci/smart-ci/nightly-coordinator.test.mjs @@ -17,9 +17,12 @@ import { NIGHTLY_DEEP_SUITES, REASONS, SKIP_REASONS, + SUITE_PREREQUISITES, + closeUnderPrerequisites, decideNightlyPlan, detectDuplicateQualification, nightlyCoordinatorMappingErrors, + nightlyCoordinatorSuiteGraphErrors, normaliseLastQualified, parseArgs, parseChangedFileList, @@ -132,15 +135,22 @@ test('a backend-only change selects the backend deep suites and not the browser assert.equal(skipped.get('e2e-cross-browser'), SKIP_REASONS.notAffected); }); -test('a frontend-only change selects the browser suites and no backend solution run', () => { +test('a frontend-only change selects the browser suites plus the backend-solution they need', () => { const receipt = decideNightlyPlan(coordinatorInput({ changedFiles: ['frontend/taskdeck-web/src/views/BoardView.vue'], })); assert.equal(receipt.verdict, 'affected'); assert.deepEqual(receipt.matchedGroups, ['frontend-src']); - assert.deepEqual(receipt.selectedSuites, ['e2e-smoke', 'e2e-cross-browser', 'frontend-coverage']); - assert.ok(!receipt.selectedSuites.includes('backend-solution')); + // `e2e-smoke` and `e2e-cross-browser` declare `needs: backend-solution` in ci-nightly.yml, so a + // receipt that promised them without it would promise evidence GitHub Actions would skip. + assert.deepEqual(receipt.selectedSuites, [ + 'backend-solution', + 'e2e-smoke', + 'e2e-cross-browser', + 'frontend-coverage', + ]); + assert.ok(!receipt.selectedSuites.includes('backend-coverage')); }); test('the weekly slot forces weekly-full even when the diff is empty', () => { @@ -345,6 +355,114 @@ test('every policy path group has a deep-suite mapping and every mapped suite ex ]); }); +/** + * The `needs:` edges of a nightly workflow, as `{ jobId: [dependency, ...] }`, read straight from + * the YAML. Deliberately a narrow line scanner rather than a YAML dependency: it understands the + * two shapes these files use (`needs: job` and a `needs:` block of `- job` items) at their exact + * indentation, so an unexpected shape shows up as a missing edge and fails the drift assertion. + */ +function workflowNeedsEdges(workflowPath) { + const edges = new Map(); + let job = null; + let inNeedsBlock = false; + for (const rawLine of readFileSync(workflowPath, 'utf8').split(/\r?\n/)) { + const jobMatch = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(rawLine); + if (jobMatch) { + job = jobMatch[1]; + edges.set(job, []); + inNeedsBlock = false; + continue; + } + if (job === null) continue; + const inlineNeeds = /^ {4}needs:\s*(\S.*)$/.exec(rawLine); + if (inlineNeeds) { + edges.get(job).push(inlineNeeds[1].replace(/^\[|\]$/g, '').split(',').map((part) => part.trim()) + .filter(Boolean)); + inNeedsBlock = false; + continue; + } + if (/^ {4}needs:\s*$/.test(rawLine)) { + inNeedsBlock = true; + continue; + } + if (inNeedsBlock) { + const item = /^ {6}- (\S+)\s*$/.exec(rawLine); + if (item) { + edges.get(job).push([item[1]]); + continue; + } + inNeedsBlock = false; + } + } + return new Map([...edges].map(([id, groups]) => [id, groups.flat()])); +} + +test('the prerequisite table matches the nightly workflows and the graph is well formed', () => { + assert.deepEqual(nightlyCoordinatorSuiteGraphErrors(), []); + + const workflows = [ + fileURLToPath(new URL('../../../.github/workflows/ci-nightly.yml', import.meta.url)), + fileURLToPath(new URL('../../../.github/workflows/nightly-quality.yml', import.meta.url)), + ]; + const actual = new Map(); + for (const workflow of workflows) { + for (const [job, needs] of workflowNeedsEdges(workflow)) { + if (!NIGHTLY_DEEP_SUITES.includes(job)) continue; + actual.set(job, needs.slice().sort()); + } + } + // Every deep suite is a real job in one of the two workflows, and nothing is missed. + assert.deepEqual([...actual.keys()].sort(), [...NIGHTLY_DEEP_SUITES].sort()); + + const declared = new Map(NIGHTLY_DEEP_SUITES.map((suite) => [ + suite, + (Object.hasOwn(SUITE_PREREQUISITES, suite) ? [...SUITE_PREREQUISITES[suite]] : []).sort(), + ])); + for (const suite of NIGHTLY_DEEP_SUITES) { + assert.deepEqual(declared.get(suite), actual.get(suite), `needs edges for ${suite}`); + } + // The scanner really did find the five documented edges, so an empty parse cannot pass above. + assert.equal([...actual.values()].filter((needs) => needs.length > 0).length, 5); +}); + +test('every receipt selection is closed under the workflow needs edges', () => { + const scenarios = [ + ['frontend/taskdeck-web/src/views/BoardView.vue'], + ['frontend/taskdeck-web/tests/e2e/board.spec.ts'], + ['deploy/docker/frontend.Dockerfile'], + ['tests/load/board-read.js'], + ['backend/src/Taskdeck.Domain/Boards/Board.cs'], + ['.mcp.json'], + ['backend/src/Taskdeck.Api/Endpoints/BoardEndpoints.cs'], + ['docs/STATUS.md'], + // A fail-closed full sweep is closed too: every suite is selected. + ['some-new-top-level-thing.bin'], + ]; + for (const changedFiles of scenarios) { + const receipt = decideNightlyPlan(coordinatorInput({ changedFiles })); + const selected = new Set(receipt.selectedSuites); + const skipped = new Set(receipt.skippedSuites.map((entry) => entry.suite)); + for (const suite of selected) { + for (const need of (Object.hasOwn(SUITE_PREREQUISITES, suite) ? SUITE_PREREQUISITES[suite] : [])) { + assert.ok(selected.has(need), `${changedFiles[0]}: ${suite} selected without ${need}`); + assert.ok(!skipped.has(need), `${changedFiles[0]}: ${need} both needed and skipped`); + } + } + } + + // The three groups the closure exists for: each names a dependent suite, none names the + // dependency, and the receipt supplies it. + for (const groupId of ['frontend-src', 'frontend-e2e', 'containers-deploy', 'load-and-evals']) { + assert.ok(!GROUP_DEEP_SUITES[groupId].includes('backend-solution'), groupId); + assert.ok(closeUnderPrerequisites(GROUP_DEEP_SUITES[groupId]).includes('backend-solution'), groupId); + } + + // Closure adds only prerequisites; it never invents an unrelated suite or drops one. + assert.deepEqual(closeUnderPrerequisites([]), []); + assert.deepEqual(closeUnderPrerequisites(['backend-coverage']), ['backend-coverage']); + assert.deepEqual(closeUnderPrerequisites([...NIGHTLY_DEEP_SUITES]).sort(), [...NIGHTLY_DEEP_SUITES].sort()); +}); + test('two calls with the same input produce byte-identical JSON and markdown', () => { const input = () => coordinatorInput({ changedFiles: [