Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions sdk/typescript/src/multiscan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ async function runCampaign(
const progress = { repository: task.id, attempt };
options.onProgress?.({ ...progress, status: "started" });
let failure: string | undefined;
let cleanup: string | undefined;
let cost: Readonly<ScanCost> | null = null;
try {
await mkdir(dirname(scanDir), { recursive: true, mode: 0o700 });
Expand Down Expand Up @@ -197,9 +198,26 @@ async function runCampaign(
if (options.signal?.aborted === true) options.signal.throwIfAborted();
failure = redactedErrorMessage(error);
} finally {
await rm(checkout, { recursive: true, force: true });
// Removing the checkout is best effort. `force` ignores a checkout that
// is already gone but not an EACCES, EPERM, or EBUSY removal, and a
// throw here would replace the outcome the try and catch just captured
// and skip the receipt below, leaving the attempt unrecorded for resume.
// The removal failure travels with that outcome instead, so the attempt
// keeps the status its scan earned and a leftover checkout is still
// reported. The next attempt removes the checkout again inside the try
// above, so a leftover that outlives this run fails there rather than
// being scanned as if it were fresh.
cleanup = await rm(checkout, { recursive: true, force: true }).then(
() => undefined,
(error: unknown) =>
`Multiscan checkout cleanup failed: ${redactedErrorMessage(error)}`,
);
Comment on lines +210 to +214

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Retry cleanup before permanently skipping completed scans

When the post-scan removal fails transiently, such as EBUSY on Windows, this handler suppresses the failure while the receipt is still marked completed. The worker then breaks, and future runs recognize the completed artifacts and skip the task before reaching the pre-attempt removal, so the claimed “next attempt” never occurs and the cloned source can remain indefinitely. Preserve the successful scan result, but independently retry removal of a leftover checkout before treating a completed task as fully skipped.

Useful? React with 👍 / 👎.

}
const status = failure === undefined ? "completed" : "failed";
const reported = [failure, cleanup].filter(
(message): message is string => message !== undefined,
);
const error = reported.length === 0 ? undefined : reported.join("; ");
await appendReceipt(
ledger,
`${JSON.stringify({
Expand All @@ -208,13 +226,13 @@ async function runCampaign(
attempt,
outputDir: scanDir,
...(cost === null ? {} : { cost }),
...(failure === undefined ? {} : { error: failure }),
...(error === undefined ? {} : { error }),
})}\n`,
);
options.onProgress?.({
...progress,
status,
...(failure === undefined ? {} : { error: failure }),
...(error === undefined ? {} : { error }),
});
if (failure === undefined) {
completed += 1;
Expand Down
113 changes: 112 additions & 1 deletion sdk/typescript/tests-ts/multiscan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ import {
symlink,
writeFile,
} from "node:fs/promises";
import * as fsPromises from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, test } from "bun:test";
import { afterEach, describe, expect, mock, test } from "bun:test";
import type { ScanResult } from "../src/result.js";
import { buildGitHubCredentialArgs, runMultiscan } from "../src/multiscan.js";
import { resolveTrustedExecutable } from "../src/trusted-executable.js";
Expand Down Expand Up @@ -398,6 +399,116 @@ describe("multiscan", () => {
expect(await results(summary.resultsPath)).toHaveLength(3);
});

// A checkout removal that fails once the scan is over used to escape the
// worker's finally, replacing the outcome the scan had already earned and
// skipping its receipt, so the attempt was never recorded for resume.
async function unremovableCheckout(
checkout: string,
scanned: () => boolean,
): Promise<() => void> {
const originalRm = fsPromises.rm;
mock.module("node:fs/promises", () => ({
...fsPromises,
rm: async (...args: Parameters<typeof originalRm>) => {
if (scanned() && String(args[0]) === checkout) {
throw Object.assign(
new Error(`EACCES: permission denied, rm '${checkout}'`),
{ code: "EACCES" },
);
}
return await originalRm(...args);
},
}));
return () => {
mock.module("node:fs/promises", () => ({
...fsPromises,
rm: originalRm,
}));
};
}

test("keeps a failed scan's outcome when its checkout cannot be removed", async () => {
const paths = await fixture();
const source = await repository(paths.root, "stubborn");
await writeFile(
paths.input,
`id,repository,revision\nstubborn,${source.path},${source.revision}\n`,
);
let scanned = false;
const restore = await unremovableCheckout(
join(paths.output, "checkouts", "stubborn"),
() => scanned,
);

try {
const summary = await runMultiscan(
options(
paths,
client(async () => {
scanned = true;
throw new Error("ORIGINAL_SCAN_FAILURE");
}),
{ maxAttempts: 1 },
),
);

expect(summary).toMatchObject({ total: 1, completed: 0, failed: 1 });
const receipts = await results(summary.resultsPath);
expect(receipts).toHaveLength(1);
expect(receipts[0]).toMatchObject({ status: "failed" });
const error = String(receipts[0]!["error"]);
expect(error).toContain("ORIGINAL_SCAN_FAILURE");
expect(error).toContain("Multiscan checkout cleanup failed");
} finally {
restore();
}
});

test("keeps a completed scan's outcome when its checkout cannot be removed", async () => {
const paths = await fixture();
const source = await repository(paths.root, "stubborn");
await writeFile(
paths.input,
`id,repository,revision\nstubborn,${source.path},${source.revision}\n`,
);
let scanned = false;
const reported: string[] = [];
const restore = await unremovableCheckout(
join(paths.output, "checkouts", "stubborn"),
() => scanned,
);

try {
const summary = await runMultiscan(
options(
paths,
client(async (_repository, scanOptions = {}) => {
scanned = true;
return await completedScan(scanOptions.outputDir!);
}),
{
maxAttempts: 1,
onProgress: ({ status, error }) => {
if (error !== undefined) reported.push(`${status}: ${error}`);
},
},
),
);

expect(summary).toMatchObject({ total: 1, completed: 1, failed: 0 });
const receipts = await results(summary.resultsPath);
expect(receipts).toHaveLength(1);
expect(receipts[0]).toMatchObject({ status: "completed" });
expect(String(receipts[0]!["error"])).toContain(
"Multiscan checkout cleanup failed",
);
expect(reported).toHaveLength(1);
expect(reported[0]).toStartWith("completed: ");
} finally {
restore();
}
});

test("rejects another supervisor and recovers a crashed owner's checkout", async () => {
const paths = await fixture();
const source = await repository(paths.root, "exclusive");
Expand Down