From 719906dab671302688b2dcc0daf15748bd4165e8 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 3 Sep 2026 16:19:54 -0400 Subject: [PATCH] Capture browser session video as a named artifact --- package.json | 3 +- .../src/browser-probe-contract.ts | 2 +- .../src/browser-actions-runner.ts | 31 ++++++- .../src/browser-artifacts.ts | 3 + .../src/browser-probe-runner.ts | 25 ++++- ...wser-actions-video-capture.browser.test.ts | 92 +++++++++++++++++++ 6 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 tests/browser-actions-video-capture.browser.test.ts diff --git a/package.json b/package.json index 3c72d3790..62f103b22 100644 --- a/package.json +++ b/package.json @@ -176,7 +176,8 @@ "test:browser": "npm run build && npm run smoke -- --group=browser", "test:all": "npm run smoke -- --all", "check": "npm run smoke -- --group=check", - "check:all": "npm run test:all && npm run cloudflare:check" + "check:all": "npm run test:all && npm run cloudflare:check", + "test:browser-video-capture": "tsx tests/browser-actions-video-capture.browser.test.ts" }, "workspaces": [ "packages/cli", diff --git a/packages/runtime-core/src/browser-probe-contract.ts b/packages/runtime-core/src/browser-probe-contract.ts index d6793aca7..673d15951 100644 --- a/packages/runtime-core/src/browser-probe-contract.ts +++ b/packages/runtime-core/src/browser-probe-contract.ts @@ -14,7 +14,7 @@ export type BrowserProbeProfileDefinition = { export const BROWSER_PROBE_BROWSER_VALUES = ["chromium"] as const -export const BROWSER_PROBE_CAPTURE_VALUES = ["console", "errors", "html", "network", "websocket", "performance", "memory", "screenshot"] as const +export const BROWSER_PROBE_CAPTURE_VALUES = ["console", "errors", "html", "network", "websocket", "performance", "memory", "screenshot", "video"] as const export const BROWSER_PROBE_CHROMIUM_PROFILE_IDS = ["desktop-chrome", "mobile-chrome", "low-end-mobile-slow-4g"] as const export const BROWSER_PROBE_THROTTLE_PROFILE_IDS = ["low-end-mobile-slow-4g"] as const export const BROWSER_GEOLOCATION_PERMISSION_STATES = ["granted", "denied", "prompt"] as const diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index f22ffe4d8..5fca45b3b 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -1,4 +1,5 @@ -import { readFile, writeFile } from "node:fs/promises" +import { copyFile, readdir, readFile, writeFile } from "node:fs/promises" +import { join } from "node:path" import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_PROBE_PROFILES, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserAdaptiveExplorationContract, browserEnvironment, browserEnvironmentDigest, browserGeolocation, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, transportFaultModel, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserAdaptiveExplorationArtifact, type BrowserAdaptiveExplorationContract, type BrowserEnvironment, type BrowserGeolocationPermissionState, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec, type TransportFaultModel } from "@automattic/wp-codebox-core" import { now, sha256 } from "@automattic/wp-codebox-core/internals" import { browserInteractionStepsFromArgs, browserStepTimeoutMs, durationStringMs, sanitizeScreenshotName } from "./browser-actions.js" @@ -107,8 +108,8 @@ export async function runBrowserActionsCommand({ const capture = runPlan.capture for (const item of capture) { - if (!["steps", "console", "errors", "html", "network", "websocket", "screenshot", "dom-snapshot"].includes(item)) { - throw new Error(`wordpress.browser-actions capture supports steps, console, errors, html, network, websocket, screenshot, dom-snapshot: ${item}`) + if (!["steps", "console", "errors", "html", "network", "websocket", "screenshot", "dom-snapshot", "video"].includes(item)) { + throw new Error(`wordpress.browser-actions capture supports steps, console, errors, html, network, websocket, screenshot, dom-snapshot, video: ${item}`) } } @@ -143,6 +144,9 @@ export async function runBrowserActionsCommand({ let finalUrl = requestedUrl let htmlSha256: string | undefined let screenshotSha256: string | undefined + const videoStagingDirectory = artifactSession.absolutePath("video-source") + let videoRecording: import("playwright").Video | null = null + let videoSaved = false const screenshots: string[] = [] const domSnapshots: Array<{ screenshot: string; snapshot: string; step?: { index: number; name?: string; kind: string }; elementCount: number; capturedElements: number; truncated: boolean }> = [] const verifierResults: NonNullable = [] @@ -194,12 +198,15 @@ export async function runBrowserActionsCommand({ if (unsupportedEnvironment.length > 0) { throw new Error(`wordpress.browser-actions browser environment is unsupported: ${unsupportedEnvironment.join(", ")}`) } - const needsEnvironmentContext = Object.keys(requestedEnvironment).length > 0 || browserPreviewNeedsContextRouting(networkPolicy) || !!storageStateImport || !!runPlan.transportFaults + // A recording is a context-level capability, so asking for video requires the + // environment context path rather than a bare page. + const needsEnvironmentContext = Object.keys(requestedEnvironment).length > 0 || browserPreviewNeedsContextRouting(networkPolicy) || !!storageStateImport || !!runPlan.transportFaults || capture.has("video") environmentRuntime = session?.runtime ?? (needsEnvironmentContext ? await createPlaywrightBrowserEnvironmentContext(browser, resolvedEnvironment, { contextOptions: { ...topology.contextOptions(), ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}), ...(runPlan.transportFaults ? { serviceWorkers: "block" as const } : {}), + ...(capture.has("video") ? { recordVideo: { dir: videoStagingDirectory } } : {}), }, }) : undefined) const context = environmentRuntime?.context ?? null @@ -208,6 +215,7 @@ export async function runBrowserActionsCommand({ } if (context && runPlan.transportFaults) installedTransportFaults = await installBrowserTransportFaults(context, runPlan.transportFaults, { policy: browserPreviewTransportFaultPolicy(networkPolicy, topology.origins.localProxyOrigin), serviceWorkersBlocked: true }) const page = activePage = environmentRuntime?.page ?? await browser.newPage() + if (capture.has("video")) videoRecording = page.video() navigationTracker = trackBrowserNavigation(page) if (onProgress) { await page.exposeFunction("__wpCodeboxProbeCheckpointEvent", (checkpoint: unknown) => { @@ -532,6 +540,20 @@ export async function runBrowserActionsCommand({ errors.push(serializeBrowserError("probe-error", routeError)) if (browserPreviewCleanupErrorIsFatal(routeError)) pendingError ??= routeError } + if (videoRecording) { + // Playwright finalizes a recording on context close and names it itself. The + // recording is adopted from its staging directory rather than through + // video.saveAs(), which requires a browser connection that cleanup has closed. + try { + const staged = (await readdir(videoStagingDirectory)).filter((entry) => entry.endsWith(".webm")).sort() + if (staged.length === 0) throw new Error("wordpress.browser-actions capture=video produced no recording") + const source = join(videoStagingDirectory, staged[0]) + await artifactSession.writeGenerated("video", "video.webm", (path) => copyFile(source, path)) + videoSaved = true + } catch (error) { + errors.push(serializeBrowserError("probe-error", error)) + } + } if (capture.has("steps")) { await artifactSession.writeJsonLines("steps", "steps.jsonl", stepRecords) } @@ -625,6 +647,7 @@ export async function runBrowserActionsCommand({ ...(transportFaultReport ? { transportFaults: browserTransportFaultSummary(transportFaultReport) } : {}), replayability: browserProbeReplayability(capture), screenshot: Boolean(screenshotSha256), + ...(capture.has("video") ? { video: videoSaved } : {}), auth: authSummary, environment: environmentEvidence, viewport, diff --git a/packages/runtime-playground/src/browser-artifacts.ts b/packages/runtime-playground/src/browser-artifacts.ts index 9703df342..672c0caff 100644 --- a/packages/runtime-playground/src/browser-artifacts.ts +++ b/packages/runtime-playground/src/browser-artifacts.ts @@ -84,6 +84,7 @@ export interface BrowserArtifactFiles { review?: string screenshot?: string screenshots?: string[] + video?: string traces?: string[] domSnapshots?: string[] verifierResults?: string[] @@ -242,6 +243,7 @@ export interface BrowserArtifactSummary { } replayability: BrowserProbeReplayability screenshot: boolean + video?: boolean visualCompare?: { status: string mismatchRatio?: number @@ -1252,6 +1254,7 @@ const BROWSER_ARTIFACT_FILE_MANIFEST: Record> = [] const checkpoints: BrowserProbeCheckpointRecord[] = [] const screenshotPath = artifactSession.absolutePath("screenshot.png") + // Playwright writes the recording itself and only finalizes it once the context + // closes, so the recording is staged in its own directory and adopted as a named + // artifact afterwards. + const videoSourceDirectory = artifactSession.absolutePath("video-source") const startedAt = now() const startedAtMs = Date.now() const progress = createBrowserProbeProgressTracker(startedAt, stallTimeoutMs) @@ -287,6 +293,8 @@ export async function runSingleBrowserProbeCommand({ let authSummary: BrowserProbeAuthSummary | undefined let capabilityDiagnostics: BrowserProbeCapabilityDiagnostics | undefined let environmentRuntime: PlaywrightBrowserEnvironmentRuntime | undefined + let videoHandle: import("playwright").Video | null = null + let videoCaptured = false let assertionResults: import("./browser-artifacts.js").BrowserStepAssertion[] = [] let pendingError: Error | undefined let artifact: BrowserProbeArtifact | undefined @@ -320,13 +328,14 @@ export async function runSingleBrowserProbeCommand({ const resolved = await resolvePlaywrightBrowserEnvironment(browserEnvironmentCell(environment), browser) const unsupported = resolved.capabilities.filter(({ fidelity }) => fidelity === "unsupported").map(({ id }) => id) if (unsupported.length > 0) throw new Error(`${command} browser environment is unsupported: ${unsupported.join(", ")}`) - environmentRuntime = await createPlaywrightBrowserEnvironmentContext(browser, resolved, { contextOptions: { ...topology.contextOptions(), ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}) } }) + environmentRuntime = await createPlaywrightBrowserEnvironmentContext(browser, resolved, { contextOptions: { ...topology.contextOptions(), ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}), ...(capture.has("video") ? { recordVideo: { dir: videoSourceDirectory } } : {}) } }) } context = environmentRuntime.context if (context && browserPreviewNeedsContextRouting(networkPolicy) && !session) { await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } page = environmentRuntime.page + if (capture.has("video")) videoHandle = page.video() if (onProgress) { await page.exposeFunction("__wpCodeboxProbeCheckpointEvent", (checkpoint: unknown) => { const normalized = normalizeBrowserProbeScriptCheckpoint(checkpoint) @@ -522,6 +531,19 @@ export async function runSingleBrowserProbeCommand({ } errors.push(serializeBrowserError("probe-error", routeError)) } + if (videoHandle) { + // The recording is adopted from its staging directory rather than through + // video.saveAs(), which requires a browser connection that cleanup has closed. + try { + const staged = (await readdir(videoSourceDirectory)).filter((entry) => entry.endsWith(".webm")).sort() + if (staged.length === 0) throw new Error("wordpress.browser-probe capture=video produced no recording") + const source = nodePathJoin(videoSourceDirectory, staged[0]) + await artifactSession.writeGenerated("video", "video.webm", (path) => copyFile(source, path)) + videoCaptured = true + } catch (error) { + errors.push(serializeBrowserError("probe-error", error)) + } + } if (captureSelection.console) { await artifactSession.writeJsonLines("console", "console.jsonl", consoleMessages) } @@ -601,6 +623,7 @@ export async function runSingleBrowserProbeCommand({ hashes: { ...(capture.has("html") ? { htmlSha256 } : {}), ...(capture.has("screenshot") ? { screenshotSha256 } : {}), + ...(capture.has("video") ? { videoCaptured } : {}), }, lifecycleArtifact, lifecycleSelectors, diff --git a/tests/browser-actions-video-capture.browser.test.ts b/tests/browser-actions-video-capture.browser.test.ts new file mode 100644 index 000000000..a2a946063 --- /dev/null +++ b/tests/browser-actions-video-capture.browser.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict" +import { mkdtemp, rm, stat } from "node:fs/promises" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import { runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js" +import { wordpressRuntimeSpec } from "../scripts/test-kit.js" + +const runtimeSpec = wordpressRuntimeSpec({ commands: ["wordpress.browser-actions"] }) + +test("browser actions capture=video records the session and adopts it as a named artifact", async () => { + const fixture = await pageFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-video-")) + try { + const result = await runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-actions", args: [] }, + plan: { + steps: [ + { kind: "navigate", url: fixture.url, waitFor: "load" }, + { kind: "click", selector: "#target" }, + ], + capture: new Set(["steps", "video"]), + stepTimeoutMs: 2_000, + totalTimeoutMs: 10_000, + networkSettleTimeoutMs: 100, + maxDomSnapshotElements: 20, + }, + }) + + assert.equal(result.artifact.summary.video, true, "the summary must report the recording") + + const recording = join(artifactRoot, "files/browser/video.webm") + const recorded = await stat(recording) + assert(recorded.isFile(), "the recording must be adopted as video.webm") + assert(recorded.size > 0, "the recording must not be empty") + } finally { + await rm(artifactRoot, { recursive: true, force: true }) + await fixture.close() + } +}) + +test("browser actions rejects an unsupported capture value", async () => { + const fixture = await pageFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-video-")) + try { + await assert.rejects( + runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-actions", args: [] }, + plan: { + steps: [{ kind: "navigate", url: fixture.url, waitFor: "load" }], + capture: new Set(["recording"]), + stepTimeoutMs: 500, + totalTimeoutMs: 2_000, + networkSettleTimeoutMs: 100, + maxDomSnapshotElements: 20, + }, + }), + /capture supports .*video/, + ) + } finally { + await rm(artifactRoot, { recursive: true, force: true }) + await fixture.close() + } +}) + +async function pageFixture() { + const httpServer = createServer((_request, response) => { + response.setHeader("content-type", "text/html") + response.end("video fixture
ready
") + }) + await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)) + const address = httpServer.address() + assert(address && typeof address === "object") + const url = `http://127.0.0.1:${address.port}` + return { + url, + server: { + serverUrl: url, + playground: { async run() { return { text: "", exitCode: 0 } } }, + async [Symbol.asyncDispose]() {}, + }, + close: () => new Promise((resolve, reject) => httpServer.close((error) => error ? reject(error) : resolve())), + } +}