diff --git a/apps/extension/src/tools/__tests__/visual-ref.test.ts b/apps/extension/src/tools/__tests__/visual-ref.test.ts index 94be024e..df686d91 100644 --- a/apps/extension/src/tools/__tests__/visual-ref.test.ts +++ b/apps/extension/src/tools/__tests__/visual-ref.test.ts @@ -4,7 +4,7 @@ import { RefStore, type VisualRefInput } from "@/session-manager/ref-store"; import { handleDownload } from "../download"; import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop"; import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction"; -import { handleGetHtml, handleScreenshot } from "../observation"; +import { handleGetHtml } from "../observation"; import type { CdpRunner } from "../shared"; import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref"; import { handleUpload } from "../upload"; @@ -117,12 +117,10 @@ describe("typed visual refs", () => { "press", "select", "get_html", - "screenshot", ])("rejects visual refs in %s before target effects", async (tool) => { const { manager, send, cdp, tabsApi } = await setup(); const params = { session_id: "test", tab_id: 4, ref: "@e1" }; const deps = { cdp, tabsApi }; - const captureVisibleTab = vi.fn(); let result: unknown; switch (tool) { case "click": @@ -143,16 +141,9 @@ describe("typed visual refs", () => { case "get_html": result = await handleGetHtml(manager, params, deps); break; - case "screenshot": - result = await handleScreenshot(manager, params, { - ...deps, - captureApi: { ...tabsApi, captureVisibleTab }, - }); - break; } expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } }); expect(send).not.toHaveBeenCalled(); - expect(captureVisibleTab).not.toHaveBeenCalled(); }); it.each([ diff --git a/apps/extension/src/tools/__tests__/visual-screenshot.test.ts b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts new file mode 100644 index 00000000..ab720a78 --- /dev/null +++ b/apps/extension/src/tools/__tests__/visual-screenshot.test.ts @@ -0,0 +1,460 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import { handleScreenshot } from "../observation"; +import type { CdpRunner } from "../shared"; +import { captureVisualScreenshot, visualScreenshotScale } from "../visual-screenshot"; +import type { VisualCandidate, VisualFramePath } from "../vom/visual-discovery"; + +const styles = { + position: "static", + visibility: "visible", + opacity: "1", + display: "block", + "overflow-x": "visible", + "overflow-y": "visible", + transform: "none", + zoom: "1", + "clip-path": "none", + "mask-image": "none", + rotate: "none", + scale: "none", + perspective: "none", + clip: "auto", + contain: "none", + "overflow-clip-margin": "0px", +}; +const rect = (x = 10, y = 20, width = 100, height = 40) => ({ x, y, width, height }); +const root = { + attachmentId: "a", + target: { tabId: 4 }, + frameId: "top", + documentElementBackendNodeId: 1, +}; +function row(id: number, tag: string, box = rect()) { + return { + node: { backend: id }, + tag, + box, + client: box, + contentSize: { width: box.width, height: box.height }, + styles: { ...styles }, + }; +} +function encode(value: unknown): unknown { + if (value === null) return { type: "null" }; + if (Array.isArray(value)) return { type: "array", value: value.map(encode) }; + if (typeof value === "object") { + const record = value as Record; + if ("backend" in record) return { type: "node", value: { backendNodeId: record.backend } }; + return { type: "object", value: Object.entries(record).map(([k, v]) => [k, encode(v)]) }; + } + return { type: typeof value, value }; +} +function png(width: number, height: number) { + const bytes = new Uint8Array(33); + bytes.set([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82]); + const view = new DataView(bytes.buffer); + view.setUint32(16, width); + view.setUint32(20, height); + return btoa(String.fromCharCode(...bytes)); +} +function fixture(child = false, oopif = false) { + const parent: VisualFramePath = { document: root }; + const frame: VisualFramePath = child + ? { + document: { + ...root, + frameId: "child", + documentElementBackendNodeId: 11, + target: oopif ? { tabId: 4, sessionId: "oopif" } : { tabId: 4 }, + }, + parent: { frame: parent, ownerBackendNodeId: 2 }, + } + : parent; + const crop = child ? rect(120, 240, 200, 80) : rect(); + const candidate: VisualCandidate = { + document: frame.document, + backendNodeId: child ? 12 : 3, + parentBackendNodeId: frame.document.documentElementBackendNodeId, + framePath: frame, + region: { status: "available", borderBox: crop, crop }, + }; + const topRows = child + ? [row(2, "iframe", rect(100, 200, 400, 200)), row(1, "html", rect(0, 0, 1200, 800))] + : [row(3, "canvas"), row(1, "html", rect(0, 0, 1200, 800))]; + if (child) topRows[0].contentSize = { width: 200, height: 100 }; + const childRows = [row(12, "canvas"), row(11, "html", rect(0, 0, 200, 100))]; + let calls = 0; + const control = { + attachmentId: "a", + detached: false, + failIdentity: false, + onShot: (_attempt: number) => {}, + wrongOwner: false, + rootChanged: false, + failRead: false, + abortRead: false, + dpr: 1, + shots: [png(crop.width, crop.height)], + }; + const controller = new AbortController(); + const send = vi.fn( + async (target: CdpTarget, method: string, params: Record = {}) => { + if (method === "DOM.getFrameOwner") return { backendNodeId: control.wrongOwner ? 999 : 2 }; + if (method === "Page.createIsolatedWorld") + return { executionContextId: params.frameId === "child" ? 11 : 1 }; + if (method === "DOM.resolveNode" && control.failIdentity) throw new Error("target gone"); + if (method === "DOM.resolveNode") + return { object: { objectId: String(params.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") { + const isChild = params.objectId === "12"; + if (control.detached) return { result: { deepSerializedValue: { type: "null" } } }; + if (!(params.functionDeclaration as string).includes("styleNames")) + return { + result: { + deepSerializedValue: { + type: "node", + value: { backendNodeId: control.rootChanged ? 999 : isChild ? 11 : 1 }, + }, + }, + }; + if (control.failRead) throw new Error("read failed"); + if (control.abortRead) controller.abort(); + return { + result: { + deepSerializedValue: encode({ + top: !isChild, + dpr: control.dpr, + rows: isChild ? childRows : topRows, + }), + }, + }; + } + if (method === "Runtime.releaseObjectGroup") return {}; + // OOPIF projection uses the full viewport for scale, independently of + // the scrollbar-excluding layout viewport used for clipping. + if (method === "Runtime.evaluate" && target.sessionId === "oopif") { + expect(params).toMatchObject({ + expression: "({ width: window.innerWidth, height: window.innerHeight })", + returnByValue: true, + }); + return { result: { value: { width: 200, height: 100 } } }; + } + if (method === "Page.getLayoutMetrics") + return { + cssLayoutViewport: { clientWidth: 1200, clientHeight: 800, pageX: 0, pageY: 0 }, + cssVisualViewport: { scale: 1, zoom: 1 }, + }; + if (method === "DOM.getBoxModel") + return { model: { content: [100, 200, 500, 200, 500, 400, 100, 400] } }; + if (method === "Page.captureScreenshot") { + const data = control.shots[Math.min(calls++, control.shots.length - 1)]; + control.onShot(calls); + return { data }; + } + throw new Error(`unexpected ${method} ${target.sessionId}`); + }, + ); + const cdp = { + send: ((tabId, method, params) => + send({ tabId }, method, params as Record)) as CdpRunner["send"], + sendToTarget: send as unknown as CdpRunner["sendToTarget"], + getAttachmentId: () => control.attachmentId, + getFrameGraph: vi.fn(async () => { + throw new Error("whole page discovery forbidden"); + }), + }; + return { candidate, cdp, send, control, controller, topRows, childRows }; +} + +describe("visual screenshot", () => { + it.each([ + [false, false], + [true, false], + [true, true], + ])("screenshots the local target child=%s oopif=%s without whole-page discovery", async (child, oopif) => { + const f = fixture(child, oopif); + const result = await captureVisualScreenshot(f.cdp, f.candidate); + expect(result).toMatchObject({ width: child ? 200 : 100, height: child ? 80 : 40 }); + expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + const names = f.send.mock.calls.map((c) => c[1]); + expect(names.filter((n) => n === "DOM.getFrameOwner")).toHaveLength(child ? 2 : 0); + expect(names.filter((n) => n === "DOM.resolveNode")).toHaveLength(child ? 4 : 2); + expect(names.some((n) => /scroll|DOMSnapshot|getFrameTree|Accessibility/.test(n))).toBe(false); + expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({ + clip: { ...f.candidate.region.crop, scale: 1 }, + }); + }); + it.each([ + false, + true, + ])("projects a nested child through a parent target oopif=%s", async (oopif) => { + const f = fixture(true, oopif); + const parent = f.candidate.framePath!; + const document = { + ...parent.document, + frameId: "grandchild", + documentElementBackendNodeId: 21, + }; + const crop = rect(130, 250, 40, 20); + const candidate: VisualCandidate = { + ...f.candidate, + document, + backendNodeId: 22, + framePath: { document, parent: { frame: parent, ownerBackendNodeId: 12 } }, + region: { status: "available", borderBox: crop, crop }, + }; + f.childRows[0].tag = "iframe"; + f.control.shots = [png(40, 20)]; + const original = f.send.getMockImplementation()!; + f.send.mockImplementation(async (target, method, params = {}) => { + if (method === "DOM.getFrameOwner" && params.frameId === "grandchild") + return { backendNodeId: 12 }; + if (method === "Runtime.callFunctionOn" && params.objectId === "22") { + if (!(params.functionDeclaration as string).includes("styleNames")) + return { + result: { deepSerializedValue: { type: "node", value: { backendNodeId: 21 } } }, + }; + return { + result: { + deepSerializedValue: encode({ + top: false, + dpr: 1, + rows: [row(22, "canvas", rect(5, 5, 20, 10)), row(21, "html", rect(0, 0, 100, 40))], + }), + }, + }; + } + if (method === "DOM.getBoxModel" && params.backendNodeId === 12) + return { + model: { + content: oopif + ? [10, 20, 110, 20, 110, 60, 10, 60] + : [120, 240, 320, 240, 320, 320, 120, 320], + }, + }; + if (method === "Page.getLayoutMetrics" && target.sessionId) + return { + cssLayoutViewport: { clientWidth: 200, clientHeight: 100, pageX: 0, pageY: 0 }, + cssVisualViewport: { scale: 1, zoom: 1 }, + }; + return original(target, method, params); + }); + expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({ + width: 40, + height: 20, + }); + expect(f.send.mock.calls.filter((c) => c[1] === "DOM.getFrameOwner")).toHaveLength(4); + expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({ + clip: crop, + }); + expect(f.cdp.getFrameGraph).not.toHaveBeenCalled(); + }); + + it("runs the live reader on only connected local ancestry", async () => { + const f = fixture(); + await captureVisualScreenshot(f.cdp, f.candidate); + const params = f.send.mock.calls.find( + (c) => + c[1] === "Runtime.callFunctionOn" && + String(c[2]?.functionDeclaration).includes("styleNames"), + )![2]!; + const read = new Function(`return (${params.functionDeclaration});`)() as ( + this: Element, + names: string[], + ) => { rows: { node: Element }[] } | null; + const canvas = document.createElement("canvas"); + document.body.append(canvas); + try { + const result = read.call(canvas, ["display"]); + expect(result?.rows.map((row) => row.node)).toEqual([ + canvas, + document.body, + document.documentElement, + ]); + canvas.remove(); + expect(read.call(canvas, ["display"])).toBeNull(); + const other = document.implementation.createHTMLDocument(); + const foreign = other.createElement("canvas"); + other.body.append(foreign); + expect(foreign.ownerDocument).toBe(other); + expect(read.call(foreign, ["display"])).toBeNull(); + } finally { + canvas.remove(); + } + }); + + it("keeps requests constant as ordinary ancestry grows", async () => { + const small = fixture(), + large = fixture(); + large.topRows.splice( + 1, + 0, + ...Array.from({ length: 100 }, (_, i) => row(100 + i, "div", rect(0, 0, 1200, 800))), + ); + await captureVisualScreenshot(small.cdp, small.candidate); + await captureVisualScreenshot(large.cdp, large.candidate); + expect(large.send.mock.calls.map((c) => c[1])).toEqual(small.send.mock.calls.map((c) => c[1])); + }); + it("preserves missing-path candidates but cannot execute them", async () => { + const f = fixture(); + const { framePath: _, ...candidate } = f.candidate; + expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({ + data: { reason: "visual_target_changed" }, + }); + expect(f.send).not.toHaveBeenCalled(); + }); + it.each([0.249, 0.25, 0.251])("compares rectangle edges with tolerance %s", async (delta) => { + const f = fixture(); + f.topRows[0].box.x += delta; + const result = await captureVisualScreenshot(f.cdp, f.candidate); + expect("code" in result).toBe(delta > 0.25); + }); + it("rejects a new clipping ancestor even if the resulting crop is identical", async () => { + const f = fixture(); + const clip = row(8, "div", rect(0, 0, 1200, 800)); + clip.styles["overflow-x"] = "hidden"; + f.topRows.splice(1, 0, clip); + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + data: { reason: "visual_target_changed" }, + }); + expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false); + }); + it("rejects changed owner and changed DOM before capture", async () => { + for (const kind of ["wrongOwner", "rootChanged"] as const) { + const f = fixture(true); + f.control[kind] = true; + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + data: { reason: "visual_target_changed" }, + }); + expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false); + } + }); + it("releases retained objects on read failure and cancellation", async () => { + for (const kind of ["failRead", "abortRead"] as const) { + const f = fixture(); + f.control[kind] = true; + const result = await captureVisualScreenshot(f.cdp, f.candidate, f.controller.signal); + expect(result).toMatchObject({ code: kind === "abortRead" ? "cancelled" : "cdp_failed" }); + expect(f.send.mock.calls.at(-1)?.[1]).toBe("Runtime.releaseObjectGroup"); + } + }); + it("plans pixels without enlarging images and retries at most once", async () => { + expect(visualScreenshotScale(4096, 2048, 2)).toBe(0.25); + expect(visualScreenshotScale(4000, 4000, 1)).toBe(0.5); + const f = fixture(); + f.control.shots = [png(3000, 3000), png(1800, 1800)]; + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + width: 1800, + height: 1800, + }); + const shots = f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot"); + expect(shots).toHaveLength(2); + expect((shots[1][2]!.clip as { scale: number }).scale).toBeLessThan(1); + const fail = fixture(); + fail.control.shots = [png(3000, 3000)]; + expect(await captureVisualScreenshot(fail.cdp, fail.candidate)).toMatchObject({ + data: { reason: "visual_pixel_budget_exceeded" }, + }); + expect(fail.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(2); + }); + it.each([ + "attachment", + "root", + "anchor", + "owner", + "unavailable", + "cancel", + ])("rejects %s changes during capture", async (change) => { + const f = fixture(true, true); + f.control.onShot = () => { + if (change === "attachment") f.control.attachmentId = "new"; + if (change === "root") f.control.rootChanged = true; + if (change === "anchor") f.control.detached = true; + if (change === "owner") f.control.wrongOwner = true; + if (change === "unavailable") f.control.failIdentity = true; + if (change === "cancel") f.controller.abort(); + }; + const result = await captureVisualScreenshot(f.cdp, f.candidate, f.controller.signal); + expect(result).toMatchObject( + change === "cancel" ? { code: "cancelled" } : { data: { reason: "visual_target_changed" } }, + ); + expect(f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(1); + }); + it("accepts post-capture layout and style updates without rereading geometry", async () => { + const f = fixture(); + f.control.onShot = () => { + f.topRows[0].box.x += 100; + f.topRows[0].styles.opacity = "0.5"; + f.control.failRead = true; + }; + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + width: 100, + height: 40, + }); + const after = f.send.mock.calls.slice( + f.send.mock.calls.findIndex((c) => c[1] === "Page.captureScreenshot") + 1, + ); + expect(after.filter((c) => c[1] === "DOM.resolveNode")).toHaveLength(1); + expect( + after.some( + (c) => + c[1] === "Page.getLayoutMetrics" || + c[1] === "DOM.getBoxModel" || + String(c[2]?.functionDeclaration).includes("styleNames"), + ), + ).toBe(false); + }); + it("remeasures geometry before a pixel-budget retry", async () => { + const f = fixture(); + f.control.shots = [png(3000, 3000), png(100, 40)]; + f.control.onShot = () => { + f.topRows[0].box.x += 100; + }; + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + data: { reason: "visual_target_changed" }, + }); + expect(f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(1); + }); + it("also checks identity after the second raster attempt", async () => { + const f = fixture(); + f.control.shots = [png(3000, 3000), png(100, 40)]; + f.control.onShot = (attempt) => { + if (attempt === 2) f.control.rootChanged = true; + }; + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + data: { reason: "visual_target_changed" }, + }); + expect(f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(2); + }); + it("rejects invalid PNG without guessing dimensions", async () => { + const f = fixture(); + f.control.shots = ["invalid"]; + expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({ + data: { reason: "screenshot_capture_failed" }, + }); + }); + it("dispatches a visual ref through the screenshot tool and restores overlays", async () => { + const f = fixture(); + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("test"); + ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]); + const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab; + const tabsApi = { get: async () => tab, query: async () => [tab] }; + const sendToTab = vi.fn(async () => ({})); + const result = await handleScreenshot( + manager, + { session_id: "test", ref: "e1" }, + { cdp: f.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab: vi.fn() }, sendToTab }, + ); + expect(result).toMatchObject({ width: 100, height: 40, format: "png" }); + expect(sendToTab).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/extension/src/tools/geometry/frame-context.ts b/apps/extension/src/tools/geometry/frame-context.ts index 03d66855..5d995190 100644 --- a/apps/extension/src/tools/geometry/frame-context.ts +++ b/apps/extension/src/tools/geometry/frame-context.ts @@ -212,11 +212,16 @@ export class GeometryContext { ownerBackendNodeId: number, ancestorClips: Polygon[], viewport: Size, + contentSize?: Size, ): Promise { const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`; let promise = this.snapshotOwners.get(key); if (!promise) { - promise = this.snapshotOwner(source.target, ownerBackendNodeId); + promise = contentSize + ? this.ownerContent(source.target, ownerBackendNodeId).then((quad) => + quad ? { quad, size: contentSize } : null, + ) + : this.snapshotOwner(source.target, ownerBackendNodeId); this.snapshotOwners.set(key, promise); } let owner: { quad: Quad; size: Size } | null; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index acb41be9..cba06aa3 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1,3 +1,8 @@ +import { parsePngDimensions } from "./png"; +import { captureVisualScreenshot } from "./visual-screenshot"; + +export { parsePngDimensions } from "./png"; + import type { CapturedSceneInput } from "./vom/facts"; // Observation handlers โ€” `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // and semantic `tool.observe` (design ยง7). Each handler resolves the target @@ -101,38 +106,6 @@ export function stripDataUrlPrefix(dataUrl: string): string { return m ? dataUrl.slice(m[0].length) : dataUrl; } -/** - * Parse a PNG's IHDR chunk and return `(width, height)`. Returns - * `null` on any malformed input so callers fall back to `0/0` instead - * of throwing. - * - * PNG layout: 8-byte signature, then a 4-byte length, 4-byte type - * ("IHDR"), then the chunk data โ€” width is bytes 16-19 BE, height is - * 20-23 BE. - */ -export function parsePngDimensions(base64: string): { width: number; height: number } | null { - try { - // atob is available in MV3 service workers. - const head = base64.length > 64 ? base64.slice(0, 64) : base64; - const bin = atob(head); - if (bin.length < 24) return null; - if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) { - return null; - } - const u32 = (off: number) => - (bin.charCodeAt(off) << 24) | - (bin.charCodeAt(off + 1) << 16) | - (bin.charCodeAt(off + 2) << 8) | - bin.charCodeAt(off + 3); - const width = u32(16) >>> 0; - const height = u32(20) >>> 0; - if (width === 0 || height === 0) return null; - return { width, height }; - } catch { - return null; - } -} - export interface ScreenshotDeps { cdp?: SharedCdpRunner; tabsApi: ChromeTabsApi; @@ -308,12 +281,17 @@ export async function handleScreenshot( if (!deps.cdp) { return { code: "cdp_failed", message: "screenshot ref capture requires CDP" }; } - if (lookupRefTarget(ctx, ref, target.tabId)?.kind === "visual-region") - return rpcError( - "unsupported", - "ref_kind_unsupported", - "visual-region screenshot execution is not available in this build", + const entry = lookupRefTarget(ctx, ref, target.tabId); + if (entry?.kind === "visual-region") { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const captured = await withExtensionOverlayHidden( + target.tabId, + () => captureVisualScreenshot(deps.cdp!, entry.candidate, signal), + deps.sendToTab, ); + if (isRpcError(captured)) return captured; + return withShotDialogs({ ...captured, format: "png", tab_id: target.tabId }); + } const node = resolveSnapshotRef(ctx, ref, target.tabId); if (isRpcError(node)) return node; if (signal?.aborted) return cancelled("screenshot"); diff --git a/apps/extension/src/tools/png.ts b/apps/extension/src/tools/png.ts new file mode 100644 index 00000000..38cde427 --- /dev/null +++ b/apps/extension/src/tools/png.ts @@ -0,0 +1,31 @@ +/** + * Parse a PNG's IHDR chunk and return `(width, height)`. Returns + * `null` on any malformed input so callers fall back to `0/0` instead + * of throwing. + * + * PNG layout: 8-byte signature, then a 4-byte length, 4-byte type + * ("IHDR"), then the chunk data โ€” width is bytes 16-19 BE, height is + * 20-23 BE. + */ +export function parsePngDimensions(base64: string): { width: number; height: number } | null { + try { + // atob is available in MV3 service workers. + const head = base64.length > 64 ? base64.slice(0, 64) : base64; + const bin = atob(head); + if (bin.length < 33) return null; + if (bin.slice(0, 8) !== "\x89PNG\r\n\x1a\n" || bin.slice(8, 16) !== "\x00\x00\x00\x0dIHDR") { + return null; + } + const u32 = (off: number) => + (bin.charCodeAt(off) << 24) | + (bin.charCodeAt(off + 1) << 16) | + (bin.charCodeAt(off + 2) << 8) | + bin.charCodeAt(off + 3); + const width = u32(16) >>> 0; + const height = u32(20) >>> 0; + if (width === 0 || height === 0) return null; + return { width, height }; + } catch { + return null; + } +} diff --git a/apps/extension/src/tools/visual-screenshot.ts b/apps/extension/src/tools/visual-screenshot.ts new file mode 100644 index 00000000..91aa20fc --- /dev/null +++ b/apps/extension/src/tools/visual-screenshot.ts @@ -0,0 +1,440 @@ +import type { CdpFrame } from "@/browser-driver/frame-graph"; +import type { RpcError } from "@/transport/types"; +import { rpcError } from "./errors"; +import { + type GeometryProjection, + projectRectToViewport, + type Size, + type ViewportRect, +} from "./geometry"; +import { type CssViewport, screenshotPageRect } from "./geometry/coordinate-types"; +import { cssViewport, GeometryContext } from "./geometry/frame-context"; +import { parsePngDimensions } from "./png"; +import { type CdpRunner, sendToCdpTarget } from "./shared"; +import { isAbortError, throwIfAborted } from "./vom/capture-abort"; +import { resolveVerifiedNode } from "./vom/document-identity"; +import type { DocumentIdentity } from "./vom/facts"; +import { VISUAL_STYLES } from "./vom/snapshot"; +import type { VisualCandidate, VisualFramePath } from "./vom/visual-discovery"; +import { + EMPTY_VISUAL_CONTEXT, + extendVisualContext, + projectVisualBox, + resolveVisualRegion, + type VisualClipSource, + type VisualContext, + visualProjectionIssue, +} from "./vom/visual-region"; + +interface LiveRow { + node: number; + tag: string; + box: ViewportRect; + client: ViewportRect; + contentSize: Size; + styles: Record; +} +interface LiveFrame { + top: boolean; + dpr: number; + rows: LiveRow[]; // Anchor first, root last. +} + +// Read only one current ancestry, including shadow hosts, in the verified isolated world. +const READ_ANCESTRY = `function(styleNames) { + if (!this.isConnected || this.ownerDocument !== document) return null; + const rows = []; + for (let node = this; node; node = node.parentElement || node.getRootNode().host) { + if (!(node instanceof Element)) return null; + const s = getComputedStyle(node), r = node.getBoundingClientRect(); + rows.push({ node, tag: node.localName, + box: { x:r.x, y:r.y, width:r.width, height:r.height }, + client: { x:r.x+node.clientLeft, y:r.y+node.clientTop, width:node.clientWidth, height:node.clientHeight }, + contentSize: { width:node.clientWidth-parseFloat(s.paddingLeft)-parseFloat(s.paddingRight), height:node.clientHeight-parseFloat(s.paddingTop)-parseFloat(s.paddingBottom) }, + styles: Object.fromEntries(styleNames.map(key => [key,s.getPropertyValue(key)])) }); + } + return { top: window === window.top, dpr: devicePixelRatio, rows }; +}`; + +interface DeepValue { + type: string; + value?: unknown; +} +/** This read returns JSON primitives plus DOM nodes (backend IDs), never remote object handles. */ +function decode(value: DeepValue): unknown { + if (value.type === "node") return (value.value as { backendNodeId?: number })?.backendNodeId; + if (value.type === "array") return (value.value as DeepValue[]).map(decode); + if (value.type === "object") + return Object.fromEntries( + (value.value as [string, DeepValue][]).map(([key, item]) => [key, decode(item)]), + ); + if (value.type === "null") return null; + if (["string", "number", "boolean"].includes(value.type)) return value.value; + throw new Error("unexpected live ancestry serialization"); +} + +function stale(message: string): RpcError { + return rpcError( + "not_found", + "visual_target_changed", + `${message}; observe again before requesting a screenshot`, + ); +} + +function sameIdentity(a: DocumentIdentity, b: DocumentIdentity): boolean { + return ( + a.attachmentId === b.attachmentId && + a.frameId === b.frameId && + a.target.tabId === b.target.tabId && + a.target.sessionId === b.target.sessionId && + a.documentElementBackendNodeId === b.documentElementBackendNodeId + ); +} + +function closeRect(a: ViewportRect, b: ViewportRect): boolean { + return [ + a.x - b.x, + a.y - b.y, + a.x + a.width - b.x - b.width, + a.y + a.height - b.y - b.height, + ].every((delta) => Number.isFinite(delta) && Math.abs(delta) <= 0.25); +} + +function sameClips(a?: VisualClipSource, b?: VisualClipSource): boolean { + while (a && b) { + if ( + !sameIdentity(a.document, b.document) || + a.backendNodeId !== b.backendNodeId || + a.x !== b.x || + a.y !== b.y || + a.overflowX !== b.overflowX || + a.overflowY !== b.overflowY || + !closeRect(a.box, b.box) + ) + return false; + a = a.parent; + b = b.parent; + } + return !a && !b; +} + +/** Scale affects raster size only, never the selected CSS region. */ +export function visualScreenshotScale(width: number, height: number, pixelsPerCss: number): number { + if (![width, height, pixelsPerCss].every((n) => Number.isFinite(n) && n > 0)) + throw new Error("invalid screenshot pixel dimensions"); + return Math.min( + 1, + 2048 / (Math.max(width, height) * pixelsPerCss), + Math.sqrt(4_000_000 / (width * height * pixelsPerCss * pixelsPerCss)), + ); +} + +async function readFrame( + cdp: CdpRunner, + document: DocumentIdentity, + anchor: number, + signal?: AbortSignal, +): Promise { + const verified = await resolveVerifiedNode(cdp, document, anchor, signal); + if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`); + try { + throwIfAborted(signal); + const reply = await sendToCdpTarget<{ + result?: { deepSerializedValue?: DeepValue }; + exceptionDetails?: unknown; + }>(cdp, document.target, "Runtime.callFunctionOn", { + objectId: verified.objectId, + objectGroup: verified.objectGroup, + functionDeclaration: READ_ANCESTRY, + arguments: [{ value: VISUAL_STYLES }], + serializationOptions: { + serialization: "deep", + additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" }, + }, + }); + throwIfAborted(signal); + if (reply.exceptionDetails || !reply.result?.deepSerializedValue) + return stale("visual ancestry unavailable"); + const result = decode(reply.result.deepSerializedValue) as LiveFrame | null; + if ( + !result?.rows?.length || + result.rows[0].node !== anchor || + result.rows.at(-1)?.node !== document.documentElementBackendNodeId || + !result.rows.every((row) => Number.isSafeInteger(row.node) && row.node > 0) + ) + return stale("visual ancestry incomplete"); + if (cdp.getAttachmentId?.(document.target.tabId) !== document.attachmentId) + return stale("visual attachment changed"); + return result; + } finally { + await sendToCdpTarget(cdp, document.target, "Runtime.releaseObjectGroup", { + objectGroup: verified.objectGroup, + }).catch(() => {}); + throwIfAborted(signal); + } +} + +/** All live reads for one screenshot use this one measurement context. */ +async function resolveVisualRegionNow( + cdp: CdpRunner, + candidate: VisualCandidate, + signal?: AbortSignal, +): Promise<{ crop: ViewportRect; viewport: CssViewport; dpr: number } | RpcError> { + throwIfAborted(signal); + if (!candidate.framePath || !sameIdentity(candidate.document, candidate.framePath.document)) + return stale("visual frame path unavailable"); + const path: { frame: VisualFramePath; anchor: number }[] = []; + const seen = new Set(); + let frame: VisualFramePath | undefined = candidate.framePath; + let anchor = candidate.backendNodeId; + while (frame) { + if ( + seen.has(frame.document.frameId) || + frame.document.target.tabId !== candidate.document.target.tabId || + cdp.getAttachmentId?.(frame.document.target.tabId) !== frame.document.attachmentId + ) + return stale("invalid visual frame path"); + seen.add(frame.document.frameId); + path.push({ frame, anchor }); + anchor = frame.parent?.ownerBackendNodeId ?? 0; + frame = frame.parent?.frame; + } + path.reverse(); + const frames: CdpFrame[] = path.map(({ frame }) => ({ + frameId: frame.document.frameId, + target: frame.document.target, + ...(frame.parent + ? { + parentFrameId: frame.parent.frame.document.frameId, + ownerBackendNodeId: frame.parent.ownerBackendNodeId, + } + : {}), + })); + // Validate each recorded edge instead of asking the driver to fill all page owners. + for (const { frame } of path) { + if (!frame.parent) continue; + throwIfAborted(signal); + const owner = await sendToCdpTarget<{ backendNodeId?: number }>( + cdp, + frame.parent.frame.document.target, + "DOM.getFrameOwner", + { frameId: frame.document.frameId }, + ); + if (owner.backendNodeId !== frame.parent.ownerBackendNodeId) + return stale("visual frame owner changed"); + } + const geometry = new GeometryContext( + cdp, + candidate.document.target.tabId, + { rootFrameId: frames[0].frameId, frames }, + signal, + ); + let context: VisualContext = EMPTY_VISUAL_CONTEXT; + let parentRead: LiveFrame | undefined; + let finalRegion: VisualCandidate["region"] | undefined; + let topDpr = 0; + for (let i = 0; i < path.length; i++) { + const { frame, anchor } = path[i]; + const live = await readFrame(cdp, frame.document, anchor, signal); + if ("code" in live) return live; + if (live.top !== (i === 0)) return stale("visual root frame changed"); + if (i === 0) topDpr = live.dpr; + const metrics = await geometry.layoutMetrics(frame.document.target); + const viewport = cssViewport(metrics); + const projections: GeometryProjection[] = []; + if (frame.parent) { + const parent = frame.parent.frame.document; + const size = parentRead!.rows[0].contentSize; + if (![size.width, size.height].every((n) => Number.isFinite(n) && n > 0)) + return stale("iframe content size unavailable"); + const parentViewport = await geometry.viewport(parent.target); + if (!parentViewport) return stale("parent viewport unavailable"); + const local = await geometry.snapshotProjection( + { target: parent.target, frameId: frame.document.frameId }, + frame.parent.ownerBackendNodeId, + [], + parentViewport, + size, + ); + const outer = await geometry.targetProjection(parent.frameId); + if (local.status !== "available" || !outer) + return stale("visual frame projection unavailable"); + projections.push(local.projection.geometry, outer); + } else projections.push({ sourceClips: [], edges: [], topViewport: viewport }); + const issue = visualProjectionIssue({ + projections, + coordinates: { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } }, + pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale, + }); + if (issue) return stale(issue); + for (let j = live.rows.length - 1; j >= 0; j--) { + const row = live.rows[j]; + const isAnchor = j === 0; + const node = { + document: frame.document, + backendNodeId: row.node, + styles: row.styles, + clientBox: projectVisualBox(row.client, projections), + }; + context = extendVisualContext( + context, + node, + !isAnchor && row.tag !== "iframe" && row.tag !== "frame", + ); + } + if (i < path.length - 1) { + const owner = live.rows[0]; + context = { + ...context, + hidden: + context.hidden || + owner.styles.visibility === "hidden" || + owner.styles.visibility === "collapse", + transformed: false, + localClip: false, + unpositionedClip: false, + }; + } else { + const row = live.rows[0]; + if (row.tag !== "canvas") return stale("visual anchor is no longer Canvas"); + let visible: ViewportRect | null = row.box; + for (const projection of projections) + visible = visible + ? projectRectToViewport( + { x: visible.x, y: visible.y, w: visible.width, h: visible.height }, + projection, + ) + : null; + const region = resolveVisualRegion({ + borderBox: projectVisualBox(row.box, projections), + frameVisibleBox: visible, + context, + visibility: row.styles.visibility, + }); + if (region.status !== "available") + return stale( + `visual region ${region.status}${region.status === "unavailable" ? `: ${region.reason}` : ""}`, + ); + finalRegion = region; + } + parentRead = live; + } + if ( + !finalRegion || + !closeRect(candidate.region.borderBox, finalRegion.borderBox) || + !closeRect(candidate.region.crop, finalRegion.crop) || + !sameClips(candidate.region.clips, finalRegion.clips) + ) + return stale("visual region changed"); + const viewport = cssViewport(await geometry.layoutMetrics(frames[0].target)); + return { crop: finalRegion.crop, viewport, dpr: topDpr }; +} + +/** Check identity only: repainting and post-capture layout changes are allowed. */ +async function verifyCapturedTarget( + cdp: CdpRunner, + candidate: VisualCandidate, + signal?: AbortSignal, +): Promise { + try { + let frame = candidate.framePath; + let anchor = candidate.backendNodeId; + // The path was validated before capture; never rebuild it from the current page. + while (frame) { + const verified = await resolveVerifiedNode(cdp, frame.document, anchor, signal); + if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`); + await sendToCdpTarget(cdp, frame.document.target, "Runtime.releaseObjectGroup", { + objectGroup: verified.objectGroup, + }).catch(() => {}); + throwIfAborted(signal); + if (frame.parent) { + const owner = await sendToCdpTarget<{ backendNodeId?: number }>( + cdp, + frame.parent.frame.document.target, + "DOM.getFrameOwner", + { frameId: frame.document.frameId }, + ); + throwIfAborted(signal); + if (owner.backendNodeId !== frame.parent.ownerBackendNodeId) + return stale("visual frame owner changed"); + anchor = frame.parent.ownerBackendNodeId; + } + frame = frame.parent?.frame; + } + throwIfAborted(signal); + return cdp.getAttachmentId?.(candidate.document.target.tabId) === + candidate.document.attachmentId + ? null + : stale("visual attachment changed"); + } catch (error) { + throwIfAborted(signal); + if (isAbortError(error)) throw error; + return stale("visual identity unavailable after capture"); + } +} + +/** One target only. No frame discovery, scrolling, AX, or snapshot recapture. */ +export async function captureVisualScreenshot( + cdp: CdpRunner, + candidate: VisualCandidate, + signal?: AbortSignal, +): Promise<{ image_base64: string; width: number; height: number } | RpcError> { + try { + let region = await resolveVisualRegionNow(cdp, candidate, signal); + if ("code" in region) return region; + let scale = visualScreenshotScale(region.crop.width, region.crop.height, region.dpr); + for (let attempt = 0; attempt < 2; attempt++) { + if (attempt > 0) { + // A new raster attempt needs fresh geometry, not the previous crop/cache. + region = await resolveVisualRegionNow(cdp, candidate, signal); + if ("code" in region) return region; + scale = Math.min( + scale, + visualScreenshotScale(region.crop.width, region.crop.height, region.dpr), + ); + } + const clip = screenshotPageRect(region.crop, region.viewport); + if (!clip) return stale("invalid screenshot coordinates"); + throwIfAborted(signal); + if ( + cdp.getAttachmentId?.(candidate.document.target.tabId) !== candidate.document.attachmentId + ) + return stale("visual attachment changed"); + const shot = await cdp.send<{ data?: string }>( + candidate.document.target.tabId, + "Page.captureScreenshot", + { format: "png", captureBeyondViewport: false, clip: { ...clip.rect, scale } }, + ); + throwIfAborted(signal); + if ( + cdp.getAttachmentId?.(candidate.document.target.tabId) !== candidate.document.attachmentId + ) + return stale("visual attachment changed"); + const identityError = await verifyCapturedTarget(cdp, candidate, signal); + if (identityError) return identityError; + const dims = shot.data ? parsePngDimensions(shot.data) : null; + if (!dims) + return rpcError( + "cdp_failed", + "screenshot_capture_failed", + "visual screenshot returned invalid PNG dimensions", + ); + const correction = visualScreenshotScale(dims.width, dims.height, 1); + if (correction === 1) return { image_base64: shot.data!, ...dims }; + scale *= correction * 0.99; + } + return rpcError( + "cdp_failed", + "visual_pixel_budget_exceeded", + "visual screenshot exceeds the pixel budget", + ); + } catch (error) { + if (isAbortError(error) || signal?.aborted) + return { code: "cancelled", message: "visual screenshot aborted" }; + return rpcError( + "cdp_failed", + "screenshot_capture_failed", + error instanceof Error ? error.message : String(error), + ); + } +} diff --git a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts index 60c7d55c..a8a0f184 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts @@ -141,6 +141,32 @@ function facts( } describe("Canvas discovery", () => { + it("shares frame address paths from existing Facts without changing candidate evidence", async () => { + const parent = await document([{ id: 2, parent: 1, tag: "iframe" }]); + const child = await document( + [ + { id: 3, parent: 1 }, + { id: 4, parent: 1 }, + ], + { + frame: { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 2, + target: { tabId: 1 }, + }, + }, + ); + const result = await discoverVisualCandidates(facts([child, parent])); + expect(result.candidates).toHaveLength(2); + const path = result.candidates[0].framePath; + expect(path).toBe(result.candidates[1].framePath); + expect(path?.document).toBe(child.identity); + expect(path?.parent?.ownerBackendNodeId).toBe(2); + expect(path?.parent?.frame.document).toBe(parent.identity); + expect(path?.parent?.frame.parent).toBeUndefined(); + }); + it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => { const empty = facts([]); expect(await discoverVisualCandidates(empty)).toMatchObject({ diff --git a/apps/extension/src/tools/vom/document-identity.ts b/apps/extension/src/tools/vom/document-identity.ts index da233943..458e2307 100644 --- a/apps/extension/src/tools/vom/document-identity.ts +++ b/apps/extension/src/tools/vom/document-identity.ts @@ -11,7 +11,7 @@ export async function verifyDocumentIdentity( identity: DocumentIdentity, signal?: AbortSignal, ): Promise<"current" | "changed" | "unavailable"> { - return verifyIdentity(cdp, identity, signal); + return (await verifyIdentity(cdp, identity, signal)).status; } /** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */ @@ -20,7 +20,24 @@ export async function verifyVisualTargetIdentity( candidate: VisualCandidate, signal?: AbortSignal, ): Promise<"current" | "changed" | "unavailable"> { - return verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId); + return (await verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId)).status; +} + +export type VerifiedNodeResult = + | { status: "changed" | "unavailable" } + | { status: "current"; objectId: string; objectGroup: string }; + +/** Caller must release objectGroup in finally after its local read. */ +export async function resolveVerifiedNode( + cdp: CdpRunner, + identity: DocumentIdentity, + backendNodeId: number, + signal?: AbortSignal, +): Promise { + const result = await verifyIdentity(cdp, identity, signal, backendNodeId, true); + return result.status === "current" && result.objectId && result.objectGroup + ? { status: "current", objectId: result.objectId, objectGroup: result.objectGroup } + : { status: result.status === "changed" ? "changed" : "unavailable" }; } async function verifyIdentity( @@ -28,15 +45,22 @@ async function verifyIdentity( identity: DocumentIdentity, signal?: AbortSignal, anchorBackendNodeId?: number, -): Promise<"current" | "changed" | "unavailable"> { + retain = false, +): Promise<{ + status: "current" | "changed" | "unavailable"; + objectId?: string; + objectGroup?: string; +}> { const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId; throwIfAborted(signal); - if (!attached()) return "changed"; + if (!attached()) return { status: "changed" }; const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`; const send = (method: string, params: object) => { throwIfAborted(signal); return sendToCdpTarget(cdp, identity.target, method, params); }; + let objectId: string | undefined; + let retained = false; try { const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", { frameId: identity.frameId, @@ -63,7 +87,8 @@ async function verifyIdentity( executionContextId: world.executionContextId, objectGroup, }); - if (!anchor.object?.objectId) return "unavailable"; + if (!anchor.object?.objectId) return { status: "unavailable" }; + objectId = anchor.object.objectId; reply = await send("Runtime.callFunctionOn", { objectId: anchor.object.objectId, functionDeclaration: @@ -72,20 +97,24 @@ async function verifyIdentity( serializationOptions, }); } - if (!attached()) return "changed"; + if (!attached()) return { status: "changed" }; const root = reply.result?.deepSerializedValue; - if (root?.type === "null") return "changed"; + if (root?.type === "null") return { status: "changed" }; const id = root?.type === "node" ? root.value?.backendNodeId : undefined; - if (id === undefined) return "unavailable"; - return id === identity.documentElementBackendNodeId ? "current" : "changed"; + if (id === undefined) return { status: "unavailable" }; + if (id !== identity.documentElementBackendNodeId) return { status: "changed" }; + throwIfAborted(signal); + retained = retain && !!objectId; + return { status: "current", ...(retained ? { objectId, objectGroup } : {}) }; } catch (error) { throwIfAborted(signal); if (isAbortError(error)) throw error; - return attached() ? "unavailable" : "changed"; + return { status: attached() ? "unavailable" : "changed" }; } finally { - await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", { - objectGroup, - }).catch(() => {}); + if (!retained) + await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", { + objectGroup, + }).catch(() => {}); throwIfAborted(signal); } } diff --git a/apps/extension/src/tools/vom/visual-discovery.ts b/apps/extension/src/tools/vom/visual-discovery.ts index 43c9b35d..fd538b79 100644 --- a/apps/extension/src/tools/vom/visual-discovery.ts +++ b/apps/extension/src/tools/vom/visual-discovery.ts @@ -20,11 +20,19 @@ import { visualProjectionIssue, } from "./visual-region"; +/** Shared observation-time addresses, not cached live geometry. */ +export interface VisualFramePath { + readonly document: DocumentIdentity; + readonly parent?: { readonly ownerBackendNodeId: number; readonly frame: VisualFramePath }; +} + export interface VisualCandidate { readonly document: DocumentIdentity; readonly backendNodeId: number; readonly parentBackendNodeId: number | null; readonly label?: string; + /** Absent when execution ancestry could not be established; discovery is still retained. */ + readonly framePath?: VisualFramePath; readonly region: Extract; } @@ -113,6 +121,7 @@ export async function discoverVisualCandidates( const candidates: VisualCandidate[] = []; const issues: VisualDiscoveryIssue[] = []; const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document])); + const framePaths = new Map(); const contexts = new Map>(); const roots = new Map(); @@ -248,6 +257,16 @@ export async function discoverVisualCandidates( root.issue ?? (!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)), }; + if (doc.identity) { + const parentPath = frame.parentFrameId ? framePaths.get(frame.parentFrameId) : undefined; + if (!frame.parentFrameId && frame.frameId === facts.rootFrameId) + framePaths.set(frame.frameId, { document: doc.identity }); + else if (parentPath && frame.ownerBackendNodeId !== undefined) + framePaths.set(frame.frameId, { + document: doc.identity, + parent: { ownerBackendNodeId: frame.ownerBackendNodeId, frame: parentPath }, + }); + } roots.set(frame.frameId, root); contexts.set(frame.frameId, new Map()); } @@ -291,6 +310,7 @@ export async function discoverVisualCandidates( const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim(); candidates.push({ document: document.identity, + framePath: framePaths.get(document.frame.frameId), backendNodeId: node.backendNodeId, parentBackendNodeId: node.parentBackendNodeId, ...(label ? { label } : {}), diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 42d726ed..97f0070b 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -25,6 +25,8 @@ export type RpcErrorReason = | "element_not_visible" | "ref_not_found" | "ref_kind_unsupported" + | "visual_target_changed" + | "visual_pixel_budget_exceeded" | "selector_not_found" | "target_not_fillable" | "fill_value_invalid" diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index 66cb2c19..88f39ebe 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -40,6 +40,8 @@ use bsk_protocol::ErrorCode; pub mod reason { pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope"; pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible"; + pub const VISUAL_TARGET_CHANGED: &str = "visual_target_changed"; + pub const VISUAL_PIXEL_BUDGET_EXCEEDED: &str = "visual_pixel_budget_exceeded"; pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported"; pub const REF_NOT_FOUND: &str = "ref_not_found"; pub const SELECTOR_NOT_FOUND: &str = "selector_not_found"; @@ -248,6 +250,18 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ), exit_code: base.exit_code, }, + (ErrorCode::NotFound, reason::VISUAL_TARGET_CHANGED) => RenderInfo { + summary: "visual target needs a fresh observation", + hint: Some( + "observe the page again and use the current visual ref; the previous identity or region cannot be used", + ), + exit_code: base.exit_code, + }, + (ErrorCode::CdpFailed, reason::VISUAL_PIXEL_BUDGET_EXCEEDED) => RenderInfo { + summary: "visual screenshot exceeds the image size limit", + hint: Some("the screenshot could not be reduced within the pixel budget"), + exit_code: base.exit_code, + }, (ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo { summary: "this tool does not support the ref target type", hint: Some( @@ -672,6 +686,16 @@ mod tests { assert!(info.summary.contains("timed out after browser dispatch")); } + #[test] + fn visual_screenshot_failures_have_specific_guidance() { + let changed = serde_json::json!({ "reason": reason::VISUAL_TARGET_CHANGED }); + let info = info_for_error(ErrorCode::NotFound, Some(&changed)); + assert!(info.hint.unwrap().contains("observe")); + let budget = serde_json::json!({ "reason": reason::VISUAL_PIXEL_BUDGET_EXCEEDED }); + let info = info_for_error(ErrorCode::CdpFailed, Some(&budget)); + assert!(info.summary.contains("image size limit")); + } + #[test] fn visual_ref_type_error_has_specific_guidance() { let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });