From e7a8f51e8c0c3b175209ea149127b259148b8e54 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 9 Sep 2026 16:33:23 +0800 Subject: [PATCH 1/3] feat(vom): discover canvas candidates and resolve visible regions --- .../src/tools/geometry/frame-context.ts | 9 +- .../vom/__tests__/capture-coordinator.test.ts | 56 ++- .../src/tools/vom/__tests__/facts.test.ts | 52 ++ .../vom/__tests__/visual-discovery.test.ts | 459 ++++++++++++++++++ .../tools/vom/__tests__/visual-region.test.ts | 192 ++++++++ .../src/tools/vom/capture-coordinator.ts | 1 + apps/extension/src/tools/vom/facts.ts | 28 +- apps/extension/src/tools/vom/normalize.ts | 25 +- apps/extension/src/tools/vom/snapshot.ts | 18 + .../src/tools/vom/visual-discovery.ts | 307 ++++++++++++ apps/extension/src/tools/vom/visual-region.ts | 316 ++++++++++++ 11 files changed, 1446 insertions(+), 17 deletions(-) create mode 100644 apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts create mode 100644 apps/extension/src/tools/vom/__tests__/visual-region.test.ts create mode 100644 apps/extension/src/tools/vom/visual-discovery.ts create mode 100644 apps/extension/src/tools/vom/visual-region.ts diff --git a/apps/extension/src/tools/geometry/frame-context.ts b/apps/extension/src/tools/geometry/frame-context.ts index 7956f2bf..03d66855 100644 --- a/apps/extension/src/tools/geometry/frame-context.ts +++ b/apps/extension/src/tools/geometry/frame-context.ts @@ -20,8 +20,13 @@ import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./c import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes"; export interface LayoutMetrics { - cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number }; - visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number }; + cssVisualViewport?: { + zoom?: number; + scale?: number; + clientWidth?: number; + clientHeight?: number; + }; + visualViewport?: { zoom?: number; scale?: number; clientWidth?: number; clientHeight?: number }; cssLayoutViewport?: { clientWidth?: number; clientHeight?: number; diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index 07747b18..0a4f9bd9 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -7,10 +7,12 @@ import type { CdpRunner } from "../../shared"; import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; import { buildSemanticGraph } from "../semantic-graph/build"; import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot"; +import { discoverVisualCandidates } from "../visual-discovery"; function fixture( options: { frames?: CdpFrame[]; + canvas?: boolean; after?: Record; fail?: string; missingIdentity?: boolean; @@ -78,7 +80,7 @@ function fixture( if (method === "Page.getLayoutMetrics") result = { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, }; if (method === "DOMSnapshot.captureSnapshot") { @@ -88,13 +90,16 @@ function fixture( strings: [ "#document", "html", - "button", + options.canvas ? "canvas" : "button", OVERLAY_HOST_MARKER_ATTR, "", "visible", "1", "static", "auto", + "block", + "none", + "0px", ], documents: frames .filter( @@ -144,9 +149,13 @@ function fixture( [0, 0, 1000, 800], [10, 20, 100, 40], ], + clientRects: [ + [0, 0, 1000, 800], + [0, 0, 100, 40], + ], styles: [ - [7, 8, 8, 5, 6], - [7, 8, 8, 5, 6], + [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11], + [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11], ], }, }; @@ -193,7 +202,7 @@ describe("captureObservationFacts", () => { if (method === "Page.getLayoutMetrics") return { visualViewport: { clientWidth: 2000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 }, } as never; const reply = await original(target, method, params); @@ -659,7 +668,7 @@ describe("OOPIF capture", () => { if (method === "Page.getLayoutMetrics") { return { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 }, }; } @@ -685,7 +694,7 @@ describe("OOPIF capture", () => { if (method === "Page.getLayoutMetrics") { return { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, }; } @@ -767,7 +776,7 @@ describe("OOPIF capture", () => { if (method === "Page.getLayoutMetrics") return { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 300, clientHeight: 200 }, }; if (method === "DOMSnapshot.captureSnapshot") @@ -829,7 +838,7 @@ describe("OOPIF capture", () => { if (method === "Page.getLayoutMetrics") { return { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, }; } @@ -919,7 +928,7 @@ function siblingCaptureFixture( if (method === "Page.getLayoutMetrics") return { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 }, }; if (method === "DOM.getBoxModel") @@ -1127,7 +1136,7 @@ describe("snapshot document provenance", () => { : method === "Page.getLayoutMetrics" ? { visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, + cssVisualViewport: { clientWidth: 1000, scale: 1 }, cssLayoutViewport: { clientWidth: 200, clientHeight: 100 }, } : {}) as T, @@ -1363,3 +1372,28 @@ describe("AX frame scheduling", () => { expect(active).toBe(0); }); }); + +describe("visual facts integration", () => { + it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => { + const { cdp, logs } = fixture({ + canvas: true, + frames: [{ frameId: "main", target: { tabId: 4 } }], + }); + const facts = await captureObservationFacts(cdp, 4); + const before = logs.length; + const result = await discoverVisualCandidates(facts); + expect(logs).toHaveLength(before); + expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(1); + expect(result.complete).toBe(true); + expect(result.candidates).toHaveLength(1); + expect(result.candidates[0]).toMatchObject({ + document: { frameId: "main", documentElementBackendNodeId: 1 }, + region: { crop: { x: 10, y: 20, width: 100, height: 40 } }, + }); + expect(facts.documents[0].index.nodes.get(2)?.layout?.clientRect).toEqual([0, 0, 100, 40]); + expect(facts.documents[0].geometry?.pageScale).toBe(1); + const baseline = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] }); + await captureObservationFacts(baseline.cdp, 4); + expect(logs.map((call) => call.method)).toEqual(baseline.logs.map((call) => call.method)); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/facts.test.ts b/apps/extension/src/tools/vom/__tests__/facts.test.ts index 2991049e..d99da011 100644 --- a/apps/extension/src/tools/vom/__tests__/facts.test.ts +++ b/apps/extension/src/tools/vom/__tests__/facts.test.ts @@ -80,6 +80,19 @@ describe("document facts", () => { boundsSpace: "snapshot-document-layout", bounds: [10, 20, 120, 40], styles: { + rotate: "auto", + scale: "auto", + perspective: "auto", + clip: "auto", + contain: "auto", + "overflow-clip-margin": "auto", + display: "auto", + "overflow-x": "auto", + "overflow-y": "auto", + transform: "auto", + zoom: "auto", + "clip-path": "auto", + "mask-image": "auto", position: "static", "pointer-events": "auto", cursor: "auto", @@ -125,3 +138,42 @@ describe("document facts", () => { expect(reads).toBeLessThanOrEqual(356); }); }); + +describe("visual ancestry facts", () => { + it("distinguishes document roots, malformed roots, missing parents and cycles", async () => { + const input = [ + { ...node(0, null), nodeType: 9 }, + { ...node(1, 0), nodeType: 1 }, + { ...node(2, 1), nodeType: 11 }, + node(3, 2), + node(4, 99), + node(5, 6), + node(6, 5), + node(7, 5), + { ...node(8, null), nodeType: 1 }, + { ...node(9, null), nodeType: 9, parentMissing: true }, + ]; + const index = await buildDocumentIndex(input.reverse()); + for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete.get(id)).toBe(true); + for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete.get(id)).toBe(false); + }); + + it("retains client offsets in their source units and does not disguise missing parent indices", async () => { + const decoded = await decodeDocument( + { + nodes: { backendNodeId: [1, 2], nodeType: [1, 1], nodeName: [0, 0], parentIndex: [99] }, + layout: { + nodeIndex: [0], + bounds: [[745, 455.625, 97, 59.5]], + clientRects: [[5, 5, 68, 38]], + }, + }, + ["div"], + ); + expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]); + expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]); + expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true); + const index = await buildDocumentIndex(decoded.nodes); + expect([...index.ancestryComplete.values()]).toEqual([false, false]); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts new file mode 100644 index 00000000..8c3c97f4 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts @@ -0,0 +1,459 @@ +import { describe, expect, it } from "vitest"; +import type { CdpFrame } from "@/browser-driver/frame-graph"; +import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; +import type { GeometryProjection } from "../../geometry"; +import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts"; +import type { FrameOwnedAxNode } from "../frame-document"; +import { normalizeDocument } from "../normalize"; +import { REQUESTED_STYLES } from "../snapshot"; +import { discoverVisualCandidates } from "../visual-discovery"; + +const defaults: Record = { + position: "static", + "pointer-events": "auto", + cursor: "auto", + visibility: "visible", + opacity: "1", + display: "block", + "overflow-x": "visible", + "overflow-y": "visible", + transform: "none", + zoom: "1", + "clip-path": "none", + "mask-image": "none", + rotate: "none", + scale: "none", + perspective: "none", + clip: "auto", + contain: "none", + "overflow-clip-margin": "0px", +}; +const projection: GeometryProjection = { + sourceClips: [], + edges: [], + topViewport: { width: 1000, height: 800 }, +}; +interface Spec { + id: number; + parent?: number; + tag?: string; + styles?: Record; + bounds?: number[] | null; + client?: number[]; + attrs?: Record; +} +async function document( + nodes: Spec[], + options: { + frame?: CdpFrame; + projection?: GeometryProjection; + pageScale?: number; + scrollY?: number; + layoutScale?: number; + } = {}, +): Promise> { + const frame = options.frame ?? { frameId: "main", target: { tabId: 1 } }; + const strings: string[] = []; + const str = (value: string) => { + const index = strings.indexOf(value); + if (index >= 0) return index; + strings.push(value); + return strings.length - 1; + }; + const input: Spec[] = [ + { id: 0, tag: "#document", bounds: null }, + { id: 1, parent: 0, tag: "html", bounds: [0, 0, 1000, 800] }, + ...nodes, + ]; + const layouts = input.filter((node) => node.bounds !== null); + const normalized = await normalizeDocument( + { + nodes: { + backendNodeId: input.map((n) => n.id), + nodeType: input.map((n) => + n.tag === "#document" ? 9 : n.tag === "#document-fragment" ? 11 : 1, + ), + parentIndex: input.map((n) => + n.parent === undefined ? -1 : input.findIndex((p) => p.id === n.parent), + ), + nodeName: input.map((n) => str(n.tag ?? "canvas")), + attributes: input.map((n) => + Object.entries(n.attrs ?? {}).flatMap(([k, v]) => [str(k), str(v)]), + ), + }, + layout: { + nodeIndex: layouts.map((n) => input.indexOf(n)), + bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]), + clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]), + styles: layouts.map((n) => + REQUESTED_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])), + ), + }, + }, + strings, + { + frameId: frame.frameId, + ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null, + target: frame.target, + projection: { + status: "available", + projection: { + source: { target: frame.target, frameId: frame.frameId }, + geometry: options.projection ?? projection, + }, + }, + coordinates: { + layoutUnitsPerCssPixel: options.layoutScale ?? 1, + scrollCss: { x: 0, y: options.scrollY ?? 0 }, + }, + pageScale: options.pageScale ?? 1, + }, + ); + return { + frame, + identity: { + attachmentId: "attached", + target: frame.target, + frameId: frame.frameId, + documentElementBackendNodeId: 1, + }, + index: normalized.index, + geometry: normalized.geometry, + domNodes: normalized.nodes, + axNodes: [], + }; +} +function facts( + documents: DocumentFacts[], + issues: ObservationFacts["issues"] = [], +): ObservationFacts { + return { + rootFrameId: "main", + documents, + viewport: { width: 1000, height: 800 }, + issues, + startedAt: 1, + finishedAt: 2, + }; +} + +describe("Canvas discovery", () => { + it.each([ + 0.8, 1, 1.25, 2, + ])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => { + const doc = await document( + [ + { + id: 2, + parent: 1, + tag: "div", + bounds: [100, 200, 64, 34].map((n) => n * scale), + client: [2, 2, 60, 30], + styles: { "overflow-x": "hidden", "overflow-y": "hidden" }, + }, + { id: 3, parent: 2, bounds: [102, 202, 120, 40].map((n) => n * scale) }, + ], + { layoutScale: scale, scrollY: 75 }, + ); + const result = await discoverVisualCandidates(facts([doc])); + expect(result.issues).toEqual([]); + const expected = { + borderBox: { x: 102, y: 127, width: 120, height: 40 }, + crop: { x: 102, y: 127, width: 60, height: 30 }, + }; + for (const box of ["borderBox", "crop"] as const) + for (const key of ["x", "y", "width", "height"] as const) + expect(result.candidates[0].region[box][key]).toBeCloseTo(expected[box][key], 8); + }); + + it("normalizes a zero-width ancestor origin before one-axis clipping", async () => { + const doc = await document( + [ + { + id: 2, + parent: 1, + tag: "div", + bounds: [200, 400, 0, 60], + client: [0, 0, 0, 30], + styles: { "overflow-y": "hidden" }, + }, + { id: 3, parent: 2, bounds: [200, 400, 240, 80] }, + ], + { layoutScale: 2, scrollY: 75 }, + ); + const result = await discoverVisualCandidates(facts([doc])); + expect(result.issues).toEqual([]); + expect(result.candidates[0].region.crop).toEqual({ x: 100, y: 125, width: 120, height: 30 }); + }); + + it("keeps unnamed, semantic, aria-hidden, inert, pointer-none and visibility override Canvas", async () => { + const doc = await document([ + { id: 2, parent: 1 }, + { id: 3, parent: 1, attrs: { "aria-label": "Chart" } }, + { + id: 4, + parent: 1, + attrs: { "aria-hidden": "true", inert: "" }, + styles: { "pointer-events": "none" }, + }, + { id: 5, parent: 1, tag: "div", styles: { visibility: "hidden" } }, + { id: 6, parent: 5, styles: { visibility: "visible" } }, + { id: 7, parent: 1, attrs: { hidden: "" }, styles: { display: "block" } }, + ]); + doc.axNodes.push({ nodeId: "canvas", backendDOMNodeId: 2 } as FrameOwnedAxNode); + const result = await discoverVisualCandidates(facts([doc])); + expect(result.candidates.map((c) => c.backendNodeId)).toEqual([2, 3, 4, 6, 7]); + expect(result.candidates[1].label).toBe("Chart"); + expect(result.complete).toBe(true); + }); + + it("excludes CSS transparent subtrees, hidden/zero/no-layout/fully clipped nodes and overlays", async () => { + const doc = await document([ + { id: 2, parent: 1, tag: "div", styles: { opacity: "0" } }, + { id: 3, parent: 2 }, + { id: 4, parent: 1, styles: { visibility: "hidden" } }, + { id: 5, parent: 1, bounds: [0, 0, 0, 40] }, + { id: 6, parent: 1, bounds: null }, + { id: 7, parent: 1, bounds: [1200, 0, 100, 50] }, + { id: 8, parent: 1, tag: "div", attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } }, + { id: 9, parent: 8, tag: "#document-fragment", bounds: null }, + { id: 10, parent: 9 }, + ]); + expect(await discoverVisualCandidates(facts([doc]))).toMatchObject({ + candidates: [], + issues: [], + complete: true, + }); + }); + + it.each([ + ["both", "hidden", "hidden", [0, 0, 60, 30], [60, 30]], + ["x", "clip", "visible", [0, 0, 60, 20], [60, 40]], + ["border", "hidden", "hidden", [5, 5, 68, 38], [64, 34]], + ])("uses client box for %s clipping", async (_name, x, y, client, expected) => { + const doc = await document([ + { + id: 2, + parent: 1, + tag: "div", + bounds: [0, 0, 78, 48], + client: client as number[], + styles: { "overflow-x": x as string, "overflow-y": y as string }, + }, + { + id: 3, + parent: 2, + bounds: x === "clip" || _name === "both" ? [0, 0, 120, 40] : [9, 9, 120, 40], + }, + ]); + const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates; + expect([candidate.region.crop.width, candidate.region.crop.height]).toEqual(expected); + expect(candidate.region.clips?.backendNodeId).toBe(2); + }); + + it("keeps Canvas border-box and clips partial viewport intersections", async () => { + const doc = await document([{ id: 2, parent: 1, bounds: [950, 20, 130, 50] }]); + const [candidate] = (await discoverVisualCandidates(facts([doc]))).candidates; + expect(candidate.region.borderBox.width).toBe(130); + expect(candidate.region.crop).toEqual({ x: 950, y: 20, width: 50, height: 50 }); + }); + + it("propagates owner ancestry across targets, without depending on semantic parents", async () => { + const parent = await document([ + { + id: 2, + parent: 1, + tag: "div", + bounds: [0, 0, 200, 100], + client: [0, 0, 200, 100], + styles: { "overflow-x": "hidden", "overflow-y": "hidden" }, + }, + { id: 3, parent: 2, tag: "iframe", bounds: [100, 0, 200, 200] }, + ]); + parent.domNodes.splice(0); // Visual discovery never reads the semantic subset. + const child = await document([{ id: 2, parent: 1, bounds: [0, 0, 200, 200] }], { + frame: { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 3, + target: { tabId: 1, sessionId: "remote" }, + }, + projection: { + ...projection, + edges: [ + { + sourceViewport: { width: 200, height: 200 }, + destinationQuad: [ + { x: 100, y: 0 }, + { x: 300, y: 0 }, + { x: 300, y: 200 }, + { x: 100, y: 200 }, + ], + }, + ], + }, + }); + const result = await discoverVisualCandidates(facts([child, parent])); + expect(result.complete).toBe(true); + expect(result.candidates[0]).toMatchObject({ + document: { frameId: "child", target: { sessionId: "remote" } }, + region: { crop: { x: 100, y: 0, width: 100, height: 100 } }, + }); + const owner = parent.index.nodes.get(3)!; + owner.layout!.styles = { ...owner.layout!.styles, visibility: "hidden" }; + expect((await discoverVisualCandidates(facts([parent, child]))).candidates).toHaveLength(0); + }); + + it.each>([ + { transform: "matrix(0,1,-1,0,0,0)" }, + { "clip-path": "circle(50%)" }, + { "mask-image": "url(mask)" }, + { zoom: "1.25", "overflow-x": "hidden" }, + ])("reports unsupported ancestor geometry: %o", async (styles) => { + const doc = await document([ + { id: 2, parent: 1, tag: "div", styles }, + { id: 3, parent: 2 }, + ]); + const result = await discoverVisualCandidates(facts([doc])); + expect(result.candidates).toHaveLength(0); + expect(result.issues[0].reason).toBe("geometry-unsupported"); + }); + + it("supports positive scale without overflow, rejects pinch and non-rectangular frame projection", async () => { + const nodes = [ + { + id: 2, + parent: 1, + styles: { transform: "matrix(1.25,0,0,1.25,0,0)" }, + bounds: [10, 20, 150, 50], + }, + ]; + expect( + (await discoverVisualCandidates(facts([await document(nodes)]))).candidates, + ).toHaveLength(1); + expect( + (await discoverVisualCandidates(facts([await document(nodes, { pageScale: 1.5 })]))).issues[0] + .reason, + ).toBe("geometry-unsupported"); + const rotated = { + ...projection, + edges: [ + { + sourceViewport: { width: 200, height: 200 }, + destinationQuad: [ + { x: 10, y: 0 }, + { x: 200, y: 10 }, + { x: 190, y: 200 }, + { x: 0, y: 190 }, + ] as [ + { x: number; y: number }, + { x: number; y: number }, + { x: number; y: number }, + { x: number; y: number }, + ], + }, + ], + }; + expect( + (await discoverVisualCandidates(facts([await document(nodes, { projection: rotated })]))) + .issues[0].reason, + ).toBe("geometry-unsupported"); + }); + + it("keeps partial capture separate from valid geometry, rejects unverified identities", async () => { + const doc = await document([{ id: 2, parent: 1 }]); + const partial = facts( + [doc], + [{ target: doc.frame.target, frameId: "main", stage: "ax", reason: "capture-unavailable" }], + ); + const result = await discoverVisualCandidates(partial); + expect(result.candidates).toHaveLength(1); + expect(result.captureIssues).toBe(partial.issues); + expect(result.complete).toBe(false); + const unverified = { ...doc, identity: undefined }; + const failed = await discoverVisualCandidates(facts([unverified])); + expect(failed.candidates).toHaveLength(0); + expect(failed.issues[0].reason).toBe("identity-unverified"); + }); + + it("does not trust orphan, cyclic, boxless or missing frame-owner ancestry", async () => { + const doc = await document([ + { id: 2, parent: 1, tag: "div" }, + { id: 3, parent: 2 }, + ]); + for (const parent of [99, 3]) { + doc.index.nodes.get(2)!.parentBackendNodeId = parent; + const index = await buildDocumentIndex([...doc.index.nodes.values()]); + const result = await discoverVisualCandidates(facts([{ ...doc, index }])); + expect(result.candidates).toHaveLength(0); + expect(result.issues[0].reason).toBe("ancestry-incomplete"); + } + const boxless = await document([ + { id: 2, parent: 1, tag: "div", bounds: null }, + { id: 3, parent: 2 }, + ]); + expect((await discoverVisualCandidates(facts([boxless]))).issues[0].reason).toBe( + "facts-unavailable", + ); + const child = { + ...doc, + frame: { ...doc.frame, frameId: "child", parentFrameId: "absent", ownerBackendNodeId: 2 }, + }; + expect((await discoverVisualCandidates(facts([child]))).issues[0].reason).toBe( + "ownership-unresolved", + ); + }); + + it("is deterministic, does not deduplicate or mutate inputs", async () => { + const input = facts([ + await document([ + { id: 2, parent: 1 }, + { id: 3, parent: 1 }, + ]), + ]); + const freeze = (value: unknown): void => { + if (!value || typeof value !== "object" || Object.isFrozen(value)) return; + if (value instanceof Map) for (const item of value.values()) freeze(item); + for (const item of Object.values(value)) freeze(item); + Object.freeze(value); + }; + freeze(input); + const first = await discoverVisualCandidates(input); + expect(await discoverVisualCandidates(input)).toEqual(first); + expect(first.candidates).toHaveLength(2); + }); + + it.each([ + 1000, 10000, 100000, + ])("memoizes deep ancestry across %i nodes and checks cancellation", async (count) => { + const doc = await document([{ id: 2, parent: 1 }]); + const template = doc.index.nodes.get(2)!; + let reads = 0; + const input = [doc.index.nodes.get(0)!, doc.index.nodes.get(1)!]; + for (let id = 2; id < count + 2; id++) { + const node = { + ...template, + backendNodeId: id, + tag: id < count / 2 ? "div" : "canvas", + parentBackendNodeId: id < count / 2 ? id - 1 : count / 2 - 1, + }; + Object.defineProperty(node, "parentBackendNodeId", { + get: () => { + reads++; + return id < count / 2 ? id - 1 : count / 2 - 1; + }, + }); + input.push(node); + } + const index = await buildDocumentIndex(input); + reads = 0; + const result = await discoverVisualCandidates(facts([{ ...doc, index }])); + expect(result.candidates).toHaveLength(count / 2 + 2); + expect(reads).toBeLessThan(count * 5); + const controller = new AbortController(); + controller.abort(); + await expect(discoverVisualCandidates(facts([doc]), controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/visual-region.test.ts b/apps/extension/src/tools/vom/__tests__/visual-region.test.ts new file mode 100644 index 00000000..30032837 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/visual-region.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "vitest"; +import { + EMPTY_VISUAL_CONTEXT, + extendVisualContext, + projectVisualBox, + resolveVisualRegion, + type VisualAncestor, +} from "../visual-region"; + +const styles = { + position: "static", + visibility: "visible", + opacity: "1", + display: "block", + transform: "none", + zoom: "1", + "overflow-x": "visible", + "overflow-y": "visible", + "clip-path": "none", + "mask-image": "none", + rotate: "none", + scale: "none", + perspective: "none", + clip: "auto", + contain: "none", + "overflow-clip-margin": "0px", +}; +const ancestor: VisualAncestor = { + document: { + attachmentId: "a", + target: { tabId: 1 }, + frameId: "main", + documentElementBackendNodeId: 1, + }, + backendNodeId: 2, + styles, + clientBox: { x: 5, y: 5, width: 60, height: 30 }, +}; +const box = { x: 0, y: 0, width: 120, height: 40 }; + +describe("shared visual region rules", () => { + it("accepts independently supplied normalized facts and preserves clipping policy", () => { + const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, { + ...ancestor, + styles: { ...styles, "overflow-x": "hidden" }, + }); + expect( + resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context }), + ).toMatchObject({ + status: "available", + crop: { x: 5, y: 0, width: 60, height: 40 }, + clips: { overflowX: "hidden", overflowY: "visible" }, + }); + const differentPolicy = extendVisualContext(EMPTY_VISUAL_CONTEXT, { + ...ancestor, + styles: { ...styles, "overflow-x": "scroll" }, + }); + expect(differentPolicy.clip).toEqual(context.clip); + expect(differentPolicy.clips).not.toEqual(context.clips); + }); + + it("distinguishes fully clipped, missing client geometry and invalid rectangles", () => { + const clipping = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } }; + const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, { + ...clipping, + clientBox: { x: 0, y: 0, width: 0, height: 20 }, + }); + expect( + resolveVisualRegion({ borderBox: box, frameVisibleBox: box, visibility: "visible", context }) + .status, + ).toBe("empty"); + const unavailable = extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...clipping, clientBox: null }); + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: box, + visibility: "visible", + context: unavailable, + }), + ).toEqual({ status: "unavailable", reason: "geometry-unavailable" }); + expect( + resolveVisualRegion({ + borderBox: { ...box, x: NaN }, + frameVisibleBox: box, + visibility: "visible", + context: EMPTY_VISUAL_CONTEXT, + }).status, + ).toBe("unavailable"); + expect(projectVisualBox({ x: 5, y: 5, width: 0, height: 20 }, [])).toEqual({ + x: 5, + y: 5, + width: 0, + height: 20, + }); + }); + + it.each>([ + { rotate: "45deg" }, + { scale: "-1 1" }, + { perspective: "100px" }, + { clip: "rect(0px, 20px, 20px, 0px)" }, + { contain: "paint" }, + { "overflow-x": "clip", "overflow-clip-margin": "10px" }, + { transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0.01,0,0,0,1)" }, + ])("refuses unsupported CSS evidence without changing DOM geometry: %o", (change) => { + const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, { + ...ancestor, + styles: { ...styles, ...change }, + }); + expect(context.issue).toBe("geometry-unsupported"); + }); + + it("accepts axis-preserving matrix3d translation and independent positive scale", () => { + for (const change of [ + { transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,10,20,0,1)" }, + { scale: "1.25" }, + ]) + expect( + extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...ancestor, styles: { ...styles, ...change } }) + .issue, + ).toBeUndefined(); + }); + + it.each([ + "matrix(1,0,0,1,0,0)", + "matrix(1,0,0,1,10,20)", + ])("accepts unscaled overflow transform %s", (transform) => { + const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, { + ...ancestor, + styles: { + ...styles, + transform, + position: "relative", + "overflow-x": "hidden", + "overflow-y": "hidden", + }, + }); + expect(context.issue).toBeUndefined(); + const child = extendVisualContext( + context, + { + ...ancestor, + styles: { + ...styles, + position: "absolute", + "overflow-x": "clip", + "overflow-y": "clip", + "overflow-clip-margin": "content-box", + }, + }, + false, + ); + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: box, + visibility: "visible", + context: child, + }), + ).toMatchObject({ status: "available", crop: { x: 5, y: 5, width: 60, height: 30 } }); + }); + + it("supports positioned overflow containers but refuses uncertain out-of-flow clipping", () => { + const overflow = { ...ancestor, styles: { ...styles, "overflow-x": "hidden" } }; + const staticParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, overflow); + expect( + extendVisualContext( + staticParent, + { ...ancestor, styles: { ...styles, position: "absolute" } }, + false, + ).issue, + ).toBe("geometry-unsupported"); + const positionedParent = extendVisualContext(EMPTY_VISUAL_CONTEXT, { + ...overflow, + styles: { ...overflow.styles, position: "relative" }, + }); + const child = extendVisualContext( + positionedParent, + { ...ancestor, styles: { ...styles, position: "absolute" } }, + false, + ); + expect(child.issue).toBeUndefined(); + expect(child.clip).toEqual(positionedParent.clip); + expect( + extendVisualContext( + positionedParent, + { ...ancestor, styles: { ...styles, position: "fixed" } }, + false, + ).issue, + ).toBe("geometry-unsupported"); + }); +}); diff --git a/apps/extension/src/tools/vom/capture-coordinator.ts b/apps/extension/src/tools/vom/capture-coordinator.ts index c5ecf53d..a6dba2d7 100644 --- a/apps/extension/src/tools/vom/capture-coordinator.ts +++ b/apps/extension/src/tools/vom/capture-coordinator.ts @@ -375,6 +375,7 @@ export async function captureObservationFacts( facts.push({ frame, identity: identities.get(document.frameId), + ...(doc?.geometry ? { geometry: doc.geometry } : {}), index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index, domNodes, axNodes, diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts index c2e4e758..9c1f5c21 100644 --- a/apps/extension/src/tools/vom/facts.ts +++ b/apps/extension/src/tools/vom/facts.ts @@ -1,7 +1,8 @@ import type { Viewport } from "@browser-skill/vom"; import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; import { isOverlayHostNode } from "@/lib/overlay-bridge"; -import type { FrameProjectionIssue } from "../geometry/coordinate-types"; +import type { GeometryProjection } from "../geometry"; +import type { FrameProjectionIssue, SnapshotCoordinates } from "../geometry/coordinate-types"; import { createCaptureCheckpoint } from "./capture-abort"; import type { CapturedNode } from "./capture-types"; import type { FrameOwnedAxNode } from "./frame-document"; @@ -10,21 +11,26 @@ import type { FrameOwnedAxNode } from "./frame-document"; export interface SnapshotLayout { readonly boundsSpace: "snapshot-document-layout"; bounds?: number[]; + /** [clientLeft, clientTop, clientWidth, clientHeight], untransformed local CSS units. */ + clientRect?: number[]; styles: Readonly>; } export interface DecodedNode extends Omit { nodeType?: number; + parentMissing?: boolean; layout?: SnapshotLayout; } export interface NodeFacts extends CapturedNode { nodeType?: number; + parentMissing?: boolean; layout?: SnapshotLayout; } export interface DocumentIndex { readonly nodes: ReadonlyMap; + readonly ancestryComplete: ReadonlyMap; readonly excludedBackendNodeIds: ReadonlySet; } @@ -41,6 +47,7 @@ export async function buildDocumentIndex( const checkpoint = createCaptureCheckpoint(signal); const nodes = new Map(); const overlayByNode = new Map(); + const ancestryComplete = new Map(); const excludedBackendNodeIds = new Set(); for (let i = 0; i < input.length; i++) { if (i % 256 === 0) { @@ -61,6 +68,7 @@ export async function buildDocumentIndex( const visiting = new Set(); let current: DecodedNode | undefined = node; let overlay = false; + let complete = false; while (current && !overlayByNode.has(current.backendNodeId)) { if (work++ % 256 === 0) { const pending = checkpoint(); @@ -76,13 +84,16 @@ export async function buildDocumentIndex( visiting.add(current.backendNodeId); path.push(current); if (current.parentBackendNodeId === null) { + complete = !current.parentMissing && current.nodeType === 9; current = undefined; break; } current = nodes.get(current.parentBackendNodeId); } - if (current && overlayByNode.has(current.backendNodeId)) + if (current && overlayByNode.has(current.backendNodeId)) { overlay = overlayByNode.get(current.backendNodeId)!; + complete = ancestryComplete.get(current.backendNodeId) === true; + } for (let i = path.length - 1; i >= 0; i--) { if (work++ % 256 === 0) { const pending = checkpoint(); @@ -91,10 +102,12 @@ export async function buildDocumentIndex( const item = path[i]; overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)); overlayByNode.set(item.backendNodeId, overlay); + complete = complete && !item.parentMissing; + ancestryComplete.set(item.backendNodeId, complete); if (overlay) excludedBackendNodeIds.add(item.backendNodeId); } } - return { nodes, excludedBackendNodeIds }; + return { nodes, ancestryComplete, excludedBackendNodeIds }; } export interface DocumentIdentity { @@ -127,7 +140,16 @@ export interface CaptureIssue { | "geometry-unavailable"; } +/** Shared per document; never retain the live GeometryContext in published facts. */ +export interface DocumentGeometry { + readonly projections: readonly GeometryProjection[]; + readonly coordinates: SnapshotCoordinates; + /** Visual viewport scale (pinch), distinct from browser UI zoom. */ + readonly pageScale?: number; +} + export interface DocumentFacts { + readonly geometry?: DocumentGeometry; readonly frame: CdpFrame; readonly identity?: DocumentIdentity; readonly index: DocumentIndex; diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 6f162dcd..43c01650 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -18,7 +18,13 @@ import { isAbortError as isCaptureAbort, throwIfAborted as throwCaptureAborted, } from "./capture-abort"; -import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts"; +import { + buildDocumentIndex, + type CaptureIssue, + type DocumentGeometry, + type DocumentIndex, + type NodeFacts, +} from "./facts"; import { decodeDocument, type SnapshotDocument, @@ -33,9 +39,11 @@ export interface FrameContext { targetProjection?: GeometryProjection | null; target: CdpTarget; coordinates: SnapshotCoordinates | null; + pageScale?: number; } export interface NormalizedDocument { + geometry?: DocumentGeometry; nodes: NodeFacts[]; index: DocumentIndex; documentElementBackendNodeId?: number; @@ -99,6 +107,20 @@ export async function normalizeDocument( (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), ), index, + ...(context.coordinates && + context.projection?.status === "available" && + context.targetProjection !== null + ? { + geometry: { + projections: [ + context.projection.projection.geometry, + ...(context.targetProjection ? [context.targetProjection] : []), + ], + coordinates: context.coordinates, + pageScale: context.pageScale, + }, + } + : {}), documentElementBackendNodeId: nodes.find( (node) => node.nodeType === 1 && @@ -313,6 +335,7 @@ export async function normalizeSnapshot( frameId: frame.frameId, ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null, projection: state, + pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale, target, ...(target.sessionId ? { targetProjection } : {}), coordinates, diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index df639c63..88da903c 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -8,6 +8,19 @@ export const REQUESTED_STYLES = [ "cursor", "visibility", "opacity", + "display", + "overflow-x", + "overflow-y", + "transform", + "zoom", + "clip-path", + "mask-image", + "rotate", + "scale", + "perspective", + "clip", + "contain", + "overflow-clip-margin", ] as const; const STYLE_COL = Object.fromEntries( REQUESTED_STYLES.map((name, index) => [name, index]), @@ -49,6 +62,7 @@ export interface SnapshotDocument { nodeIndex?: number[]; styles?: number[][]; bounds?: number[][]; + clientRects?: number[][]; paintOrders?: number[]; }; } @@ -177,6 +191,7 @@ export async function decodeDocument( : { boundsSpace: "snapshot-document-layout" as const, bounds: dl?.bounds?.[li], + ...(dl?.clientRects?.[li] ? { clientRect: dl.clientRects[li] } : {}), styles, }; @@ -200,6 +215,9 @@ export async function decodeDocument( backendNodeId, nodeType: dn.nodeType?.[n], parentBackendNodeId, + ...(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null) + ? { parentMissing: true } + : {}), tag, attrs, diff --git a/apps/extension/src/tools/vom/visual-discovery.ts b/apps/extension/src/tools/vom/visual-discovery.ts new file mode 100644 index 00000000..70ebe422 --- /dev/null +++ b/apps/extension/src/tools/vom/visual-discovery.ts @@ -0,0 +1,307 @@ +import { type CdpTarget } from "@/browser-driver/frame-graph"; +import type { ViewportRect } from "../geometry"; +import { createCaptureCheckpoint } from "./capture-abort"; +import type { + CaptureIssue, + DocumentFacts, + DocumentIdentity, + NodeFacts, + ObservationFacts, +} from "./facts"; +import type { FrameOwnedAxNode } from "./frame-document"; +import { + EMPTY_VISUAL_CONTEXT, + extendVisualContext, + projectVisualBox, + resolveVisualRegion, + type VisualContext, + type VisualIssueReason, + type VisualRegionResult, + visualProjectionIssue, +} from "./visual-region"; + +export interface VisualCandidate { + readonly document: DocumentIdentity; + readonly backendNodeId: number; + readonly parentBackendNodeId: number | null; + readonly label?: string; + readonly region: Extract; +} + +export interface VisualDiscoveryIssue { + readonly target: CdpTarget; + readonly frameId: string; + readonly backendNodeId?: number; + readonly reason: VisualIssueReason; +} + +export interface VisualDiscoveryResult { + readonly candidates: readonly VisualCandidate[]; + readonly issues: readonly VisualDiscoveryIssue[]; + /** Preserve upstream failures, including those that do not invalidate Canvas geometry. */ + readonly captureIssues: readonly CaptureIssue[]; + readonly complete: boolean; +} + +type Document = DocumentFacts; +interface NodeContext { + self: VisualContext; + children: VisualContext; +} + +function viewportRect( + rect: { x: number; y: number; w: number; h: number } | null | undefined, +): ViewportRect | null { + return rect ? { x: rect.x, y: rect.y, width: rect.w, height: rect.h } : null; +} + +/** Client offsets are unscaled local CSS units. Only ordinary, untransformed + * overflow chains consume this adapter; other combinations are rejected by policy. */ +function clientBox(node: NodeFacts, document: Document): ViewportRect | null { + const client = node.layout?.clientRect; + const bounds = node.layout?.bounds; + const border = + node.localRect ?? + (bounds?.length === 4 && bounds.every(Number.isFinite) && document.geometry + ? { + x: + bounds[0] / document.geometry.coordinates.layoutUnitsPerCssPixel - + document.geometry.coordinates.scrollCss.x, + y: + bounds[1] / document.geometry.coordinates.layoutUnitsPerCssPixel - + document.geometry.coordinates.scrollCss.y, + } + : null); + if ( + !client || + client.length !== 4 || + !client.every(Number.isFinite) || + client[2] < 0 || + client[3] < 0 || + !border || + !document.geometry + ) + return null; + return projectVisualBox( + { x: border.x + client[0], y: border.y + client[1], width: client[2], height: client[3] }, + document.geometry.projections, + ); +} + +/** Pure Facts consumer: no CDP, ref registration, selection, or semantic keep/drop. + * Both frame and node ancestry are memoized in this call, using iterative walks. */ +export async function discoverVisualCandidates( + facts: ObservationFacts, + signal?: AbortSignal, +): Promise { + const checkpoint = createCaptureCheckpoint(signal); + let work = 0; + const candidates: VisualCandidate[] = []; + const issues: VisualDiscoveryIssue[] = []; + const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document])); + const contexts = new Map>(); + const roots = new Map(); + + async function nodeContext(document: Document, id: number): Promise { + const cache = contexts.get(document.frame.frameId)!; + const path: NodeFacts[] = []; + let node = document.index.nodes.get(id); + while (node && !cache.has(node.backendNodeId)) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if ( + !document.index.ancestryComplete.get(node.backendNodeId) || + path.length >= document.index.nodes.size + ) { + const incomplete = { + ...roots.get(document.frame.frameId)!, + issue: roots.get(document.frame.frameId)!.issue ?? ("ancestry-incomplete" as const), + }; + cache.set(node.backendNodeId, { self: incomplete, children: incomplete }); + break; + } + path.push(node); + node = + node.parentBackendNodeId === null + ? undefined + : document.index.nodes.get(node.parentBackendNodeId); + } + let parent = node + ? cache.get(node.backendNodeId)!.children + : roots.get(document.frame.frameId)!; + for (let i = path.length - 1; i >= 0; i--) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const current = path[i]; + let self = parent; + let children = parent; + if (document.index.excludedBackendNodeIds.has(current.backendNodeId)) { + self = children = { ...parent, hidden: true }; + } else if (current.nodeType === 1) { + if (!current.layout) { + // An element without layout is not evidence that descendants are hidden + // (e.g. a boxless ancestor). Keep the missing style evidence explicit. + self = children = { ...parent, issue: parent.issue ?? "facts-unavailable" }; + } else if (document.identity) { + const ancestor = { + document: document.identity, + backendNodeId: current.backendNodeId, + styles: current.layout.styles, + clientBox: clientBox(current, document), + }; + children = extendVisualContext( + parent, + ancestor, + current.tag !== "iframe" && current.tag !== "frame", + ); + self = current.tag === "canvas" ? extendVisualContext(parent, ancestor, false) : children; + } + } + const result = { self, children }; + cache.set(current.backendNodeId, result); + parent = children; + } + return ( + cache.get(id) ?? { + self: { ...parent, issue: "ancestry-incomplete" }, + children: { ...parent, issue: "ancestry-incomplete" }, + } + ); + } + + // Process parent documents first, independent of snapshot/target completion order. + for (const document of facts.documents) { + const path: Document[] = []; + const visiting = new Set(); + let current: Document | undefined = document; + while (current && !roots.has(current.frame.frameId)) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (visiting.has(current.frame.frameId)) { + roots.set(current.frame.frameId, { + ...EMPTY_VISUAL_CONTEXT, + issue: "ownership-unresolved", + }); + contexts.set(current.frame.frameId, new Map()); + break; + } + visiting.add(current.frame.frameId); + path.push(current); + current = current.frame.parentFrameId + ? documents.get(current.frame.parentFrameId) + : undefined; + } + for (let i = path.length - 1; i >= 0; i--) { + const doc = path[i]; + const { frame } = doc; + let root: VisualContext = roots.get(frame.frameId) ?? EMPTY_VISUAL_CONTEXT; + if (frame.parentFrameId) { + const parent = documents.get(frame.parentFrameId); + if ( + !parent || + !roots.has(parent.frame.frameId) || + frame.ownerBackendNodeId === undefined || + !parent.identity + ) { + root = { ...root, issue: "ownership-unresolved" }; + } else { + const owner = parent.index.nodes.get(frame.ownerBackendNodeId); + const ownerState = await nodeContext(parent, frame.ownerBackendNodeId); + root = { + ...ownerState.children, + // CSS visibility inside a child document cannot override its hidden owner. + hidden: + ownerState.children.hidden || + owner?.layout?.styles.visibility === "hidden" || + owner?.layout?.styles.visibility === "collapse", + // Frame scaling is already interpreted by the shared content-quad projection. + transformed: false, + localClip: false, + unpositionedClip: false, + }; + } + } else if (frame.frameId !== facts.rootFrameId) + root = { ...root, issue: "ownership-unresolved" }; + root = { + ...root, + issue: + root.issue ?? + (!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)), + }; + roots.set(frame.frameId, root); + contexts.set(frame.frameId, new Map()); + } + } + + for (const document of facts.documents) { + for (const node of document.index.nodes.values()) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (node.tag !== "canvas" || document.index.excludedBackendNodeIds.has(node.backendNodeId)) + continue; + // No layout/zero size is positive evidence of no screenshot area for this node. + if (!node.layout) continue; + const bounds = node.layout.bounds; + if ( + bounds?.length === 4 && + bounds.every(Number.isFinite) && + (bounds[2] <= 0 || bounds[3] <= 0) + ) + continue; + const context = (await nodeContext(document, node.backendNodeId)).self; + const local = viewportRect(node.localRect); + const borderBox = + local && document.geometry ? projectVisualBox(local, document.geometry.projections) : null; + const region = resolveVisualRegion({ + borderBox, + frameVisibleBox: viewportRect(node.rect), + context, + visibility: node.layout.styles.visibility, + }); + if (region.status === "unavailable") { + issues.push({ + target: document.frame.target, + frameId: document.frame.frameId, + backendNodeId: node.backendNodeId, + reason: region.reason, + }); + } else if (region.status === "available" && document.identity) { + const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim(); + candidates.push({ + document: document.identity, + backendNodeId: node.backendNodeId, + parentBackendNodeId: node.parentBackendNodeId, + ...(label ? { label } : {}), + region, + }); + } + } + } + // AX-only/missing documents have no Canvas nodes to carry the missing evidence. + const reportedFrames = new Set(issues.map((issue) => issue.frameId)); + for (const document of facts.documents) { + const reason = roots.get(document.frame.frameId)?.issue; + if ( + reason && + !reportedFrames.has(document.frame.frameId) && + (!document.identity || !document.index.nodes.size) + ) + issues.push({ target: document.frame.target, frameId: document.frame.frameId, reason }); + } + const pending = checkpoint(); + if (pending) await pending; + return { + candidates, + issues, + captureIssues: facts.issues, + complete: issues.length === 0 && facts.issues.length === 0, + }; +} diff --git a/apps/extension/src/tools/vom/visual-region.ts b/apps/extension/src/tools/vom/visual-region.ts new file mode 100644 index 00000000..bf0318d2 --- /dev/null +++ b/apps/extension/src/tools/vom/visual-region.ts @@ -0,0 +1,316 @@ +import { + type GeometryProjection, + type Polygon, + projectPolygon, + rectPolygon, + regionBounds, + type ViewportRect, +} from "../geometry"; +import type { DocumentGeometry, DocumentIdentity } from "./facts"; + +export type VisualIssueReason = + | "ancestry-incomplete" + | "facts-unavailable" + | "geometry-unavailable" + | "geometry-unsupported" + | "identity-unverified" + | "ownership-unresolved"; + +/** Top viewport CSS coordinates. Unconstrained axes have infinite bounds. */ +export interface VisualClip { + readonly left: number; + readonly top: number; + readonly right: number; + readonly bottom: number; +} + +/** Shared persistent chain: one entry per clipping ancestor, not per Canvas. */ +export interface VisualClipSource { + readonly document: DocumentIdentity; + readonly backendNodeId: number; + readonly x: boolean; + readonly y: boolean; + readonly overflowX: string; + readonly overflowY: string; + readonly box: ViewportRect; + readonly parent?: VisualClipSource; +} + +export interface VisualContext { + readonly clip: VisualClip; + readonly clips?: VisualClipSource; + readonly hidden: boolean; + /** A local CSS scale/zoom makes snapshot client offsets ambiguous. */ + readonly transformed: boolean; + readonly localClip?: boolean; + /** Overflow between an absolute target and its nearest positioned ancestor. */ + readonly unpositionedClip?: boolean; + readonly issue?: VisualIssueReason; +} + +export const EMPTY_VISUAL_CONTEXT: VisualContext = { + clip: { left: -Infinity, top: -Infinity, right: Infinity, bottom: Infinity }, + hidden: false, + transformed: false, +}; + +function axisRect(polygon: Polygon): boolean { + if (polygon.length !== 4 || polygon.some((p) => !Number.isFinite(p.x) || !Number.isFinite(p.y))) + return false; + const [a, b, c, d] = polygon; + return a.y === b.y && b.x === c.x && c.y === d.y && d.x === a.x && b.x > a.x && d.y > a.y; +} + +/** Check evidence before projectRectToViewport can reduce a polygon to its bounds. */ +export function visualProjectionIssue( + geometry: DocumentGeometry | undefined, +): VisualIssueReason | undefined { + if (!geometry || !geometry.projections.length || geometry.pageScale === undefined) + return "geometry-unavailable"; + if (!Number.isFinite(geometry.pageScale) || geometry.pageScale <= 0) + return "geometry-unavailable"; + if (geometry.pageScale !== 1) return "geometry-unsupported"; + const { layoutUnitsPerCssPixel, scrollCss } = geometry.coordinates; + if ( + !Number.isFinite(layoutUnitsPerCssPixel) || + layoutUnitsPerCssPixel <= 0 || + ![scrollCss.x, scrollCss.y].every(Number.isFinite) + ) + return "geometry-unavailable"; + for (const projection of geometry.projections) { + if ( + ![projection.topViewport.width, projection.topViewport.height].every( + (n) => Number.isFinite(n) && n > 0, + ) + ) + return "geometry-unavailable"; + if (projection.sourceClips.some((clip) => !axisRect(clip))) return "geometry-unsupported"; + for (const edge of projection.edges) { + if ( + ![edge.sourceViewport.width, edge.sourceViewport.height].every( + (n) => Number.isFinite(n) && n > 0, + ) + ) + return "geometry-unavailable"; + if (!axisRect(edge.destinationQuad) || edge.destinationClips?.some((clip) => !axisRect(clip))) + return "geometry-unsupported"; + } + } +} + +/** Uses the same projection math as DOM geometry, without clipping away box origins. */ +export function projectVisualBox( + box: ViewportRect, + projections: readonly GeometryProjection[], +): ViewportRect | null { + let polygon: Polygon = rectPolygon({ x: box.x, y: box.y, w: box.width, h: box.height }); + for (const projection of projections) + for (const edge of projection.edges) polygon = projectPolygon(polygon, edge); + // Preserve zero-size client boxes: they can prove a fully clipped axis. + if (box.width === 0 || box.height === 0) { + const [a, , c] = polygon; + return { x: a.x, y: a.y, width: c.x - a.x, height: c.y - a.y }; + } + return regionBounds([polygon]); +} + +function axisTransformScale(value: string): [number, number] | null { + if (value === "none") return [1, 1]; + const match = /^matrix(3d)?\(([^)]+)\)$/.exec(value); + if (!match) return null; + const values = match[2].split(",").map((part) => Number(part.trim())); + if (!values.every(Number.isFinite)) return null; + if (match[1]) + return values.length === 16 && + values[0] > 0 && + values[5] > 0 && + values[10] === 1 && + values[15] === 1 && + [1, 2, 3, 4, 6, 7, 8, 9, 11, 14].every((index) => values[index] === 0) + ? [values[0], values[5]] + : null; + return values.length === 6 && values[0] > 0 && values[3] > 0 && values[1] === 0 && values[2] === 0 + ? [values[0], values[3]] + : null; +} + +export interface VisualAncestor { + readonly document: DocumentIdentity; + readonly backendNodeId: number; + readonly styles: Readonly>; + /** Already in top viewport CSS units, from a source-specific adapter. */ + readonly clientBox?: ViewportRect | null; +} + +/** One ancestor step, reusable by discovery and the future live local adapter. */ +export function extendVisualContext( + parent: VisualContext, + node: VisualAncestor, + clipContents = true, +): VisualContext { + const styles = node.styles; + const opacity = styles.opacity?.trim() ? Number(styles.opacity) : NaN; + const zoom = styles.zoom === "normal" ? 1 : styles.zoom?.trim() ? Number(styles.zoom) : NaN; + const scale = + styles.scale === "none" ? [1] : (styles.scale ?? "").trim().split(/\s+/).map(Number); + const scaleSupported = + scale.length >= 1 && + scale.length <= 3 && + scale.every((n) => Number.isFinite(n) && n > 0) && + (scale.length < 3 || scale[2] === 1); + const transformScale = axisTransformScale(styles.transform ?? ""); + // Identity/translation changes origins, which bounds already capture, but + // leaves client offsets in the same units. Only scaling makes them ambiguous. + const transformed = + parent.transformed || + transformScale?.some((n) => n !== 1) === true || + zoom !== 1 || + scale.some((n) => n !== 1); + const hidden = parent.hidden || opacity === 0 || styles.display === "none"; + let issue = parent.issue; + if ( + !Number.isFinite(opacity) || + !Number.isFinite(zoom) || + zoom <= 0 || + !styles.display || + !styles.visibility + ) + issue ??= "facts-unavailable"; + if (!styles.transform || !styles["clip-path"] || !styles["mask-image"]) + issue ??= "facts-unavailable"; + else if (!transformScale || styles["clip-path"] !== "none" || styles["mask-image"] !== "none") + issue ??= "geometry-unsupported"; + + if ( + !["rotate", "scale", "perspective", "clip", "contain", "overflow-clip-margin"].every( + (key) => styles[key], + ) + ) + issue ??= "facts-unavailable"; + else if ( + !scaleSupported || + !["none", "0deg"].includes(styles.rotate) || + styles.perspective !== "none" || + styles.clip !== "auto" || + /(?:^|\s)(paint|strict|content)(?:$|\s)/.test(styles.contain) + ) + issue ??= "geometry-unsupported"; + // Do not pretend every DOM ancestor clips out-of-flow descendants. Support + // the ordinary positioned-container case; other containing-block cases stay explicit. + if ( + (styles.position === "absolute" && parent.unpositionedClip) || + (styles.position === "fixed" && parent.localClip) + ) + issue ??= "geometry-unsupported"; + const overflow = [styles["overflow-x"], styles["overflow-y"]]; + if (clipContents && overflow.includes("clip") && styles["overflow-clip-margin"] !== "0px") + issue ??= "geometry-unsupported"; + if ( + overflow.some( + (value) => !["visible", "hidden", "clip", "scroll", "auto", "overlay"].includes(value), + ) + ) + issue ??= "facts-unavailable"; + const [x, y] = overflow.map((value) => value !== "visible"); + let clip = parent.clip; + let clips = parent.clips; + if (clipContents && (x || y)) { + if (transformed) issue ??= "geometry-unsupported"; + const box = node.clientBox; + if ( + !box || + ![box.x, box.y, box.width, box.height].every(Number.isFinite) || + box.width < 0 || + box.height < 0 + ) + issue ??= "geometry-unavailable"; + else { + clip = { + left: x ? Math.max(clip.left, box.x) : clip.left, + right: x ? Math.min(clip.right, box.x + box.width) : clip.right, + top: y ? Math.max(clip.top, box.y) : clip.top, + bottom: y ? Math.min(clip.bottom, box.y + box.height) : clip.bottom, + }; + clips = { + document: node.document, + backendNodeId: node.backendNodeId, + x, + y, + overflowX: styles["overflow-x"], + overflowY: styles["overflow-y"], + box, + parent: clips, + }; + } + } + const clipsHere = clipContents && (x || y); + return { + hidden, + transformed, + issue, + clip, + clips, + localClip: parent.localClip || clipsHere, + unpositionedClip: + styles.position !== "static" || + styles.transform !== "none" || + /(?:^|\s)layout(?:$|\s)/.test(styles.contain ?? "") + ? false + : parent.unpositionedClip || clipsHere, + }; +} + +export type VisualRegionResult = + | { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource } + | { status: "empty" } + | { status: "unavailable"; reason: VisualIssueReason }; + +/** Box and frameVisibleBox must come from the same box kind (Canvas border-box). + * Frame/viewport clipping is supplied by the shared geometry adapter. */ +export function resolveVisualRegion(input: { + borderBox: ViewportRect | null; + frameVisibleBox: ViewportRect | null; + context: VisualContext; + visibility: string; +}): VisualRegionResult { + const { borderBox, frameVisibleBox, context, visibility } = input; + if (context.hidden || visibility === "hidden" || visibility === "collapse") + return { status: "empty" }; + if (context.issue) return { status: "unavailable", reason: context.issue }; + if (visibility !== "visible") return { status: "unavailable", reason: "facts-unavailable" }; + if (!borderBox) return { status: "unavailable", reason: "geometry-unavailable" }; + if (!frameVisibleBox) return { status: "empty" }; + if ( + ![ + borderBox.x, + borderBox.y, + borderBox.width, + borderBox.height, + frameVisibleBox.x, + frameVisibleBox.y, + frameVisibleBox.width, + frameVisibleBox.height, + ].every(Number.isFinite) + ) + return { status: "unavailable", reason: "geometry-unavailable" }; + if ( + borderBox.width <= 0 || + borderBox.height <= 0 || + frameVisibleBox.width < 0 || + frameVisibleBox.height < 0 || + Object.values(context.clip).some(Number.isNaN) + ) + return { status: "unavailable", reason: "geometry-unavailable" }; + const x = Math.max(frameVisibleBox.x, context.clip.left); + const y = Math.max(frameVisibleBox.y, context.clip.top); + const right = Math.min(frameVisibleBox.x + frameVisibleBox.width, context.clip.right); + const bottom = Math.min(frameVisibleBox.y + frameVisibleBox.height, context.clip.bottom); + return right > x && bottom > y + ? { + status: "available", + borderBox, + crop: { x, y, width: right - x, height: bottom - y }, + clips: context.clips, + } + : { status: "empty" }; +} From 26066de2803e5abd77259a353818507bbfc46854 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 9 Sep 2026 18:27:02 +0800 Subject: [PATCH 2/3] fix(vom): fix border radius incorrectly reject canvas --- .../vom/__tests__/capture-coordinator.test.ts | 85 +++++++++++++++++-- .../src/tools/vom/__tests__/facts.test.ts | 27 ++---- .../vom/__tests__/visual-discovery.test.ts | 51 ++++++++++- .../tools/vom/__tests__/visual-region.test.ts | 58 +++++++++++++ .../src/tools/vom/capture-coordinator.ts | 22 ++++- apps/extension/src/tools/vom/facts.ts | 19 +++-- apps/extension/src/tools/vom/normalize.ts | 16 +++- apps/extension/src/tools/vom/snapshot.ts | 25 ++++-- .../src/tools/vom/visual-discovery.ts | 28 ++++-- apps/extension/src/tools/vom/visual-region.ts | 2 + 10 files changed, 275 insertions(+), 58 deletions(-) diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index 0a4f9bd9..0dde3b9e 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -5,8 +5,8 @@ import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import { captureVomObservation } from "../../observation"; import type { CdpRunner } from "../../shared"; import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; -import { buildSemanticGraph } from "../semantic-graph/build"; -import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot"; +import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph"; +import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot"; import { discoverVisualCandidates } from "../visual-discovery"; function fixture( @@ -85,7 +85,8 @@ function fixture( }; if (method === "DOMSnapshot.captureSnapshot") { snapshots++; - expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES); + const requested = (params as { computedStyles: readonly string[] }).computedStyles; + expect([REQUESTED_STYLES, VISUAL_STYLES]).toContainEqual(requested); result = { strings: [ "#document", @@ -153,10 +154,14 @@ function fixture( [0, 0, 1000, 800], [0, 0, 100, 40], ], - styles: [ - [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11], - [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11], - ], + styles: Array.from({ length: 2 }, () => + requested.map( + (name) => + [7, 8, 8, 5, 6, 9, 5, 5, 10, 6, 10, 10, 10, 10, 10, 8, 10, 11][ + VISUAL_STYLES.indexOf(name as (typeof VISUAL_STYLES)[number]) + ], + ), + ), }, }; }), @@ -1374,12 +1379,76 @@ describe("AX frame scheduling", () => { }); describe("visual facts integration", () => { + it("defaults to base facts across targets and shares semantic behavior with visual capture", async () => { + const base = fixture({ canvas: true }); + const visual = fixture({ canvas: true }); + const baseFacts = await captureObservationFacts(base.cdp, 4); + const visualFacts = await captureObservationFacts(visual.cdp, 4, undefined, undefined, { + includeVisualFacts: true, + }); + expect(baseFacts.visualFactsCollected).toBe(false); + expect(visualFacts.visualFactsCollected).toBe(true); + expect(baseFacts.documents).toHaveLength(4); + expect(visualFacts.documents).toHaveLength(4); + for (const doc of baseFacts.documents) { + expect(doc.geometry).toBeUndefined(); + expect(doc.index.ancestryComplete).toBeUndefined(); + for (const node of doc.index.nodes.values()) { + expect(node.parentMissing).toBeUndefined(); + expect(node.layout?.clientRect).toBeUndefined(); + if (node.layout) expect(Object.keys(node.layout.styles)).toEqual([...REQUESTED_STYLES]); + } + } + // This fixture has no child frame projection measurements. Enabling visual + // collection must preserve that missing evidence, not invent geometry. + expect( + visualFacts.documents.find((doc) => doc.frame.frameId === "main")?.geometry, + ).toBeDefined(); + expect(visualFacts.documents.filter((doc) => doc.geometry)).toHaveLength(1); + for (const doc of visualFacts.documents) { + expect(doc.index.ancestryComplete).toBeDefined(); + for (const node of doc.index.nodes.values()) + if (node.layout) { + expect(Object.keys(node.layout.styles)).toEqual([...VISUAL_STYLES]); + expect(Object.keys(node.layout.styles).some((key) => key.includes("radius"))).toBe(false); + } + } + for (const [capture, expected] of [ + [base, REQUESTED_STYLES], + [visual, VISUAL_STYLES], + ] as const) { + const calls = capture.logs.filter((c) => c.method === "DOMSnapshot.captureSnapshot"); + expect(calls).toHaveLength(2); + for (const call of calls) expect(call.params.computedStyles).toEqual(expected); + } + expect(base.logs.map((c) => [c.target, c.method])).toEqual( + visual.logs.map((c) => [c.target, c.method]), + ); + const scene = (input: typeof baseFacts) => + buildSemanticVomScene({ + documents: semanticCapture(input).documents, + viewport: input.viewport, + rootFrameId: input.rootFrameId, + excludedBackendNodeIds: semanticCapture(input).captured.excludedBackendNodeIds, + }); + expect(scene(baseFacts)).toEqual(scene(visualFacts)); + const before = base.logs.length; + expect(await discoverVisualCandidates(baseFacts)).toMatchObject({ + complete: false, + candidates: [], + issues: [{ reason: "visual-facts-not-collected" }], + }); + expect(base.logs).toHaveLength(before); + }); + it("passes actual snapshot styles, client units, projection and identity to pure discovery without additional CDP", async () => { const { cdp, logs } = fixture({ canvas: true, frames: [{ frameId: "main", target: { tabId: 4 } }], }); - const facts = await captureObservationFacts(cdp, 4); + const facts = await captureObservationFacts(cdp, 4, undefined, undefined, { + includeVisualFacts: true, + }); const before = logs.length; const result = await discoverVisualCandidates(facts); expect(logs).toHaveLength(before); diff --git a/apps/extension/src/tools/vom/__tests__/facts.test.ts b/apps/extension/src/tools/vom/__tests__/facts.test.ts index d99da011..c9b3367b 100644 --- a/apps/extension/src/tools/vom/__tests__/facts.test.ts +++ b/apps/extension/src/tools/vom/__tests__/facts.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import { createCaptureCheckpoint } from "../capture-abort"; import { buildDocumentIndex, type DecodedNode } from "../facts"; -import { decodeDocument, REQUESTED_STYLES } from "../snapshot"; +import { decodeDocument, REQUESTED_STYLES, VISUAL_SNAPSHOT } from "../snapshot"; function node(id: number, parent: number | null): DecodedNode { return { @@ -80,19 +80,6 @@ describe("document facts", () => { boundsSpace: "snapshot-document-layout", bounds: [10, 20, 120, 40], styles: { - rotate: "auto", - scale: "auto", - perspective: "auto", - clip: "auto", - contain: "auto", - "overflow-clip-margin": "auto", - display: "auto", - "overflow-x": "auto", - "overflow-y": "auto", - transform: "auto", - zoom: "auto", - "clip-path": "auto", - "mask-image": "auto", position: "static", "pointer-events": "auto", cursor: "auto", @@ -153,9 +140,9 @@ describe("visual ancestry facts", () => { { ...node(8, null), nodeType: 1 }, { ...node(9, null), nodeType: 9, parentMissing: true }, ]; - const index = await buildDocumentIndex(input.reverse()); - for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete.get(id)).toBe(true); - for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete.get(id)).toBe(false); + const index = await buildDocumentIndex(input.reverse(), undefined, true); + for (const id of [0, 1, 2, 3]) expect(index.ancestryComplete!.get(id)).toBe(true); + for (const id of [4, 5, 6, 7, 8, 9]) expect(index.ancestryComplete!.get(id)).toBe(false); }); it("retains client offsets in their source units and does not disguise missing parent indices", async () => { @@ -169,11 +156,13 @@ describe("visual ancestry facts", () => { }, }, ["div"], + undefined, + VISUAL_SNAPSHOT, ); expect(decoded.nodes[0].layout?.clientRect).toEqual([5, 5, 68, 38]); expect(decoded.nodes[0].layout?.bounds).toEqual([745, 455.625, 97, 59.5]); expect(decoded.nodes.every((node) => node.parentMissing)).toBe(true); - const index = await buildDocumentIndex(decoded.nodes); - expect([...index.ancestryComplete.values()]).toEqual([false, false]); + const index = await buildDocumentIndex(decoded.nodes, undefined, true); + expect([...index.ancestryComplete!.values()]).toEqual([false, false]); }); }); diff --git a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts index 8c3c97f4..60c7d55c 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-discovery.test.ts @@ -5,7 +5,7 @@ import type { GeometryProjection } from "../../geometry"; import { buildDocumentIndex, type DocumentFacts, type ObservationFacts } from "../facts"; import type { FrameOwnedAxNode } from "../frame-document"; import { normalizeDocument } from "../normalize"; -import { REQUESTED_STYLES } from "../snapshot"; +import { VISUAL_SNAPSHOT, VISUAL_STYLES } from "../snapshot"; import { discoverVisualCandidates } from "../visual-discovery"; const defaults: Record = { @@ -86,7 +86,7 @@ async function document( bounds: layouts.map((n) => n.bounds ?? [10, 20, 120, 40]), clientRects: layouts.map((n) => n.client ?? [0, 0, 1000, 800]), styles: layouts.map((n) => - REQUESTED_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])), + VISUAL_STYLES.map((key) => str({ ...defaults, ...n.styles }[key])), ), }, }, @@ -108,6 +108,8 @@ async function document( }, pageScale: options.pageScale ?? 1, }, + undefined, + VISUAL_SNAPSHOT, ); return { frame, @@ -128,6 +130,7 @@ function facts( issues: ObservationFacts["issues"] = [], ): ObservationFacts { return { + visualFactsCollected: true, rootFrameId: "main", documents, viewport: { width: 1000, height: 800 }, @@ -138,6 +141,46 @@ function facts( } describe("Canvas discovery", () => { + it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => { + const empty = facts([]); + expect(await discoverVisualCandidates(empty)).toMatchObject({ + complete: true, + candidates: [], + issues: [], + }); + expect(await discoverVisualCandidates({ ...empty, visualFactsCollected: false })).toMatchObject( + { complete: false, candidates: [], issues: [{ reason: "visual-facts-not-collected" }] }, + ); + const controller = new AbortController(); + controller.abort(); + await expect( + discoverVisualCandidates({ ...empty, visualFactsCollected: false }, controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }); + }); + + it("discovers Canvas and frame contents without corner radius facts", async () => { + const own = await document([{ id: 2, parent: 1, styles: {} }]); + expect(await discoverVisualCandidates(facts([own]))).toMatchObject({ + candidates: [{ backendNodeId: 2 }], + complete: true, + issues: [], + }); + const parent = await document([{ id: 2, parent: 1, tag: "iframe", styles: {} }]); + const child = await document([{ id: 3, parent: 1 }], { + frame: { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 2, + target: { tabId: 1 }, + }, + }); + expect(await discoverVisualCandidates(facts([parent, child]))).toMatchObject({ + candidates: [{ backendNodeId: 3 }], + complete: true, + issues: [], + }); + }); + it.each([ 0.8, 1, 1.25, 2, ])("combines layout-unit bounds with CSS client clips at scale %s", async (scale) => { @@ -383,7 +426,7 @@ describe("Canvas discovery", () => { ]); for (const parent of [99, 3]) { doc.index.nodes.get(2)!.parentBackendNodeId = parent; - const index = await buildDocumentIndex([...doc.index.nodes.values()]); + const index = await buildDocumentIndex([...doc.index.nodes.values()], undefined, true); const result = await discoverVisualCandidates(facts([{ ...doc, index }])); expect(result.candidates).toHaveLength(0); expect(result.issues[0].reason).toBe("ancestry-incomplete"); @@ -445,7 +488,7 @@ describe("Canvas discovery", () => { }); input.push(node); } - const index = await buildDocumentIndex(input); + const index = await buildDocumentIndex(input, undefined, true); reads = 0; const result = await discoverVisualCandidates(facts([{ ...doc, index }])); expect(result.candidates).toHaveLength(count / 2 + 2); diff --git a/apps/extension/src/tools/vom/__tests__/visual-region.test.ts b/apps/extension/src/tools/vom/__tests__/visual-region.test.ts index 30032837..dc9ccce2 100644 --- a/apps/extension/src/tools/vom/__tests__/visual-region.test.ts +++ b/apps/extension/src/tools/vom/__tests__/visual-region.test.ts @@ -39,6 +39,64 @@ const ancestor: VisualAncestor = { const box = { x: 0, y: 0, width: 120, height: 40 }; describe("shared visual region rules", () => { + it.each([ + "visible", + "hidden", + "auto", + "scroll", + "clip", + ])("keeps %s screenshot bounds independent of rounded styling", (overflow) => { + for (const clipContents of [true, false]) { + const node = { + ...ancestor, + styles: { ...styles, "overflow-x": overflow, "overflow-y": overflow }, + }; + const baseline = extendVisualContext(EMPTY_VISUAL_CONTEXT, node, clipContents); + for (const radius of ["0px", "8px", "50%", "4px 8px"]) { + const rounded = extendVisualContext( + EMPTY_VISUAL_CONTEXT, + { + ...node, + styles: { + ...node.styles, + "border-top-left-radius": radius, + "border-top-right-radius": radius, + "border-bottom-left-radius": radius, + "border-bottom-right-radius": radius, + }, + }, + clipContents, + ); + expect(rounded).toEqual(baseline); + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: box, + visibility: "visible", + context: rounded, + }), + ).toMatchObject({ status: "available" }); + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: box, + visibility: "hidden", + context: rounded, + }), + ).toEqual({ status: "empty" }); + if (clipContents && overflow !== "visible") + expect( + resolveVisualRegion({ + borderBox: box, + frameVisibleBox: { x: 80, y: 0, width: 20, height: 20 }, + visibility: "visible", + context: rounded, + }), + ).toEqual({ status: "empty" }); + } + } + }); + it("accepts independently supplied normalized facts and preserves clipping policy", () => { const context = extendVisualContext(EMPTY_VISUAL_CONTEXT, { ...ancestor, diff --git a/apps/extension/src/tools/vom/capture-coordinator.ts b/apps/extension/src/tools/vom/capture-coordinator.ts index a6dba2d7..0a90a037 100644 --- a/apps/extension/src/tools/vom/capture-coordinator.ts +++ b/apps/extension/src/tools/vom/capture-coordinator.ts @@ -22,7 +22,12 @@ import { type FrameOwnedAxNode, } from "./frame-document"; import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize"; -import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot"; +import { + BASIC_SNAPSHOT, + describeSnapshotFrames, + type SnapshotReply, + VISUAL_SNAPSHOT, +} from "./snapshot"; interface TargetBatch { target: CdpTarget; @@ -69,7 +74,9 @@ export async function captureObservationFacts( tabId: number, signal?: AbortSignal, pageUrl?: string, + options: { includeVisualFacts?: boolean } = {}, ): Promise> { + const profile = options.includeVisualFacts ? VISUAL_SNAPSHOT : BASIC_SNAPSHOT; const startedAt = Date.now(); throwCaptureAborted(signal); let graph: CdpFrameGraph | undefined; @@ -125,7 +132,7 @@ export async function captureObservationFacts( throwCaptureAborted(signal); batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId); const snapshot = await scoped.send(tabId, "DOMSnapshot.captureSnapshot", { - computedStyles: REQUESTED_STYLES, + computedStyles: profile.computedStyles, includePaintOrder: true, includeDOMRects: true, }); @@ -146,6 +153,7 @@ export async function captureObservationFacts( issues, signal, observed.rootFrameId, + profile, ); const formsAvailable = await enrichFormControlStates( scoped, @@ -384,7 +392,15 @@ export async function captureObservationFacts( throwCaptureAborted(signal); if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length)) throw firstFailure; - return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() }; + return { + visualFactsCollected: profile.includeVisualFacts, + rootFrameId, + viewport, + documents: facts, + issues, + startedAt, + finishedAt: Date.now(), + }; } /** Existing semantic consumers get a narrow view; no raw snapshot diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts index 9c1f5c21..a512ed18 100644 --- a/apps/extension/src/tools/vom/facts.ts +++ b/apps/extension/src/tools/vom/facts.ts @@ -30,7 +30,7 @@ export interface NodeFacts extends CapturedNode { export interface DocumentIndex { readonly nodes: ReadonlyMap; - readonly ancestryComplete: ReadonlyMap; + readonly ancestryComplete?: ReadonlyMap; readonly excludedBackendNodeIds: ReadonlySet; } @@ -43,11 +43,12 @@ export interface DecodedDocument { export async function buildDocumentIndex( input: readonly T[], signal?: AbortSignal, + includeVisualFacts = false, ): Promise> { const checkpoint = createCaptureCheckpoint(signal); const nodes = new Map(); const overlayByNode = new Map(); - const ancestryComplete = new Map(); + const ancestryComplete = includeVisualFacts ? new Map() : undefined; const excludedBackendNodeIds = new Set(); for (let i = 0; i < input.length; i++) { if (i % 256 === 0) { @@ -84,7 +85,7 @@ export async function buildDocumentIndex( visiting.add(current.backendNodeId); path.push(current); if (current.parentBackendNodeId === null) { - complete = !current.parentMissing && current.nodeType === 9; + if (ancestryComplete) complete = !current.parentMissing && current.nodeType === 9; current = undefined; break; } @@ -92,7 +93,7 @@ export async function buildDocumentIndex( } if (current && overlayByNode.has(current.backendNodeId)) { overlay = overlayByNode.get(current.backendNodeId)!; - complete = ancestryComplete.get(current.backendNodeId) === true; + if (ancestryComplete) complete = ancestryComplete.get(current.backendNodeId) === true; } for (let i = path.length - 1; i >= 0; i--) { if (work++ % 256 === 0) { @@ -102,12 +103,14 @@ export async function buildDocumentIndex( const item = path[i]; overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)); overlayByNode.set(item.backendNodeId, overlay); - complete = complete && !item.parentMissing; - ancestryComplete.set(item.backendNodeId, complete); + if (ancestryComplete) { + complete = complete && !item.parentMissing; + ancestryComplete.set(item.backendNodeId, complete); + } if (overlay) excludedBackendNodeIds.add(item.backendNodeId); } } - return { nodes, ancestryComplete, excludedBackendNodeIds }; + return { nodes, ...(ancestryComplete ? { ancestryComplete } : {}), excludedBackendNodeIds }; } export interface DocumentIdentity { @@ -158,6 +161,8 @@ export interface DocumentFacts { } export interface ObservationFacts { + /** Whether visual collection was enabled; partial capture failures remain in issues. */ + readonly visualFactsCollected: boolean; readonly rootFrameId: string; readonly viewport: Viewport; readonly documents: readonly DocumentFacts[]; diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 43c01650..4375cd5a 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -26,8 +26,10 @@ import { type NodeFacts, } from "./facts"; import { + BASIC_SNAPSHOT, decodeDocument, type SnapshotDocument, + type SnapshotProfile, type SnapshotReply, snapshotFrameId, } from "./snapshot"; @@ -55,8 +57,9 @@ export async function normalizeDocument( strings: string[], context: FrameContext, signal?: AbortSignal, + profile: SnapshotProfile = BASIC_SNAPSHOT, ): Promise { - const decoded = await decodeDocument(doc, strings, signal); + const decoded = await decodeDocument(doc, strings, signal, profile); const checkpoint = createCaptureCheckpoint(signal); const nodes: NodeFacts[] = []; for (let i = 0; i < decoded.nodes.length; i++) { @@ -101,13 +104,14 @@ export async function normalizeDocument( (Number.parseFloat(opacity) || 0) > 0, }); } - const index = await buildDocumentIndex(nodes, signal); + const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts); return { nodes: nodes.filter( (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), ), index, - ...(context.coordinates && + ...(profile.includeVisualFacts && + context.coordinates && context.projection?.status === "available" && context.targetProjection !== null ? { @@ -144,6 +148,7 @@ export async function normalizeSnapshot( issues: CaptureIssue[], signal?: AbortSignal, rootFrameId?: string, + profile: SnapshotProfile = BASIC_SNAPSHOT, ): Promise { const strings = snapshot.strings ?? []; const raw = snapshot.documents ?? []; @@ -335,12 +340,15 @@ export async function normalizeSnapshot( frameId: frame.frameId, ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null, projection: state, - pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale, + ...(profile.includeVisualFacts + ? { pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale } + : {}), target, ...(target.sessionId ? { targetProjection } : {}), coordinates, }, signal, + profile, ); result.push({ ...normalized, frame }); } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 88da903c..3409541e 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -8,6 +8,9 @@ export const REQUESTED_STYLES = [ "cursor", "visibility", "opacity", +] as const; +export const VISUAL_STYLES = [ + ...REQUESTED_STYLES, "display", "overflow-x", "overflow-y", @@ -22,9 +25,14 @@ export const REQUESTED_STYLES = [ "contain", "overflow-clip-margin", ] as const; -const STYLE_COL = Object.fromEntries( - REQUESTED_STYLES.map((name, index) => [name, index]), -) as Record<(typeof REQUESTED_STYLES)[number], number>; + +/** The same profile owns request columns, decoding and visual-only derived facts. */ +export const BASIC_SNAPSHOT = { + includeVisualFacts: false, + computedStyles: REQUESTED_STYLES, +} as const; +export const VISUAL_SNAPSHOT = { includeVisualFacts: true, computedStyles: VISUAL_STYLES } as const; +export type SnapshotProfile = typeof BASIC_SNAPSHOT | typeof VISUAL_SNAPSHOT; /** Sparse array format Chrome uses for infrequently-set per-node fields. */ interface SparseArray { index: number[]; @@ -117,6 +125,7 @@ export async function decodeDocument( doc: SnapshotDocument, strings: string[], signal?: AbortSignal, + profile: SnapshotProfile = BASIC_SNAPSHOT, ): Promise { const checkpoint = createCaptureCheckpoint(signal); const dn = doc.nodes; @@ -184,14 +193,17 @@ export async function decodeDocument( const li = layoutByNode.get(n); const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []); const styles: Record = {}; - for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]); + for (let i = 0; i < profile.computedStyles.length; i++) + styles[profile.computedStyles[i]] = str(strings, styleRow[i]); const layout = li === undefined ? undefined : { boundsSpace: "snapshot-document-layout" as const, bounds: dl?.bounds?.[li], - ...(dl?.clientRects?.[li] ? { clientRect: dl.clientRects[li] } : {}), + ...(profile.includeVisualFacts && dl?.clientRects?.[li] + ? { clientRect: dl.clientRects[li] } + : {}), styles, }; @@ -215,7 +227,8 @@ export async function decodeDocument( backendNodeId, nodeType: dn.nodeType?.[n], parentBackendNodeId, - ...(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null) + ...(profile.includeVisualFacts && + (dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null)) ? { parentMissing: true } : {}), diff --git a/apps/extension/src/tools/vom/visual-discovery.ts b/apps/extension/src/tools/vom/visual-discovery.ts index 70ebe422..43c9b35d 100644 --- a/apps/extension/src/tools/vom/visual-discovery.ts +++ b/apps/extension/src/tools/vom/visual-discovery.ts @@ -28,12 +28,19 @@ export interface VisualCandidate { readonly region: Extract; } -export interface VisualDiscoveryIssue { - readonly target: CdpTarget; - readonly frameId: string; - readonly backendNodeId?: number; - readonly reason: VisualIssueReason; -} +export type VisualDiscoveryIssue = + | { + readonly reason: "visual-facts-not-collected"; + readonly target?: never; + readonly frameId?: never; + readonly backendNodeId?: never; + } + | { + readonly target: CdpTarget; + readonly frameId: string; + readonly backendNodeId?: number; + readonly reason: VisualIssueReason; + }; export interface VisualDiscoveryResult { readonly candidates: readonly VisualCandidate[]; @@ -95,6 +102,13 @@ export async function discoverVisualCandidates( signal?: AbortSignal, ): Promise { const checkpoint = createCaptureCheckpoint(signal); + if (!facts.visualFactsCollected) + return { + candidates: [], + complete: false, + captureIssues: facts.issues, + issues: [{ reason: "visual-facts-not-collected" }], + }; let work = 0; const candidates: VisualCandidate[] = []; const issues: VisualDiscoveryIssue[] = []; @@ -112,7 +126,7 @@ export async function discoverVisualCandidates( if (pending) await pending; } if ( - !document.index.ancestryComplete.get(node.backendNodeId) || + !document.index.ancestryComplete?.get(node.backendNodeId) || path.length >= document.index.nodes.size ) { const incomplete = { diff --git a/apps/extension/src/tools/vom/visual-region.ts b/apps/extension/src/tools/vom/visual-region.ts index bf0318d2..6f8f3a8c 100644 --- a/apps/extension/src/tools/vom/visual-region.ts +++ b/apps/extension/src/tools/vom/visual-region.ts @@ -260,6 +260,8 @@ export function extendVisualContext( }; } +/** A page screenshot rectangle, not an exact mask of visible Canvas pixels. + * Rounded corners are rendered by the browser and do not invalidate this range. */ export type VisualRegionResult = | { status: "available"; borderBox: ViewportRect; crop: ViewportRect; clips?: VisualClipSource } | { status: "empty" } From 345e1ac45f6676bea318b6c559467080d292b8e4 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 9 Sep 2026 20:06:30 +0800 Subject: [PATCH 3/3] refactor(vom): preserve complete canvas candidates with exact deduplication --- .../vom/__tests__/capture-coordinator.test.ts | 10 ++ .../tools/vom/__tests__/visual-dedup.test.ts | 147 ++++++++++++++++++ apps/extension/src/tools/vom/visual-dedup.ts | 64 ++++++++ 3 files changed, 221 insertions(+) create mode 100644 apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts create mode 100644 apps/extension/src/tools/vom/visual-dedup.ts diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index 0dde3b9e..c0aea5e6 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -7,6 +7,7 @@ import type { CdpRunner } from "../../shared"; import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; import { buildSemanticGraph, buildSemanticVomScene } from "../semantic-graph"; import { REQUESTED_STYLES, type SnapshotReply, VISUAL_STYLES } from "../snapshot"; +import { deduplicateVisualCandidates } from "../visual-dedup"; import { discoverVisualCandidates } from "../visual-discovery"; function fixture( @@ -1459,6 +1460,15 @@ describe("visual facts integration", () => { document: { frameId: "main", documentElementBackendNodeId: 1 }, region: { crop: { x: 10, y: 20, width: 100, height: 40 } }, }); + const deduplicated = await deduplicateVisualCandidates(result); + expect(deduplicated.candidates).toEqual(result.candidates); + expect(deduplicated.candidates[0]).toBe(result.candidates[0]); + expect(deduplicated).toMatchObject({ + candidateCount: 1, + deduplicatedCount: 0, + dedupDegraded: false, + }); + expect(logs).toHaveLength(before); expect(facts.documents[0].index.nodes.get(2)?.layout?.clientRect).toEqual([0, 0, 100, 40]); expect(facts.documents[0].geometry?.pageScale).toBe(1); const baseline = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] }); diff --git a/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts b/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts new file mode 100644 index 00000000..2337d992 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/visual-dedup.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from "vitest"; +import { deduplicateVisualCandidates, MAX_VISUAL_DEDUP_KEYS } from "../visual-dedup"; +import type { VisualCandidate, VisualDiscoveryResult } from "../visual-discovery"; + +function candidate(id: number, width = 100, parent = 1): VisualCandidate { + const crop = { x: 0, y: 0, width, height: 100 }; + return { + document: { + attachmentId: "a", + target: { tabId: 1 }, + frameId: "main", + documentElementBackendNodeId: 1, + }, + backendNodeId: id, + parentBackendNodeId: parent, + region: { status: "available", borderBox: crop, crop }, + }; +} +function discovery(candidates: VisualCandidate[]): VisualDiscoveryResult { + return { candidates, complete: true, issues: [], captureIssues: [] }; +} + +describe("visual candidate deduplication", () => { + it("preserves small, unnamed and labelled candidates and their original evidence", async () => { + const small = { ...candidate(3, 1), label: ' A "label"\nšŸ˜€' }; + const large = candidate(2, 1000); + const input = discovery([small, large]); + Object.freeze(input.candidates); + Object.freeze(small); + Object.freeze(large); + for (let i = 0; i < 2; i++) { + const result = await deduplicateVisualCandidates(input); + expect(result.candidates).toEqual([small, large]); + expect(result.candidates[0]).toBe(small); + expect(result.candidates[1]).toBe(large); + expect(result).toMatchObject({ + candidateCount: 2, + deduplicatedCount: 0, + dedupDegraded: false, + }); + } + }); + + it("only deduplicates exact crops within the same full DOM identity and direct parent", async () => { + const base = candidate(20); + const variants = [ + candidate(30, 100, 2), + { ...candidate(40), parentBackendNodeId: null }, + ...["x", "y", "width", "height"].map((axis) => ({ + ...base, + region: { + ...base.region, + crop: { + ...base.region.crop, + [axis]: base.region.crop[axis as keyof typeof base.region.crop] + 0.001, + }, + }, + })), + ...[ + { frameId: "child" }, + { target: { tabId: 2 } }, + { target: { tabId: 1, sessionId: "remote" } }, + { documentElementBackendNodeId: 9 }, + { attachmentId: "b" }, + ].map((identity) => ({ ...base, document: { ...base.document, ...identity } })), + ]; + const duplicate = { ...candidate(10), label: "Not a new target" }; + const result = await deduplicateVisualCandidates(discovery([base, ...variants, duplicate])); + expect(result.candidates).toEqual([base, ...variants]); + expect(result.candidates[0]).toBe(base); + expect(result.deduplicatedCount).toBe(1); + }); + + it.each([ + 50_001, 100_000, + ])("keeps all %s distinct candidates including the last tiny Canvas", async (count) => { + const input = Array.from({ length: count }, (_, i) => candidate(i + 1, 100, i + 1)); + input[count - 1] = candidate(count, 1, count); + // A remembered key still deduplicates; an unremembered key passes through. + input.push({ ...input[0] }, { ...input[count - 1] }); + const result = await deduplicateVisualCandidates(discovery(input)); + expect(result.candidates).toEqual([...input.slice(0, count), input[count + 1]]); + expect(result.candidates[count - 1]).toBe(input[count - 1]); + expect(result).toMatchObject({ + candidateCount: count + 2, + deduplicatedCount: 1, + dedupDegraded: true, + }); + }); + + it("does not report degradation when every key fits", async () => { + const input = Array.from({ length: MAX_VISUAL_DEDUP_KEYS }, (_, i) => candidate(i, 1, i)); + input.push({ ...input[0] }); + const result = await deduplicateVisualCandidates(discovery(input)); + expect(result.candidates).toHaveLength(MAX_VISUAL_DEDUP_KEYS); + expect(result.dedupDegraded).toBe(false); + expect(result.deduplicatedCount).toBe(1); + }); + + it("preserves upstream unknowns without counting issues as Canvas nodes", async () => { + const input: VisualDiscoveryResult = { + ...discovery([candidate(2)]), + complete: false, + issues: [{ reason: "geometry-unavailable", target: { tabId: 1 }, frameId: "missing" }], + captureIssues: [{ stage: "ax", reason: "capture-unavailable", target: { tabId: 1 } }], + }; + const result = await deduplicateVisualCandidates(input); + expect(result.complete).toBe(false); + expect(result.candidateCount).toBe(1); + expect(result.issues).toBe(input.issues); + expect(result.captureIssues).toBe(input.captureIssues); + const absent = await deduplicateVisualCandidates({ + ...input, + candidates: [], + issues: [{ reason: "visual-facts-not-collected" }], + }); + expect(absent).toMatchObject({ + complete: false, + candidateCount: 0, + issues: [{ reason: "visual-facts-not-collected" }], + }); + expect(await deduplicateVisualCandidates(discovery([]))).toMatchObject({ + complete: true, + candidates: [], + candidateCount: 0, + dedupDegraded: false, + }); + }); + + it("drains cancellation through the shared checkpoint", async () => { + const controller = new AbortController(); + controller.abort(); + await expect( + deduplicateVisualCandidates(discovery([]), controller.signal), + ).rejects.toMatchObject({ name: "AbortError" }); + const during = new AbortController(); + const input = discovery(Array.from({ length: 100_000 }, (_, i) => candidate(i, 100, i))); + const timer = setTimeout(() => during.abort(), 0); + try { + await expect(deduplicateVisualCandidates(input, during.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + } finally { + clearTimeout(timer); + } + }); +}); diff --git a/apps/extension/src/tools/vom/visual-dedup.ts b/apps/extension/src/tools/vom/visual-dedup.ts new file mode 100644 index 00000000..149030e4 --- /dev/null +++ b/apps/extension/src/tools/vom/visual-dedup.ts @@ -0,0 +1,64 @@ +import { createCaptureCheckpoint } from "./capture-abort"; +import type { VisualCandidate, VisualDiscoveryResult } from "./visual-discovery"; + +export const MAX_VISUAL_DEDUP_KEYS = 50_000; + +export interface VisualDedupResult extends VisualDiscoveryResult { + /** Known valid candidates in this capture, not the total Canvas count on the page. */ + readonly candidateCount: number; + /** Candidates actually removed by exact coverage deduplication. */ + readonly deduplicatedCount: number; + readonly dedupDegraded: boolean; +} + +/** Keep the first real anchor for each exact DOM/parent/crop key, in encounter order. + * Once the key set is full, unknown keys pass through: capacity never drops candidates. */ +export async function deduplicateVisualCandidates( + discovery: VisualDiscoveryResult, + signal?: AbortSignal, +): Promise { + const checkpoint = createCaptureCheckpoint(signal); + const keys = new Set(); + const candidates: VisualCandidate[] = []; + let deduplicatedCount = 0; + let dedupDegraded = false; + for (let i = 0; i < discovery.candidates.length; i++) { + if (i % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const candidate = discovery.candidates[i]; + const { + document, + region: { crop }, + } = candidate; + const key = JSON.stringify([ + document.attachmentId, + document.target.tabId, + document.target.sessionId ?? null, + document.frameId, + document.documentElementBackendNodeId, + candidate.parentBackendNodeId, + crop.x, + crop.y, + crop.width, + crop.height, + ]); + if (keys.has(key)) { + deduplicatedCount++; + continue; + } + if (keys.size < MAX_VISUAL_DEDUP_KEYS) keys.add(key); + else dedupDegraded = true; + candidates.push(candidate); + } + const pending = checkpoint(); + if (pending) await pending; + return { + ...discovery, + candidates, + candidateCount: discovery.candidates.length, + deduplicatedCount, + dedupDegraded, + }; +}