From 7a10be21cb6a7e96306caacba25fbac93f8d2b9c Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Fri, 21 Aug 2026 20:56:24 +0800 Subject: [PATCH 1/4] feat(interaction): add scroll-to-element primitive --- .../src/tools/__tests__/scroll.test.ts | 145 ++++++++++++++++++ apps/extension/src/tools/dispatcher.ts | 14 ++ apps/extension/src/tools/interaction.ts | 2 +- apps/extension/src/tools/scroll.ts | 69 +++++++++ apps/extension/src/transport/types.ts | 19 +++ crates/bsk-cli/skill/SKILL.md | 1 + crates/bsk-cli/src/cli/mod.rs | 6 + crates/bsk-cli/src/cli/scroll.rs | 117 ++++++++++++++ crates/bsk-cli/src/daemon/ipc.rs | 1 + crates/bsk-cli/src/main.rs | 1 + crates/bsk-cli/tests/cli_parse.rs | 9 ++ crates/bsk-cli/tests/tools_m7_ipc.rs | 52 ++++++- .../schema/tool_scroll_to_params.json | 42 +++++ .../schema/tool_scroll_to_result.json | 123 +++++++++++++++ crates/bsk-protocol/src/bin/dump-schema.rs | 2 + crates/bsk-protocol/src/method.rs | 6 + crates/bsk-protocol/src/tools/mod.rs | 2 + crates/bsk-protocol/src/tools/scroll.rs | 65 ++++++++ skill/SKILL.md | 1 + 19 files changed, 672 insertions(+), 5 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/scroll.test.ts create mode 100644 apps/extension/src/tools/scroll.ts create mode 100644 crates/bsk-cli/src/cli/scroll.rs create mode 100644 crates/bsk-protocol/schema/tool_scroll_to_params.json create mode 100644 crates/bsk-protocol/schema/tool_scroll_to_result.json create mode 100644 crates/bsk-protocol/src/tools/scroll.rs diff --git a/apps/extension/src/tools/__tests__/scroll.test.ts b/apps/extension/src/tools/__tests__/scroll.test.ts new file mode 100644 index 00000000..6ed86e20 --- /dev/null +++ b/apps/extension/src/tools/__tests__/scroll.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it, vi } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import type { CdpRunner } from "@/tools/shared"; +import { handleScrollTo } from "../scroll"; + +function fakeAgentWindow(ids: number[]) { + let i = 0; + return { + create: vi.fn(async () => { + const id = ids[i++]; + if (id === undefined) throw new Error("ran out of fake ids"); + return id; + }), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }; +} + +function makeFakeCdp(handlers: Record unknown>) { + const sent: Array<{ tabId: number; method: string; params?: object }> = []; + const sendImpl = async (tabId: number, method: string, params?: object) => { + sent.push({ tabId, method, params }); + const handler = handlers[method]; + if (!handler && method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (!handler) throw new Error(`unexpected CDP call ${method}`); + return handler(params); + }; + const cdp: CdpRunner = { + send: vi.fn(sendImpl) as unknown as CdpRunner["send"], + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + return { cdp, tabsApi, sent }; +} + +describe("handleScrollTo", () => { + it("scrolls a ref into view and returns its visible top-viewport bounds", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + }); + + const result = await handleScrollTo( + manager, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result).toMatchObject({ + tab_id: 4, + used_ref: "e3", + x: 10, + y: 20, + width: 100, + height: 40, + }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.scrollIntoViewIfNeeded", + "DOM.getContentQuads", + "Page.getLayoutMetrics", + ]); + }); + + it("scrolls OOPIF refs through their parent frame and CDP session", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getBoxModel": () => ({ + model: { content: [204, 306, 604, 306, 604, 506, 204, 506] }, + }), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const targetCalls: Array<{ sessionId?: string; method: string }> = []; + fake.cdp.sendToTarget = vi.fn(async (target, method) => { + targetCalls.push({ sessionId: target.sessionId, method }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const result = await handleScrollTo( + manager, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result).toMatchObject({ x: 224, y: 346, width: 200, height: 80 }); + expect(targetCalls).toEqual([ + { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, + { sessionId: "child-session", method: "DOM.getContentQuads" }, + { sessionId: "child-session", method: "Page.getLayoutMetrics" }, + ]); + }); + + it("does not issue CDP calls after an early cancellation", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await manager.start("aa11"); + const abort = new AbortController(); + abort.abort(); + const fake = makeFakeCdp({}); + + const result = await handleScrollTo( + manager, + { session_id: "aa11", selector: "#target" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + + expect(result).toMatchObject({ code: "cancelled" }); + expect(fake.cdp.send).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 25b5f3ac..8afdd0f0 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -26,6 +26,7 @@ import type { ResponseFrame, RpcError, ScreenshotParams, + ScrollToParams, SelectParams, SnapshotParams, WaitForNavigationParams, @@ -52,6 +53,7 @@ import { handleSnapshot, } from "./observation"; import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record"; +import { handleScrollTo } from "./scroll"; import { handleSessionStart, handleSessionStop, @@ -471,6 +473,17 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.scroll_to": + return this.withHoverReleaseForRequest( + req.params as ScrollToParams, + () => + handleScrollTo( + this.sessions, + req.params as ScrollToParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.fill": return this.withHoverReleaseForRequest( req.params as FillParams, @@ -718,6 +731,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.scroll_to": case "tool.fill": case "tool.press": case "tool.select": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabdf..3ea1373a 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise { * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ -async function resolveBackendNode( +export async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, diff --git a/apps/extension/src/tools/scroll.ts b/apps/extension/src/tools/scroll.ts new file mode 100644 index 00000000..523f758d --- /dev/null +++ b/apps/extension/src/tools/scroll.ts @@ -0,0 +1,69 @@ +import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import type { SessionManager } from "@/session-manager/manager"; +import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { resolveNodeGeometry } from "./frame-geometry"; +import { resolveBackendNode } from "./interaction"; +import { + type CdpRunner, + type ChromeTabsApi, + chromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, +} from "./shared"; + +export interface ScrollToDeps { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + signal?: AbortSignal; +} + +let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; +function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { + if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; + return defaultDeps; +} + +export async function handleScrollTo( + manager: SessionManager, + params: ScrollToParams, + deps: ScrollToDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "scroll-to"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "scroll-to"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "scroll-to"); + if (isRpcError(node)) return node; + if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; + + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const geometry = await resolveNodeGeometry( + deps.cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: true }, + ); + if (isRpcError(geometry)) return geometry; + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + x: geometry.topBounds.x, + y: geometry.topBounds.y, + width: geometry.topBounds.width, + height: geometry.topBounds.height, + }); +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index d53825a1..73384ccc 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -444,6 +444,25 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface ScrollToParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface ScrollToResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + width: number; + height: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e1cffb92..ee68f3f8 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -202,6 +202,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | | `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | +| `bsk scroll-to ` | Scroll an element and its frame owners into the visible viewport | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) | From 61c97c9cb5c2c2777d228e3f6a3847b3aaf84103 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Thu, 10 Sep 2026 00:05:48 +0800 Subject: [PATCH 2/4] fix(interaction): honor scroll cancellation and visible bounds --- .../src/tools/__tests__/dispatcher.test.ts | 11 +- .../tools/__tests__/scroll.browser.test.ts | 348 +++++++++++++++ .../src/tools/__tests__/scroll.test.ts | 395 ++++++++++++------ apps/extension/src/tools/scroll-visibility.ts | 142 +++++++ apps/extension/src/tools/scroll.ts | 134 ++++-- apps/extension/src/transport/types.ts | 1 + crates/bsk-cli/skill/SKILL.md | 7 +- crates/bsk-cli/src/cli/scroll.rs | 25 +- crates/bsk-cli/tests/cli_parse.rs | 32 ++ .../bsk-cli/tests/session_user_interrupt.rs | 36 +- .../schema/tool_scroll_to_result.json | 2 +- crates/bsk-protocol/src/tools/scroll.rs | 4 +- skill/SKILL.md | 7 +- 13 files changed, 955 insertions(+), 189 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/scroll.browser.test.ts create mode 100644 apps/extension/src/tools/scroll-visibility.ts diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 1390ee35..5407a2cc 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -602,6 +602,7 @@ describe("ToolDispatcher", () => { it.each([ "focus", "blur", + "scroll_to", ] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => { const tab = { id: 7, windowId: 4242, active: true }; vi.stubGlobal("chrome", { @@ -654,7 +655,15 @@ describe("ToolDispatcher", () => { }); expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); - expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" }); + if (action === "scroll_to") { + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", { + objectGroup: expect.any(String), + }); + } else { + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { + objectId: "focus-target", + }); + } expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( 7, expect.objectContaining({ enabled: false }), diff --git a/apps/extension/src/tools/__tests__/scroll.browser.test.ts b/apps/extension/src/tools/__tests__/scroll.browser.test.ts new file mode 100644 index 00000000..deb1a416 --- /dev/null +++ b/apps/extension/src/tools/__tests__/scroll.browser.test.ts @@ -0,0 +1,348 @@ +// @vitest-environment node +// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile. +import { describe, expect, it } from "vitest"; +import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import type { ViewportRect } from "../geometry"; +import { handleScrollTo } from "../scroll"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] }; + +async function withBrowser( + run: (h: Awaited>) => Promise, + zoom = 1, +) { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + await withChrome( + { executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom }, + async (send: Send) => run(await harness(send)), + ); +} + +async function harness(send: Send) { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + await send("Page.bringToFront", {}, rootSession); + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + const calls: { method: string; sessionId?: string }[] = []; + const after = { command: (_method: string) => {} }; + const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; + const forward = async (target: CdpTarget, method: string, params?: object): Promise => { + calls.push({ method, sessionId: target.sessionId }); + const result = await send(method, params, sessionFor(target)); + after.command(method); + return result; + }; + const cdp: CdpRunner = { + send: (_tab, method, params) => forward({ tabId: 4 }, method, params), + sendToTarget: forward, + }; + const evaluate = async (expression: string, frame?: CdpFrame): Promise => { + const session = frame ? sessionFor(frame.target) : rootSession; + const world = frame + ? await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "scroll-oracle", + }, + session, + ) + : undefined; + const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { + expression, + contextId: world?.executionContextId, + returnByValue: true, + awaitPromise: true, + }, + session, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + const rememberRef = (frame: CdpFrame, backendNodeId: number) => + ctx.refStore.set("e1", backendNodeId, { + tabId: 4, + frameId: frame.frameId, + cdpSessionId: frame.target.sessionId, + }); + const ref = async (frame: CdpFrame, selector = "#probe") => { + const { executionContextId } = await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "scroll-oracle", + }, + sessionFor(frame.target), + ); + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { + expression: `document.querySelector(${JSON.stringify(selector)})`, + contextId: executionContextId, + }, + sessionFor(frame.target), + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionFor(frame.target), + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target)); + rememberRef(frame, node.backendNodeId); + }; + const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) => + handleScrollTo( + manager, + { session_id: "aa11", ...target }, + { + cdp, + signal, + tabsApi: { + get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], + }, + }, + ); + const loadFrames = async () => { + const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>( + "Target.getTargets", + ); + const sessions = [rootSession]; + for (const target of targetInfos.filter((t) => t.type === "iframe")) { + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId: target.targetId, + flatten: true, + }); + sessions.push(sessionId); + } + const frames: (CdpFrame & { name: string })[] = []; + for (const session of sessions) { + const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session); + const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) }; + const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { + frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" }); + for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); + }; + visit(frameTree); + } + for (const frame of frames) { + if (!frame.parentFrameId) continue; + const parent = frames.find((f) => f.frameId === frame.parentFrameId)!; + const owner = await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: frame.frameId }, + sessionFor(parent.target), + ); + frame.ownerBackendNodeId = owner.backendNodeId; + } + const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; + cdp.getFrameGraph = async () => graph; + return frames; + }; + return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after }; +} + +function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null { + const x = Math.max(a.x, b.x), + y = Math.max(a.y, b.y); + const right = Math.min(a.x + a.width, b.x + b.width), + bottom = Math.min(a.y + a.height, b.y + b.height); + return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null; +} + +function expectRect(actual: unknown, expected: ViewportRect) { + expect(actual).not.toHaveProperty("code"); + for (const key of ["x", "y", "width", "height"] as const) + expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2); +} + +describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => { + it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => { + await withBrowser(async (h) => { + for (const html of [ + '
', + '
', + '
', + ]) { + await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`); + const result = await h.scroll({ selector: "#target" }); + const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({ + rect: document.querySelector('#target').getBoundingClientRect().toJSON(), + clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON() + })`); + expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect); + } + }); + }); + + it.each([ + "visibility:hidden", + "opacity:0", + "display:none", + "content-visibility:hidden", + ])("rejects %s targets", async (style) => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML = '
'`, + ); + expect(await h.scroll({ selector: "#target" })).toMatchObject({ + code: "permission_denied", + data: { reason: "element_not_visible" }, + }); + }); + }); + + it("rejects targets fully outside an overflow:clip ancestor", async () => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML = '
'`, + ); + expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" }); + }); + }); + + it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => { + await withBrowser(async (h) => { + await h.evaluate(`(() => { + const host = document.createElement('div'); document.body.append(host); + const root = host.attachShadow({mode:'${mode}'}); + root.innerHTML = '
'; + window.probe = root.querySelector('#probe'); + })()`); + const frames = await h.loadFrames(); + const { result } = await h.send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { expression: "window.probe" }, + h.rootSession, + ); + // Build the ref from the actual closed-root node, as an observation does. + const { node } = await h.send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + h.rootSession, + ); + await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession); + h.rememberRef(frames[0], node.backendNodeId); + expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 }); + }); + }); + + it("cancels after scrolling and releases the measurement object without follow-up reads", async () => { + await withBrowser(async (h) => { + await h.evaluate("document.body.innerHTML = ''"); + const abort = new AbortController(); + h.after.command = (method) => { + if (method === "DOM.resolveNode") abort.abort(); + }; + expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({ + code: "cancelled", + }); + expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false); + }); + }); + + it.each([ + { fixture: "snapshot-coordinates", zoom: 1 }, + { fixture: "snapshot-coordinates", zoom: 1.25 }, + { fixture: "oopif-scrollbars", zoom: 1 }, + { fixture: "oopif-scrollbars", zoom: 0.8 }, + ])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => { + const { createEvalServer } = await import( + new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href + ); + const server = createEvalServer(); + const { baseUrl } = await server.start(); + try { + await withBrowser(async (h) => { + await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession); + await expect + .poll( + () => + server + .snapshot("scroll") + .events.some( + (e: { type: string; data: { root?: boolean } }) => + e.type === "geometry.ready" && e.data.root, + ), + { timeout: 10_000 }, + ) + .toBe(true); + const frames = await h.loadFrames(); + expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3); + for (const frame of frames) { + await h.ref(frame); + const result = await h.scroll({ ref: "e1" }); + // Independent DOM oracle: CSS border boxes plus this fixture's known + // axis-aligned iframe scales. Re-read after each scroll changes layout. + let rect = await h.evaluate( + "document.querySelector('#probe').getBoundingClientRect().toJSON()", + frame, + ); + let current = frame; + while (true) { + const viewport = await h.evaluate( + "({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })", + current, + ); + rect = intersection(rect, viewport)!; + if (!current.parentFrameId) break; + const parent = frames.find((f) => f.frameId === current.parentFrameId)!; + const owner = await h.evaluate<{ x: number; y: number; scale: number }>( + `(() => { + const iframe = document.querySelector('iframe[name="${current.name}"]'); + const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe); + const scale = r.width / iframe.offsetWidth; + return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale, + y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale }; + })()`, + parent, + ); + rect = { + x: owner.x + rect.x * owner.scale, + y: owner.y + rect.y * owner.scale, + width: rect.width * owner.scale, + height: rect.height * owner.scale, + }; + current = parent; + } + expectRect(result, rect); + } + const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!; + await h.ref(child); + await h.evaluate( + `document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`, + ); + expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" }); + }, zoom); + } finally { + await server.stop(); + } + }, 30_000); +}); diff --git a/apps/extension/src/tools/__tests__/scroll.test.ts b/apps/extension/src/tools/__tests__/scroll.test.ts index 69e5daec..e41a119e 100644 --- a/apps/extension/src/tools/__tests__/scroll.test.ts +++ b/apps/extension/src/tools/__tests__/scroll.test.ts @@ -1,63 +1,121 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; import { SessionManager } from "@/session-manager/manager"; -import type { CdpRunner } from "@/tools/shared"; +import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types"; +import type { ViewportRect } from "../geometry"; import { handleScrollTo } from "../scroll"; +import type { CdpRunner } from "../shared"; -function fakeAgentWindow(ids: number[]) { - let i = 0; - return { - create: vi.fn(async () => { - const id = ids[i++]; - if (id === undefined) throw new Error("ran out of fake ids"); - return id; - }), - remove: vi.fn(async () => {}), - ensureActiveTab: vi.fn(async () => {}), - }; +interface Call { + target: CdpTarget; + method: string; + params?: Record; + aborted: boolean; } -function makeFakeCdp(handlers: Record unknown>) { - const sent: Array<{ tabId: number; method: string; params?: object }> = []; - const sendImpl = async (tabId: number, method: string, params?: object) => { - sent.push({ tabId, method, params }); - const handler = handlers[method]; - if (!handler && method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; - } - if (!handler) throw new Error(`unexpected CDP call ${method}`); - return handler(params); +async function fixture(frame: "top" | "same-target" | "oopif" = "top") { + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + ...(frame !== "top" ? { frameId: "child" } : {}), + ...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}), + }); + const abort = new AbortController(); + const graph: CdpFrameGraph = { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) }, + }, + ], + }; + const rectangles = new Map([ + [1234, { x: 10, y: 20, width: 100, height: 40 }], + [99, { x: 204, y: 306, width: 400, height: 200 }], + ]); + const calls: Call[] = []; + const hooks = { + after: (_call: Call) => {}, + reply: (_call: Call): object | undefined => undefined, + }; + const send = async (target: CdpTarget, method: string, params?: object): Promise => { + const call = { + target, + method, + params: params as Call["params"], + aborted: abort.signal.aborted, + }; + calls.push(call); + const override = hooks.reply(call); + hooks.after(call); + if (override) return override as T; + const args = call.params ?? {}; + const replies: Record = { + "DOM.getDocument": { root: { nodeId: 1 } }, + "DOM.querySelector": { nodeId: 2 }, + "DOM.describeNode": { node: { backendNodeId: 1234 } }, + "DOM.scrollIntoViewIfNeeded": {}, + "DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } }, + "Runtime.callFunctionOn": { + result: { + value: String(args.functionDeclaration).includes("IntersectionObserver") + ? rectangles.get(Number(args.objectId)) + : { width: 200, height: 100 }, + }, + }, + "Runtime.releaseObject": {}, + "Runtime.releaseObjectGroup": {}, + "Runtime.evaluate": { result: { value: { width: 200, height: 100 } } }, + "Page.getLayoutMetrics": { + cssLayoutViewport: { + clientWidth: target.sessionId ? 185 : 1280, + clientHeight: target.sessionId ? 89 : 720, + }, + }, + "DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } }, + }; + if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`); + return replies[method] as T; }; const cdp: CdpRunner = { - send: vi.fn(sendImpl) as unknown as CdpRunner["send"], + send: (tabId, method, params) => send({ tabId }, method, params), + sendToTarget: send, + getFrameGraph: vi.fn(async () => { + hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted }); + return graph; + }), trackSessionTab: vi.fn(), }; const tabsApi = { - get: vi.fn( - async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, - ), + get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), }; - return { cdp, tabsApi, sent }; -} - -describe("handleScrollTo", () => { - it("scrolls a ref into view and returns its visible top-viewport bounds", async () => { - const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await manager.start("aa11"); - ctx.refStore.set("e3", 1234, { tabId: 4 }); - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), - }); - - const result = await handleScrollTo( + const run = (params: Partial = {}) => + handleScrollTo( manager, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi }, + { session_id: "aa11", ref: "@e3", ...params }, + { cdp, tabsApi, signal: abort.signal }, ); + return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run }; +} + +afterEach(() => vi.restoreAllMocks()); - if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); - expect(result).toMatchObject({ +describe("handleScrollTo", () => { + it("scrolls a ref and returns renderer-clipped bounds", async () => { + const f = await fixture(); + expect(await f.run()).toMatchObject({ tab_id: 4, used_ref: "e3", x: 10, @@ -65,85 +123,188 @@ describe("handleScrollTo", () => { width: 100, height: 40, }); - expect(fake.sent.map((call) => call.method)).toEqual([ - "DOM.scrollIntoViewIfNeeded", - "DOM.getContentQuads", - "Page.getLayoutMetrics", - ]); - }); - - it("scrolls OOPIF refs through their parent frame and CDP session", async () => { - const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - const ctx = await manager.start("aa11"); - ctx.refStore.set("e3", 1234, { - tabId: 4, - frameId: "child-frame", - cdpSessionId: "child-session", + expect(f.calls[0]).toMatchObject({ + method: "DOM.scrollIntoViewIfNeeded", + params: { backendNodeId: 1234 }, }); - const fake = makeFakeCdp({ - "DOM.scrollIntoViewIfNeeded": () => ({}), - "DOM.getBoxModel": () => ({ - model: { content: [204, 306, 604, 306, 604, 506, 204, 506] }, - }), + const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup; + expect(f.calls.at(-1)).toMatchObject({ + method: "Runtime.releaseObjectGroup", + params: { objectGroup: group }, }); - fake.cdp.getFrameGraph = vi.fn(async () => ({ - rootFrameId: "main", - frames: [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "child-frame", - parentFrameId: "main", - ownerBackendNodeId: 99, - target: { tabId: 4, sessionId: "child-session" }, - }, - ], - })); - const targetCalls: Array<{ sessionId?: string; method: string }> = []; - fake.cdp.sendToTarget = vi.fn(async (target, method) => { - targetCalls.push({ sessionId: target.sessionId, method }); - if (method === "DOM.scrollIntoViewIfNeeded") return {}; - if (method === "DOM.getContentQuads") { - return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; - } - if (method === "Runtime.evaluate") { - return { result: { value: { width: 200, height: 100 } } }; - } - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; - } - throw new Error(`unexpected child CDP call ${method}`); - }) as CdpRunner["sendToTarget"]; + }); - const result = await handleScrollTo( - manager, - { session_id: "aa11", ref: "@e3" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi }, - ); + it("resolves a selector and tracks the attached tab even when it is missing", async () => { + const f = await fixture(); + expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({ + used_selector: "#target", + }); + f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined); + expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({ + code: "not_found", + data: { reason: "selector_not_found" }, + }); + expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4); + }); - if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); - expect(result).toMatchObject({ x: 224, y: 346, width: 200, height: 80 }); - expect(targetCalls).toEqual([ - { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, - { sessionId: "child-session", method: "DOM.getContentQuads" }, - { sessionId: "child-session", method: "Page.getLayoutMetrics" }, - { sessionId: "child-session", method: "Runtime.evaluate" }, - ]); + it.each([ + "same-target", + "oopif", + ] as const)("projects %s refs and scrolls their owners", async (frame) => { + const f = await fixture(frame); + expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 }); + expect( + f.calls + .filter((call) => call.method === "DOM.scrollIntoViewIfNeeded") + .map((call) => call.params!.backendNodeId), + ).toEqual([99, 1234]); + expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1); + if (frame === "oopif") + expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe( + "child-session", + ); }); - it("does not issue CDP calls after an early cancellation", async () => { - const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); - await manager.start("aa11"); - const abort = new AbortController(); - abort.abort(); - const fake = makeFakeCdp({}); + it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => { + const f = await fixture("oopif"); + f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 }); + expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 }); + }); - const result = await handleScrollTo( - manager, - { session_id: "aa11", selector: "#target" }, - { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, - ); + it("clips against the iframe owner's ordinary DOM ancestors", async () => { + const f = await fixture("oopif"); + f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 }); + expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 }); + }); + + it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => { + const f = await fixture("oopif"); + f.rectangles.set(node, null); + expect(await f.run()).toMatchObject({ + code: "permission_denied", + data: { reason: "element_not_visible" }, + }); + expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + }); + + it.each([ + { ref: undefined }, + { selector: "#target" }, + { timeout_ms: 0 }, + ])("rejects invalid params %j", async (params) => { + const f = await fixture(); + expect(await f.run(params)).toMatchObject({ code: "invalid_params" }); + expect(f.calls).toEqual([]); + }); + + it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => { + const f = await fixture(); + if (mode === "stale") f.ctx.refStore.clear(); + else f.ctx.refStore.set("e3", 1234, { tabId: 8 }); + expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } }); + expect(f.calls).toEqual([]); + }); + + it("enforces the Agent Window boundary", async () => { + const f = await fixture(); + f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]); + f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab); + expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" }); + expect(f.calls).toEqual([]); + }); + + it.each([ + "getFrameGraph", + "DOM.scrollIntoViewIfNeeded", + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Page.getLayoutMetrics", + "Runtime.evaluate", + "DOM.getBoxModel", + ])("honors cancellation during %s and only cleans up afterwards", async (method) => { + const f = await fixture("oopif"); + f.hooks.after = (call) => { + if (call.method === method) f.abort.abort(); + }; + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect( + f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"), + ).toEqual([]); + }); + + it.each([ + "DOM.getDocument", + "DOM.querySelector", + "DOM.describeNode", + ])("cancels selector resolution during %s", async (method) => { + const f = await fixture(); + f.hooks.after = (call) => { + if (call.method === method) f.abort.abort(); + }; + expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({ + code: "cancelled", + }); + expect(f.calls.filter((call) => call.aborted)).toEqual([]); + }); + + it("does not start after early cancellation", async () => { + const f = await fixture(); + f.abort.abort(); + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect(f.calls).toEqual([]); + expect(f.tabsApi.query).not.toHaveBeenCalled(); + }); + + it("does not issue fallback scrolling after cancellation", async () => { + const f = await fixture(); + f.hooks.reply = ({ method }) => { + if (method === "DOM.scrollIntoViewIfNeeded") { + f.abort.abort(); + throw new Error("scroll failed"); + } + return undefined; + }; + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]); + }); + + it("stops target scrolling after a deadline expires while scrolling its owner", async () => { + const f = await fixture("oopif"); + const now = vi.spyOn(Date, "now").mockReturnValue(100); + f.hooks.after = ({ method }) => { + if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201); + }; + expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" }); + expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1); + }); + + it("reports measurement exceptions and releases objects even if cleanup fails", async () => { + const f = await fixture(); + f.hooks.reply = ({ method }) => { + if (method === "Runtime.callFunctionOn") + return { exceptionDetails: { text: "measurement failed" } }; + if (method === "Runtime.releaseObjectGroup") throw new Error("target closed"); + return undefined; + }; + expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" }); + expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + }); - expect(result).toMatchObject({ code: "cancelled" }); - expect(fake.cdp.send).not.toHaveBeenCalled(); + it("attaches dialogs observed during scrolling", async () => { + const f = await fixture(); + f.cdp.dialogCursor = () => 5; + const dialogs: JavaScriptDialogInfo[] = [ + { + sequence: 6, + tab_id: 4, + type: "alert", + message: "scrolled", + handled: "accepted", + url: "https://example.test", + }, + ]; + f.cdp.dialogsSince = vi.fn(() => dialogs); + expect(await f.run()).toHaveProperty("dialogs", dialogs); + expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5); }); }); diff --git a/apps/extension/src/tools/scroll-visibility.ts b/apps/extension/src/tools/scroll-visibility.ts new file mode 100644 index 00000000..a015f6d2 --- /dev/null +++ b/apps/extension/src/tools/scroll-visibility.ts @@ -0,0 +1,142 @@ +import { cdpTargetKey } from "@/browser-driver/frame-graph"; +import type { NodeAddress } from "./frame-geometry"; +import { + clipPolygon, + projectRegionToViewport, + type Region, + rectPolygon, + regionBounds, + type ViewportRect, +} from "./geometry"; +import { GeometryContext } from "./geometry/frame-context"; +import { type CdpRunner, sendToCdpTarget } from "./shared"; + +// Ask the renderer to apply containing-block clips (including shadow DOM), +// instead of treating layout quads as visible pixels. Each document is measured +// locally; the existing frame projections supply the top-viewport coordinates. +// IntersectionObserver reports a bounding rectangle, not an occlusion/hit test. +const VISIBLE_RECT = `function(timeoutMs) { + const element = this; + const visible = () => element.isConnected && element.checkVisibility({ + checkOpacity: true, checkVisibilityCSS: true, contentVisibilityAuto: true + }); + if (typeof element.checkVisibility !== 'function' || !visible()) return null; + return new Promise((resolve, reject) => { + const observer = new IntersectionObserver(([entry]) => { + clearTimeout(timer); + observer.disconnect(); + const rect = entry.intersectionRect; + resolve(visible() && entry.isIntersecting && rect.width > 0 && rect.height > 0 + ? { x: rect.x, y: rect.y, width: rect.width, height: rect.height } + : null); + }, { root: element.ownerDocument }); + const timer = setTimeout(() => { + observer.disconnect(); + reject(new Error('scroll-to visibility measurement timed out')); + }, timeoutMs); + observer.observe(element); + }); +}`; + +async function visibleRect( + cdp: CdpRunner, + address: NodeAddress, + deadline: number, +): Promise { + const resolved = await sendToCdpTarget<{ object?: { objectId?: string } }>( + cdp, + address.target, + "DOM.resolveNode", + { backendNodeId: address.backendNodeId }, + ); + if (!resolved.object?.objectId) throw new Error("scroll-to could not resolve the target element"); + const reply = await sendToCdpTarget<{ + result?: { value?: ViewportRect | null }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; + }>(cdp, address.target, "Runtime.callFunctionOn", { + objectId: resolved.object.objectId, + functionDeclaration: VISIBLE_RECT, + // Bound this read even if a background document stops rendering. Its + // observer always disconnects; cancellation cannot leave a live observer. + arguments: [{ value: Math.max(1, Math.min(1_000, deadline - Date.now())) }], + awaitPromise: true, + returnByValue: true, + }); + if (reply.exceptionDetails) { + const details = reply.exceptionDetails; + throw new Error(details.exception?.description ?? details.text ?? "visibility script failed"); + } + const rect = reply.result?.value; + if (rect === null) return null; + if ( + !rect || + ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) || + rect.width <= 0 || + rect.height <= 0 + ) { + throw new Error("scroll-to visibility measurement returned invalid bounds"); + } + return rect; +} + +async function projectRect( + context: GeometryContext, + address: NodeAddress, + rect: ViewportRect, +): Promise { + let region: Region = [rectPolygon({ x: rect.x, y: rect.y, w: rect.width, h: rect.height })]; + if (!address.frameId) return region; + const frame = await context.frame(address.frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) { + throw new Error("scroll-to frame no longer belongs to its target"); + } + const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined; + if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) { + // DOM rectangles are document-local. Same-target iframe quads are already + // target-local, so cross this one boundary before the target projection. + const viewport = await context.viewport(frame.target); + if (frame.ownerBackendNodeId === undefined || !viewport) { + throw new Error("scroll-to could not resolve the frame viewport"); + } + const local = await context.snapshotProjection(address, frame.ownerBackendNodeId, [], viewport); + if (local.status !== "available") throw new Error("scroll-to could not project the frame"); + region = projectRegionToViewport(region, local.projection.geometry); + } + const projection = await context.targetProjection(frame.frameId); + if (!projection) throw new Error("scroll-to could not project the target viewport"); + return projectRegionToViewport(region, projection); +} + +/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */ +export async function scrollVisibleBounds( + cdp: CdpRunner, + tabId: number, + address: NodeAddress, + deadline: number, +): Promise { + const context = new GeometryContext(cdp, tabId); + const rect = await visibleRect(cdp, address, deadline); + if (!rect) return null; + let region = await projectRect(context, address, rect); + if (address.frameId) { + const ancestry = await context.ancestry(address.frameId); + if (!ancestry) throw new Error("scroll-to could not resolve the frame ancestry"); + for (const child of ancestry) { + const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined; + if (!parent || child.ownerBackendNodeId === undefined) { + throw new Error("scroll-to could not resolve the frame owner"); + } + const owner = { + target: parent.target, + frameId: parent.frameId, + backendNodeId: child.ownerBackendNodeId, + }; + // A visible child document can still have a hidden or clipped iframe. + const ownerRect = await visibleRect(cdp, owner, deadline); + if (!ownerRect) return null; + const clips = await projectRect(context, owner, ownerRect); + region = region.flatMap((polygon) => clips.map((clip) => clipPolygon(polygon, clip))); + } + } + return regionBounds(region); +} diff --git a/apps/extension/src/tools/scroll.ts b/apps/extension/src/tools/scroll.ts index 523f758d..9ba79836 100644 --- a/apps/extension/src/tools/scroll.ts +++ b/apps/extension/src/tools/scroll.ts @@ -1,9 +1,12 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types"; import { attachDialogs, markDialogCursor } from "./dialogs"; -import { resolveNodeGeometry } from "./frame-geometry"; +import { cdpError, rpcError } from "./errors"; +import { scrollElementAndFramesIntoView } from "./frame-geometry"; import { resolveBackendNode } from "./interaction"; +import { scrollVisibleBounds } from "./scroll-visibility"; import { type CdpRunner, type ChromeTabsApi, @@ -12,6 +15,7 @@ import { isRpcError, lookupSession, resolveTargetTab, + sendToCdpTarget, } from "./shared"; export interface ScrollToDeps { @@ -34,36 +38,100 @@ export async function handleScrollTo( const ctxOrErr = lookupSession(manager, params, "scroll-to"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; - if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); - if (isRpcError(target)) return target; - const denied = enforceAgentWindow(ctx, target, "scroll-to"); - if (denied) return denied; - const dialogCursor = markDialogCursor(deps.cdp, target.tabId); - const node = await resolveBackendNode(deps.cdp, ctx, target, params, "scroll-to"); - if (isRpcError(node)) return node; - if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; - - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const geometry = await resolveNodeGeometry( - deps.cdp, - target.tabId, - { - target: node.cdpTarget, - backendNodeId: node.backendNodeId, - ...(node.frameId ? { frameId: node.frameId } : {}), - }, - { scrollIntoView: true }, - ); - if (isRpcError(geometry)) return geometry; - - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, - x: geometry.topBounds.x, - y: geometry.topBounds.y, - width: geometry.topBounds.width, - height: geometry.topBounds.height, - }); + const timeout = params.timeout_ms ?? 30_000; + if (!Number.isInteger(timeout) || timeout <= 0) { + return { code: "invalid_params", message: "timeout_ms must be a positive integer" }; + } + const deadline = Date.now() + timeout; + const checkActive = () => { + if (deps.signal?.aborted) throw new DOMException("scroll-to aborted", "AbortError"); + if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError"); + }; + const objectGroup = `bsk-scroll-${crypto.randomUUID()}`; + const objectTargets = new Map(); + const send = async (target: CdpTarget, method: string, args?: object): Promise => { + checkActive(); + // Also own objects allocated by the shared scroll fallback. Record the + // target before awaiting so cancellation cannot skip their cleanup. + if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target); + const result = await sendToCdpTarget( + deps.cdp, + target, + method, + method === "DOM.resolveNode" ? { ...args, objectGroup } : args, + ); + checkActive(); + return result; + }; + let graph: ReturnType> | undefined; + const cdp: CdpRunner = { + send: (tabId, method, args) => send({ tabId }, method, args), + sendToTarget: send, + trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp), + getFrameGraph: deps.cdp.getFrameGraph + ? async (tabId) => { + checkActive(); + graph ??= deps.cdp.getFrameGraph!(tabId); + const result = await graph; + checkActive(); + return result; + } + : undefined, + }; + try { + checkActive(); + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + checkActive(); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "scroll-to"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(cdp, ctx, target, params, "scroll-to"); + checkActive(); + if (isRpcError(node)) return node; + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollError = await scrollElementAndFramesIntoView( + cdp, + target.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + checkActive(); + if (scrollError) return scrollError; + const bounds = await scrollVisibleBounds( + cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + frameId: node.frameId, + }, + deadline, + ); + checkActive(); + if (!bounds) + return rpcError( + "permission_denied", + "element_not_visible", + "scroll-to target has no visible area", + ); + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + ...bounds, + }); + } catch (error) { + if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; + if (Date.now() >= deadline) return { code: "timeout", message: "scroll-to timed out" }; + return cdpError(error); + } finally { + // Cleanup bypasses the cancellation guard and cannot replace the result. + for (const target of objectTargets.values()) { + await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch( + () => {}, + ); + } + } } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 17c17c95..5020b292 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -482,6 +482,7 @@ export interface ScrollToResult { tab_id: number; used_ref?: string; used_selector?: string; + /** Clipped border-box bounds in top-level viewport CSS pixels; not an occlusion test. */ x: number; y: number; width: number; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 9abf71ef..c48b13e2 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -59,7 +59,12 @@ bsk click|hover|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` -`bsk scroll-to ` scrolls an element and its frame owners into the visible viewport. +`bsk scroll-to --session ` scrolls an element and its frame owners into view. +Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document. +The result is the visible border-box portion's bounds in top-level viewport CSS pixels after +ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with +`element_not_visible`. The result does not guarantee that other elements do not cover the target. +For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether it was focused. Use these for UI states triggered by focus changes. diff --git a/crates/bsk-cli/src/cli/scroll.rs b/crates/bsk-cli/src/cli/scroll.rs index ebe11150..ba79983f 100644 --- a/crates/bsk-cli/src/cli/scroll.rs +++ b/crates/bsk-cli/src/cli/scroll.rs @@ -7,9 +7,10 @@ use bsk_protocol::Method; use bsk_protocol::tools::{ScrollToParams, ScrollToResult}; use clap::Args; +use crate::cli::dialogs::print_dialog_summaries; use crate::cli::ensure_daemon::ensure_daemon; use crate::cli::error::{CliError, Format}; -use crate::cli::interaction::looks_like_ref; +use crate::cli::interaction::split_target; use crate::cli::navigate::parse_timeout_ms; #[derive(Debug, Clone, Args)] @@ -34,8 +35,8 @@ pub struct ScrollToArgs { } pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> { - let info = ensure_daemon().context("ensure daemon is running")?; let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; + let info = ensure_daemon().context("ensure daemon is running")?; let params = ScrollToParams { session_id: args.session, ref_, @@ -63,30 +64,12 @@ pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> { "scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})", reply.tab_id, reply.x, reply.y, reply.width, reply.height ); + print_dialog_summaries(&reply.dialogs); } } Ok(()) } -fn split_target( - positional: Option, - explicit_ref: Option, - explicit_selector: Option, -) -> Result<(Option, Option), CliError> { - match (positional, explicit_ref, explicit_selector) { - (None, None, None) => Err(CliError::Local(anyhow::anyhow!( - "missing target: pass , --ref @eN, or --selector " - ))), - (None, Some(r), None) => Ok((Some(r), None)), - (None, None, Some(s)) => Ok((None, Some(s))), - (Some(target), None, None) if looks_like_ref(&target) => Ok((Some(target), None)), - (Some(target), None, None) => Ok((None, Some(target))), - _ => Err(CliError::Local(anyhow::anyhow!( - "pass exactly one of: , --ref, or --selector" - ))), - } -} - fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String { used_ref .map(|r| format!("@{r}")) diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs index 44ae864a..11666e4c 100644 --- a/crates/bsk-cli/tests/cli_parse.rs +++ b/crates/bsk-cli/tests/cli_parse.rs @@ -264,6 +264,38 @@ fn parses_scroll_to_target() { assert_eq!(args.target.as_deref(), Some("@e2")); } +#[test] +fn parses_scroll_to_explicit_target_tab_and_timeout() { + for flag in ["--ref", "--selector"] { + let cli = parse(&[ + "bsk", + "scroll-to", + flag, + "e3", + "--session", + "s1", + "--tab-id", + "42", + "--timeout", + "5s", + ]); + let Command::ScrollTo(args) = cli.command else { + panic!("expected scroll-to command"); + }; + assert_eq!(args.tab_id, Some(42)); + assert_eq!(args.timeout, 5_000); + assert_eq!( + if flag == "--ref" { + args.ref_ + } else { + args.selector + } + .as_deref(), + Some("e3") + ); + } +} + #[test] fn parses_focus_target() { let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]); diff --git a/crates/bsk-cli/tests/session_user_interrupt.rs b/crates/bsk-cli/tests/session_user_interrupt.rs index 9c5288cc..5d3cbd36 100644 --- a/crates/bsk-cli/tests/session_user_interrupt.rs +++ b/crates/bsk-cli/tests/session_user_interrupt.rs @@ -293,6 +293,15 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() { /// catch a tool dispatched arbitrarily later. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() { + assert_idle_interrupt_rejects(Method::ToolClick).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_interrupt_rejects_scroll_to_before_forwarding() { + assert_idle_interrupt_rejects(Method::ToolScrollTo).await; +} + +async fn assert_idle_interrupt_rejects(method: Method) { let (handle, sock) = spawn_daemon().await; let mut ws = connect_ext(handle.ws_addr()).await; let _ = handshake_as_ext(&mut ws).await; @@ -301,10 +310,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() // Fake extension: answer tool.session_start, observe other // requests but never reply. We expect the daemon to reject the - // tool call BEFORE it ever forwards a tool.click frame. + // tool call BEFORE it ever forwards a browser-input frame. let ws_sink_for_responder = Arc::clone(&ws_sink); - let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let observed_click_count_clone = Arc::clone(&observed_click_count); + let observed_call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed_call_count_clone = Arc::clone(&observed_call_count); + let forwarded_method = method.clone(); let responder = tokio::spawn(async move { let mut ws_stream = ws_stream; while let Some(Ok(msg)) = ws_stream.next().await { @@ -329,8 +339,8 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() g.send(Message::Text(serde_json::to_string(&reply).unwrap())) .await .unwrap(); - } else if req.method == Method::ToolClick { - observed_click_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } else if req.method == forwarded_method { + observed_call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); // Deliberately do NOT reply — daemon should reject before // ever forwarding to us. let _ = req; @@ -377,12 +387,12 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() let state = handle.state(); wait_for_session_interrupt_pending(&state, &start.session_id).await; - // The CLI now issues a tool.click (mutating). The daemon must + // The CLI now issues a browser-input (mutating). The daemon must // reject it WITHOUT forwarding to the extension. - let click_outcome = ipc + let outcome = ipc .call::<_, serde_json::Value>( - "click-after-interrupt", - Method::ToolClick, + "input-after-interrupt", + method, Some(json!({ "session_id": start.session_id, "ref": "fake-ref-1", @@ -391,7 +401,7 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() ) .await .unwrap(); - let err = click_outcome.expect_err("tool.click must be rejected"); + let err = outcome.expect_err("browser-input must be rejected"); assert_eq!( err.code, ErrorCode::UserAborted, @@ -399,11 +409,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() err ); - // Crucially: the extension never saw a tool.click frame. + // Crucially: the extension never saw a browser-input frame. assert_eq!( - observed_click_count.load(std::sync::atomic::Ordering::SeqCst), + observed_call_count.load(std::sync::atomic::Ordering::SeqCst), 0, - "tool.click must NOT have been forwarded to the extension" + "browser-input must NOT have been forwarded to the extension" ); responder.abort(); diff --git a/crates/bsk-protocol/schema/tool_scroll_to_result.json b/crates/bsk-protocol/schema/tool_scroll_to_result.json index 04cd87d7..cad27108 100644 --- a/crates/bsk-protocol/schema/tool_scroll_to_result.json +++ b/crates/bsk-protocol/schema/tool_scroll_to_result.json @@ -41,7 +41,7 @@ "format": "double" }, "x": { - "description": "Visible target bounds in top-level viewport CSS pixels.", + "description": "Bounding rectangle of the element's visible border-box portion, in top-level viewport CSS pixels, after ancestor and viewport clipping. Partial visibility is sufficient. This is not an occlusion or hit test.", "type": "number", "format": "double" }, diff --git a/crates/bsk-protocol/src/tools/scroll.rs b/crates/bsk-protocol/src/tools/scroll.rs index 5908769f..28f3491a 100644 --- a/crates/bsk-protocol/src/tools/scroll.rs +++ b/crates/bsk-protocol/src/tools/scroll.rs @@ -34,7 +34,9 @@ pub struct ScrollToResult { pub used_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub used_selector: Option, - /// Visible target bounds in top-level viewport CSS pixels. + /// Bounding rectangle of the element's visible border-box portion, in + /// top-level viewport CSS pixels, after ancestor and viewport clipping. + /// Partial visibility is sufficient. This is not an occlusion or hit test. pub x: f64, pub y: f64, pub width: f64, diff --git a/skill/SKILL.md b/skill/SKILL.md index 9abf71ef..c48b13e2 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -59,7 +59,12 @@ bsk click|hover|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` -`bsk scroll-to ` scrolls an element and its frame owners into the visible viewport. +`bsk scroll-to --session ` scrolls an element and its frame owners into view. +Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document. +The result is the visible border-box portion's bounds in top-level viewport CSS pixels after +ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with +`element_not_visible`. The result does not guarantee that other elements do not cover the target. +For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether it was focused. Use these for UI states triggered by focus changes. From 77e3f5e1f338465c3f507cdec3a26168da8048f1 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Thu, 10 Sep 2026 00:05:52 +0800 Subject: [PATCH 3/4] feat(dsh-plugin): expose scroll-to action --- packages/dsh-plugin-browserskill/README.md | 2 +- .../dsh-plugin-browserskill/skill/SKILL.md | 7 +- .../src/browser-tools.ts | 5 +- .../src/phase-one-tools-interaction.ts | 85 ++++++++++++++++++- .../tests/tools.test.ts | 28 +++++- 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index e02453a0..42e141b9 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 6ca293ad..55a744ec 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,7 +51,12 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur +Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the +visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial +visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other +elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document. + +Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index 3204eed8..d002a9a1 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + - "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + + "Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " + + "press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " + "values; press requires key and may optionally focus target first.", actions: { click: "interact.click", hover: "interact.hover", + "scroll-to": "interact.scroll-to", focus: "interact.focus", blur: "interact.blur", fill: "interact.fill", diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 858fcc2f..50480215 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools"; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; -/** Add hover, focus, blur and select without bypassing session ownership or observation. */ +/** Add interaction primitives without bypassing session ownership or observation. */ export function registerPhaseOneInteractionTools( deps: ToolDeps, register: ToolRegistrar, @@ -100,6 +100,89 @@ export function registerPhaseOneInteractionTools( }), ); + register( + defineTool({ + name: "interact.scroll-to", + description: + "Scroll an element and its frame owners into view. Returns the visible portion's " + + "bounds in top-level viewport CSS pixels; fails if no visible area remains. " + + "Use a fresh ref for iframe targets; selectors search the main document.", + parameters: { + target: { + type: "string", + required: true, + description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + x: { type: "number", required: true }, + y: { type: "number", required: true }, + width: { type: "number", required: true }, + height: { type: "number", required: true }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: + `[session ${value.session}] scrolled into view on tab ${value.tabId}: ` + + `(${value.x}, ${value.y}, ${value.width}, ${value.height})`, + }, + ], + }, + async execute(args, exec) { + requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)"); + const cmdArgs = ["scroll-to", "--session", sessionId]; + appendTabId(cmdArgs, args.tabId); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + "scroll-to", + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + x: number; + y: number; + width: number; + height: number; + }; + return { + session: sessionId, + tabId: reply.tab_id, + x: reply.x, + y: reply.y, + width: reply.width, + height: reply.height, + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + "scroll-to", + args.target, + "--session", + args.session ?? "(current)", + ]), + description: "Scroll an element into view", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + for (const action of ["focus", "blur"] as const) { register( defineTool({ diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index e1441997..ab7686cb 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,7 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.scroll-to": ["browser_interact", "scroll-to"], "interact.focus": ["browser_interact", "focus"], "interact.blur": ["browser_interact", "blur"], "interact.fill": ["browser_interact", "fill"], @@ -202,7 +203,7 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], + browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -540,6 +541,7 @@ describe("interaction tools", () => { it.each([ "focus", "blur", + "scroll-to", ] as const)("interact.%s validates arguments and session ownership before running", async (action) => { const { tools, calls } = setup({ "session start": START_REPLY("s1") }); await startSession(tools); @@ -555,6 +557,30 @@ describe("interaction tools", () => { expect(calls).toHaveLength(1); }); + it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + "scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 }, + }); + await startSession(tools); + expect( + await tools + .get("interact.scroll-to") + ?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()), + ).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 }); + expect(calls[1].args).toEqual([ + "scroll-to", + "--session", + "s1", + "--tab-id", + "7", + "--timeout", + "150000ms", + target, + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + it("interact.fill passes value and --no-clear", async () => { const { tools, calls } = setup(responses); await startSession(tools); From fd0e9de61bddee5696dd0c1b076214e39fe06dda Mon Sep 17 00:00:00 2001 From: drakezhang Date: Thu, 10 Sep 2026 10:15:57 +0800 Subject: [PATCH 4/4] docs: document scroll-to primitive contract --- CHANGELOG.md | 7 ++ README.md | 3 + README.zh-CN.md | 2 + crates/bsk-cli/skill/SKILL.md | 4 +- docs/architecture.md | 5 + docs/scroll-to.md | 131 +++++++++++++++++++++ packages/dsh-plugin-browserskill/README.md | 3 + skill/SKILL.md | 4 +- 8 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 docs/scroll-to.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4669d4fa..b0900e71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number. +## [Unreleased] + +### Added + +- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin, + with ancestor-clipped visible bounds, iframe support and cooperative cancellation + ## [0.2.1] - 2026-09-09 ### Changed diff --git a/README.md b/README.md index 5cf4e813..919a220a 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,9 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected ## For Developers +The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol +and plugin entry points, visible bounds and interruption behavior. + The repository is a Cargo + pnpm workspace: - `crates/bsk-cli` — `bsk` CLI and local daemon diff --git a/README.zh-CN.md b/README.zh-CN.md index b0c8582f..2d2bd3c7 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -177,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务； ## 面向开发者 +[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口，以及可见区域、错误和中断语义。 + 本仓库是 Cargo + pnpm workspace： - `crates/bsk-cli` — `bsk` CLI 与本地 daemon diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index c48b13e2..765b91cc 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -55,7 +55,7 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|focus|blur|fill|select|press ... --session +bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` @@ -63,7 +63,7 @@ bsk observe --session # after navigation or a meaningful DOM ch Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document. The result is the visible border-box portion's bounds in top-level viewport CSS pixels after ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with -`element_not_visible`. The result does not guarantee that other elements do not cover the target. +`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements. For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether diff --git a/docs/architecture.md b/docs/architecture.md index e55f5923..d7887dbc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,6 +84,11 @@ Shared Rust types + JSON Schema generation. TypeScript mirrors frame shapes in 4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`. 5. Response travels CLI ← daemon ← extension; CLI prints result and exits. +The [scroll-to primitive reference](scroll-to.md) documents `tool.scroll_to`, +including its CLI/plugin mappings, visible-bounds contract and cancellation +behavior. It follows the same routing path and is classified as a browser +mutation for session queueing and user-interruption gating. + ## Session and sandbox model - **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window** diff --git a/docs/scroll-to.md b/docs/scroll-to.md new file mode 100644 index 00000000..f1b3838b --- /dev/null +++ b/docs/scroll-to.md @@ -0,0 +1,131 @@ +# Scroll-to primitive + +`scroll-to` brings an existing element and its containing frames into view, then +returns the visible portion's bounds. Use it to reveal a target before inspecting +the page or taking a screenshot. It accepts an element target rather than a pixel +distance or scroll direction. + +| Entry point | Name | +| --- | --- | +| CLI | `bsk scroll-to` | +| Wire protocol | `tool.scroll_to` | +| DeepSeek Harness | `browser_interact` with `action: "scroll-to"` | + +## CLI + +Use a session you started and a fresh ref from an observation. The ids below are +examples; substitute the session, tab and ref returned by your own calls. + +```sh +bsk scroll-to @e3 --session abcd --tab-id 42 --timeout 5s --json +bsk scroll-to --selector '#details' --session abcd +``` + +Supply exactly one target: a positional ref or selector, `--ref`, or `--selector`. +Refs accept `@e3` and `e3`. CSS selectors search only the main document; use refs +for elements in iframes (including out-of-process iframes) or shadow roots. +Refs belong to one session and tab and may become stale after page changes. + +`--session` is required. `--tab-id` defaults to the Agent Window's active tab. +`--timeout` defaults to `30s` and must be positive; durations such as `5000ms` and +`5s` are accepted. The command operates on Agent Window tabs, including user tabs +explicitly borrowed into that window. + +Example JSON output: + +```json +{ + "tab_id": 42, + "used_ref": "e3", + "x": 10, + "y": 20, + "width": 100, + "height": 80 +} +``` + +`used_ref` or `used_selector` identifies the resolved target. The optional +`dialogs` array reports JavaScript dialogs handled during the call. + +## Result contract + +`x` and `y` are the upper-left corner, and `width` and `height` are the size of +the visible border-box portion's bounding rectangle. All four values use +**top-level viewport CSS pixels**, after scrolling and clipping by ancestor +containers and frame viewports. They are not document or screenshot pixel +coordinates. + +Partial visibility is sufficient: a 400px-high element clipped by an 80px-high +scroll container can succeed with `height: 80`. Hidden or fully clipped targets +fail. The rectangle is not an occlusion or hit test: another element can cover +the target, and the rectangle's center is not a guaranteed click point. Layout +can change after the result; use fresh refs for subsequent interactions. + +## Protocol and plugin + +The CLI sends a normal request through the daemon to the extension: + +```json +{ + "id": "scroll-1", + "method": "tool.scroll_to", + "params": { + "session_id": "abcd", + "ref": "@e3", + "tab_id": 42, + "timeout_ms": 5000 + } +} +``` + +Protocol callers must supply exactly one nonempty `ref` or `selector`. +`tab_id` is optional; `timeout_ms` is a positive integer and defaults to 30000. +These target constraints are enforced by the handler. See the generated +[parameter schema](../crates/bsk-protocol/schema/tool_scroll_to_params.json) and +[result schema](../crates/bsk-protocol/schema/tool_scroll_to_result.json) for field +types. The wire response wraps the result above in `{ "id": "scroll-1", "result": ... }`. + +The plugin exposes the same action with its usual camelCase parameters: + +```json +{ + "action": "scroll-to", + "session": "abcd", + "target": "@e3", + "tabId": 42, + "timeoutMs": 5000 +} +``` + +Call this through `browser_interact` using a plugin-owned session. Its result +contains `session`, `tabId`, `x`, `y`, `width` and `height` with the same bounds +semantics. Omitted `session` uses the plugin's current session. + +## Errors and interruption + +The wire response uses the normal `error.code`, `error.message` and optional +`error.data.reason` fields. Common cases are: + +| Code | `data.reason` | Meaning | +| --- | --- | --- | +| `invalid_params` | — | Missing/conflicting target or invalid timeout/tab id | +| `not_found` | `ref_not_found` | Ref is unknown, expired or belongs to another tab | +| `not_found` | `selector_not_found` | Main-document selector matched no element | +| `permission_denied` | `agent_window_scope` | Target tab has not been borrowed into the Agent Window | +| `permission_denied` | `element_not_visible` | No visible area remains after scrolling | +| `cancelled` | — | The call was cancelled | +| `timeout` | — | The action deadline expired | +| `cdp_failed` | varies | Browser command, frame geometry or visibility measurement failed | + +The protocol classifies scroll-to as a browser mutation, so it uses the existing +per-session queue and user-interruption gate. The extension releases any retained +hover state before dispatch. Cancellation and deadline checks surround each CDP +step, including frame lookup, scrolling and measurement. Once detected, they +prevent further scrolling and a successful response; remote-object cleanup still +runs. Already performed scrolling is not rolled back, so inspect the current +page before retrying an interrupted call. + +The implementation is scoped to scroll-to: it reuses frame scrolling and +coordinate projection, with a separate visibility measurement that accounts for +ancestor clipping and hidden frame owners. Shared click, hover and screenshot +geometry is unchanged. diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index 42e141b9..b2d13bf6 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -58,6 +58,9 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | +For `browser_interact` with `action: "scroll-to"`, see the +[scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors. + Arbitrary page-script evaluation and interaction recording are not supported. ## Multi-session model diff --git a/skill/SKILL.md b/skill/SKILL.md index c48b13e2..765b91cc 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -55,7 +55,7 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|focus|blur|fill|select|press ... --session +bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` @@ -63,7 +63,7 @@ bsk observe --session # after navigation or a meaningful DOM ch Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document. The result is the visible border-box portion's bounds in top-level viewport CSS pixels after ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with -`element_not_visible`. The result does not guarantee that other elements do not cover the target. +`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements. For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether