diff --git a/CHANGELOG.md b/CHANGELOG.md index 4669d4fa..b0900e71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number. +## [Unreleased] + +### Added + +- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin, + with ancestor-clipped visible bounds, iframe support and cooperative cancellation + ## [0.2.1] - 2026-09-09 ### Changed diff --git a/README.md b/README.md index 5cf4e813..919a220a 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,9 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected ## For Developers +The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol +and plugin entry points, visible bounds and interruption behavior. + The repository is a Cargo + pnpm workspace: - `crates/bsk-cli` — `bsk` CLI and local daemon diff --git a/README.zh-CN.md b/README.zh-CN.md index b0c8582f..2d2bd3c7 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -177,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务； ## 面向开发者 +[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口，以及可见区域、错误和中断语义。 + 本仓库是 Cargo + pnpm workspace： - `crates/bsk-cli` — `bsk` CLI 与本地 daemon diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 1390ee35..5407a2cc 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -602,6 +602,7 @@ describe("ToolDispatcher", () => { it.each([ "focus", "blur", + "scroll_to", ] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => { const tab = { id: 7, windowId: 4242, active: true }; vi.stubGlobal("chrome", { @@ -654,7 +655,15 @@ describe("ToolDispatcher", () => { }); expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); - expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" }); + if (action === "scroll_to") { + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", { + objectGroup: expect.any(String), + }); + } else { + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { + objectId: "focus-target", + }); + } expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( 7, expect.objectContaining({ enabled: false }), diff --git a/apps/extension/src/tools/__tests__/scroll.browser.test.ts b/apps/extension/src/tools/__tests__/scroll.browser.test.ts new file mode 100644 index 00000000..deb1a416 --- /dev/null +++ b/apps/extension/src/tools/__tests__/scroll.browser.test.ts @@ -0,0 +1,348 @@ +// @vitest-environment node +// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile. +import { describe, expect, it } from "vitest"; +import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import type { ViewportRect } from "../geometry"; +import { handleScrollTo } from "../scroll"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] }; + +async function withBrowser( + run: (h: Awaited>) => Promise, + zoom = 1, +) { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + await withChrome( + { executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom }, + async (send: Send) => run(await harness(send)), + ); +} + +async function harness(send: Send) { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + await send("Page.bringToFront", {}, rootSession); + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + const calls: { method: string; sessionId?: string }[] = []; + const after = { command: (_method: string) => {} }; + const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession; + const forward = async (target: CdpTarget, method: string, params?: object): Promise => { + calls.push({ method, sessionId: target.sessionId }); + const result = await send(method, params, sessionFor(target)); + after.command(method); + return result; + }; + const cdp: CdpRunner = { + send: (_tab, method, params) => forward({ tabId: 4 }, method, params), + sendToTarget: forward, + }; + const evaluate = async (expression: string, frame?: CdpFrame): Promise => { + const session = frame ? sessionFor(frame.target) : rootSession; + const world = frame + ? await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "scroll-oracle", + }, + session, + ) + : undefined; + const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { + expression, + contextId: world?.executionContextId, + returnByValue: true, + awaitPromise: true, + }, + session, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + const rememberRef = (frame: CdpFrame, backendNodeId: number) => + ctx.refStore.set("e1", backendNodeId, { + tabId: 4, + frameId: frame.frameId, + cdpSessionId: frame.target.sessionId, + }); + const ref = async (frame: CdpFrame, selector = "#probe") => { + const { executionContextId } = await send<{ executionContextId: number }>( + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "scroll-oracle", + }, + sessionFor(frame.target), + ); + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { + expression: `document.querySelector(${JSON.stringify(selector)})`, + contextId: executionContextId, + }, + sessionFor(frame.target), + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionFor(frame.target), + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target)); + rememberRef(frame, node.backendNodeId); + }; + const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) => + handleScrollTo( + manager, + { session_id: "aa11", ...target }, + { + cdp, + signal, + tabsApi: { + get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], + }, + }, + ); + const loadFrames = async () => { + const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>( + "Target.getTargets", + ); + const sessions = [rootSession]; + for (const target of targetInfos.filter((t) => t.type === "iframe")) { + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId: target.targetId, + flatten: true, + }); + sessions.push(sessionId); + } + const frames: (CdpFrame & { name: string })[] = []; + for (const session of sessions) { + const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session); + const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) }; + const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => { + frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" }); + for (const child of tree.childFrames ?? []) visit(child, tree.frame.id); + }; + visit(frameTree); + } + for (const frame of frames) { + if (!frame.parentFrameId) continue; + const parent = frames.find((f) => f.frameId === frame.parentFrameId)!; + const owner = await send<{ backendNodeId: number }>( + "DOM.getFrameOwner", + { frameId: frame.frameId }, + sessionFor(parent.target), + ); + frame.ownerBackendNodeId = owner.backendNodeId; + } + const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames }; + cdp.getFrameGraph = async () => graph; + return frames; + }; + return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after }; +} + +function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null { + const x = Math.max(a.x, b.x), + y = Math.max(a.y, b.y); + const right = Math.min(a.x + a.width, b.x + b.width), + bottom = Math.min(a.y + a.height, b.y + b.height); + return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null; +} + +function expectRect(actual: unknown, expected: ViewportRect) { + expect(actual).not.toHaveProperty("code"); + for (const key of ["x", "y", "width", "height"] as const) + expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2); +} + +describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => { + it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => { + await withBrowser(async (h) => { + for (const html of [ + '
', + '
', + '
', + ]) { + await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`); + const result = await h.scroll({ selector: "#target" }); + const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({ + rect: document.querySelector('#target').getBoundingClientRect().toJSON(), + clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON() + })`); + expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect); + } + }); + }); + + it.each([ + "visibility:hidden", + "opacity:0", + "display:none", + "content-visibility:hidden", + ])("rejects %s targets", async (style) => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML = '
'`, + ); + expect(await h.scroll({ selector: "#target" })).toMatchObject({ + code: "permission_denied", + data: { reason: "element_not_visible" }, + }); + }); + }); + + it("rejects targets fully outside an overflow:clip ancestor", async () => { + await withBrowser(async (h) => { + await h.evaluate( + `document.body.innerHTML = '
'`, + ); + expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" }); + }); + }); + + it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => { + await withBrowser(async (h) => { + await h.evaluate(`(() => { + const host = document.createElement('div'); document.body.append(host); + const root = host.attachShadow({mode:'${mode}'}); + root.innerHTML = '
'; + window.probe = root.querySelector('#probe'); + })()`); + const frames = await h.loadFrames(); + const { result } = await h.send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { expression: "window.probe" }, + h.rootSession, + ); + // Build the ref from the actual closed-root node, as an observation does. + const { node } = await h.send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + h.rootSession, + ); + await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession); + h.rememberRef(frames[0], node.backendNodeId); + expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 }); + }); + }); + + it("cancels after scrolling and releases the measurement object without follow-up reads", async () => { + await withBrowser(async (h) => { + await h.evaluate("document.body.innerHTML = ''"); + const abort = new AbortController(); + h.after.command = (method) => { + if (method === "DOM.resolveNode") abort.abort(); + }; + expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({ + code: "cancelled", + }); + expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false); + }); + }); + + it.each([ + { fixture: "snapshot-coordinates", zoom: 1 }, + { fixture: "snapshot-coordinates", zoom: 1.25 }, + { fixture: "oopif-scrollbars", zoom: 1 }, + { fixture: "oopif-scrollbars", zoom: 0.8 }, + ])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => { + const { createEvalServer } = await import( + new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href + ); + const server = createEvalServer(); + const { baseUrl } = await server.start(); + try { + await withBrowser(async (h) => { + await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession); + await expect + .poll( + () => + server + .snapshot("scroll") + .events.some( + (e: { type: string; data: { root?: boolean } }) => + e.type === "geometry.ready" && e.data.root, + ), + { timeout: 10_000 }, + ) + .toBe(true); + const frames = await h.loadFrames(); + expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3); + for (const frame of frames) { + await h.ref(frame); + const result = await h.scroll({ ref: "e1" }); + // Independent DOM oracle: CSS border boxes plus this fixture's known + // axis-aligned iframe scales. Re-read after each scroll changes layout. + let rect = await h.evaluate( + "document.querySelector('#probe').getBoundingClientRect().toJSON()", + frame, + ); + let current = frame; + while (true) { + const viewport = await h.evaluate( + "({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })", + current, + ); + rect = intersection(rect, viewport)!; + if (!current.parentFrameId) break; + const parent = frames.find((f) => f.frameId === current.parentFrameId)!; + const owner = await h.evaluate<{ x: number; y: number; scale: number }>( + `(() => { + const iframe = document.querySelector('iframe[name="${current.name}"]'); + const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe); + const scale = r.width / iframe.offsetWidth; + return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale, + y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale }; + })()`, + parent, + ); + rect = { + x: owner.x + rect.x * owner.scale, + y: owner.y + rect.y * owner.scale, + width: rect.width * owner.scale, + height: rect.height * owner.scale, + }; + current = parent; + } + expectRect(result, rect); + } + const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!; + await h.ref(child); + await h.evaluate( + `document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`, + ); + expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" }); + }, zoom); + } finally { + await server.stop(); + } + }, 30_000); +}); diff --git a/apps/extension/src/tools/__tests__/scroll.test.ts b/apps/extension/src/tools/__tests__/scroll.test.ts new file mode 100644 index 00000000..e41a119e --- /dev/null +++ b/apps/extension/src/tools/__tests__/scroll.test.ts @@ -0,0 +1,310 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types"; +import type { ViewportRect } from "../geometry"; +import { handleScrollTo } from "../scroll"; +import type { CdpRunner } from "../shared"; + +interface Call { + target: CdpTarget; + method: string; + params?: Record; + aborted: boolean; +} + +async function fixture(frame: "top" | "same-target" | "oopif" = "top") { + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + ...(frame !== "top" ? { frameId: "child" } : {}), + ...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}), + }); + const abort = new AbortController(); + const graph: CdpFrameGraph = { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) }, + }, + ], + }; + const rectangles = new Map([ + [1234, { x: 10, y: 20, width: 100, height: 40 }], + [99, { x: 204, y: 306, width: 400, height: 200 }], + ]); + const calls: Call[] = []; + const hooks = { + after: (_call: Call) => {}, + reply: (_call: Call): object | undefined => undefined, + }; + const send = async (target: CdpTarget, method: string, params?: object): Promise => { + const call = { + target, + method, + params: params as Call["params"], + aborted: abort.signal.aborted, + }; + calls.push(call); + const override = hooks.reply(call); + hooks.after(call); + if (override) return override as T; + const args = call.params ?? {}; + const replies: Record = { + "DOM.getDocument": { root: { nodeId: 1 } }, + "DOM.querySelector": { nodeId: 2 }, + "DOM.describeNode": { node: { backendNodeId: 1234 } }, + "DOM.scrollIntoViewIfNeeded": {}, + "DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } }, + "Runtime.callFunctionOn": { + result: { + value: String(args.functionDeclaration).includes("IntersectionObserver") + ? rectangles.get(Number(args.objectId)) + : { width: 200, height: 100 }, + }, + }, + "Runtime.releaseObject": {}, + "Runtime.releaseObjectGroup": {}, + "Runtime.evaluate": { result: { value: { width: 200, height: 100 } } }, + "Page.getLayoutMetrics": { + cssLayoutViewport: { + clientWidth: target.sessionId ? 185 : 1280, + clientHeight: target.sessionId ? 89 : 720, + }, + }, + "DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } }, + }; + if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`); + return replies[method] as T; + }; + const cdp: CdpRunner = { + send: (tabId, method, params) => send({ tabId }, method, params), + sendToTarget: send, + getFrameGraph: vi.fn(async () => { + hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted }); + return graph; + }), + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + const run = (params: Partial = {}) => + handleScrollTo( + manager, + { session_id: "aa11", ref: "@e3", ...params }, + { cdp, tabsApi, signal: abort.signal }, + ); + return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("handleScrollTo", () => { + it("scrolls a ref and returns renderer-clipped bounds", async () => { + const f = await fixture(); + expect(await f.run()).toMatchObject({ + tab_id: 4, + used_ref: "e3", + x: 10, + y: 20, + width: 100, + height: 40, + }); + expect(f.calls[0]).toMatchObject({ + method: "DOM.scrollIntoViewIfNeeded", + params: { backendNodeId: 1234 }, + }); + const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup; + expect(f.calls.at(-1)).toMatchObject({ + method: "Runtime.releaseObjectGroup", + params: { objectGroup: group }, + }); + }); + + it("resolves a selector and tracks the attached tab even when it is missing", async () => { + const f = await fixture(); + expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({ + used_selector: "#target", + }); + f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined); + expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({ + code: "not_found", + data: { reason: "selector_not_found" }, + }); + expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4); + }); + + it.each([ + "same-target", + "oopif", + ] as const)("projects %s refs and scrolls their owners", async (frame) => { + const f = await fixture(frame); + expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 }); + expect( + f.calls + .filter((call) => call.method === "DOM.scrollIntoViewIfNeeded") + .map((call) => call.params!.backendNodeId), + ).toEqual([99, 1234]); + expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1); + if (frame === "oopif") + expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe( + "child-session", + ); + }); + + it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => { + const f = await fixture("oopif"); + f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 }); + expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 }); + }); + + it("clips against the iframe owner's ordinary DOM ancestors", async () => { + const f = await fixture("oopif"); + f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 }); + expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 }); + }); + + it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => { + const f = await fixture("oopif"); + f.rectangles.set(node, null); + expect(await f.run()).toMatchObject({ + code: "permission_denied", + data: { reason: "element_not_visible" }, + }); + expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + }); + + it.each([ + { ref: undefined }, + { selector: "#target" }, + { timeout_ms: 0 }, + ])("rejects invalid params %j", async (params) => { + const f = await fixture(); + expect(await f.run(params)).toMatchObject({ code: "invalid_params" }); + expect(f.calls).toEqual([]); + }); + + it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => { + const f = await fixture(); + if (mode === "stale") f.ctx.refStore.clear(); + else f.ctx.refStore.set("e3", 1234, { tabId: 8 }); + expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } }); + expect(f.calls).toEqual([]); + }); + + it("enforces the Agent Window boundary", async () => { + const f = await fixture(); + f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]); + f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab); + expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" }); + expect(f.calls).toEqual([]); + }); + + it.each([ + "getFrameGraph", + "DOM.scrollIntoViewIfNeeded", + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Page.getLayoutMetrics", + "Runtime.evaluate", + "DOM.getBoxModel", + ])("honors cancellation during %s and only cleans up afterwards", async (method) => { + const f = await fixture("oopif"); + f.hooks.after = (call) => { + if (call.method === method) f.abort.abort(); + }; + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect( + f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"), + ).toEqual([]); + }); + + it.each([ + "DOM.getDocument", + "DOM.querySelector", + "DOM.describeNode", + ])("cancels selector resolution during %s", async (method) => { + const f = await fixture(); + f.hooks.after = (call) => { + if (call.method === method) f.abort.abort(); + }; + expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({ + code: "cancelled", + }); + expect(f.calls.filter((call) => call.aborted)).toEqual([]); + }); + + it("does not start after early cancellation", async () => { + const f = await fixture(); + f.abort.abort(); + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect(f.calls).toEqual([]); + expect(f.tabsApi.query).not.toHaveBeenCalled(); + }); + + it("does not issue fallback scrolling after cancellation", async () => { + const f = await fixture(); + f.hooks.reply = ({ method }) => { + if (method === "DOM.scrollIntoViewIfNeeded") { + f.abort.abort(); + throw new Error("scroll failed"); + } + return undefined; + }; + expect(await f.run()).toMatchObject({ code: "cancelled" }); + expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]); + }); + + it("stops target scrolling after a deadline expires while scrolling its owner", async () => { + const f = await fixture("oopif"); + const now = vi.spyOn(Date, "now").mockReturnValue(100); + f.hooks.after = ({ method }) => { + if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201); + }; + expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" }); + expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1); + }); + + it("reports measurement exceptions and releases objects even if cleanup fails", async () => { + const f = await fixture(); + f.hooks.reply = ({ method }) => { + if (method === "Runtime.callFunctionOn") + return { exceptionDetails: { text: "measurement failed" } }; + if (method === "Runtime.releaseObjectGroup") throw new Error("target closed"); + return undefined; + }; + expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" }); + expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup"); + }); + + it("attaches dialogs observed during scrolling", async () => { + const f = await fixture(); + f.cdp.dialogCursor = () => 5; + const dialogs: JavaScriptDialogInfo[] = [ + { + sequence: 6, + tab_id: 4, + type: "alert", + message: "scrolled", + handled: "accepted", + url: "https://example.test", + }, + ]; + f.cdp.dialogsSince = vi.fn(() => dialogs); + expect(await f.run()).toHaveProperty("dialogs", dialogs); + expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 6b1023d1..0415ca8e 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -29,6 +29,7 @@ import type { ResponseFrame, RpcError, ScreenshotParams, + ScrollToParams, SelectParams, SnapshotParams, UploadParams, @@ -71,6 +72,7 @@ import { handleRecordStop, type RecordRuntimeDeps, } from "./record"; +import { handleScrollTo } from "./scroll"; import { handleSessionStart, handleSessionStop, @@ -525,6 +527,17 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.scroll_to": + return this.withHoverReleaseForRequest( + req.params as ScrollToParams, + () => + handleScrollTo( + this.sessions, + req.params as ScrollToParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.focus": return this.withHoverReleaseForRequest( req.params as FocusParams, @@ -817,6 +830,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.scroll_to": case "tool.focus": case "tool.blur": case "tool.fill": diff --git a/apps/extension/src/tools/scroll-visibility.ts b/apps/extension/src/tools/scroll-visibility.ts new file mode 100644 index 00000000..a015f6d2 --- /dev/null +++ b/apps/extension/src/tools/scroll-visibility.ts @@ -0,0 +1,142 @@ +import { cdpTargetKey } from "@/browser-driver/frame-graph"; +import type { NodeAddress } from "./frame-geometry"; +import { + clipPolygon, + projectRegionToViewport, + type Region, + rectPolygon, + regionBounds, + type ViewportRect, +} from "./geometry"; +import { GeometryContext } from "./geometry/frame-context"; +import { type CdpRunner, sendToCdpTarget } from "./shared"; + +// Ask the renderer to apply containing-block clips (including shadow DOM), +// instead of treating layout quads as visible pixels. Each document is measured +// locally; the existing frame projections supply the top-viewport coordinates. +// IntersectionObserver reports a bounding rectangle, not an occlusion/hit test. +const VISIBLE_RECT = `function(timeoutMs) { + const element = this; + const visible = () => element.isConnected && element.checkVisibility({ + checkOpacity: true, checkVisibilityCSS: true, contentVisibilityAuto: true + }); + if (typeof element.checkVisibility !== 'function' || !visible()) return null; + return new Promise((resolve, reject) => { + const observer = new IntersectionObserver(([entry]) => { + clearTimeout(timer); + observer.disconnect(); + const rect = entry.intersectionRect; + resolve(visible() && entry.isIntersecting && rect.width > 0 && rect.height > 0 + ? { x: rect.x, y: rect.y, width: rect.width, height: rect.height } + : null); + }, { root: element.ownerDocument }); + const timer = setTimeout(() => { + observer.disconnect(); + reject(new Error('scroll-to visibility measurement timed out')); + }, timeoutMs); + observer.observe(element); + }); +}`; + +async function visibleRect( + cdp: CdpRunner, + address: NodeAddress, + deadline: number, +): Promise { + const resolved = await sendToCdpTarget<{ object?: { objectId?: string } }>( + cdp, + address.target, + "DOM.resolveNode", + { backendNodeId: address.backendNodeId }, + ); + if (!resolved.object?.objectId) throw new Error("scroll-to could not resolve the target element"); + const reply = await sendToCdpTarget<{ + result?: { value?: ViewportRect | null }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; + }>(cdp, address.target, "Runtime.callFunctionOn", { + objectId: resolved.object.objectId, + functionDeclaration: VISIBLE_RECT, + // Bound this read even if a background document stops rendering. Its + // observer always disconnects; cancellation cannot leave a live observer. + arguments: [{ value: Math.max(1, Math.min(1_000, deadline - Date.now())) }], + awaitPromise: true, + returnByValue: true, + }); + if (reply.exceptionDetails) { + const details = reply.exceptionDetails; + throw new Error(details.exception?.description ?? details.text ?? "visibility script failed"); + } + const rect = reply.result?.value; + if (rect === null) return null; + if ( + !rect || + ![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) || + rect.width <= 0 || + rect.height <= 0 + ) { + throw new Error("scroll-to visibility measurement returned invalid bounds"); + } + return rect; +} + +async function projectRect( + context: GeometryContext, + address: NodeAddress, + rect: ViewportRect, +): Promise { + let region: Region = [rectPolygon({ x: rect.x, y: rect.y, w: rect.width, h: rect.height })]; + if (!address.frameId) return region; + const frame = await context.frame(address.frameId); + if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) { + throw new Error("scroll-to frame no longer belongs to its target"); + } + const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined; + if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) { + // DOM rectangles are document-local. Same-target iframe quads are already + // target-local, so cross this one boundary before the target projection. + const viewport = await context.viewport(frame.target); + if (frame.ownerBackendNodeId === undefined || !viewport) { + throw new Error("scroll-to could not resolve the frame viewport"); + } + const local = await context.snapshotProjection(address, frame.ownerBackendNodeId, [], viewport); + if (local.status !== "available") throw new Error("scroll-to could not project the frame"); + region = projectRegionToViewport(region, local.projection.geometry); + } + const projection = await context.targetProjection(frame.frameId); + if (!projection) throw new Error("scroll-to could not project the target viewport"); + return projectRegionToViewport(region, projection); +} + +/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */ +export async function scrollVisibleBounds( + cdp: CdpRunner, + tabId: number, + address: NodeAddress, + deadline: number, +): Promise { + const context = new GeometryContext(cdp, tabId); + const rect = await visibleRect(cdp, address, deadline); + if (!rect) return null; + let region = await projectRect(context, address, rect); + if (address.frameId) { + const ancestry = await context.ancestry(address.frameId); + if (!ancestry) throw new Error("scroll-to could not resolve the frame ancestry"); + for (const child of ancestry) { + const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined; + if (!parent || child.ownerBackendNodeId === undefined) { + throw new Error("scroll-to could not resolve the frame owner"); + } + const owner = { + target: parent.target, + frameId: parent.frameId, + backendNodeId: child.ownerBackendNodeId, + }; + // A visible child document can still have a hidden or clipped iframe. + const ownerRect = await visibleRect(cdp, owner, deadline); + if (!ownerRect) return null; + const clips = await projectRect(context, owner, ownerRect); + region = region.flatMap((polygon) => clips.map((clip) => clipPolygon(polygon, clip))); + } + } + return regionBounds(region); +} diff --git a/apps/extension/src/tools/scroll.ts b/apps/extension/src/tools/scroll.ts new file mode 100644 index 00000000..9ba79836 --- /dev/null +++ b/apps/extension/src/tools/scroll.ts @@ -0,0 +1,137 @@ +import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph"; +import type { SessionManager } from "@/session-manager/manager"; +import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { cdpError, rpcError } from "./errors"; +import { scrollElementAndFramesIntoView } from "./frame-geometry"; +import { resolveBackendNode } from "./interaction"; +import { scrollVisibleBounds } from "./scroll-visibility"; +import { + type CdpRunner, + type ChromeTabsApi, + chromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, + sendToCdpTarget, +} from "./shared"; + +export interface ScrollToDeps { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + signal?: AbortSignal; +} + +let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; +function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { + if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; + return defaultDeps; +} + +export async function handleScrollTo( + manager: SessionManager, + params: ScrollToParams, + deps: ScrollToDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "scroll-to"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const timeout = params.timeout_ms ?? 30_000; + if (!Number.isInteger(timeout) || timeout <= 0) { + return { code: "invalid_params", message: "timeout_ms must be a positive integer" }; + } + const deadline = Date.now() + timeout; + const checkActive = () => { + if (deps.signal?.aborted) throw new DOMException("scroll-to aborted", "AbortError"); + if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError"); + }; + const objectGroup = `bsk-scroll-${crypto.randomUUID()}`; + const objectTargets = new Map(); + const send = async (target: CdpTarget, method: string, args?: object): Promise => { + checkActive(); + // Also own objects allocated by the shared scroll fallback. Record the + // target before awaiting so cancellation cannot skip their cleanup. + if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target); + const result = await sendToCdpTarget( + deps.cdp, + target, + method, + method === "DOM.resolveNode" ? { ...args, objectGroup } : args, + ); + checkActive(); + return result; + }; + let graph: ReturnType> | undefined; + const cdp: CdpRunner = { + send: (tabId, method, args) => send({ tabId }, method, args), + sendToTarget: send, + trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp), + getFrameGraph: deps.cdp.getFrameGraph + ? async (tabId) => { + checkActive(); + graph ??= deps.cdp.getFrameGraph!(tabId); + const result = await graph; + checkActive(); + return result; + } + : undefined, + }; + try { + checkActive(); + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + checkActive(); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "scroll-to"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(cdp, ctx, target, params, "scroll-to"); + checkActive(); + if (isRpcError(node)) return node; + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollError = await scrollElementAndFramesIntoView( + cdp, + target.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + checkActive(); + if (scrollError) return scrollError; + const bounds = await scrollVisibleBounds( + cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + frameId: node.frameId, + }, + deadline, + ); + checkActive(); + if (!bounds) + return rpcError( + "permission_denied", + "element_not_visible", + "scroll-to target has no visible area", + ); + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + ...bounds, + }); + } catch (error) { + if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; + if (Date.now() >= deadline) return { code: "timeout", message: "scroll-to timed out" }; + return cdpError(error); + } finally { + // Cleanup bypasses the cancellation guard and cannot replace the result. + for (const target of objectTargets.values()) { + await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch( + () => {}, + ); + } + } +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 5891e400..5020b292 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -470,6 +470,26 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface ScrollToParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface ScrollToResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + /** Clipped border-box bounds in top-level viewport CSS pixels; not an occlusion test. */ + x: number; + y: number; + width: number; + height: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FocusParams { session_id: string; ref?: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 21fe7a17..765b91cc 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -55,10 +55,17 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|focus|blur|fill|select|press ... --session +bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` +`bsk scroll-to --session ` scrolls an element and its frame owners into view. +Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document. +The result is the visible border-box portion's bounds in top-level viewport CSS pixels after +ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with +`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements. +For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. + `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether it was focused. Use these for UI states triggered by focus changes. @@ -117,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read session start|stop|list browsers status doctor update logs navigate navigate-back navigate-forward reload wait-for-navigation wait-ms observe snapshot get-html screenshot console network -click hover focus blur fill select press evaluate +click hover scroll-to focus blur fill select press evaluate tab list|create|close|select|borrow|return window resize emulate upload download request-help record start|stop ``` diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index c42e9749..4d0f9d2e 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -28,6 +28,7 @@ pub mod record_recovery; pub mod record_state; pub mod render_error; pub mod screenshot; +pub mod scroll; pub mod session; pub mod snapshot; pub mod status; @@ -55,6 +56,7 @@ use crate::cli::network::NetworkArgs; use crate::cli::observe::ObserveArgs; use crate::cli::record::RecordCmd; use crate::cli::screenshot::ScreenshotArgs; +use crate::cli::scroll::ScrollToArgs; use crate::cli::session::SessionCmd; use crate::cli::snapshot::SnapshotArgs; use crate::cli::tab::TabCmd; @@ -175,6 +177,10 @@ pub enum Command { /// Hover a snapshot ref or CSS selector. Hover(HoverArgs), + /// Scroll a snapshot ref or CSS selector into the visible viewport. + #[command(name = "scroll-to")] + ScrollTo(ScrollToArgs), + /// Focus a snapshot ref or CSS selector. Focus(FocusArgs), diff --git a/crates/bsk-cli/src/cli/scroll.rs b/crates/bsk-cli/src/cli/scroll.rs new file mode 100644 index 00000000..ba79983f --- /dev/null +++ b/crates/bsk-cli/src/cli/scroll.rs @@ -0,0 +1,100 @@ +//! `bsk scroll-to` — bring a target element into the visible viewport. + +use std::time::Duration; + +use anyhow::Context; +use bsk_protocol::Method; +use bsk_protocol::tools::{ScrollToParams, ScrollToResult}; +use clap::Args; + +use crate::cli::dialogs::print_dialog_summaries; +use crate::cli::ensure_daemon::ensure_daemon; +use crate::cli::error::{CliError, Format}; +use crate::cli::interaction::split_target; +use crate::cli::navigate::parse_timeout_ms; + +#[derive(Debug, Clone, Args)] +pub struct ScrollToArgs { + /// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used. + pub target: Option, + + #[arg(long = "ref")] + pub ref_: Option, + + #[arg(long = "selector")] + pub selector: Option, + + #[arg(long)] + pub session: String, + + #[arg(long = "tab-id")] + pub tab_id: Option, + + #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> { + let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; + let info = ensure_daemon().context("ensure daemon is running")?; + let params = ScrollToParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + timeout_ms: Some(args.timeout), + }; + let reply: ScrollToResult = crate::cli::business_rpc::call( + info.sock_path, + "scroll-to-1", + Method::ToolScrollTo, + Some(params), + ipc_timeout(args.timeout), + )?; + match format { + Format::Json => println!( + "{}", + serde_json::to_string_pretty(&reply) + .map_err(|e| CliError::Local(anyhow::anyhow!(e)))? + ), + Format::Human => { + let target = + format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref()); + println!( + "scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})", + reply.tab_id, reply.x, reply.y, reply.width, reply.height + ); + print_dialog_summaries(&reply.dialogs); + } + } + Ok(()) +} + +fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String { + used_ref + .map(|r| format!("@{r}")) + .or_else(|| used_selector.map(str::to_string)) + .unwrap_or_else(|| "?".into()) +} + +fn ipc_timeout(timeout_ms: u32) -> Duration { + Duration::from_millis(u64::from(timeout_ms)) + .checked_add(Duration::from_secs(15)) + .unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_target_detects_refs_and_selectors() { + let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap(); + assert_eq!(ref_.as_deref(), Some("@e3")); + assert!(selector.is_none()); + + let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap(); + assert!(ref_.is_none()); + assert_eq!(selector.as_deref(), Some("#target")); + } +} diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 26ef6405..440a7f50 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -253,6 +253,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolReload | Method::ToolClick | Method::ToolHover + | Method::ToolScrollTo | Method::ToolFocus | Method::ToolBlur | Method::ToolFill diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index fe44a37c..19aff7e5 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -94,6 +94,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format), + Command::ScrollTo(args) => cli::scroll::dispatch(args, format), Command::Focus(args) => cli::interaction::dispatch_focus(args, format), Command::Blur(args) => cli::interaction::dispatch_blur(args, format), Command::Fill(args) => cli::interaction::dispatch_fill(args, format), diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs index 0e7a7a94..11666e4c 100644 --- a/crates/bsk-cli/tests/cli_parse.rs +++ b/crates/bsk-cli/tests/cli_parse.rs @@ -255,6 +255,47 @@ fn parses_hover_with_settle() { assert_eq!(args.settle, 300); } +#[test] +fn parses_scroll_to_target() { + let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]); + let Command::ScrollTo(args) = cli.command else { + panic!("expected scroll-to command"); + }; + assert_eq!(args.target.as_deref(), Some("@e2")); +} + +#[test] +fn parses_scroll_to_explicit_target_tab_and_timeout() { + for flag in ["--ref", "--selector"] { + let cli = parse(&[ + "bsk", + "scroll-to", + flag, + "e3", + "--session", + "s1", + "--tab-id", + "42", + "--timeout", + "5s", + ]); + let Command::ScrollTo(args) = cli.command else { + panic!("expected scroll-to command"); + }; + assert_eq!(args.tab_id, Some(42)); + assert_eq!(args.timeout, 5_000); + assert_eq!( + if flag == "--ref" { + args.ref_ + } else { + args.selector + } + .as_deref(), + Some("e3") + ); + } +} + #[test] fn parses_focus_target() { let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]); diff --git a/crates/bsk-cli/tests/session_user_interrupt.rs b/crates/bsk-cli/tests/session_user_interrupt.rs index 9c5288cc..5d3cbd36 100644 --- a/crates/bsk-cli/tests/session_user_interrupt.rs +++ b/crates/bsk-cli/tests/session_user_interrupt.rs @@ -293,6 +293,15 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() { /// catch a tool dispatched arbitrarily later. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() { + assert_idle_interrupt_rejects(Method::ToolClick).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_interrupt_rejects_scroll_to_before_forwarding() { + assert_idle_interrupt_rejects(Method::ToolScrollTo).await; +} + +async fn assert_idle_interrupt_rejects(method: Method) { let (handle, sock) = spawn_daemon().await; let mut ws = connect_ext(handle.ws_addr()).await; let _ = handshake_as_ext(&mut ws).await; @@ -301,10 +310,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() // Fake extension: answer tool.session_start, observe other // requests but never reply. We expect the daemon to reject the - // tool call BEFORE it ever forwards a tool.click frame. + // tool call BEFORE it ever forwards a browser-input frame. let ws_sink_for_responder = Arc::clone(&ws_sink); - let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let observed_click_count_clone = Arc::clone(&observed_click_count); + let observed_call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed_call_count_clone = Arc::clone(&observed_call_count); + let forwarded_method = method.clone(); let responder = tokio::spawn(async move { let mut ws_stream = ws_stream; while let Some(Ok(msg)) = ws_stream.next().await { @@ -329,8 +339,8 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() g.send(Message::Text(serde_json::to_string(&reply).unwrap())) .await .unwrap(); - } else if req.method == Method::ToolClick { - observed_click_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } else if req.method == forwarded_method { + observed_call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); // Deliberately do NOT reply — daemon should reject before // ever forwarding to us. let _ = req; @@ -377,12 +387,12 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() let state = handle.state(); wait_for_session_interrupt_pending(&state, &start.session_id).await; - // The CLI now issues a tool.click (mutating). The daemon must + // The CLI now issues a browser-input (mutating). The daemon must // reject it WITHOUT forwarding to the extension. - let click_outcome = ipc + let outcome = ipc .call::<_, serde_json::Value>( - "click-after-interrupt", - Method::ToolClick, + "input-after-interrupt", + method, Some(json!({ "session_id": start.session_id, "ref": "fake-ref-1", @@ -391,7 +401,7 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() ) .await .unwrap(); - let err = click_outcome.expect_err("tool.click must be rejected"); + let err = outcome.expect_err("browser-input must be rejected"); assert_eq!( err.code, ErrorCode::UserAborted, @@ -399,11 +409,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() err ); - // Crucially: the extension never saw a tool.click frame. + // Crucially: the extension never saw a browser-input frame. assert_eq!( - observed_click_count.load(std::sync::atomic::Ordering::SeqCst), + observed_call_count.load(std::sync::atomic::Ordering::SeqCst), 0, - "tool.click must NOT have been forwarded to the extension" + "browser-input must NOT have been forwarded to the extension" ); responder.abort(); diff --git a/crates/bsk-cli/tests/tools_m7_ipc.rs b/crates/bsk-cli/tests/tools_m7_ipc.rs index 1a6e6e83..e8c2c42c 100644 --- a/crates/bsk-cli/tests/tools_m7_ipc.rs +++ b/crates/bsk-cli/tests/tools_m7_ipc.rs @@ -18,6 +18,7 @@ use bsk_protocol::tools::{ PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams, SessionStartResult, WaitUntil, }; +use bsk_protocol::tools::{ScrollToParams, ScrollToResult}; use bsk_protocol::{ BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError, }; @@ -468,6 +469,51 @@ async fn blur_round_trips_previous_focus_state() { handle.shutdown().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn scroll_to_round_trips_visible_bounds() { + let (handle, sock) = spawn_daemon().await; + let mut ws = connect_ext(handle.ws_addr()).await; + let _ = do_handshake(&mut ws).await; + + run_extension(ws, |req| { + assert_eq!(req.method, Method::ToolScrollTo); + let params = req.params.clone().unwrap(); + let parsed: ScrollToParams = serde_json::from_value(params).unwrap(); + assert_eq!(parsed.ref_.as_deref(), Some("@e3")); + ResponseBody::Ok( + serde_json::to_value(ScrollToResult { + tab_id: 9, + used_ref: Some("e3".into()), + used_selector: None, + x: 10.0, + y: 20.0, + width: 100.0, + height: 40.0, + dialogs: vec![], + }) + .unwrap(), + ) + }); + + let session_id = ipc_session_start(&sock).await; + let result: ScrollToResult = ipc_tool_call( + &sock, + Method::ToolScrollTo, + ScrollToParams { + session_id, + ref_: Some("@e3".into()), + selector: None, + tab_id: None, + timeout_ms: Some(5_000), + }, + ) + .await + .expect("scroll-to ok"); + assert_eq!(result.used_ref.as_deref(), Some("e3")); + assert_eq!(result.width, 100.0); + handle.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn fill_round_trips_clear_before_default() { let (handle, sock) = spawn_daemon().await; diff --git a/crates/bsk-protocol/schema/tool_scroll_to_params.json b/crates/bsk-protocol/schema/tool_scroll_to_params.json new file mode 100644 index 00000000..77ada1de --- /dev/null +++ b/crates/bsk-protocol/schema/tool_scroll_to_params.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ScrollToParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "ref": { + "description": "Optional `@e` ref allocated by the last observation. Mutually exclusive with `selector`.", + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "description": "Target tab. Defaults to the Agent Window's active tab.", + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 1.0 + } + } +} diff --git a/crates/bsk-protocol/schema/tool_scroll_to_result.json b/crates/bsk-protocol/schema/tool_scroll_to_result.json new file mode 100644 index 00000000..cad27108 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_scroll_to_result.json @@ -0,0 +1,123 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ScrollToResult", + "type": "object", + "required": [ + "height", + "tab_id", + "width", + "x", + "y" + ], + "properties": { + "dialogs": { + "type": "array", + "items": { + "$ref": "#/definitions/JavaScriptDialogInfo" + } + }, + "height": { + "type": "number", + "format": "double" + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + }, + "width": { + "type": "number", + "format": "double" + }, + "x": { + "description": "Bounding rectangle of the element's visible border-box portion, in top-level viewport CSS pixels, after ancestor and viewport clipping. Partial visibility is sufficient. This is not an occlusion or hit test.", + "type": "number", + "format": "double" + }, + "y": { + "type": "number", + "format": "double" + } + }, + "definitions": { + "JavaScriptDialogHandledAction": { + "description": "How the extension resolved the dialog so CDP could continue.", + "type": "string", + "enum": [ + "accepted", + "dismissed" + ] + }, + "JavaScriptDialogInfo": { + "description": "One observed + handled JavaScript dialog during a tool call.", + "type": "object", + "required": [ + "handled", + "message", + "sequence", + "tab_id", + "type" + ], + "properties": { + "default_prompt": { + "type": [ + "string", + "null" + ] + }, + "handled": { + "$ref": "#/definitions/JavaScriptDialogHandledAction" + }, + "has_browser_handler": { + "type": [ + "boolean", + "null" + ] + }, + "message": { + "type": "string" + }, + "sequence": { + "description": "Monotonic per-tab sequence for ordering within a session.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "type": { + "$ref": "#/definitions/JavaScriptDialogType" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "JavaScriptDialogType": { + "description": "Native JS dialog kind reported by CDP.", + "type": "string", + "enum": [ + "alert", + "confirm", + "prompt", + "beforeunload" + ] + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index f327d312..fc60243a 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -79,6 +79,8 @@ fn main() { dump!(ClickResult, "tool_click_result"); dump!(HoverParams, "tool_hover_params"); dump!(HoverResult, "tool_hover_result"); + dump!(ScrollToParams, "tool_scroll_to_params"); + dump!(ScrollToResult, "tool_scroll_to_result"); dump!(FocusParams, "tool_focus_params"); dump!(FocusResult, "tool_focus_result"); dump!(BlurParams, "tool_blur_params"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 5ba61143..4fb4cf6d 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -72,6 +72,8 @@ pub enum Method { ToolClick, #[serde(rename = "tool.hover")] ToolHover, + #[serde(rename = "tool.scroll_to")] + ToolScrollTo, #[serde(rename = "tool.focus")] ToolFocus, #[serde(rename = "tool.blur")] @@ -172,6 +174,7 @@ impl Method { | Method::ToolNavigateForward | Method::ToolReload | Method::ToolClick + | Method::ToolScrollTo | Method::ToolFocus | Method::ToolBlur | Method::ToolFill @@ -311,6 +314,7 @@ mod tests { assert!(Method::ToolNavigateForward.is_mutating()); assert!(Method::ToolReload.is_mutating()); assert!(Method::ToolClick.is_mutating()); + assert!(Method::ToolScrollTo.is_mutating()); assert!(Method::ToolFocus.is_mutating()); assert!(Method::ToolBlur.is_mutating()); assert!(Method::ToolFill.is_mutating()); @@ -357,6 +361,7 @@ mod tests { assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); + assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane); @@ -368,6 +373,7 @@ mod tests { assert!(Method::ToolHover.requires_interrupt_gate()); assert!(Method::ToolObserve.requires_interrupt_gate()); assert!(Method::ToolClick.requires_interrupt_gate()); + assert!(Method::ToolScrollTo.requires_interrupt_gate()); assert!(Method::ToolFocus.requires_interrupt_gate()); assert!(Method::ToolBlur.requires_interrupt_gate()); assert!(!Method::Cancel.requires_interrupt_gate()); diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs index 9e841bbf..035f3289 100644 --- a/crates/bsk-protocol/src/tools/mod.rs +++ b/crates/bsk-protocol/src/tools/mod.rs @@ -14,6 +14,7 @@ mod record_common; mod record_v2; mod record_v3; pub mod script; +pub mod scroll; pub mod session; pub mod tabs; pub mod waits; @@ -30,6 +31,7 @@ pub use network::*; pub use observation::*; pub use record::*; pub use script::*; +pub use scroll::*; pub use session::*; pub use tabs::*; pub use waits::*; diff --git a/crates/bsk-protocol/src/tools/scroll.rs b/crates/bsk-protocol/src/tools/scroll.rs new file mode 100644 index 00000000..28f3491a --- /dev/null +++ b/crates/bsk-protocol/src/tools/scroll.rs @@ -0,0 +1,67 @@ +//! Element scrolling primitive (`tool.scroll_to`). + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::JavaScriptDialogInfo; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ScrollToParams { + pub session_id: String, + /// Optional `@e` ref allocated by the last observation. + /// Mutually exclusive with `selector`. + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Target tab. Defaults to the Agent Window's active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ScrollToResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + /// Bounding rectangle of the element's visible border-box portion, in + /// top-level viewport CSS pixels, after ancestor and viewport clipping. + /// Partial visibility is sufficient. This is not an occlusion or hit test. + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn params_serialise_ref_field_name() { + let params = ScrollToParams { + session_id: "abcd".into(), + ref_: Some("@e3".into()), + selector: None, + tab_id: Some(42), + timeout_ms: Some(5_000), + }; + let value = serde_json::to_value(¶ms).unwrap(); + assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3")); + assert!(value.get("ref_").is_none()); + let round: ScrollToParams = serde_json::from_value(value).unwrap(); + assert_eq!(round, params); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index e55f5923..d7887dbc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,6 +84,11 @@ Shared Rust types + JSON Schema generation. TypeScript mirrors frame shapes in 4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`. 5. Response travels CLI ← daemon ← extension; CLI prints result and exits. +The [scroll-to primitive reference](scroll-to.md) documents `tool.scroll_to`, +including its CLI/plugin mappings, visible-bounds contract and cancellation +behavior. It follows the same routing path and is classified as a browser +mutation for session queueing and user-interruption gating. + ## Session and sandbox model - **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window** diff --git a/docs/scroll-to.md b/docs/scroll-to.md new file mode 100644 index 00000000..f1b3838b --- /dev/null +++ b/docs/scroll-to.md @@ -0,0 +1,131 @@ +# Scroll-to primitive + +`scroll-to` brings an existing element and its containing frames into view, then +returns the visible portion's bounds. Use it to reveal a target before inspecting +the page or taking a screenshot. It accepts an element target rather than a pixel +distance or scroll direction. + +| Entry point | Name | +| --- | --- | +| CLI | `bsk scroll-to` | +| Wire protocol | `tool.scroll_to` | +| DeepSeek Harness | `browser_interact` with `action: "scroll-to"` | + +## CLI + +Use a session you started and a fresh ref from an observation. The ids below are +examples; substitute the session, tab and ref returned by your own calls. + +```sh +bsk scroll-to @e3 --session abcd --tab-id 42 --timeout 5s --json +bsk scroll-to --selector '#details' --session abcd +``` + +Supply exactly one target: a positional ref or selector, `--ref`, or `--selector`. +Refs accept `@e3` and `e3`. CSS selectors search only the main document; use refs +for elements in iframes (including out-of-process iframes) or shadow roots. +Refs belong to one session and tab and may become stale after page changes. + +`--session` is required. `--tab-id` defaults to the Agent Window's active tab. +`--timeout` defaults to `30s` and must be positive; durations such as `5000ms` and +`5s` are accepted. The command operates on Agent Window tabs, including user tabs +explicitly borrowed into that window. + +Example JSON output: + +```json +{ + "tab_id": 42, + "used_ref": "e3", + "x": 10, + "y": 20, + "width": 100, + "height": 80 +} +``` + +`used_ref` or `used_selector` identifies the resolved target. The optional +`dialogs` array reports JavaScript dialogs handled during the call. + +## Result contract + +`x` and `y` are the upper-left corner, and `width` and `height` are the size of +the visible border-box portion's bounding rectangle. All four values use +**top-level viewport CSS pixels**, after scrolling and clipping by ancestor +containers and frame viewports. They are not document or screenshot pixel +coordinates. + +Partial visibility is sufficient: a 400px-high element clipped by an 80px-high +scroll container can succeed with `height: 80`. Hidden or fully clipped targets +fail. The rectangle is not an occlusion or hit test: another element can cover +the target, and the rectangle's center is not a guaranteed click point. Layout +can change after the result; use fresh refs for subsequent interactions. + +## Protocol and plugin + +The CLI sends a normal request through the daemon to the extension: + +```json +{ + "id": "scroll-1", + "method": "tool.scroll_to", + "params": { + "session_id": "abcd", + "ref": "@e3", + "tab_id": 42, + "timeout_ms": 5000 + } +} +``` + +Protocol callers must supply exactly one nonempty `ref` or `selector`. +`tab_id` is optional; `timeout_ms` is a positive integer and defaults to 30000. +These target constraints are enforced by the handler. See the generated +[parameter schema](../crates/bsk-protocol/schema/tool_scroll_to_params.json) and +[result schema](../crates/bsk-protocol/schema/tool_scroll_to_result.json) for field +types. The wire response wraps the result above in `{ "id": "scroll-1", "result": ... }`. + +The plugin exposes the same action with its usual camelCase parameters: + +```json +{ + "action": "scroll-to", + "session": "abcd", + "target": "@e3", + "tabId": 42, + "timeoutMs": 5000 +} +``` + +Call this through `browser_interact` using a plugin-owned session. Its result +contains `session`, `tabId`, `x`, `y`, `width` and `height` with the same bounds +semantics. Omitted `session` uses the plugin's current session. + +## Errors and interruption + +The wire response uses the normal `error.code`, `error.message` and optional +`error.data.reason` fields. Common cases are: + +| Code | `data.reason` | Meaning | +| --- | --- | --- | +| `invalid_params` | — | Missing/conflicting target or invalid timeout/tab id | +| `not_found` | `ref_not_found` | Ref is unknown, expired or belongs to another tab | +| `not_found` | `selector_not_found` | Main-document selector matched no element | +| `permission_denied` | `agent_window_scope` | Target tab has not been borrowed into the Agent Window | +| `permission_denied` | `element_not_visible` | No visible area remains after scrolling | +| `cancelled` | — | The call was cancelled | +| `timeout` | — | The action deadline expired | +| `cdp_failed` | varies | Browser command, frame geometry or visibility measurement failed | + +The protocol classifies scroll-to as a browser mutation, so it uses the existing +per-session queue and user-interruption gate. The extension releases any retained +hover state before dispatch. Cancellation and deadline checks surround each CDP +step, including frame lookup, scrolling and measurement. Once detected, they +prevent further scrolling and a successful response; remote-object cleanup still +runs. Already performed scrolling is not rolled back, so inspect the current +page before retrying an interrupted call. + +The implementation is scoped to scroll-to: it reuses frame scrolling and +coordinate projection, with a separate visibility measurement that accounts for +ancestor clipping and hidden frame owners. Shared click, hover and screenshot +geometry is unchanged. diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index e02453a0..b2d13bf6 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,10 +54,13 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | +For `browser_interact` with `action: "scroll-to"`, see the +[scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors. + Arbitrary page-script evaluation and interaction recording are not supported. ## Multi-session model diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 6ca293ad..55a744ec 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,7 +51,12 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur +Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the +visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial +visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other +elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document. + +Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index 3204eed8..d002a9a1 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + - "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + + "Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " + + "press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " + "values; press requires key and may optionally focus target first.", actions: { click: "interact.click", hover: "interact.hover", + "scroll-to": "interact.scroll-to", focus: "interact.focus", blur: "interact.blur", fill: "interact.fill", diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 858fcc2f..50480215 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools"; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; -/** Add hover, focus, blur and select without bypassing session ownership or observation. */ +/** Add interaction primitives without bypassing session ownership or observation. */ export function registerPhaseOneInteractionTools( deps: ToolDeps, register: ToolRegistrar, @@ -100,6 +100,89 @@ export function registerPhaseOneInteractionTools( }), ); + register( + defineTool({ + name: "interact.scroll-to", + description: + "Scroll an element and its frame owners into view. Returns the visible portion's " + + "bounds in top-level viewport CSS pixels; fails if no visible area remains. " + + "Use a fresh ref for iframe targets; selectors search the main document.", + parameters: { + target: { + type: "string", + required: true, + description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + x: { type: "number", required: true }, + y: { type: "number", required: true }, + width: { type: "number", required: true }, + height: { type: "number", required: true }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: + `[session ${value.session}] scrolled into view on tab ${value.tabId}: ` + + `(${value.x}, ${value.y}, ${value.width}, ${value.height})`, + }, + ], + }, + async execute(args, exec) { + requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)"); + const cmdArgs = ["scroll-to", "--session", sessionId]; + appendTabId(cmdArgs, args.tabId); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + "scroll-to", + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + x: number; + y: number; + width: number; + height: number; + }; + return { + session: sessionId, + tabId: reply.tab_id, + x: reply.x, + y: reply.y, + width: reply.width, + height: reply.height, + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + "scroll-to", + args.target, + "--session", + args.session ?? "(current)", + ]), + description: "Scroll an element into view", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + for (const action of ["focus", "blur"] as const) { register( defineTool({ diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index e1441997..ab7686cb 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,7 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.scroll-to": ["browser_interact", "scroll-to"], "interact.focus": ["browser_interact", "focus"], "interact.blur": ["browser_interact", "blur"], "interact.fill": ["browser_interact", "fill"], @@ -202,7 +203,7 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], + browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -540,6 +541,7 @@ describe("interaction tools", () => { it.each([ "focus", "blur", + "scroll-to", ] as const)("interact.%s validates arguments and session ownership before running", async (action) => { const { tools, calls } = setup({ "session start": START_REPLY("s1") }); await startSession(tools); @@ -555,6 +557,30 @@ describe("interaction tools", () => { expect(calls).toHaveLength(1); }); + it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + "scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 }, + }); + await startSession(tools); + expect( + await tools + .get("interact.scroll-to") + ?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()), + ).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 }); + expect(calls[1].args).toEqual([ + "scroll-to", + "--session", + "s1", + "--tab-id", + "7", + "--timeout", + "150000ms", + target, + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + it("interact.fill passes value and --no-clear", async () => { const { tools, calls } = setup(responses); await startSession(tools); diff --git a/skill/SKILL.md b/skill/SKILL.md index 21fe7a17..765b91cc 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -55,10 +55,17 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|focus|blur|fill|select|press ... --session +bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` +`bsk scroll-to --session ` scrolls an element and its frame owners into view. +Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document. +The result is the visible border-box portion's bounds in top-level viewport CSS pixels after +ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with +`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements. +For a specific tab or deadline: `bsk scroll-to @e3 --session --tab-id 42 --timeout 5s`. + `bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether it was focused. Use these for UI states triggered by focus changes. @@ -117,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read session start|stop|list browsers status doctor update logs navigate navigate-back navigate-forward reload wait-for-navigation wait-ms observe snapshot get-html screenshot console network -click hover focus blur fill select press evaluate +click hover scroll-to focus blur fill select press evaluate tab list|create|close|select|borrow|return window resize emulate upload download request-help record start|stop ```