From 0055e90f918a1ccdc1ea22101a6ae759043e72ba Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Fri, 21 Aug 2026 19:14:40 +0800 Subject: [PATCH 1/3] feat(interaction): add explicit focus and blur primitives --- .../src/tools/__tests__/interaction.test.ts | 152 +++++++++++++++++ apps/extension/src/tools/dispatcher.ts | 36 +++- apps/extension/src/tools/interaction.ts | 158 +++++++++++++++++- apps/extension/src/transport/types.ts | 33 ++++ crates/bsk-cli/skill/SKILL.md | 4 + crates/bsk-cli/src/cli/interaction.rs | 126 +++++++++++++- crates/bsk-cli/src/cli/mod.rs | 10 +- crates/bsk-cli/src/daemon/ipc.rs | 2 + crates/bsk-cli/src/main.rs | 2 + crates/bsk-cli/tests/cli_parse.rs | 18 ++ crates/bsk-cli/tests/tools_m7_ipc.rs | 98 ++++++++++- .../bsk-protocol/schema/tool_blur_params.json | 42 +++++ .../bsk-protocol/schema/tool_blur_result.json | 112 +++++++++++++ .../schema/tool_focus_params.json | 42 +++++ .../schema/tool_focus_result.json | 107 ++++++++++++ crates/bsk-protocol/src/bin/dump-schema.rs | 4 + crates/bsk-protocol/src/method.rs | 12 ++ crates/bsk-protocol/src/tools/interaction.rs | 111 +++++++++++- skill/SKILL.md | 4 + 19 files changed, 1058 insertions(+), 15 deletions(-) create mode 100644 crates/bsk-protocol/schema/tool_blur_params.json create mode 100644 crates/bsk-protocol/schema/tool_blur_result.json create mode 100644 crates/bsk-protocol/schema/tool_focus_params.json create mode 100644 crates/bsk-protocol/schema/tool_focus_result.json diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index b78877ad..7a8d83c3 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { + handleBlur, handleClick, handleFill, + handleFocus, handleHover, handlePress, handleSelect, @@ -540,6 +542,156 @@ describe("handleHover", () => { }); }); +describe("handleFocus and handleBlur", () => { + it("focuses a ref and verifies the deep active element", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.focus": () => ({}), + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ result: { value: { focused: true } } }), + }); + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.scrollIntoViewIfNeeded", + "DOM.focus", + "DOM.resolveNode", + "Runtime.callFunctionOn", + ]); + }); + + it("focuses an OOPIF ref in its CDP session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const targetCalls: Array<{ sessionId?: string; method: string }> = []; + fake.cdp.sendToTarget = vi.fn(async (target, method) => { + targetCalls.push({ sessionId: target.sessionId, method }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.focus") return {}; + if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } }; + if (method === "Runtime.callFunctionOn") return { result: { value: { focused: true } } }; + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.focused).toBe(true); + expect(targetCalls).toEqual([ + { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, + { sessionId: "child-session", method: "DOM.focus" }, + { sessionId: "child-session", method: "DOM.resolveNode" }, + { sessionId: "child-session", method: "Runtime.callFunctionOn" }, + ]); + }); + + it("does not focus after cancellation during scrolling", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const abort = new AbortController(); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => { + abort.abort(); + return {}; + }, + }); + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + + expect(res).toMatchObject({ code: "cancelled" }); + expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false); + }); + + it("blurs a ref and returns its previous focus state", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ + result: { value: { ok: true, was_focused: true, focused: false } }, + }), + }); + + const res = await handleBlur( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ + tab_id: 4, + used_ref: "e3", + was_focused: true, + focused: false, + }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.resolveNode", + "Runtime.callFunctionOn", + ]); + }); + + it("rejects a target that does not implement blur", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ + result: { value: { ok: false, was_focused: false, focused: false } }, + }), + }); + + const res = await handleBlur( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ }); + }); +}); + describe("handleFill", () => { it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 25b5f3ac..ea87f4e8 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -2,11 +2,13 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { Transport } from "@/transport/transport"; import type { + BlurParams, ClickParams, ConsoleParams, EmulateParams, EvaluateParams, FillParams, + FocusParams, GetHtmlParams, HoverParams, HoverResult, @@ -35,7 +37,15 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; +import { + handleBlur, + handleClick, + handleFill, + handleFocus, + handleHover, + handlePress, + handleSelect, +} from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -471,6 +481,28 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.focus": + return this.withHoverReleaseForRequest( + req.params as FocusParams, + () => + handleFocus( + this.sessions, + req.params as FocusParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); + case "tool.blur": + return this.withHoverReleaseForRequest( + req.params as BlurParams, + () => + handleBlur( + this.sessions, + req.params as BlurParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.fill": return this.withHoverReleaseForRequest( req.params as FillParams, @@ -718,6 +750,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.focus": + case "tool.blur": case "tool.fill": case "tool.press": case "tool.select": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabdf..83068990 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -1,5 +1,4 @@ -// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and -// `tool.select`. +// DOM interaction tools — click, hover, focus/blur, fill, press, and select. // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). @@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { + BlurParams, + BlurResult, ClickParams, ClickResult, FillParams, FillResult, + FocusParams, + FocusResult, HoverParams, HoverResult, KeyModifier, @@ -215,6 +218,157 @@ async function resolveBackendNode( } } +type FocusCheck = { focused: boolean }; +type BlurMutation = + | { ok: true; was_focused: boolean; focused: boolean } + | { ok: false; was_focused: boolean; focused: boolean }; + +const DEEP_FOCUS_CHECK = `function() { + const deepActiveElement = (root) => { + let active = root && root.activeElement; + while (active && active.shadowRoot && active.shadowRoot.activeElement) { + active = active.shadowRoot.activeElement; + } + return active; + }; + return { focused: deepActiveElement(this.ownerDocument) === this }; +}`; + +const BLUR_TARGET = `function() { + const deepActiveElement = (root) => { + let active = root && root.activeElement; + while (active && active.shadowRoot && active.shadowRoot.activeElement) { + active = active.shadowRoot.activeElement; + } + return active; + }; + const wasFocused = deepActiveElement(this.ownerDocument) === this; + if (typeof this.blur !== 'function') { + return { ok: false, was_focused: wasFocused, focused: wasFocused }; + } + this.blur(); + return { + ok: true, + was_focused: wasFocused, + focused: deepActiveElement(this.ownerDocument) === this, + }; +}`; + +// --------------------------------------------------------------------------- +// tool.focus / tool.blur +// --------------------------------------------------------------------------- + +export async function handleFocus( + manager: SessionManager, + params: FocusParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "focus"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "focus"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "focus"); + if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollElementAndFramesIntoView( + deps.cdp, + target.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + if (scrollErr) return scrollErr; + const abortedBeforeFocus = throwIfAborted(deps.signal); + if (abortedBeforeFocus) return abortedBeforeFocus; + + try { + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); + if (isRpcError(objectIdOrErr)) return objectIdOrErr; + const evaluated = await nodeCdp.send<{ result?: { value?: FocusCheck } }>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId: objectIdOrErr, + functionDeclaration: DEEP_FOCUS_CHECK, + returnByValue: true, + }, + ); + if (evaluated.result?.value?.focused !== true) { + return { code: "invalid_params", message: "target element did not become focused" }; + } + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + focused: true, + }); + } catch (err) { + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } +} + +export async function handleBlur( + manager: SessionManager, + params: BlurParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "blur"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "blur"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "blur"); + if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const abortedBeforeBlur = throwIfAborted(deps.signal); + if (abortedBeforeBlur) return abortedBeforeBlur; + + try { + const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); + if (isRpcError(objectIdOrErr)) return objectIdOrErr; + const evaluated = await nodeCdp.send<{ result?: { value?: BlurMutation } }>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId: objectIdOrErr, + functionDeclaration: BLUR_TARGET, + returnByValue: true, + }, + ); + const mutation = evaluated.result?.value; + if (!mutation?.ok) { + return { code: "invalid_params", message: "target element does not support blur()" }; + } + if (mutation.focused) { + return { code: "cdp_failed", message: "target element remained focused after blur()" }; + } + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + was_focused: mutation.was_focused, + focused: false, + }); + } catch (err) { + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } +} + // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index d53825a1..5162e22f 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -444,6 +444,39 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface FocusParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface FocusResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + focused: boolean; + dialogs?: JavaScriptDialogInfo[]; +} + +export interface BlurParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface BlurResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + was_focused: boolean; + focused: boolean; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e1cffb92..e6c47924 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -70,6 +70,8 @@ bsk navigate --session bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces bsk snapshot --session → static aria tree fallback when VOM is insufficient bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking +bsk focus @e4 --session → explicitly enter focus-driven UI states +bsk blur @e4 --session → explicitly leave focus-driven UI states bsk click @e4 --session → or bsk fill, bsk select, bsk press bsk observe --session → again after navigation / DOM change ``` @@ -202,6 +204,8 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | | `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | +| `bsk focus ` | Focus an element and verify it became the deep active element | +| `bsk blur ` | Remove focus from an element and report whether it was focused | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) | From ac9e7948bf599e0923e07580a361fe0b115c8e81 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 9 Sep 2026 22:47:42 +0800 Subject: [PATCH 2/3] fix(interaction): verify focus changes and honor cancellation --- .../src/tools/__tests__/dispatcher.test.ts | 64 ++++ .../src/tools/__tests__/focus.browser.test.ts | 166 ++++++++++ .../src/tools/__tests__/focus.test.ts | 293 ++++++++++++++++++ .../src/tools/__tests__/interaction.test.ts | 21 +- apps/extension/src/tools/interaction.ts | 288 ++++++++++------- .../schema/tool_focus_result.json | 2 +- crates/bsk-protocol/src/tools/interaction.rs | 2 +- 7 files changed, 707 insertions(+), 129 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/focus.browser.test.ts create mode 100644 apps/extension/src/tools/__tests__/focus.test.ts diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index b1e2bf00..1390ee35 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -599,6 +599,70 @@ describe("ToolDispatcher", () => { expect(onSessionsChanged).toHaveBeenCalledTimes(2); }); + it.each([ + "focus", + "blur", + ] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => { + const tab = { id: 7, windowId: 4242, active: true }; + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async () => tab), + query: vi.fn(async () => [tab]), + sendMessage: vi.fn(async () => undefined), + }, + }); + const sessions = new SessionManager({ + agentWindow: { + create: async () => 4242, + remove: async () => {}, + ensureActiveTab: async () => 7, + }, + }); + const ctx = await sessions.start("aa11"); + ctx.refStore.set("e1", 12, { tabId: 7 }); + const { transport, sent, deliver } = fakeTransport(); + const onBrowserControlResumed = vi.fn(); + let resolveNode: ((value: object) => void) | undefined; + const cdp = { + send: vi.fn(async (_tabId: number, method: string) => { + if (method === "DOM.resolveNode") + return new Promise((resolve) => { + resolveNode = resolve; + }); + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + return {}; + }), + } as unknown as TestDispatcherCdp; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed }); + const hover = dispatcher as unknown as { + rememberHover: (sessionId: string, result: object) => void; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + }; + hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + await hover.setHoverBypass("aa11", 7, true); + dispatcher.start(); + deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" })); + await vi.waitFor(() => expect(resolveNode).toBeDefined()); + expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); + deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } }); + resolveNode!({ object: { objectId: "focus-target" } }); + await vi.waitFor(() => expect(sent).toHaveLength(2)); + expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } }); + expect(sent).toContainEqual({ + id: "r-1", + error: expect.objectContaining({ code: "cancelled" }), + }); + expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything()); + expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" }); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + expect(dispatcher.inflightAbortControllers.size).toBe(0); + dispatcher.stop(); + }); + it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => { vi.stubGlobal("chrome", { tabs: { diff --git a/apps/extension/src/tools/__tests__/focus.browser.test.ts b/apps/extension/src/tools/__tests__/focus.browser.test.ts new file mode 100644 index 00000000..d7e91d10 --- /dev/null +++ b/apps/extension/src/tools/__tests__/focus.browser.test.ts @@ -0,0 +1,166 @@ +// @vitest-environment node +// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile. +import { describe, expect, it } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import { handleBlur, handleFocus } from "../interaction"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; + +async function withFocusBrowser( + run: (h: { + evaluate: (expression: string) => Promise; + ref: () => Promise; + focus: (signal?: AbortSignal) => ReturnType; + blur: (signal?: AbortSignal) => ReturnType; + calls: string[]; + afterCommand: { current: (method: string) => void }; + }) => Promise, +) { + // Reuse the existing isolated Chrome launcher; no new browser dependency. + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + await withChrome( + { executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + await send("Page.bringToFront", {}, sessionId); + const evaluate = async (expression: string) => { + const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { expression, returnByValue: true }, + sessionId, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await evaluate( + `document.body.innerHTML = ''; window.probe = document.querySelector('#target');`, + ); + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + const calls: string[] = []; + const afterCommand = { current: (_method: string) => {} }; + const cdp: CdpRunner = { + send: async (_tabId, method, params) => { + calls.push(method); + const result = await send(method, params, sessionId); + afterCommand.current(method); + return result as never; + }, + }; + const deps = { + cdp, + tabsApi: { + get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], + }, + }; + await run({ + evaluate, + calls, + afterCommand, + ref: async () => { + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { expression: "window.probe" }, + sessionId, + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionId, + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId); + ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 }); + }, + focus: (signal) => + handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }), + blur: (signal) => + handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }), + }); + }, + ); +} + +describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => { + it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => { + await withFocusBrowser(async (h) => { + if (mode !== "light") + await h.evaluate(`(() => { + const host = document.createElement('div'); document.body.append(host); + const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested); + const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input); + window.probe = input; + })()`); + await h.ref(); + expect(await h.focus()).toMatchObject({ focused: true }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true); + expect(await h.blur()).toMatchObject({ was_focused: true, focused: false }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false); + expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2); + }); + }); + + it("detects focus changes from event-handler microtasks", async () => { + await withFocusBrowser(async (h) => { + await h.ref(); + await h.evaluate( + "window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})", + ); + expect(await h.focus()).toMatchObject({ code: "cdp_failed" }); + expect(await h.focus()).toMatchObject({ focused: true }); + await h.evaluate( + "window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})", + ); + expect(await h.blur()).toMatchObject({ code: "cdp_failed" }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true); + }); + }); + + it("cancels before blur while still releasing the resolved object", async () => { + await withFocusBrowser(async (h) => { + await h.ref(); + await h.focus(); + const abort = new AbortController(); + h.afterCommand.current = (method) => { + if (method === "DOM.resolveNode") abort.abort(); + }; + expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true); + expect(h.calls.at(-1)).toBe("Runtime.releaseObject"); + }); + }); + + it("reports a page exception with its cause", async () => { + await withFocusBrowser(async (h) => { + await h.ref(); + await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }"); + expect(await h.blur()).toMatchObject({ + code: "cdp_failed", + message: expect.stringContaining("page blur failed"), + }); + expect(h.calls.at(-1)).toBe("Runtime.releaseObject"); + }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/focus.test.ts b/apps/extension/src/tools/__tests__/focus.test.ts new file mode 100644 index 00000000..60d50938 --- /dev/null +++ b/apps/extension/src/tools/__tests__/focus.test.ts @@ -0,0 +1,293 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import { handleBlur, handleFocus } from "../interaction"; +import type { CdpRunner } from "../shared"; + +interface ScriptParams { + functionDeclaration: string; +} +interface Call { + target: CdpTarget; + method: string; +} + +async function setup(element?: HTMLElement, childSession = false) { + if (!element) { + document.body.innerHTML = ''; + element = document.querySelector("#target")!; + } + const targetElement = element; + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e1", 12, { + tabId: 4, + ...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}), + }); + const calls: Call[] = []; + const afterCommand = vi.fn<(call: Call) => void>(); + const script = vi.fn(async (params: ScriptParams) => { + try { + const fn = new Function(`return (${params.functionDeclaration})`)(); + return { result: { value: await fn.call(targetElement) } }; + } catch (error) { + return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } }; + } + }); + const focus = vi.fn(() => targetElement.focus()); + const release = vi.fn(async () => ({})); + const send = async (target: CdpTarget, method: string, params?: object) => { + const call = { target, method }; + calls.push(call); + try { + switch (method) { + case "DOM.getDocument": + return { root: { nodeId: 1 } }; + case "DOM.querySelector": + return { nodeId: 2 }; + case "DOM.describeNode": + return { node: { backendNodeId: 12 } }; + case "DOM.scrollIntoViewIfNeeded": + return {}; + case "DOM.resolveNode": + return { object: { objectId: "focus-target" } }; + case "DOM.focus": + focus(); + return {}; + case "Runtime.callFunctionOn": + return await script(params as ScriptParams); + case "Runtime.releaseObject": + return await release(); + default: + throw new Error(`unexpected ${method}`); + } + } finally { + afterCommand(call); + } + }; + const cdp: CdpRunner = { + send: (tabId, method, params) => send({ tabId }, method, params) as never, + sendToTarget: send as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + ...(childSession + ? [ + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ] + : []), + ], + })), + }; + const tabsApi = { + get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + return { + element: targetElement, + calls, + afterCommand, + script, + focus, + release, + cdp, + tabsApi, + run: ( + action: "focus" | "blur", + signal?: AbortSignal, + params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" }, + ) => + (action === "focus" ? handleFocus : handleBlur)( + manager, + { session_id: "aa11", ...params }, + { cdp, tabsApi, signal }, + ), + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + document.body.innerHTML = ""; +}); + +describe("focus and blur state verification", () => { + it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => { + const host = document.createElement("div"); + document.body.append(host); + const outer = host.attachShadow({ mode }); + const innerHost = document.createElement("div"); + outer.append(innerHost); + const inner = innerHost.attachShadow({ mode }); + const element = document.createElement("input"); + inner.append(element); + const h = await setup(element); + expect(await h.run("focus")).toMatchObject({ focused: true }); + expect(inner.activeElement).toBe(element); + expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false }); + expect(inner.activeElement).toBeNull(); + expect(h.release).toHaveBeenCalledTimes(2); + }); + + it("keeps DOM focus usable when the document is in the background", async () => { + const h = await setup(); + vi.spyOn(document, "hasFocus").mockReturnValue(false); + expect(await h.run("focus")).toMatchObject({ focused: true }); + expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false }); + }); + + it("blurring an unfocused target preserves the other element's focus", async () => { + const h = await setup(); + const other = document.querySelector("#other")!; + other.focus(); + expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false }); + expect(document.activeElement).toBe(other); + }); + + it("detects focus redirected by a focus handler's microtask", async () => { + const h = await setup(); + h.element.addEventListener("focus", () => + queueMicrotask(() => document.querySelector("#other")!.focus()), + ); + expect(await h.run("focus")).toMatchObject({ + code: "cdp_failed", + message: "target element did not become focused", + }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it("does not report blur success when a microtask restores focus", async () => { + const h = await setup(); + h.element.focus(); + h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus())); + expect(await h.run("blur")).toMatchObject({ + code: "cdp_failed", + message: "target element remained focused after blur()", + }); + expect(document.activeElement).toBe(h.element); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => { + const h = await setup(); + h.element.remove(); + expect(await h.run(action)).toMatchObject({ code: "cdp_failed" }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it("preserves a script exception's cause", async () => { + const h = await setup(); + vi.spyOn(h.element, "blur").mockImplementation(() => { + throw new Error("page blur failed"); + }); + expect(await h.run("blur")).toMatchObject({ + code: "cdp_failed", + message: expect.stringContaining("page blur failed"), + }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => { + const h = await setup(); + h.script.mockResolvedValueOnce({ result: { value: null } }); + expect(await h.run(action)).toMatchObject({ + code: "cdp_failed", + message: expect.stringContaining("unexpected result"), + }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it.each([ + "focus", + "blur", + ] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => { + const h = await setup(undefined, true); + if (action === "blur") h.element.focus(); + expect(await h.run(action)).toMatchObject({ focused: action === "focus" }); + const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded"); + expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true); + expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject"); + }); + + it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => { + const h = await setup(); + h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab); + expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({ + code: "permission_denied", + }); + expect(h.calls).toEqual([]); + }); + + it("does not replace a successful result when navigation disposed the remote object", async () => { + const h = await setup(); + h.release.mockRejectedValue(new Error("Cannot find context")); + expect(await h.run("focus")).toMatchObject({ focused: true }); + }); +}); + +describe("focus and blur cancellation", () => { + const boundaries = [ + "DOM.getDocument", + "DOM.querySelector", + "DOM.describeNode", + "DOM.resolveNode", + "Runtime.callFunctionOn", + ]; + for (const action of ["focus", "blur"] as const) { + it.each( + action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries, + )(`${action} stops after cancellation during %s`, async (method) => { + const h = await setup(); + const abort = new AbortController(); + let cancelledAfter = 0; + h.afterCommand.mockImplementation((call) => { + if (call.method === method && !abort.signal.aborted) { + cancelledAfter = h.calls.length; + abort.abort(); + } + }); + expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({ + code: "cancelled", + }); + expect(abort.signal.aborted).toBe(true); + expect( + h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"), + ).toBe(true); + if (h.calls.some((call) => call.method === "DOM.resolveNode")) + expect(h.release).toHaveBeenCalledOnce(); + }); + + it(`${action} stops if cancellation arrives while resolving the tab`, async () => { + const h = await setup(); + const abort = new AbortController(); + h.tabsApi.query.mockImplementation(async () => { + abort.abort(); + return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]; + }); + expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" }); + expect(h.calls).toEqual([]); + }); + } + + it("does not scroll frame owners after cancellation in an OOPIF", async () => { + const h = await setup(undefined, true); + const abort = new AbortController(); + h.afterCommand.mockImplementation((call) => { + if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort(); + }); + expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" }); + expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1); + expect(h.focus).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 1638d315..00799d71 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -553,7 +553,7 @@ describe("handleFocus and handleBlur", () => { "DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.focus": () => ({}), "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), - "Runtime.callFunctionOn": () => ({ result: { value: { focused: true } } }), + "Runtime.callFunctionOn": () => ({ result: { value: true } }), }); const res = await handleFocus( @@ -566,9 +566,10 @@ describe("handleFocus and handleBlur", () => { expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true }); expect(fake.sent.map((call) => call.method)).toEqual([ "DOM.scrollIntoViewIfNeeded", - "DOM.focus", "DOM.resolveNode", + "DOM.focus", "Runtime.callFunctionOn", + "Runtime.releaseObject", ]); }); @@ -599,9 +600,9 @@ describe("handleFocus and handleBlur", () => { fake.cdp.sendToTarget = vi.fn(async (target, method) => { targetCalls.push({ sessionId: target.sessionId, method }); if (method === "DOM.scrollIntoViewIfNeeded") return {}; - if (method === "DOM.focus") return {}; + if (method === "DOM.focus" || method === "Runtime.releaseObject") return {}; if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } }; - if (method === "Runtime.callFunctionOn") return { result: { value: { focused: true } } }; + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; throw new Error(`unexpected child CDP call ${method}`); }) as CdpRunner["sendToTarget"]; @@ -615,9 +616,10 @@ describe("handleFocus and handleBlur", () => { expect(res.focused).toBe(true); expect(targetCalls).toEqual([ { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, - { sessionId: "child-session", method: "DOM.focus" }, { sessionId: "child-session", method: "DOM.resolveNode" }, + { sessionId: "child-session", method: "DOM.focus" }, { sessionId: "child-session", method: "Runtime.callFunctionOn" }, + { sessionId: "child-session", method: "Runtime.releaseObject" }, ]); }); @@ -649,9 +651,10 @@ describe("handleFocus and handleBlur", () => { ctx.refStore.set("e3", 1234, { tabId: 4 }); const fake = makeFakeCdp({ "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), - "Runtime.callFunctionOn": () => ({ - result: { value: { ok: true, was_focused: true, focused: false } }, - }), + "Runtime.callFunctionOn": vi + .fn() + .mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } }) + .mockResolvedValueOnce({ result: { value: false } }), }); const res = await handleBlur( @@ -670,6 +673,8 @@ describe("handleFocus and handleBlur", () => { expect(fake.sent.map((call) => call.method)).toEqual([ "DOM.resolveNode", "Runtime.callFunctionOn", + "Runtime.callFunctionOn", + "Runtime.releaseObject", ]); }); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index a980a161..6d2d9404 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -230,154 +230,204 @@ export async function resolveBackendNode( } } -type FocusCheck = { focused: boolean }; -type BlurMutation = - | { ok: true; was_focused: boolean; focused: boolean } - | { ok: false; was_focused: boolean; focused: boolean }; - -const DEEP_FOCUS_CHECK = `function() { - const deepActiveElement = (root) => { - let active = root && root.activeElement; - while (active && active.shadowRoot && active.shadowRoot.activeElement) { - active = active.shadowRoot.activeElement; - } - return active; - }; - return { focused: deepActiveElement(this.ownerDocument) === this }; +// Check the target's own root, then its hosts: closed shadow roots are not +// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs. +const FOCUS_CHECK = `function() { + if (!this.isConnected) return false; + let element = this; + while (element) { + const root = element.getRootNode(); + if (root.activeElement !== element) return false; + element = root.host; + } + return true; }`; const BLUR_TARGET = `function() { - const deepActiveElement = (root) => { - let active = root && root.activeElement; - while (active && active.shadowRoot && active.shadowRoot.activeElement) { - active = active.shadowRoot.activeElement; - } - return active; - }; - const wasFocused = deepActiveElement(this.ownerDocument) === this; - if (typeof this.blur !== 'function') { - return { ok: false, was_focused: wasFocused, focused: wasFocused }; - } + if (!this.isConnected) throw new Error('blur target is detached'); + const wasFocused = (${FOCUS_CHECK}).call(this); + if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused }; this.blur(); + return { ok: true, was_focused: wasFocused }; +}`; + +interface FocusScriptReply { + result?: { value?: T }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +function checkFocusAbort(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError"); +} + +// Only focus/blur use this view. Guard every CDP boundary, including commands +// inside selector resolution and frame scrolling, without changing other tools. +function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner { + const send = (target: CdpTarget, method: string, params?: object): Promise => { + checkFocusAbort(signal); + return cdpRunnerForTarget(cdp, target).send(target.tabId, method, params); + }; return { - ok: true, - was_focused: wasFocused, - focused: deepActiveElement(this.ownerDocument) === this, + send: (tabId, method, params) => send({ tabId }, method, params), + sendToTarget: send, + trackSessionTab: cdp.trackSessionTab?.bind(cdp), + getFrameGraph: cdp.getFrameGraph + ? (tabId) => { + checkFocusAbort(signal); + return cdp.getFrameGraph!(tabId); + } + : undefined, }; -}`; +} // --------------------------------------------------------------------------- // tool.focus / tool.blur // --------------------------------------------------------------------------- -export async function handleFocus( +export function handleFocus( manager: SessionManager, params: FocusParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { - const ctxOrErr = lookupSession(manager, params, "focus"); - if (isRpcError(ctxOrErr)) return ctxOrErr; - const ctx = ctxOrErr; - const aborted = throwIfAborted(deps.signal); - if (aborted) return aborted; - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); - if (isRpcError(target)) return target; - const denied = enforceAgentWindow(ctx, target, "focus"); - if (denied) return denied; - const dialogCursor = markDialogCursor(deps.cdp, target.tabId); - const node = await resolveBackendNode(deps.cdp, ctx, target, params, "focus"); - if (isRpcError(node)) return node; - const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); - - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const scrollErr = await scrollElementAndFramesIntoView( - deps.cdp, - target.tabId, - node.cdpTarget, - node.backendNodeId, - node.frameId, - ); - if (scrollErr) return scrollErr; - const abortedBeforeFocus = throwIfAborted(deps.signal); - if (abortedBeforeFocus) return abortedBeforeFocus; - - try { - await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); - const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); - if (isRpcError(objectIdOrErr)) return objectIdOrErr; - const evaluated = await nodeCdp.send<{ result?: { value?: FocusCheck } }>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId: objectIdOrErr, - functionDeclaration: DEEP_FOCUS_CHECK, - returnByValue: true, - }, - ); - if (evaluated.result?.value?.focused !== true) { - return { code: "invalid_params", message: "target element did not become focused" }; - } - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, - focused: true, - }); - } catch (err) { - return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; - } + return changeFocus(manager, params, deps, "focus"); } -export async function handleBlur( +export function handleBlur( manager: SessionManager, params: BlurParams, deps: InteractionDeps = getDefaultDeps(), ): Promise { - const ctxOrErr = lookupSession(manager, params, "blur"); - if (isRpcError(ctxOrErr)) return ctxOrErr; - const ctx = ctxOrErr; - const aborted = throwIfAborted(deps.signal); - if (aborted) return aborted; - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); - if (isRpcError(target)) return target; - const denied = enforceAgentWindow(ctx, target, "blur"); - if (denied) return denied; - const dialogCursor = markDialogCursor(deps.cdp, target.tabId); - const node = await resolveBackendNode(deps.cdp, ctx, target, params, "blur"); - if (isRpcError(node)) return node; - const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const abortedBeforeBlur = throwIfAborted(deps.signal); - if (abortedBeforeBlur) return abortedBeforeBlur; + return changeFocus(manager, params, deps, "blur"); +} +function changeFocus( + manager: SessionManager, + params: FocusParams, + deps: InteractionDeps, + action: "focus", +): Promise; +function changeFocus( + manager: SessionManager, + params: BlurParams, + deps: InteractionDeps, + action: "blur", +): Promise; +async function changeFocus( + manager: SessionManager, + params: FocusParams | BlurParams, + deps: InteractionDeps, + action: "focus" | "blur", +): Promise { + const ctx = lookupSession(manager, params, action); + if (isRpcError(ctx)) return ctx; + const cdp = focusCdp(deps.cdp, deps.signal); + let objectId: string | undefined; + let cleanupCdp: CdpRunner | undefined; + let tabId: number | undefined; try { - const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); - if (isRpcError(objectIdOrErr)) return objectIdOrErr; - const evaluated = await nodeCdp.send<{ result?: { value?: BlurMutation } }>( - target.tabId, - "Runtime.callFunctionOn", - { - objectId: objectIdOrErr, - functionDeclaration: BLUR_TARGET, - returnByValue: true, - }, - ); - const mutation = evaluated.result?.value; - if (!mutation?.ok) { - return { code: "invalid_params", message: "target element does not support blur()" }; + checkFocusAbort(deps.signal); + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + checkFocusAbort(deps.signal); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, action); + if (denied) return denied; + tabId = target.tabId; + const dialogCursor = markDialogCursor(deps.cdp, tabId); + const node = await resolveBackendNode(cdp, ctx, target, params, action); + checkFocusAbort(deps.signal); + if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget); + cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + deps.cdp.trackSessionTab?.(ctx.sessionId, tabId); + + if (action === "focus") { + const scrollErr = await scrollElementAndFramesIntoView( + cdp, + tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + checkFocusAbort(deps.signal); + if (scrollErr) return scrollErr; } - if (mutation.focused) { - return { code: "cdp_failed", message: "target element remained focused after blur()" }; + const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId); + // Keep the handle before checking cancellation so finally can release it. + if (!isRpcError(resolved)) objectId = resolved; + checkFocusAbort(deps.signal); + if (isRpcError(resolved)) return resolved; + + const runScript = async (functionDeclaration: string): Promise => { + const reply = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration, + returnByValue: true, + }, + ); + checkFocusAbort(deps.signal); + if (reply.exceptionDetails) { + const details = reply.exceptionDetails; + throw new Error( + `${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`, + ); + } + return reply.result?.value; + }; + + let wasFocused: boolean | undefined; + if (action === "focus") { + await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + checkFocusAbort(deps.signal); + } else { + const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET); + if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") { + throw new Error("blur script returned an unexpected result"); + } + if (!mutation.ok) { + return { code: "invalid_params", message: "target element does not support blur()" }; + } + wasFocused = mutation.was_focused; } - return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - tab_id: target.tabId, + // Use a separate call so microtasks from focus/blur handlers finish before + // we report success. In particular, a blur handler can restore focus. + const focused = await runScript(FOCUS_CHECK); + if (typeof focused !== "boolean") { + throw new Error(`${action} verification returned an unexpected result`); + } + if (focused !== (action === "focus")) { + throw new Error( + action === "focus" + ? "target element did not become focused" + : "target element remained focused after blur()", + ); + } + return attachDialogs(deps.cdp, tabId, dialogCursor, { + tab_id: tabId, used_ref: node.usedRef, used_selector: node.usedSelector, - was_focused: mutation.was_focused, - focused: false, + focused, + ...(action === "blur" ? { was_focused: wasFocused! } : {}), }); } catch (err) { - return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + return ( + throwIfAborted(deps.signal) ?? { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + } + ); + } finally { + if (objectId !== undefined && cleanupCdp && tabId !== undefined) { + // Cleanup must also run after cancellation. Navigation may have already + // disposed the object, which must not replace the operation's result. + try { + await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId }); + } catch { + // The target or execution context may no longer exist. + } + } } } diff --git a/crates/bsk-protocol/schema/tool_focus_result.json b/crates/bsk-protocol/schema/tool_focus_result.json index 28d7233f..196501da 100644 --- a/crates/bsk-protocol/schema/tool_focus_result.json +++ b/crates/bsk-protocol/schema/tool_focus_result.json @@ -14,7 +14,7 @@ } }, "focused": { - "description": "Whether the target is the document's deep active element after the call.", + "description": "Whether the target holds DOM focus in its document or shadow root after the call.", "type": "boolean" }, "tab_id": { diff --git a/crates/bsk-protocol/src/tools/interaction.rs b/crates/bsk-protocol/src/tools/interaction.rs index b0b27dac..20004ed6 100644 --- a/crates/bsk-protocol/src/tools/interaction.rs +++ b/crates/bsk-protocol/src/tools/interaction.rs @@ -164,7 +164,7 @@ pub struct FocusResult { pub used_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub used_selector: Option, - /// Whether the target is the document's deep active element after the call. + /// Whether the target holds DOM focus in its document or shadow root after the call. pub focused: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dialogs: Vec, From 20e87700064529c5b509650447c86a2c80c93c95 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 9 Sep 2026 22:47:47 +0800 Subject: [PATCH 3/3] feat(dsh-plugin): expose focus and blur actions --- packages/dsh-plugin-browserskill/README.md | 2 +- .../dsh-plugin-browserskill/skill/SKILL.md | 4 +- .../src/browser-tools.ts | 6 +- .../src/observation.ts | 4 + .../src/phase-one-tools-interaction.ts | 84 ++++++++++++++++++- .../tests/tools.test.ts | 58 ++++++++++++- 6 files changed, 151 insertions(+), 7 deletions(-) diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index 83b3a6af..e02453a0 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 464a24ea..6ca293ad 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,8 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a -hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are +Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur +explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index b69a83d6..3204eed8 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " + - "press. click/hover/fill/select require target; fill also requires value; select requires " + + "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + + "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + "values; press requires key and may optionally focus target first.", actions: { click: "interact.click", hover: "interact.hover", + focus: "interact.focus", + blur: "interact.blur", fill: "interact.fill", select: "interact.select", press: "interact.press", diff --git a/packages/dsh-plugin-browserskill/src/observation.ts b/packages/dsh-plugin-browserskill/src/observation.ts index efdd9c15..31f13c3d 100644 --- a/packages/dsh-plugin-browserskill/src/observation.ts +++ b/packages/dsh-plugin-browserskill/src/observation.ts @@ -562,6 +562,10 @@ export function actionForLabel(label: string): string { return "clicking"; case "hover": return "hovering"; + case "focus": + return "focusing"; + case "blur": + return "blurring"; case "fill": return "filling"; case "select": diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 3c7c4f90..858fcc2f 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools"; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; -/** Add hover and select without bypassing session ownership or observation. */ +/** Add hover, focus, blur and select without bypassing session ownership or observation. */ export function registerPhaseOneInteractionTools( deps: ToolDeps, register: ToolRegistrar, @@ -100,6 +100,88 @@ export function registerPhaseOneInteractionTools( }), ); + for (const action of ["focus", "blur"] as const) { + register( + defineTool({ + name: `interact.${action}`, + description: + action === "focus" + ? "Focus an element and verify its DOM focus state." + : "Remove focus from an element and report whether it was focused.", + parameters: { + target: { + type: "string", + required: true, + description: "Snapshot ref (@e3 / e3) or CSS selector of the element.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + focused: { type: "boolean", required: true }, + wasFocused: { + type: "boolean", + description: "Whether the target was focused before blur.", + }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: + `[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` + + (value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`), + }, + ], + }, + async execute(args, exec) { + requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`); + const cmdArgs = [action, "--session", sessionId]; + appendTabId(cmdArgs, args.tabId); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + action, + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + focused: boolean; + was_focused?: boolean; + }; + return { + session: sessionId, + tabId: reply.tab_id, + focused: reply.focused, + ...(action === "blur" ? { wasFocused: reply.was_focused } : {}), + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + action, + args.target, + "--session", + args.session ?? "(current)", + ]), + description: action === "focus" ? "Focus an element" : "Remove focus from an element", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + } + register( defineTool({ name: "interact.select", diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index 7faf3fa5..e1441997 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,8 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.focus": ["browser_interact", "focus"], + "interact.blur": ["browser_interact", "blur"], "interact.fill": ["browser_interact", "fill"], "interact.select": ["browser_interact", "select"], "interact.press": ["browser_interact", "press"], @@ -200,7 +202,7 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "fill", "select", "press"], + browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -499,6 +501,60 @@ describe("interaction tools", () => { ]); }); + it.each([ + "focus", + "blur", + ] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + [action]: { + tab_id: 7, + focused: action === "focus", + ...(action === "blur" ? { was_focused: true } : {}), + }, + }); + await startSession(tools); + const target = action === "focus" ? "@e3" : "#field"; + const value = await tools + .get(`interact.${action}`) + ?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()); + expect(value).toEqual({ + session: "s1", + tabId: 7, + focused: action === "focus", + ...(action === "blur" ? { wasFocused: true } : {}), + }); + expect(calls[1].args).toEqual([ + action, + "--session", + "s1", + "--tab-id", + "7", + "--timeout", + "150000ms", + target, + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + + it.each([ + "focus", + "blur", + ] as const)("interact.%s validates arguments and session ownership before running", async (action) => { + const { tools, calls } = setup({ "session start": START_REPLY("s1") }); + await startSession(tools); + const tool = tools.get(`interact.${action}`)!; + await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i); + await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i); + await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow( + /greater than zero/i, + ); + await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow( + /session/i, + ); + expect(calls).toHaveLength(1); + }); + it("interact.fill passes value and --no-clear", async () => { const { tools, calls } = setup(responses); await startSession(tools);