diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index b1e2bf00..1390ee35 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -599,6 +599,70 @@ describe("ToolDispatcher", () => { expect(onSessionsChanged).toHaveBeenCalledTimes(2); }); + it.each([ + "focus", + "blur", + ] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => { + const tab = { id: 7, windowId: 4242, active: true }; + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async () => tab), + query: vi.fn(async () => [tab]), + sendMessage: vi.fn(async () => undefined), + }, + }); + const sessions = new SessionManager({ + agentWindow: { + create: async () => 4242, + remove: async () => {}, + ensureActiveTab: async () => 7, + }, + }); + const ctx = await sessions.start("aa11"); + ctx.refStore.set("e1", 12, { tabId: 7 }); + const { transport, sent, deliver } = fakeTransport(); + const onBrowserControlResumed = vi.fn(); + let resolveNode: ((value: object) => void) | undefined; + const cdp = { + send: vi.fn(async (_tabId: number, method: string) => { + if (method === "DOM.resolveNode") + return new Promise((resolve) => { + resolveNode = resolve; + }); + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + return {}; + }), + } as unknown as TestDispatcherCdp; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed }); + const hover = dispatcher as unknown as { + rememberHover: (sessionId: string, result: object) => void; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + }; + hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + await hover.setHoverBypass("aa11", 7, true); + dispatcher.start(); + deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" })); + await vi.waitFor(() => expect(resolveNode).toBeDefined()); + expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); + deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } }); + resolveNode!({ object: { objectId: "focus-target" } }); + await vi.waitFor(() => expect(sent).toHaveLength(2)); + expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } }); + expect(sent).toContainEqual({ + id: "r-1", + error: expect.objectContaining({ code: "cancelled" }), + }); + expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything()); + expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" }); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + expect(dispatcher.inflightAbortControllers.size).toBe(0); + dispatcher.stop(); + }); + it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => { vi.stubGlobal("chrome", { tabs: { diff --git a/apps/extension/src/tools/__tests__/focus.browser.test.ts b/apps/extension/src/tools/__tests__/focus.browser.test.ts new file mode 100644 index 00000000..d7e91d10 --- /dev/null +++ b/apps/extension/src/tools/__tests__/focus.browser.test.ts @@ -0,0 +1,166 @@ +// @vitest-environment node +// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile. +import { describe, expect, it } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import { handleBlur, handleFocus } from "../interaction"; +import type { CdpRunner } from "../shared"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; + +async function withFocusBrowser( + run: (h: { + evaluate: (expression: string) => Promise; + ref: () => Promise; + focus: (signal?: AbortSignal) => ReturnType; + blur: (signal?: AbortSignal) => ReturnType; + calls: string[]; + afterCommand: { current: (method: string) => void }; + }) => Promise, +) { + // Reuse the existing isolated Chrome launcher; no new browser dependency. + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + await withChrome( + { executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + await send("Page.bringToFront", {}, sessionId); + const evaluate = async (expression: string) => { + const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { expression, returnByValue: true }, + sessionId, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await evaluate( + `document.body.innerHTML = ''; window.probe = document.querySelector('#target');`, + ); + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + const calls: string[] = []; + const afterCommand = { current: (_method: string) => {} }; + const cdp: CdpRunner = { + send: async (_tabId, method, params) => { + calls.push(method); + const result = await send(method, params, sessionId); + afterCommand.current(method); + return result as never; + }, + }; + const deps = { + cdp, + tabsApi: { + get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab, + query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab], + }, + }; + await run({ + evaluate, + calls, + afterCommand, + ref: async () => { + const { result } = await send<{ result: { objectId: string } }>( + "Runtime.evaluate", + { expression: "window.probe" }, + sessionId, + ); + const { node } = await send<{ node: { backendNodeId: number } }>( + "DOM.describeNode", + { objectId: result.objectId }, + sessionId, + ); + await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId); + ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 }); + }, + focus: (signal) => + handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }), + blur: (signal) => + handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }), + }); + }, + ); +} + +describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => { + it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => { + await withFocusBrowser(async (h) => { + if (mode !== "light") + await h.evaluate(`(() => { + const host = document.createElement('div'); document.body.append(host); + const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested); + const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input); + window.probe = input; + })()`); + await h.ref(); + expect(await h.focus()).toMatchObject({ focused: true }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true); + expect(await h.blur()).toMatchObject({ was_focused: true, focused: false }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false); + expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2); + }); + }); + + it("detects focus changes from event-handler microtasks", async () => { + await withFocusBrowser(async (h) => { + await h.ref(); + await h.evaluate( + "window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})", + ); + expect(await h.focus()).toMatchObject({ code: "cdp_failed" }); + expect(await h.focus()).toMatchObject({ focused: true }); + await h.evaluate( + "window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})", + ); + expect(await h.blur()).toMatchObject({ code: "cdp_failed" }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true); + }); + }); + + it("cancels before blur while still releasing the resolved object", async () => { + await withFocusBrowser(async (h) => { + await h.ref(); + await h.focus(); + const abort = new AbortController(); + h.afterCommand.current = (method) => { + if (method === "DOM.resolveNode") abort.abort(); + }; + expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" }); + expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true); + expect(h.calls.at(-1)).toBe("Runtime.releaseObject"); + }); + }); + + it("reports a page exception with its cause", async () => { + await withFocusBrowser(async (h) => { + await h.ref(); + await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }"); + expect(await h.blur()).toMatchObject({ + code: "cdp_failed", + message: expect.stringContaining("page blur failed"), + }); + expect(h.calls.at(-1)).toBe("Runtime.releaseObject"); + }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/focus.test.ts b/apps/extension/src/tools/__tests__/focus.test.ts new file mode 100644 index 00000000..60d50938 --- /dev/null +++ b/apps/extension/src/tools/__tests__/focus.test.ts @@ -0,0 +1,293 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import { handleBlur, handleFocus } from "../interaction"; +import type { CdpRunner } from "../shared"; + +interface ScriptParams { + functionDeclaration: string; +} +interface Call { + target: CdpTarget; + method: string; +} + +async function setup(element?: HTMLElement, childSession = false) { + if (!element) { + document.body.innerHTML = ''; + element = document.querySelector("#target")!; + } + const targetElement = element; + const manager = new SessionManager({ + agentWindow: { + create: async () => 100, + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e1", 12, { + tabId: 4, + ...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}), + }); + const calls: Call[] = []; + const afterCommand = vi.fn<(call: Call) => void>(); + const script = vi.fn(async (params: ScriptParams) => { + try { + const fn = new Function(`return (${params.functionDeclaration})`)(); + return { result: { value: await fn.call(targetElement) } }; + } catch (error) { + return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } }; + } + }); + const focus = vi.fn(() => targetElement.focus()); + const release = vi.fn(async () => ({})); + const send = async (target: CdpTarget, method: string, params?: object) => { + const call = { target, method }; + calls.push(call); + try { + switch (method) { + case "DOM.getDocument": + return { root: { nodeId: 1 } }; + case "DOM.querySelector": + return { nodeId: 2 }; + case "DOM.describeNode": + return { node: { backendNodeId: 12 } }; + case "DOM.scrollIntoViewIfNeeded": + return {}; + case "DOM.resolveNode": + return { object: { objectId: "focus-target" } }; + case "DOM.focus": + focus(); + return {}; + case "Runtime.callFunctionOn": + return await script(params as ScriptParams); + case "Runtime.releaseObject": + return await release(); + default: + throw new Error(`unexpected ${method}`); + } + } finally { + afterCommand(call); + } + }; + const cdp: CdpRunner = { + send: (tabId, method, params) => send({ tabId }, method, params) as never, + sendToTarget: send as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + ...(childSession + ? [ + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ] + : []), + ], + })), + }; + const tabsApi = { + get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + return { + element: targetElement, + calls, + afterCommand, + script, + focus, + release, + cdp, + tabsApi, + run: ( + action: "focus" | "blur", + signal?: AbortSignal, + params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" }, + ) => + (action === "focus" ? handleFocus : handleBlur)( + manager, + { session_id: "aa11", ...params }, + { cdp, tabsApi, signal }, + ), + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + document.body.innerHTML = ""; +}); + +describe("focus and blur state verification", () => { + it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => { + const host = document.createElement("div"); + document.body.append(host); + const outer = host.attachShadow({ mode }); + const innerHost = document.createElement("div"); + outer.append(innerHost); + const inner = innerHost.attachShadow({ mode }); + const element = document.createElement("input"); + inner.append(element); + const h = await setup(element); + expect(await h.run("focus")).toMatchObject({ focused: true }); + expect(inner.activeElement).toBe(element); + expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false }); + expect(inner.activeElement).toBeNull(); + expect(h.release).toHaveBeenCalledTimes(2); + }); + + it("keeps DOM focus usable when the document is in the background", async () => { + const h = await setup(); + vi.spyOn(document, "hasFocus").mockReturnValue(false); + expect(await h.run("focus")).toMatchObject({ focused: true }); + expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false }); + }); + + it("blurring an unfocused target preserves the other element's focus", async () => { + const h = await setup(); + const other = document.querySelector("#other")!; + other.focus(); + expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false }); + expect(document.activeElement).toBe(other); + }); + + it("detects focus redirected by a focus handler's microtask", async () => { + const h = await setup(); + h.element.addEventListener("focus", () => + queueMicrotask(() => document.querySelector("#other")!.focus()), + ); + expect(await h.run("focus")).toMatchObject({ + code: "cdp_failed", + message: "target element did not become focused", + }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it("does not report blur success when a microtask restores focus", async () => { + const h = await setup(); + h.element.focus(); + h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus())); + expect(await h.run("blur")).toMatchObject({ + code: "cdp_failed", + message: "target element remained focused after blur()", + }); + expect(document.activeElement).toBe(h.element); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => { + const h = await setup(); + h.element.remove(); + expect(await h.run(action)).toMatchObject({ code: "cdp_failed" }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it("preserves a script exception's cause", async () => { + const h = await setup(); + vi.spyOn(h.element, "blur").mockImplementation(() => { + throw new Error("page blur failed"); + }); + expect(await h.run("blur")).toMatchObject({ + code: "cdp_failed", + message: expect.stringContaining("page blur failed"), + }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => { + const h = await setup(); + h.script.mockResolvedValueOnce({ result: { value: null } }); + expect(await h.run(action)).toMatchObject({ + code: "cdp_failed", + message: expect.stringContaining("unexpected result"), + }); + expect(h.release).toHaveBeenCalledOnce(); + }); + + it.each([ + "focus", + "blur", + ] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => { + const h = await setup(undefined, true); + if (action === "blur") h.element.focus(); + expect(await h.run(action)).toMatchObject({ focused: action === "focus" }); + const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded"); + expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true); + expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject"); + }); + + it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => { + const h = await setup(); + h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab); + expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({ + code: "permission_denied", + }); + expect(h.calls).toEqual([]); + }); + + it("does not replace a successful result when navigation disposed the remote object", async () => { + const h = await setup(); + h.release.mockRejectedValue(new Error("Cannot find context")); + expect(await h.run("focus")).toMatchObject({ focused: true }); + }); +}); + +describe("focus and blur cancellation", () => { + const boundaries = [ + "DOM.getDocument", + "DOM.querySelector", + "DOM.describeNode", + "DOM.resolveNode", + "Runtime.callFunctionOn", + ]; + for (const action of ["focus", "blur"] as const) { + it.each( + action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries, + )(`${action} stops after cancellation during %s`, async (method) => { + const h = await setup(); + const abort = new AbortController(); + let cancelledAfter = 0; + h.afterCommand.mockImplementation((call) => { + if (call.method === method && !abort.signal.aborted) { + cancelledAfter = h.calls.length; + abort.abort(); + } + }); + expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({ + code: "cancelled", + }); + expect(abort.signal.aborted).toBe(true); + expect( + h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"), + ).toBe(true); + if (h.calls.some((call) => call.method === "DOM.resolveNode")) + expect(h.release).toHaveBeenCalledOnce(); + }); + + it(`${action} stops if cancellation arrives while resolving the tab`, async () => { + const h = await setup(); + const abort = new AbortController(); + h.tabsApi.query.mockImplementation(async () => { + abort.abort(); + return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]; + }); + expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" }); + expect(h.calls).toEqual([]); + }); + } + + it("does not scroll frame owners after cancellation in an OOPIF", async () => { + const h = await setup(undefined, true); + const abort = new AbortController(); + h.afterCommand.mockImplementation((call) => { + if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort(); + }); + expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" }); + expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1); + expect(h.focus).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index a7a4382f..00799d71 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { + handleBlur, handleClick, handleFill, + handleFocus, handleHover, handlePress, handleSelect, @@ -542,6 +544,161 @@ describe("handleHover", () => { }); }); +describe("handleFocus and handleBlur", () => { + it("focuses a ref and verifies the deep active element", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.focus": () => ({}), + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ result: { value: true } }), + }); + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.scrollIntoViewIfNeeded", + "DOM.resolveNode", + "DOM.focus", + "Runtime.callFunctionOn", + "Runtime.releaseObject", + ]); + }); + + it("focuses an OOPIF ref in its CDP session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const targetCalls: Array<{ sessionId?: string; method: string }> = []; + fake.cdp.sendToTarget = vi.fn(async (target, method) => { + targetCalls.push({ sessionId: target.sessionId, method }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.focus" || method === "Runtime.releaseObject") return {}; + if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } }; + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.focused).toBe(true); + expect(targetCalls).toEqual([ + { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, + { sessionId: "child-session", method: "DOM.resolveNode" }, + { sessionId: "child-session", method: "DOM.focus" }, + { sessionId: "child-session", method: "Runtime.callFunctionOn" }, + { sessionId: "child-session", method: "Runtime.releaseObject" }, + ]); + }); + + it("does not focus after cancellation during scrolling", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const abort = new AbortController(); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => { + abort.abort(); + return {}; + }, + }); + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + + expect(res).toMatchObject({ code: "cancelled" }); + expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false); + }); + + it("blurs a ref and returns its previous focus state", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": vi + .fn() + .mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } }) + .mockResolvedValueOnce({ result: { value: false } }), + }); + + const res = await handleBlur( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ + tab_id: 4, + used_ref: "e3", + was_focused: true, + focused: false, + }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Runtime.callFunctionOn", + "Runtime.releaseObject", + ]); + }); + + it("rejects a target that does not implement blur", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ + result: { value: { ok: false, was_focused: false, focused: false } }, + }), + }); + + const res = await handleBlur( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ }); + }); +}); + function successfulFillScript(params: unknown) { const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string }; const args = script.arguments ?? []; diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index bd12db7c..6b1023d1 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -2,12 +2,14 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { Transport } from "@/transport/transport"; import type { + BlurParams, ClickParams, ConsoleParams, DownloadParams, EmulateParams, EvaluateParams, FillParams, + FocusParams, GetHtmlParams, HoverParams, HoverResult, @@ -39,7 +41,15 @@ import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { classifyCdpError } from "./errors"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; +import { + handleBlur, + handleClick, + handleFill, + handleFocus, + handleHover, + handlePress, + handleSelect, +} from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -515,6 +525,28 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.focus": + return this.withHoverReleaseForRequest( + req.params as FocusParams, + () => + handleFocus( + this.sessions, + req.params as FocusParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); + case "tool.blur": + return this.withHoverReleaseForRequest( + req.params as BlurParams, + () => + handleBlur( + this.sessions, + req.params as BlurParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.fill": return this.withHoverReleaseForRequest( req.params as FillParams, @@ -785,6 +817,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.focus": + case "tool.blur": case "tool.fill": case "tool.press": case "tool.select": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 9e36c2d2..6d2d9404 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -1,5 +1,4 @@ -// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and -// `tool.select`. +// DOM interaction tools — click, hover, focus/blur, fill, press, and select. // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). @@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { + BlurParams, + BlurResult, ClickParams, ClickResult, FillParams, FillResult, + FocusParams, + FocusResult, HoverParams, HoverResult, KeyModifier, @@ -227,6 +230,207 @@ export async function resolveBackendNode( } } +// Check the target's own root, then its hosts: closed shadow roots are not +// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs. +const FOCUS_CHECK = `function() { + if (!this.isConnected) return false; + let element = this; + while (element) { + const root = element.getRootNode(); + if (root.activeElement !== element) return false; + element = root.host; + } + return true; +}`; + +const BLUR_TARGET = `function() { + if (!this.isConnected) throw new Error('blur target is detached'); + const wasFocused = (${FOCUS_CHECK}).call(this); + if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused }; + this.blur(); + return { ok: true, was_focused: wasFocused }; +}`; + +interface FocusScriptReply { + result?: { value?: T }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +function checkFocusAbort(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError"); +} + +// Only focus/blur use this view. Guard every CDP boundary, including commands +// inside selector resolution and frame scrolling, without changing other tools. +function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner { + const send = (target: CdpTarget, method: string, params?: object): Promise => { + checkFocusAbort(signal); + return cdpRunnerForTarget(cdp, target).send(target.tabId, method, params); + }; + return { + send: (tabId, method, params) => send({ tabId }, method, params), + sendToTarget: send, + trackSessionTab: cdp.trackSessionTab?.bind(cdp), + getFrameGraph: cdp.getFrameGraph + ? (tabId) => { + checkFocusAbort(signal); + return cdp.getFrameGraph!(tabId); + } + : undefined, + }; +} + +// --------------------------------------------------------------------------- +// tool.focus / tool.blur +// --------------------------------------------------------------------------- + +export function handleFocus( + manager: SessionManager, + params: FocusParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + return changeFocus(manager, params, deps, "focus"); +} + +export function handleBlur( + manager: SessionManager, + params: BlurParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + return changeFocus(manager, params, deps, "blur"); +} + +function changeFocus( + manager: SessionManager, + params: FocusParams, + deps: InteractionDeps, + action: "focus", +): Promise; +function changeFocus( + manager: SessionManager, + params: BlurParams, + deps: InteractionDeps, + action: "blur", +): Promise; +async function changeFocus( + manager: SessionManager, + params: FocusParams | BlurParams, + deps: InteractionDeps, + action: "focus" | "blur", +): Promise { + const ctx = lookupSession(manager, params, action); + if (isRpcError(ctx)) return ctx; + const cdp = focusCdp(deps.cdp, deps.signal); + let objectId: string | undefined; + let cleanupCdp: CdpRunner | undefined; + let tabId: number | undefined; + try { + checkFocusAbort(deps.signal); + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + checkFocusAbort(deps.signal); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, action); + if (denied) return denied; + tabId = target.tabId; + const dialogCursor = markDialogCursor(deps.cdp, tabId); + const node = await resolveBackendNode(cdp, ctx, target, params, action); + checkFocusAbort(deps.signal); + if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget); + cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + deps.cdp.trackSessionTab?.(ctx.sessionId, tabId); + + if (action === "focus") { + const scrollErr = await scrollElementAndFramesIntoView( + cdp, + tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + checkFocusAbort(deps.signal); + if (scrollErr) return scrollErr; + } + const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId); + // Keep the handle before checking cancellation so finally can release it. + if (!isRpcError(resolved)) objectId = resolved; + checkFocusAbort(deps.signal); + if (isRpcError(resolved)) return resolved; + + const runScript = async (functionDeclaration: string): Promise => { + const reply = await nodeCdp.send>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId, + functionDeclaration, + returnByValue: true, + }, + ); + checkFocusAbort(deps.signal); + if (reply.exceptionDetails) { + const details = reply.exceptionDetails; + throw new Error( + `${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`, + ); + } + return reply.result?.value; + }; + + let wasFocused: boolean | undefined; + if (action === "focus") { + await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + checkFocusAbort(deps.signal); + } else { + const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET); + if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") { + throw new Error("blur script returned an unexpected result"); + } + if (!mutation.ok) { + return { code: "invalid_params", message: "target element does not support blur()" }; + } + wasFocused = mutation.was_focused; + } + // Use a separate call so microtasks from focus/blur handlers finish before + // we report success. In particular, a blur handler can restore focus. + const focused = await runScript(FOCUS_CHECK); + if (typeof focused !== "boolean") { + throw new Error(`${action} verification returned an unexpected result`); + } + if (focused !== (action === "focus")) { + throw new Error( + action === "focus" + ? "target element did not become focused" + : "target element remained focused after blur()", + ); + } + return attachDialogs(deps.cdp, tabId, dialogCursor, { + tab_id: tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + focused, + ...(action === "blur" ? { was_focused: wasFocused! } : {}), + }); + } catch (err) { + return ( + throwIfAborted(deps.signal) ?? { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + } + ); + } finally { + if (objectId !== undefined && cleanupCdp && tabId !== undefined) { + // Cleanup must also run after cancellation. Navigation may have already + // disposed the object, which must not replace the operation's result. + try { + await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId }); + } catch { + // The target or execution context may no longer exist. + } + } + } +} + export async function resolveActionTarget( cdp: CdpRunner, ctx: SessionContext, diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 2187cd1a..5891e400 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -470,6 +470,39 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface FocusParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface FocusResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + focused: boolean; + dialogs?: JavaScriptDialogInfo[]; +} + +export interface BlurParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface BlurResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + was_focused: boolean; + focused: boolean; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index f27ab2f7..21fe7a17 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -55,10 +55,13 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|fill|select|press ... --session +bsk click|hover|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` +`bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether +it was focused. Use these for UI states triggered by focus changes. + Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also make them stale. Observe again before the next interaction. @@ -114,7 +117,7 @@ This list of names is complete. Never invent a command outside it; read session start|stop|list browsers status doctor update logs navigate navigate-back navigate-forward reload wait-for-navigation wait-ms observe snapshot get-html screenshot console network -click hover fill select press evaluate +click hover focus blur fill select press evaluate tab list|create|close|select|borrow|return window resize emulate upload download request-help record start|stop ``` diff --git a/crates/bsk-cli/src/cli/interaction.rs b/crates/bsk-cli/src/cli/interaction.rs index 075f9ac4..742db123 100644 --- a/crates/bsk-cli/src/cli/interaction.rs +++ b/crates/bsk-cli/src/cli/interaction.rs @@ -1,4 +1,5 @@ -//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools). +//! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press` +//! interaction tools. //! //! The `` positional accepts either a snapshot ref (`@e3`, //! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when @@ -10,8 +11,9 @@ use std::time::Duration; use anyhow::Context; use bsk_protocol::Method; use bsk_protocol::tools::{ - ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier, - MouseButton, PressParams, PressResult, SelectParams, SelectResult, + BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams, + FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult, + SelectParams, SelectResult, }; use clap::{Args, ValueEnum}; @@ -274,6 +276,124 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> { Ok(()) } +// --------------------------------------------------------------------------- +// bsk focus / bsk blur +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Args)] +pub struct FocusArgs { + /// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used. + pub target: Option, + + #[arg(long = "ref")] + pub ref_: Option, + + #[arg(long = "selector")] + pub selector: Option, + + #[arg(long)] + pub session: String, + + #[arg(long = "tab-id")] + pub tab_id: Option, + + #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> { + let info = ensure_daemon().context("ensure daemon is running")?; + let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; + let params = FocusParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + timeout_ms: Some(args.timeout), + }; + let reply: FocusResult = call( + info.sock_path, + Method::ToolFocus, + params, + "focus-1", + args.timeout, + )?; + match format { + Format::Json => println!( + "{}", + serde_json::to_string_pretty(&reply) + .map_err(|e| CliError::Local(anyhow::anyhow!(e)))? + ), + Format::Human => { + let target = + format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref()); + println!( + "focus ok tab={} target={target} focused={}", + reply.tab_id, reply.focused + ); + print_dialog_summaries(&reply.dialogs); + } + } + Ok(()) +} + +#[derive(Debug, Clone, Args)] +pub struct BlurArgs { + /// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used. + pub target: Option, + + #[arg(long = "ref")] + pub ref_: Option, + + #[arg(long = "selector")] + pub selector: Option, + + #[arg(long)] + pub session: String, + + #[arg(long = "tab-id")] + pub tab_id: Option, + + #[arg(long, default_value = "30s", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> { + let info = ensure_daemon().context("ensure daemon is running")?; + let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; + let params = BlurParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + timeout_ms: Some(args.timeout), + }; + let reply: BlurResult = call( + info.sock_path, + Method::ToolBlur, + params, + "blur-1", + args.timeout, + )?; + match format { + Format::Json => println!( + "{}", + serde_json::to_string_pretty(&reply) + .map_err(|e| CliError::Local(anyhow::anyhow!(e)))? + ), + Format::Human => { + let target = + format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref()); + println!( + "blur ok tab={} target={target} was_focused={} focused={}", + reply.tab_id, reply.was_focused, reply.focused + ); + print_dialog_summaries(&reply.dialogs); + } + } + Ok(()) +} + // --------------------------------------------------------------------------- // bsk fill // --------------------------------------------------------------------------- diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 81400457..c42e9749 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -47,7 +47,9 @@ use crate::cli::evaluate::EvaluateArgs; use crate::cli::get_html::GetHtmlArgs; use crate::cli::human_loop::RequestHelpArgs; use crate::cli::install_skill::InstallSkillArgs; -use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs}; +use crate::cli::interaction::{ + BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs, +}; use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs}; use crate::cli::network::NetworkArgs; use crate::cli::observe::ObserveArgs; @@ -173,6 +175,12 @@ pub enum Command { /// Hover a snapshot ref or CSS selector. Hover(HoverArgs), + /// Focus a snapshot ref or CSS selector. + Focus(FocusArgs), + + /// Remove focus from a snapshot ref or CSS selector. + Blur(BlurArgs), + /// Fill an input / textarea / contenteditable. Fill(FillArgs), diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 45b64ff2..26ef6405 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -253,6 +253,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolReload | Method::ToolClick | Method::ToolHover + | Method::ToolFocus + | Method::ToolBlur | Method::ToolFill | Method::ToolPress | Method::ToolSelect diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index 6ac54682..fe44a37c 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -94,6 +94,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format), + Command::Focus(args) => cli::interaction::dispatch_focus(args, format), + Command::Blur(args) => cli::interaction::dispatch_blur(args, format), Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format), diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs index 7a52f657..0e7a7a94 100644 --- a/crates/bsk-cli/tests/cli_parse.rs +++ b/crates/bsk-cli/tests/cli_parse.rs @@ -255,6 +255,24 @@ fn parses_hover_with_settle() { assert_eq!(args.settle, 300); } +#[test] +fn parses_focus_target() { + let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]); + let Command::Focus(args) = cli.command else { + panic!("expected focus command"); + }; + assert_eq!(args.target.as_deref(), Some("@e2")); +} + +#[test] +fn parses_blur_selector() { + let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]); + let Command::Blur(args) = cli.command else { + panic!("expected blur command"); + }; + assert_eq!(args.selector.as_deref(), Some("#search")); +} + #[test] fn rejects_zero_click_count() { assert!( diff --git a/crates/bsk-cli/tests/tools_m7_ipc.rs b/crates/bsk-cli/tests/tools_m7_ipc.rs index ee74b5e5..1a6e6e83 100644 --- a/crates/bsk-cli/tests/tools_m7_ipc.rs +++ b/crates/bsk-cli/tests/tools_m7_ipc.rs @@ -1,5 +1,4 @@ -//! M7 integration: drive the 7 new tool RPCs (navigate / -//! navigate_back / navigate_forward / reload / click / fill / press) +//! M7 integration: drive interaction and navigation tool RPCs //! through the IPC + per-session queue + fake extension and assert //! the wire shapes line up. The extension stub mirrors each method's //! `*Result` so we exercise the daemon's serialise → forward → @@ -13,10 +12,11 @@ use bsk::daemon::{self, DaemonConfig}; use bsk::ipc_client::IpcClient; use bsk_protocol::system::{HandshakeParams, HandshakeResult}; use bsk_protocol::tools::{ - ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams, - NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams, - NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams, - SelectResult, SessionStartParams, SessionStartResult, WaitUntil, + BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams, + FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult, + NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams, + PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams, + SessionStartResult, WaitUntil, }; use bsk_protocol::{ BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError, @@ -382,6 +382,92 @@ async fn click_round_trips_ref_and_modifiers() { handle.shutdown().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn focus_round_trips_verified_state() { + let (handle, sock) = spawn_daemon().await; + let mut ws = connect_ext(handle.ws_addr()).await; + let _ = do_handshake(&mut ws).await; + + run_extension(ws, |req| { + assert_eq!(req.method, Method::ToolFocus); + let params = req.params.clone().unwrap(); + let p: FocusParams = serde_json::from_value(params).unwrap(); + assert_eq!(p.ref_.as_deref(), Some("@e3")); + ResponseBody::Ok( + serde_json::to_value(FocusResult { + tab_id: 9, + used_ref: Some("e3".into()), + used_selector: None, + focused: true, + dialogs: vec![], + }) + .unwrap(), + ) + }); + + let session_id = ipc_session_start(&sock).await; + let result: FocusResult = ipc_tool_call( + &sock, + Method::ToolFocus, + FocusParams { + session_id, + ref_: Some("@e3".into()), + selector: None, + tab_id: None, + timeout_ms: Some(5_000), + }, + ) + .await + .expect("focus ok"); + assert!(result.focused); + assert_eq!(result.used_ref.as_deref(), Some("e3")); + handle.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn blur_round_trips_previous_focus_state() { + let (handle, sock) = spawn_daemon().await; + let mut ws = connect_ext(handle.ws_addr()).await; + let _ = do_handshake(&mut ws).await; + + run_extension(ws, |req| { + assert_eq!(req.method, Method::ToolBlur); + let params = req.params.clone().unwrap(); + let p: BlurParams = serde_json::from_value(params).unwrap(); + assert_eq!(p.selector.as_deref(), Some("#search")); + ResponseBody::Ok( + serde_json::to_value(BlurResult { + tab_id: 9, + used_ref: None, + used_selector: Some("#search".into()), + was_focused: true, + focused: false, + dialogs: vec![], + }) + .unwrap(), + ) + }); + + let session_id = ipc_session_start(&sock).await; + let result: BlurResult = ipc_tool_call( + &sock, + Method::ToolBlur, + BlurParams { + session_id, + ref_: None, + selector: Some("#search".into()), + tab_id: None, + timeout_ms: Some(5_000), + }, + ) + .await + .expect("blur ok"); + assert!(result.was_focused); + assert!(!result.focused); + assert_eq!(result.used_selector.as_deref(), Some("#search")); + handle.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn fill_round_trips_clear_before_default() { let (handle, sock) = spawn_daemon().await; diff --git a/crates/bsk-protocol/schema/tool_blur_params.json b/crates/bsk-protocol/schema/tool_blur_params.json new file mode 100644 index 00000000..525886eb --- /dev/null +++ b/crates/bsk-protocol/schema/tool_blur_params.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BlurParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "ref": { + "description": "Optional `@e` ref allocated by the last observation. Mutually exclusive with `selector`.", + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "description": "Target tab. Defaults to the Agent Window's active tab.", + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 1.0 + } + } +} diff --git a/crates/bsk-protocol/schema/tool_blur_result.json b/crates/bsk-protocol/schema/tool_blur_result.json new file mode 100644 index 00000000..fa2e353b --- /dev/null +++ b/crates/bsk-protocol/schema/tool_blur_result.json @@ -0,0 +1,112 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BlurResult", + "type": "object", + "required": [ + "focused", + "tab_id", + "was_focused" + ], + "properties": { + "dialogs": { + "type": "array", + "items": { + "$ref": "#/definitions/JavaScriptDialogInfo" + } + }, + "focused": { + "description": "Always false for a successful blur; included for machine verification.", + "type": "boolean" + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + }, + "was_focused": { + "description": "Whether the target held focus before the call.", + "type": "boolean" + } + }, + "definitions": { + "JavaScriptDialogHandledAction": { + "description": "How the extension resolved the dialog so CDP could continue.", + "type": "string", + "enum": [ + "accepted", + "dismissed" + ] + }, + "JavaScriptDialogInfo": { + "description": "One observed + handled JavaScript dialog during a tool call.", + "type": "object", + "required": [ + "handled", + "message", + "sequence", + "tab_id", + "type" + ], + "properties": { + "default_prompt": { + "type": [ + "string", + "null" + ] + }, + "handled": { + "$ref": "#/definitions/JavaScriptDialogHandledAction" + }, + "has_browser_handler": { + "type": [ + "boolean", + "null" + ] + }, + "message": { + "type": "string" + }, + "sequence": { + "description": "Monotonic per-tab sequence for ordering within a session.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "type": { + "$ref": "#/definitions/JavaScriptDialogType" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "JavaScriptDialogType": { + "description": "Native JS dialog kind reported by CDP.", + "type": "string", + "enum": [ + "alert", + "confirm", + "prompt", + "beforeunload" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_focus_params.json b/crates/bsk-protocol/schema/tool_focus_params.json new file mode 100644 index 00000000..9009bec2 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_focus_params.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "FocusParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "ref": { + "description": "Optional `@e` ref allocated by the last observation. Mutually exclusive with `selector`.", + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "description": "Target tab. Defaults to the Agent Window's active tab.", + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 1.0 + } + } +} diff --git a/crates/bsk-protocol/schema/tool_focus_result.json b/crates/bsk-protocol/schema/tool_focus_result.json new file mode 100644 index 00000000..196501da --- /dev/null +++ b/crates/bsk-protocol/schema/tool_focus_result.json @@ -0,0 +1,107 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "FocusResult", + "type": "object", + "required": [ + "focused", + "tab_id" + ], + "properties": { + "dialogs": { + "type": "array", + "items": { + "$ref": "#/definitions/JavaScriptDialogInfo" + } + }, + "focused": { + "description": "Whether the target holds DOM focus in its document or shadow root after the call.", + "type": "boolean" + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + } + }, + "definitions": { + "JavaScriptDialogHandledAction": { + "description": "How the extension resolved the dialog so CDP could continue.", + "type": "string", + "enum": [ + "accepted", + "dismissed" + ] + }, + "JavaScriptDialogInfo": { + "description": "One observed + handled JavaScript dialog during a tool call.", + "type": "object", + "required": [ + "handled", + "message", + "sequence", + "tab_id", + "type" + ], + "properties": { + "default_prompt": { + "type": [ + "string", + "null" + ] + }, + "handled": { + "$ref": "#/definitions/JavaScriptDialogHandledAction" + }, + "has_browser_handler": { + "type": [ + "boolean", + "null" + ] + }, + "message": { + "type": "string" + }, + "sequence": { + "description": "Monotonic per-tab sequence for ordering within a session.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "type": { + "$ref": "#/definitions/JavaScriptDialogType" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "JavaScriptDialogType": { + "description": "Native JS dialog kind reported by CDP.", + "type": "string", + "enum": [ + "alert", + "confirm", + "prompt", + "beforeunload" + ] + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index 6599dce5..f327d312 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -79,6 +79,10 @@ fn main() { dump!(ClickResult, "tool_click_result"); dump!(HoverParams, "tool_hover_params"); dump!(HoverResult, "tool_hover_result"); + dump!(FocusParams, "tool_focus_params"); + dump!(FocusResult, "tool_focus_result"); + dump!(BlurParams, "tool_blur_params"); + dump!(BlurResult, "tool_blur_result"); dump!(FillParams, "tool_fill_params"); dump!(FillResult, "tool_fill_result"); dump!(PressParams, "tool_press_params"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 740498ca..5ba61143 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -72,6 +72,10 @@ pub enum Method { ToolClick, #[serde(rename = "tool.hover")] ToolHover, + #[serde(rename = "tool.focus")] + ToolFocus, + #[serde(rename = "tool.blur")] + ToolBlur, #[serde(rename = "tool.fill")] ToolFill, #[serde(rename = "tool.press")] @@ -168,6 +172,8 @@ impl Method { | Method::ToolNavigateForward | Method::ToolReload | Method::ToolClick + | Method::ToolFocus + | Method::ToolBlur | Method::ToolFill | Method::ToolPress | Method::ToolSelect @@ -305,6 +311,8 @@ mod tests { assert!(Method::ToolNavigateForward.is_mutating()); assert!(Method::ToolReload.is_mutating()); assert!(Method::ToolClick.is_mutating()); + assert!(Method::ToolFocus.is_mutating()); + assert!(Method::ToolBlur.is_mutating()); assert!(Method::ToolFill.is_mutating()); assert!(Method::ToolPress.is_mutating()); assert!(Method::ToolSelect.is_mutating()); @@ -349,6 +357,8 @@ mod tests { assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); + assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation); + assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane); } @@ -358,6 +368,8 @@ mod tests { assert!(Method::ToolHover.requires_interrupt_gate()); assert!(Method::ToolObserve.requires_interrupt_gate()); assert!(Method::ToolClick.requires_interrupt_gate()); + assert!(Method::ToolFocus.requires_interrupt_gate()); + assert!(Method::ToolBlur.requires_interrupt_gate()); assert!(!Method::Cancel.requires_interrupt_gate()); } } diff --git a/crates/bsk-protocol/src/tools/interaction.rs b/crates/bsk-protocol/src/tools/interaction.rs index 2d2e808f..20004ed6 100644 --- a/crates/bsk-protocol/src/tools/interaction.rs +++ b/crates/bsk-protocol/src/tools/interaction.rs @@ -1,5 +1,5 @@ -//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`, -//! `tool.select`). +//! DOM interaction tools (`tool.click`, `tool.hover`, `tool.focus`, +//! `tool.blur`, `tool.fill`, `tool.press`, `tool.select`). //! //! Element-targeted tools accept either a snapshot `ref` (`@e` form, //! normalised against the session's RefStore) **or** a CSS selector @@ -131,6 +131,82 @@ pub struct HoverResult { pub dialogs: Vec, } +// --------------------------------------------------------------------------- +// focus / blur +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct FocusParams { + pub session_id: String, + /// Optional `@e` ref allocated by the last observation. + /// Mutually exclusive with `selector`. + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Target tab. Defaults to the Agent Window's active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct FocusResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + /// Whether the target holds DOM focus in its document or shadow root after the call. + pub focused: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct BlurParams { + pub session_id: String, + /// Optional `@e` ref allocated by the last observation. + /// Mutually exclusive with `selector`. + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Target tab. Defaults to the Agent Window's active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct BlurResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + /// Whether the target held focus before the call. + pub was_focused: bool, + /// Always false for a successful blur; included for machine verification. + pub focused: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + // --------------------------------------------------------------------------- // fill // --------------------------------------------------------------------------- @@ -325,6 +401,37 @@ mod tests { assert_eq!(round, p); } + #[test] + fn focus_params_serialise_ref_field_name() { + let p = FocusParams { + session_id: "abcd".into(), + ref_: Some("@e4".into()), + selector: None, + tab_id: Some(42), + timeout_ms: Some(5_000), + }; + let v = serde_json::to_value(&p).unwrap(); + assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e4")); + assert!(v.get("ref_").is_none()); + let round: FocusParams = serde_json::from_value(v).unwrap(); + assert_eq!(round, p); + } + + #[test] + fn blur_result_round_trips_focus_state() { + let result = BlurResult { + tab_id: 42, + used_ref: Some("e4".into()), + used_selector: None, + was_focused: true, + focused: false, + dialogs: vec![], + }; + let value = serde_json::to_value(&result).unwrap(); + let round: BlurResult = serde_json::from_value(value).unwrap(); + assert_eq!(round, result); + } + #[test] fn modifiers_render_as_lowercase_strings() { let v = serde_json::to_value(KeyModifier::Ctrl).unwrap(); diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index 83b3a6af..e02453a0 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 464a24ea..6ca293ad 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,8 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a -hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are +Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur +explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index b69a83d6..3204eed8 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " + - "press. click/hover/fill/select require target; fill also requires value; select requires " + + "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + + "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + "values; press requires key and may optionally focus target first.", actions: { click: "interact.click", hover: "interact.hover", + focus: "interact.focus", + blur: "interact.blur", fill: "interact.fill", select: "interact.select", press: "interact.press", diff --git a/packages/dsh-plugin-browserskill/src/observation.ts b/packages/dsh-plugin-browserskill/src/observation.ts index efdd9c15..31f13c3d 100644 --- a/packages/dsh-plugin-browserskill/src/observation.ts +++ b/packages/dsh-plugin-browserskill/src/observation.ts @@ -562,6 +562,10 @@ export function actionForLabel(label: string): string { return "clicking"; case "hover": return "hovering"; + case "focus": + return "focusing"; + case "blur": + return "blurring"; case "fill": return "filling"; case "select": diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 3c7c4f90..858fcc2f 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools"; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; -/** Add hover and select without bypassing session ownership or observation. */ +/** Add hover, focus, blur and select without bypassing session ownership or observation. */ export function registerPhaseOneInteractionTools( deps: ToolDeps, register: ToolRegistrar, @@ -100,6 +100,88 @@ export function registerPhaseOneInteractionTools( }), ); + for (const action of ["focus", "blur"] as const) { + register( + defineTool({ + name: `interact.${action}`, + description: + action === "focus" + ? "Focus an element and verify its DOM focus state." + : "Remove focus from an element and report whether it was focused.", + parameters: { + target: { + type: "string", + required: true, + description: "Snapshot ref (@e3 / e3) or CSS selector of the element.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + focused: { type: "boolean", required: true }, + wasFocused: { + type: "boolean", + description: "Whether the target was focused before blur.", + }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: + `[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` + + (value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`), + }, + ], + }, + async execute(args, exec) { + requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`); + const cmdArgs = [action, "--session", sessionId]; + appendTabId(cmdArgs, args.tabId); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + action, + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + focused: boolean; + was_focused?: boolean; + }; + return { + session: sessionId, + tabId: reply.tab_id, + focused: reply.focused, + ...(action === "blur" ? { wasFocused: reply.was_focused } : {}), + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + action, + args.target, + "--session", + args.session ?? "(current)", + ]), + description: action === "focus" ? "Focus an element" : "Remove focus from an element", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + } + register( defineTool({ name: "interact.select", diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index 7faf3fa5..e1441997 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,8 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.focus": ["browser_interact", "focus"], + "interact.blur": ["browser_interact", "blur"], "interact.fill": ["browser_interact", "fill"], "interact.select": ["browser_interact", "select"], "interact.press": ["browser_interact", "press"], @@ -200,7 +202,7 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "fill", "select", "press"], + browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -499,6 +501,60 @@ describe("interaction tools", () => { ]); }); + it.each([ + "focus", + "blur", + ] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + [action]: { + tab_id: 7, + focused: action === "focus", + ...(action === "blur" ? { was_focused: true } : {}), + }, + }); + await startSession(tools); + const target = action === "focus" ? "@e3" : "#field"; + const value = await tools + .get(`interact.${action}`) + ?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()); + expect(value).toEqual({ + session: "s1", + tabId: 7, + focused: action === "focus", + ...(action === "blur" ? { wasFocused: true } : {}), + }); + expect(calls[1].args).toEqual([ + action, + "--session", + "s1", + "--tab-id", + "7", + "--timeout", + "150000ms", + target, + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + + it.each([ + "focus", + "blur", + ] as const)("interact.%s validates arguments and session ownership before running", async (action) => { + const { tools, calls } = setup({ "session start": START_REPLY("s1") }); + await startSession(tools); + const tool = tools.get(`interact.${action}`)!; + await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i); + await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i); + await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow( + /greater than zero/i, + ); + await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow( + /session/i, + ); + expect(calls).toHaveLength(1); + }); + it("interact.fill passes value and --no-clear", async () => { const { tools, calls } = setup(responses); await startSession(tools); diff --git a/skill/SKILL.md b/skill/SKILL.md index f27ab2f7..21fe7a17 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -55,10 +55,13 @@ Use this default loop: ```text bsk navigate --session bsk observe --session -bsk click|hover|fill|select|press ... --session +bsk click|hover|focus|blur|fill|select|press ... --session bsk observe --session # after navigation or a meaningful DOM change ``` +`bsk focus ` explicitly focuses a target; `bsk blur ` removes focus and reports whether +it was focused. Use these for UI states triggered by focus changes. + Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also make them stale. Observe again before the next interaction. @@ -114,7 +117,7 @@ This list of names is complete. Never invent a command outside it; read session start|stop|list browsers status doctor update logs navigate navigate-back navigate-forward reload wait-for-navigation wait-ms observe snapshot get-html screenshot console network -click hover fill select press evaluate +click hover focus blur fill select press evaluate tab list|create|close|select|borrow|return window resize emulate upload download request-help record start|stop ```