From a4b654caaa7b32da9386fff70b77aadcbff309e9 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 23 Jul 2026 01:05:24 +0200 Subject: [PATCH 1/4] Improve DocBot error handling and interaction screenshots --- boat/doc-collector/src/ai/documentarian.ts | 3 + boat/doc-collector/src/ai/tools.ts | 11 +- boat/doc-collector/src/cli.ts | 1 + boat/doc-collector/src/config.ts | 2 + boat/doc-collector/src/docbot.ts | 29 ++- .../src/interaction-screenshots.ts | 145 ++++++++++++++ boat/doc-collector/src/screenshots.ts | 31 +-- docs/doc-collection/basics.md | 1 + docs/doc-collection/interactive-mode.md | 12 ++ src/ai/researcher/locators.ts | 2 +- src/utils/html-diff.ts | 26 ++- tests/unit/doc-collector.test.ts | 184 +++++++++++++++++- tests/unit/html-diff.test.ts | 21 ++ 13 files changed, 440 insertions(+), 28 deletions(-) create mode 100644 boat/doc-collector/src/interaction-screenshots.ts diff --git a/boat/doc-collector/src/ai/documentarian.ts b/boat/doc-collector/src/ai/documentarian.ts index 72555f6..70d77fa 100644 --- a/boat/doc-collector/src/ai/documentarian.ts +++ b/boat/doc-collector/src/ai/documentarian.ts @@ -94,6 +94,9 @@ class Documentarian { if ((interaction.changes?.newElements || 0) > 0) { return true; } + if ((interaction.changes?.removedElements || 0) > 0) { + return true; + } return (interaction.discoveredUrls || []).length > 0; }); } diff --git a/boat/doc-collector/src/ai/tools.ts b/boat/doc-collector/src/ai/tools.ts index 9b1ccc8..b8da065 100644 --- a/boat/doc-collector/src/ai/tools.ts +++ b/boat/doc-collector/src/ai/tools.ts @@ -49,7 +49,7 @@ export interface InteractionScreenshot { relativePath: string; } -export type CaptureInteractionState = (state: WebPageState, transition: DocStateTransition) => Promise; +export type CaptureInteractionState = (beforeState: WebPageState, state: WebPageState, transition: DocStateTransition) => Promise; const DEFAULT_MAX_PRIMARY_CANDIDATES = 3; const DEFAULT_MAX_INTERACTIONS = 5; @@ -151,13 +151,13 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager }); if (captureState && isMeaningfulStateTransition(transition)) { - const screenshot = await captureState(afterState, transition); + const screenshot = await captureState(beforeState, afterState, transition); if (screenshot) { transition.screenshot = screenshot; } } - if (urlChanged || ariaChanges.newCount > 0) { + if (urlChanged || ariaChanges.newCount > 0 || ariaChanges.removedCount > 0) { await restoreInteractionState(explorer, restoreUrl); } @@ -192,11 +192,12 @@ async function restoreInteractionState(explorer: Explorer, restoreUrl: string, p } function buildTransition(candidate: InteractionCandidate, beforeState: WebPageState, afterState: WebPageState, changes: InteractionChanges): DocStateTransition { + const existingUrls = new Set(collectLinks(beforeState).map((link) => link.url)); const transition: DocStateTransition = { action: describeAction(candidate), before: summarizeInteractiveState(beforeState), after: summarizeInteractiveState(afterState), - discoveredUrls: collectLinks(afterState).map((link) => link.url), + discoveredUrls: [...new Set(collectLinks(afterState).map((link) => link.url).filter((url) => !existingUrls.has(url)))], newCapabilities: collectDiscoveryNotes(afterState, changes), element: buildInteractionElement(candidate), changes, @@ -239,7 +240,7 @@ function isMeaningfulStateTransition(transition: DocStateTransition): boolean { if (transition.targetUrl || transition.changes?.urlChanged) { return true; } - return (transition.changes?.newElements || 0) > 0; + return (transition.changes?.newElements || 0) > 0 || (transition.changes?.removedElements || 0) > 0; } function buildInteractionElement(candidate: InteractionCandidate): InteractionElement { diff --git a/boat/doc-collector/src/cli.ts b/boat/doc-collector/src/cli.ts index 2fba9f7..3c57677 100644 --- a/boat/doc-collector/src/cli.ts +++ b/boat/doc-collector/src/cli.ts @@ -91,6 +91,7 @@ export function createDocsCommands(name = 'docs'): Command { output: 'docs', screenshot: true, interactive: false, + ignoreErrors: true, collapseDynamicPages: true, scope: 'site', includePaths: [], diff --git a/boat/doc-collector/src/config.ts b/boat/doc-collector/src/config.ts index b92036f..650ab83 100644 --- a/boat/doc-collector/src/config.ts +++ b/boat/doc-collector/src/config.ts @@ -115,6 +115,7 @@ class DocbotConfigParser { output: 'docs', screenshot: true, interactive: false, + ignoreErrors: true, collapseDynamicPages: true, scope: 'site', includePaths: [], @@ -151,6 +152,7 @@ interface DocbotConfig { maxPages?: number; output?: string; screenshot?: boolean; + ignoreErrors?: boolean | string[]; prompt?: string; collapseDynamicPages?: boolean; scope?: 'site' | 'section' | 'subtree'; diff --git a/boat/doc-collector/src/docbot.ts b/boat/doc-collector/src/docbot.ts index 956629a..426aa4a 100644 --- a/boat/doc-collector/src/docbot.ts +++ b/boat/doc-collector/src/docbot.ts @@ -113,11 +113,11 @@ class DocBot { force: true, }); const pagePath = this.getPageFilePath(state.url); - const documentation = await this.documentarian.document(state, research, async (interactionState, transition) => { + const documentation = await this.documentarian.document(state, research, async (beforeState, interactionState, transition) => { if (!this.shouldUseScreenshots()) { return null; } - return captureInteractionScreenshot(this.explorBot.getExplorer(), interactionState, transition, { + return captureInteractionScreenshot(this.explorBot.getExplorer(), beforeState, interactionState, transition, { pageFilePath: pagePath, screenshotsDir: this.getScreenshotsDir(), config: this.config, @@ -163,6 +163,9 @@ class DocBot { } } catch (error) { const reason = error instanceof Error ? error.message : String(error); + if (!this.shouldIgnoreError(error)) { + throw error; + } tag('warning').log(`Skipping ${target}: ${reason}`); skipped.push({ url: target, @@ -410,6 +413,28 @@ class DocBot { return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`; } + private shouldIgnoreError(error: unknown): boolean { + const ignoreErrors = this.config.docs?.ignoreErrors; + if (ignoreErrors === undefined || ignoreErrors === true) return true; + if (ignoreErrors === false) return false; + + const details = [error instanceof Error ? error.name : '', error instanceof Error ? error.message : String(error)]; + if (typeof error === 'object' && error && 'code' in error) { + details.push(String(error.code)); + } + const normalized = details + .join(' ') + .toLowerCase() + .replaceAll(/[\W_]+/g, ' '); + return ignoreErrors.some((pattern) => { + const normalizedPattern = pattern + .trim() + .toLowerCase() + .replaceAll(/[\W_]+/g, ' '); + return normalizedPattern.length > 0 && normalized.includes(normalizedPattern); + }); + } + private countInteractiveElements(research: string): number { const matches = [...research.matchAll(/\((\d+) elements?\)/g)]; return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0); diff --git a/boat/doc-collector/src/interaction-screenshots.ts b/boat/doc-collector/src/interaction-screenshots.ts new file mode 100644 index 0000000..c905660 --- /dev/null +++ b/boat/doc-collector/src/interaction-screenshots.ts @@ -0,0 +1,145 @@ +import type { Page } from 'playwright'; +import type { WebPageState } from '../../../src/state-manager.ts'; +import { collectInteractiveNodes, detectFocusArea } from '../../../src/utils/aria.ts'; +import { htmlDiff } from '../../../src/utils/html-diff.ts'; + +const TARGET_ATTRIBUTE = 'data-docbot-change-target'; +const REGION_PADDING = 16; + +export async function captureInteractionStateScreenshot(page: Page, beforeState: WebPageState, state: WebPageState, filePath: string): Promise { + await removeVisualAnnotations(page); + if (await captureNewFocusArea(page, beforeState, state, filePath)) return true; + + const selectors = await getChangedDomSelectors(beforeState, state); + try { + if (selectors.length === 1) return await captureChangedContainer(page, selectors[0], filePath); + await markChangedElements(page, beforeState, state, selectors); + return await captureMarkedRegion(page, filePath); + } catch { + return false; + } finally { + await clearChangeMarkers(page); + } +} + +async function removeVisualAnnotations(page: Page): Promise { + try { + await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => { + for (const element of elements) element.remove(); + }); + } catch {} +} + +async function captureNewFocusArea(page: Page, beforeState: WebPageState, state: WebPageState, filePath: string): Promise { + const before = detectFocusArea(beforeState.ariaSnapshot || null); + const after = detectFocusArea(state.ariaSnapshot || null); + if (!after.detected || (before.detected && before.type === after.type && before.name === after.name)) return false; + + try { + await page.locator('[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible').last().screenshot({ path: filePath }); + return true; + } catch { + return false; + } +} + +async function getChangedDomSelectors(beforeState: WebPageState, state: WebPageState): Promise { + if (!beforeState.html || !state.html) return []; + + try { + const [added, removed] = await Promise.all([htmlDiff(beforeState.html, state.html, undefined, { includeTextChanges: true }), htmlDiff(state.html, beforeState.html, undefined, { includeTextChanges: true })]); + return [...new Set([...added.parts, ...removed.parts].map((part) => part.container).filter((selector) => selector !== 'body' && selector !== 'html'))]; + } catch { + return []; + } +} + +async function captureChangedContainer(page: Page, selector: string, filePath: string): Promise { + const locator = page.locator(selector).first(); + const isDocumentRoot = await locator.evaluate((element) => { + const visibleContent = [...document.body.children].filter((child) => { + if (['SCRIPT', 'STYLE', 'TEMPLATE'].includes(child.tagName)) return false; + if (child.hasAttribute('data-explorbot-annotation')) return false; + const box = child.getBoundingClientRect(); + return box.width > 0 && box.height > 0; + }); + return visibleContent.length === 1 && visibleContent[0] === element; + }); + if (isDocumentRoot) return false; + + await locator.screenshot({ path: filePath }); + return true; +} + +async function markChangedElements(page: Page, beforeState: WebPageState, state: WebPageState, selectors: string[]): Promise { + for (const node of getChangedAriaNodes(beforeState.ariaSnapshot || null, state.ariaSnapshot || null)) { + if (!node.name) continue; + try { + const locators = await page.getByRole(node.role, { name: node.name, exact: true }).all(); + if (locators.length !== 1) continue; + await locators[0].evaluate((element, attribute) => element.setAttribute(attribute, 'true'), TARGET_ATTRIBUTE); + } catch {} + } + + for (const selector of selectors) { + try { + await page.locator(selector).first().evaluate((element, attribute) => element.setAttribute(attribute, 'true'), TARGET_ATTRIBUTE); + } catch {} + } +} + +async function captureMarkedRegion(page: Page, filePath: string): Promise { + const clip = await page.evaluate( + ({ targetAttribute, padding }) => { + const elements = [...document.querySelectorAll(`[${targetAttribute}]`)]; + if (elements.length === 0) return null; + + let region: Element | null = elements[0]; + while (region && !elements.every((element) => region?.contains(element))) region = region.parentElement; + if (!region || region === document.body || region === document.documentElement) return null; + + const boxes = elements.map((element) => element.getBoundingClientRect()).filter((box) => box.width > 0 && box.height > 0); + if (boxes.length === 0) return null; + const left = Math.max(0, Math.min(...boxes.map((box) => box.left)) + window.scrollX - padding); + const top = Math.max(0, Math.min(...boxes.map((box) => box.top)) + window.scrollY - padding); + const right = Math.min(document.documentElement.scrollWidth, Math.max(...boxes.map((box) => box.right)) + window.scrollX + padding); + const bottom = Math.min(document.documentElement.scrollHeight, Math.max(...boxes.map((box) => box.bottom)) + window.scrollY + padding); + return { x: left, y: top, width: right - left, height: bottom - top }; + }, + { targetAttribute: TARGET_ATTRIBUTE, padding: REGION_PADDING } + ); + if (!clip || clip.width < 1 || clip.height < 1) return false; + + await page.screenshot({ path: filePath, clip }); + return true; +} + +async function clearChangeMarkers(page: Page): Promise { + try { + await page.evaluate((targetAttribute) => { + for (const element of document.querySelectorAll(`[${targetAttribute}]`)) element.removeAttribute(targetAttribute); + }, TARGET_ATTRIBUTE); + } catch {} +} + +function getChangedAriaNodes(before: string | null, after: string | null): AriaTarget[] { + const remaining = collectInteractiveNodes(before).map((node) => JSON.stringify(node)); + const changed: AriaTarget[] = []; + + for (const node of collectInteractiveNodes(after)) { + const serialized = JSON.stringify(node); + const match = remaining.indexOf(serialized); + if (match >= 0) { + remaining.splice(match, 1); + continue; + } + if (typeof node.role !== 'string') continue; + changed.push({ role: node.role as AriaTarget['role'], name: typeof node.name === 'string' ? node.name : '' }); + } + return changed; +} + +interface AriaTarget { + role: Parameters[0]; + name: string; +} diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index 81c5ec8..f78b28b 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -3,10 +3,10 @@ import path from 'node:path'; import { parseResearchSections } from '../../../src/ai/researcher/parser.ts'; import type Explorer from '../../../src/explorer.ts'; import type { WebPageState } from '../../../src/state-manager.ts'; -import { detectFocusArea } from '../../../src/utils/aria.ts'; import { safeFilename, sanitizeFilename } from '../../../src/utils/strings.ts'; import type { DocStateTransition } from './ai/tools.ts'; import type { DocbotConfig } from './config.ts'; +import { captureInteractionStateScreenshot } from './interaction-screenshots.ts'; const DEFAULT_MAX_SECTION_SCREENSHOTS = 8; @@ -61,7 +61,7 @@ export function getScreenshotSections(research: string): ScreenshotSection[] { return sections; } -export async function captureInteractionScreenshot(explorer: Explorer, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { +export async function captureInteractionScreenshot(explorer: Explorer, beforeState: WebPageState, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { const page = explorer.page; if (!page) { return null; @@ -70,18 +70,11 @@ export async function captureInteractionScreenshot(explorer: Explorer, state: We mkdirSync(options.screenshotsDir, { recursive: true }); const pageName = sanitizeFilename(state.url || 'page') || 'page'; const stateName = sanitizeFilename(transition.targetState?.label || transition.action) || 'state'; - const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}`, '.png')); - const focus = detectFocusArea(state.ariaSnapshot || null); - - try { - if (focus.detected) { - await page.locator('[role="dialog"], [role="alertdialog"], [aria-modal="true"]').last().screenshot({ path: filePath }); - } else { - await page.screenshot({ path: filePath }); - } - } catch { - return null; - } + const stateId = state.id ? `_${state.id}` : ''; + const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}${stateId}`, '.png')); + let captured = await captureInteractionStateScreenshot(page, beforeState, state, filePath); + if (!captured) captured = await captureViewport(page, filePath); + if (!captured) return null; return { title: transition.targetState?.label || transition.action, @@ -91,6 +84,16 @@ export async function captureInteractionScreenshot(explorer: Explorer, state: We }; } +async function captureViewport(page: any, filePath: string): Promise { + try { + await page.screenshot({ path: filePath }); + return true; + } catch { + return false; + } +} + + async function captureFullPageScreenshot(page: any, pageName: string, options: DocumentationScreenshotOptions): Promise { const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_page`, '.png')); try { diff --git a/docs/doc-collection/basics.md b/docs/doc-collection/basics.md index c3477a0..37e47ad 100644 --- a/docs/doc-collection/basics.md +++ b/docs/doc-collection/basics.md @@ -39,6 +39,7 @@ export default { maxPages: 100, // how many pages to document output: 'docs', // subfolder inside your output dir screenshot: true, // capture page and section screenshots + ignoreErrors: true, // skip pages that fail instead of stopping the crawl }, }; ``` diff --git a/docs/doc-collection/interactive-mode.md b/docs/doc-collection/interactive-mode.md index 7bf2e67..26ec8fb 100644 --- a/docs/doc-collection/interactive-mode.md +++ b/docs/doc-collection/interactive-mode.md @@ -87,3 +87,15 @@ docs: { ``` Set `screenshot: false` to turn captures off entirely. This also disables screenshot-assisted research, which makes the run cheaper and faster but text-only. + +Interaction screenshots focus on the visible area changed by an action. DocBot first captures a newly opened dialog, then combines ARIA and DOM differences to find the smallest live container that covers the change. If the change belongs to the whole document or cannot be located safely, it captures the current viewport. + +## Error handling + +`ignoreErrors` controls page-level crawl failures. `true` keeps the current best-effort behavior and skips every failed page, `false` stops the crawl on the first error, and an array skips only errors whose code, name, or message contains one of the listed strings. + +```ts +docs: { + ignoreErrors: ['timeout', 'navigation interrupted'], +} +``` diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index 520a86f..e9a0ea7 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -228,7 +228,7 @@ export function WithLocators(Base: T) { const eidxList = section.elements.map((el) => el.eidx).filter(Boolean) as string[]; if (eidxList.length < 2) continue; - const ancestor = await this.explorer.runWithBrowserRecovery('recoverContainerFromChildren', () => WebElement.commonAncestor(this.explorer.playwrightHelper.page, eidxList)); + const ancestor = await this.explorer.withPage((page) => WebElement.commonAncestor(page, eidxList)); if (!ancestor) continue; const candidates: string[] = []; diff --git a/src/utils/html-diff.ts b/src/utils/html-diff.ts index 2d90c98..b666a0c 100644 --- a/src/utils/html-diff.ts +++ b/src/utils/html-diff.ts @@ -190,7 +190,7 @@ export function computeHtmlFingerprint(html: string): string[] { /** * Compares two HTML documents and returns differences along with a diff subtree. */ -export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlConfig?: HtmlConfig): Promise { +export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlConfig?: HtmlConfig, options: { includeTextChanges?: boolean } = {}): Promise { const originalDocument = parseDocument(originalHtml, htmlConfig); const modifiedDocument = parseDocument(modifiedHtml, htmlConfig); @@ -203,7 +203,7 @@ export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlC const similarity = calculateSimilarity(originalLines, modifiedLines); const { added, removed } = findDifferences(originalLines, modifiedLines); - const parts = await buildDiffParts(originalDocument, modifiedDocument); + const parts = await buildDiffParts(originalDocument, modifiedDocument, options.includeTextChanges === true); const structuralAdditions = parts.flatMap((p) => p.added.filter((a) => a.startsWith('ELEMENT:'))); const allAdded = [...added, ...structuralAdditions]; @@ -237,9 +237,27 @@ function parseDocument(html: string, htmlConfig?: HtmlConfig): DocumentNode { function sanitizeDocumentTreeForDiff(document: parse5TreeAdapter.Document): void { // Remove standard non-semantic tags stripElementsByTag(document, new Set([...NON_SEMANTIC_TAGS, 'iframe'])); + stripExplorbotArtifacts(document); pruneDocumentHeadForDiff(document); } +function stripExplorbotArtifacts(node: ParentNodeLike): void { + if (!node.childNodes) return; + + for (let i = node.childNodes.length - 1; i >= 0; i--) { + const child = node.childNodes[i]; + if (!('tagName' in child) || !child.tagName) continue; + + const element = child as ElementNode; + if (element.attrs?.some((attr) => attr.name === 'data-explorbot-annotation')) { + node.childNodes.splice(i, 1); + continue; + } + element.attrs = element.attrs?.filter((attr) => !attr.name.startsWith('data-explorbot-')) ?? []; + stripExplorbotArtifacts(element); + } +} + /** * Prune non-essential elements from document head for diff purposes. * Only keeps title element, removes everything else. @@ -448,7 +466,7 @@ function findStableContainer(topLevelPath: string, originalMap: NodeMap, modifie return { path: 'html[1]/body[1]', selector: 'body' }; } -async function buildDiffParts(originalDocument: DocumentNode, modifiedDocument: DocumentNode): Promise { +async function buildDiffParts(originalDocument: DocumentNode, modifiedDocument: DocumentNode, includeTextChanges: boolean): Promise { const originalMap = collectElementMap(originalDocument); const modifiedMap = collectElementMap(modifiedDocument); @@ -465,7 +483,7 @@ async function buildDiffParts(originalDocument: DocumentNode, modifiedDocument: continue; } - if (attributesDiffer(element, originalElement)) { + if (attributesDiffer(element, originalElement) || (includeTextChanges && directTextDiffer(element, originalElement))) { changedPaths.push(path); } } diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index caa56ef..416a222 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -6,7 +6,7 @@ import { DocBot } from '../../boat/doc-collector/src/docbot.ts'; import { normalizeAction, renderPageDocumentation, renderSpecIndex } from '../../boat/doc-collector/src/docs-renderer.ts'; import { getDocPageKey, shouldCrawlDocPath } from '../../boat/doc-collector/src/path-filter.ts'; import { extractResearchNavigationTargets } from '../../boat/doc-collector/src/research-navigation.ts'; -import { captureDocumentationScreenshots, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; +import { captureDocumentationScreenshots, captureInteractionScreenshot, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; describe('doc-collector path filter', () => { @@ -434,9 +434,189 @@ describe('doc-collector screenshots', () => { expect(screenshots[1].relativePath).toBe('../screenshots/users_sign_in_navigation.png'); expect(captured.map((item) => item.selector)).toEqual([undefined, '.mainnav-menu']); }); + + it('captures the smallest live container covering ARIA changes', async () => { + const calls: Array<{ clip?: { x: number; y: number; width: number; height: number } }> = []; + let evaluations = 0; + const clip = { x: 20, y: 30, width: 400, height: 300 }; + const page = { + async screenshot(options: any) { + calls.push(options); + }, + getByRole() { + return { + async all() { + return [ + { + async evaluate() {}, + }, + ]; + }, + }; + }, + async evaluate() { + evaluations++; + return evaluations === 1 ? clip : undefined; + }, + }; + + const result = await captureInteractionScreenshot( + { page } as any, + { url: '/dashboard', ariaSnapshot: '- button "Old"' }, + { url: '/dashboard', ariaSnapshot: '- button "Old"\n- link "New item"' }, + { action: 'Clicked tab: Details', before: 'before', after: 'after', targetState: { kind: 'section', label: 'Details', url: '/dashboard' }, element: { role: 'tab', name: 'Details', section: 'Tabs', container: '.tabs' } }, + { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } + ); + + expect(result?.kind).toBe('state'); + expect(calls[0].clip).toEqual(clip); + }); + + it('uses the reverse DOM diff to capture a container after content is removed', async () => { + const calls: Array<{ selector?: string }> = []; + const markedSelectors: string[] = []; + const page = { + async screenshot(options: any) { + calls.push(options); + }, + locator(selector: string) { + return { + first() { + return { + async evaluate() { + markedSelectors.push(selector); + }, + async screenshot() { + calls.push({ selector }); + }, + }; + }, + }; + }, + async evaluate() { + return undefined; + }, + }; + + const result = await captureInteractionScreenshot( + { page } as any, + { url: '/search', html: '
One
Two
' }, + { url: '/search', html: '
One
' }, + { action: 'Clicked button: Clear', before: 'before', after: 'after' }, + { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } + ); + + expect(result?.kind).toBe('state'); + expect(markedSelectors.length).toBeGreaterThan(0); + expect(calls).toEqual([{ selector: markedSelectors[0] }]); + }); + + it('falls back to the viewport when changes only share the document root', async () => { + const calls: Array<{ viewport?: boolean }> = []; + const page = { + async screenshot() { + calls.push({ viewport: true }); + }, + getByRole: () => ({ all: async () => [{ evaluate: async () => {} }] }), + async evaluate() { + return false; + }, + }; + + const result = await captureInteractionScreenshot( + { page } as any, + { url: '/dashboard', ariaSnapshot: '- link "Report"' }, + { url: '/reports/1', ariaSnapshot: '- heading "Report"\n- button "Download"' }, + { action: 'Clicked link: Report', before: 'before', after: 'after' }, + { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } + ); + + expect(result?.kind).toBe('state'); + expect(calls).toEqual([{ viewport: true }]); + }); + + it('uses the viewport when the only changed container is the document root', async () => { + const calls: string[] = []; + const page = { + async screenshot() { + calls.push('viewport'); + }, + locator(selector: string) { + return { + async evaluateAll() {}, + first() { + return { + async evaluate() { + return true; + }, + async screenshot() { + calls.push(selector); + }, + }; + }, + }; + }, + }; + + await captureInteractionScreenshot( + { page } as any, + { url: '/products', html: '

Products

Catalog
' }, + { url: '/account', html: '

Account

' }, + { action: 'Clicked link: Account', before: 'before', after: 'after' }, + { pageFilePath: 'output/docs/pages/products.md', screenshotsDir: 'output/docs/screenshots', config: {} } + ); + + expect(calls).toEqual(['viewport']); + }); + + it('captures a newly visible dialog before calculating other changes', async () => { + const calls: string[] = []; + const page = { + locator(selector: string) { + return { + last() { + return { + async screenshot() { + calls.push(selector); + }, + }; + }, + }; + }, + }; + + const result = await captureInteractionScreenshot( + { page } as any, + { url: '/dashboard', ariaSnapshot: '- button "Open"' }, + { url: '/dashboard', ariaSnapshot: '- dialog "Settings"\n - button "Close"' }, + { action: 'Clicked button: Open', before: 'before', after: 'after' }, + { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } + ); + + expect(result?.kind).toBe('state'); + expect(calls).toEqual(['[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible']); + }); }); describe('doc-collector scope and signal', () => { + it('supports all, none, and selected page error ignoring', () => { + const bot = new DocBot(); + + (bot as any).config = { docs: { ignoreErrors: true } }; + expect((bot as any).shouldIgnoreError('Navigation timeout')).toBe(true); + + (bot as any).config = { docs: { ignoreErrors: false } }; + expect((bot as any).shouldIgnoreError('Navigation timeout')).toBe(false); + + (bot as any).config = { docs: { ignoreErrors: ['timeout', 'connection refused'] } }; + expect((bot as any).shouldIgnoreError(new Error('Navigation TIMEOUT after 30s'))).toBe(true); + expect((bot as any).shouldIgnoreError(Object.assign(new Error('Navigation failed'), { code: 'ERR_CONNECTION_REFUSED' }))).toBe(true); + expect((bot as any).shouldIgnoreError(new Error('Page crashed'))).toBe(false); + + (bot as any).config = { docs: { ignoreErrors: [''] } }; + expect((bot as any).shouldIgnoreError(new Error('Page crashed'))).toBe(false); + }); + it('keeps subtree scope around the start page', () => { const bot = new DocBot(); (bot as any).config = { docs: { scope: 'subtree' } }; @@ -837,7 +1017,7 @@ describe('documentarian interactive mode', () => { { action: 'Clicked link: Item A', before: '1', after: '2', targetUrl: '/items/a' }, { action: 'Clicked button: Save', before: '1', after: '2', changes: { urlChanged: false, newElements: 2, removedElements: 0 } }, { action: 'Clicked tab: Merged', before: '1', after: '2', discoveredUrls: ['/branches/merged'] }, - { action: 'Clicked button: No change', before: '1', after: '1', changes: { urlChanged: false, newElements: 0, removedElements: 0 } }, + { action: 'Clicked button: No change', before: '1', after: '1', discoveredUrls: [], changes: { urlChanged: false, newElements: 0, removedElements: 0 } }, ]); expect(interactions).toHaveLength(3); diff --git a/tests/unit/html-diff.test.ts b/tests/unit/html-diff.test.ts index 1f39c05..8baf8bd 100644 --- a/tests/unit/html-diff.test.ts +++ b/tests/unit/html-diff.test.ts @@ -6,6 +6,17 @@ function allSubtrees(result: { parts: { subtree: string }[] }): string { } describe('HTML Diff', () => { + it('ignores Explorbot attributes and visual annotation overlays', async () => { + const original = '
'; + const modified = '
e9
'; + + const result = await htmlDiff(original, modified); + + expect(result.parts).toEqual([]); + expect(result.added).toEqual([]); + expect(result.removed).toEqual([]); + }); + it('should detect no changes in identical HTML', async () => { const html1 = ` @@ -194,6 +205,16 @@ describe('HTML Diff', () => { expect(result.removed).toContain('BUTTON:Submit'); }); + it('can locate a text-only change for screenshot regions', async () => { + const original = '

Unavailable

'; + const modified = '

Available now

'; + + const result = await htmlDiff(original, modified, undefined, { includeTextChanges: true }); + + expect(result.parts).toHaveLength(1); + expect(result.parts[0].container).toBe('#card'); + }); + it('should sanitize scripts and non-semantic nodes from diff output', async () => { const original = ` From 65ea2b90c9465f207ffbc7b21faa7edb3b5e4b37 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 23 Jul 2026 01:07:46 +0200 Subject: [PATCH 2/4] Fix --- boat/doc-collector/src/ai/tools.ts | 8 +++++++- boat/doc-collector/src/interaction-screenshots.ts | 5 ++++- boat/doc-collector/src/screenshots.ts | 1 - 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/boat/doc-collector/src/ai/tools.ts b/boat/doc-collector/src/ai/tools.ts index b8da065..f0b4671 100644 --- a/boat/doc-collector/src/ai/tools.ts +++ b/boat/doc-collector/src/ai/tools.ts @@ -197,7 +197,13 @@ function buildTransition(candidate: InteractionCandidate, beforeState: WebPageSt action: describeAction(candidate), before: summarizeInteractiveState(beforeState), after: summarizeInteractiveState(afterState), - discoveredUrls: [...new Set(collectLinks(afterState).map((link) => link.url).filter((url) => !existingUrls.has(url)))], + discoveredUrls: [ + ...new Set( + collectLinks(afterState) + .map((link) => link.url) + .filter((url) => !existingUrls.has(url)) + ), + ], newCapabilities: collectDiscoveryNotes(afterState, changes), element: buildInteractionElement(candidate), changes, diff --git a/boat/doc-collector/src/interaction-screenshots.ts b/boat/doc-collector/src/interaction-screenshots.ts index c905660..a9a40ff 100644 --- a/boat/doc-collector/src/interaction-screenshots.ts +++ b/boat/doc-collector/src/interaction-screenshots.ts @@ -83,7 +83,10 @@ async function markChangedElements(page: Page, beforeState: WebPageState, state: for (const selector of selectors) { try { - await page.locator(selector).first().evaluate((element, attribute) => element.setAttribute(attribute, 'true'), TARGET_ATTRIBUTE); + await page + .locator(selector) + .first() + .evaluate((element, attribute) => element.setAttribute(attribute, 'true'), TARGET_ATTRIBUTE); } catch {} } } diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index f78b28b..c6beb96 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -93,7 +93,6 @@ async function captureViewport(page: any, filePath: string): Promise { } } - async function captureFullPageScreenshot(page: any, pageName: string, options: DocumentationScreenshotOptions): Promise { const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_page`, '.png')); try { From 09d68573db282a540c2326bb065ff0eaf84bcb6b Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 23 Jul 2026 15:57:33 +0200 Subject: [PATCH 3/4] Upd logic to library --- boat/doc-collector/src/ai/tools.ts | 9 +- boat/doc-collector/src/docbot.ts | 25 +- .../src/interaction-screenshots.ts | 240 +++++++++--------- boat/doc-collector/src/screenshots.ts | 16 +- bun.lock | 7 + docs/doc-collection/interactive-mode.md | 4 +- package.json | 3 + src/utils/html-diff.ts | 26 +- tests/unit/doc-collector.test.ts | 178 ++----------- tests/unit/html-diff.test.ts | 21 -- tests/unit/interaction-screenshots.test.ts | 146 +++++++++++ 11 files changed, 334 insertions(+), 341 deletions(-) create mode 100644 tests/unit/interaction-screenshots.test.ts diff --git a/boat/doc-collector/src/ai/tools.ts b/boat/doc-collector/src/ai/tools.ts index f0b4671..658643c 100644 --- a/boat/doc-collector/src/ai/tools.ts +++ b/boat/doc-collector/src/ai/tools.ts @@ -49,7 +49,10 @@ export interface InteractionScreenshot { relativePath: string; } -export type CaptureInteractionState = (beforeState: WebPageState, state: WebPageState, transition: DocStateTransition) => Promise; +export interface CaptureInteractionState { + before(): Promise; + after(beforeScreenshot: Buffer | null, state: WebPageState, transition: DocStateTransition): Promise; +} const DEFAULT_MAX_PRIMARY_CANDIDATES = 3; const DEFAULT_MAX_INTERACTIONS = 5; @@ -130,6 +133,8 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager return null; } + const beforeScreenshot = await captureState?.before(); + const executed = await attemptInteraction(explorer, candidate); if (!executed) { return null; @@ -151,7 +156,7 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager }); if (captureState && isMeaningfulStateTransition(transition)) { - const screenshot = await captureState(beforeState, afterState, transition); + const screenshot = await captureState.after(beforeScreenshot ?? null, afterState, transition); if (screenshot) { transition.screenshot = screenshot; } diff --git a/boat/doc-collector/src/docbot.ts b/boat/doc-collector/src/docbot.ts index 426aa4a..e394cc4 100644 --- a/boat/doc-collector/src/docbot.ts +++ b/boat/doc-collector/src/docbot.ts @@ -6,11 +6,12 @@ import { normalizeUrl } from '../../../src/state-manager.ts'; import { tag } from '../../../src/utils/logger.ts'; import { sanitizeFilename } from '../../../src/utils/strings.ts'; import { Documentarian, type PageDocumentation } from './ai/documentarian.ts'; +import type { DocStateTransition } from './ai/tools.ts'; import { type DocbotConfig, DocbotConfigParser } from './config.ts'; import { type DocumentedPage, type SkippedPage, renderPageDocumentation, renderSpecIndex } from './docs-renderer.ts'; import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts'; import { extractResearchNavigationTargets } from './research-navigation.ts'; -import { type DocumentationScreenshot, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts'; +import { type DocumentationScreenshot, captureBeforeInteraction, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts'; import { renderMermaidBody } from './state-diagram.ts'; class DocBot { @@ -113,16 +114,18 @@ class DocBot { force: true, }); const pagePath = this.getPageFilePath(state.url); - const documentation = await this.documentarian.document(state, research, async (beforeState, interactionState, transition) => { - if (!this.shouldUseScreenshots()) { - return null; - } - return captureInteractionScreenshot(this.explorBot.getExplorer(), beforeState, interactionState, transition, { - pageFilePath: pagePath, - screenshotsDir: this.getScreenshotsDir(), - config: this.config, - }); - }); + const captureState = this.shouldUseScreenshots() + ? { + before: () => captureBeforeInteraction(this.explorBot.getExplorer()), + after: (beforeScreenshot: Buffer | null, interactionState: WebPageState, transition: DocStateTransition) => + captureInteractionScreenshot(this.explorBot.getExplorer(), beforeScreenshot, interactionState, transition, { + pageFilePath: pagePath, + screenshotsDir: this.getScreenshotsDir(), + config: this.config, + }), + } + : undefined; + const documentation = await this.documentarian.document(state, research, captureState); const lowSignalReason = this.getLowSignalReason(documentation, research); if (lowSignalReason) { skipped.push({ diff --git a/boat/doc-collector/src/interaction-screenshots.ts b/boat/doc-collector/src/interaction-screenshots.ts index a9a40ff..09684c7 100644 --- a/boat/doc-collector/src/interaction-screenshots.ts +++ b/boat/doc-collector/src/interaction-screenshots.ts @@ -1,148 +1,160 @@ +import { writeFileSync } from 'node:fs'; +import pixelmatch from 'pixelmatch'; import type { Page } from 'playwright'; -import type { WebPageState } from '../../../src/state-manager.ts'; -import { collectInteractiveNodes, detectFocusArea } from '../../../src/utils/aria.ts'; -import { htmlDiff } from '../../../src/utils/html-diff.ts'; +import { PNG } from 'pngjs'; -const TARGET_ATTRIBUTE = 'data-docbot-change-target'; -const REGION_PADDING = 16; +const REGION_PADDING = 30; +const SCREENSHOT_OPTIONS = { animations: 'disabled', caret: 'hide' } as const; -export async function captureInteractionStateScreenshot(page: Page, beforeState: WebPageState, state: WebPageState, filePath: string): Promise { +export async function captureInteractionBefore(page: Page): Promise { await removeVisualAnnotations(page); - if (await captureNewFocusArea(page, beforeState, state, filePath)) return true; - - const selectors = await getChangedDomSelectors(beforeState, state); try { - if (selectors.length === 1) return await captureChangedContainer(page, selectors[0], filePath); - await markChangedElements(page, beforeState, state, selectors); - return await captureMarkedRegion(page, filePath); + return await page.screenshot(SCREENSHOT_OPTIONS); } catch { - return false; - } finally { - await clearChangeMarkers(page); + return null; } } -async function removeVisualAnnotations(page: Page): Promise { - try { - await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => { - for (const element of elements) element.remove(); - }); - } catch {} -} - -async function captureNewFocusArea(page: Page, beforeState: WebPageState, state: WebPageState, filePath: string): Promise { - const before = detectFocusArea(beforeState.ariaSnapshot || null); - const after = detectFocusArea(state.ariaSnapshot || null); - if (!after.detected || (before.detected && before.type === after.type && before.name === after.name)) return false; +export async function captureInteractionAfter(page: Page, beforeScreenshot: Buffer | null, filePath: string, detectUnmarkedOverlay = false): Promise { + if (!beforeScreenshot) return 'failed'; + await removeVisualAnnotations(page); try { - await page.locator('[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible').last().screenshot({ path: filePath }); - return true; + const afterScreenshot = await page.screenshot(SCREENSHOT_OPTIONS); + const before = PNG.sync.read(beforeScreenshot); + const after = PNG.sync.read(afterScreenshot); + if (before.width !== after.width || before.height !== after.height) return 'failed'; + const changedPixels = findChangedPixelBounds(before, after); + if (!changedPixels) return 'unchanged'; + const fullViewportChanged = changedPixels.x === 0 && changedPixels.y === 0 && changedPixels.width === after.width && changedPixels.height === after.height; + const changedRegion = addPadding(changedPixels, after.width, after.height); + const overlayRegion = fullViewportChanged ? await findOverlayRegion(page, after, detectUnmarkedOverlay) : null; + saveRegion(after, overlayRegion || changedRegion, filePath); + return 'captured'; } catch { - return false; + return 'failed'; } } -async function getChangedDomSelectors(beforeState: WebPageState, state: WebPageState): Promise { - if (!beforeState.html || !state.html) return []; - - try { - const [added, removed] = await Promise.all([htmlDiff(beforeState.html, state.html, undefined, { includeTextChanges: true }), htmlDiff(state.html, beforeState.html, undefined, { includeTextChanges: true })]); - return [...new Set([...added.parts, ...removed.parts].map((part) => part.container).filter((selector) => selector !== 'body' && selector !== 'html'))]; - } catch { - return []; - } +export function findChangedRegion(beforeScreenshot: Buffer, afterScreenshot: Buffer, padding = REGION_PADDING): ScreenshotRegion | null { + const before = PNG.sync.read(beforeScreenshot); + const after = PNG.sync.read(afterScreenshot); + if (before.width !== after.width || before.height !== after.height) return null; + const changedPixels = findChangedPixelBounds(before, after); + return changedPixels ? addPadding(changedPixels, before.width, before.height, padding) : null; } -async function captureChangedContainer(page: Page, selector: string, filePath: string): Promise { - const locator = page.locator(selector).first(); - const isDocumentRoot = await locator.evaluate((element) => { - const visibleContent = [...document.body.children].filter((child) => { - if (['SCRIPT', 'STYLE', 'TEMPLATE'].includes(child.tagName)) return false; - if (child.hasAttribute('data-explorbot-annotation')) return false; - const box = child.getBoundingClientRect(); - return box.width > 0 && box.height > 0; - }); - return visibleContent.length === 1 && visibleContent[0] === element; - }); - if (isDocumentRoot) return false; - - await locator.screenshot({ path: filePath }); - return true; +function saveRegion(after: PNG, region: ScreenshotRegion, filePath: string): void { + const cropped = new PNG({ width: region.width, height: region.height }); + PNG.bitblt(after, cropped, region.x, region.y, region.width, region.height, 0, 0); + writeFileSync(filePath, PNG.sync.write(cropped)); } -async function markChangedElements(page: Page, beforeState: WebPageState, state: WebPageState, selectors: string[]): Promise { - for (const node of getChangedAriaNodes(beforeState.ariaSnapshot || null, state.ariaSnapshot || null)) { - if (!node.name) continue; - try { - const locators = await page.getByRole(node.role, { name: node.name, exact: true }).all(); - if (locators.length !== 1) continue; - await locators[0].evaluate((element, attribute) => element.setAttribute(attribute, 'true'), TARGET_ATTRIBUTE); - } catch {} +function findChangedPixelBounds(before: PNG, after: PNG): ScreenshotRegion | null { + const diff = Buffer.alloc(before.width * before.height * 4); + const changedPixels = pixelmatch(before.data, after.data, diff, before.width, before.height, { diffMask: true }); + if (changedPixels === 0) return null; + + let left = before.width; + let top = before.height; + let right = 0; + let bottom = 0; + + for (let y = 0; y < before.height; y++) { + for (let x = 0; x < before.width; x++) { + if (diff[(y * before.width + x) * 4 + 3] === 0) continue; + left = Math.min(left, x); + top = Math.min(top, y); + right = Math.max(right, x); + bottom = Math.max(bottom, y); + } } - for (const selector of selectors) { - try { - await page - .locator(selector) - .first() - .evaluate((element, attribute) => element.setAttribute(attribute, 'true'), TARGET_ATTRIBUTE); - } catch {} - } + return { x: left, y: top, width: right - left + 1, height: bottom - top + 1 }; } -async function captureMarkedRegion(page: Page, filePath: string): Promise { - const clip = await page.evaluate( - ({ targetAttribute, padding }) => { - const elements = [...document.querySelectorAll(`[${targetAttribute}]`)]; - if (elements.length === 0) return null; - - let region: Element | null = elements[0]; - while (region && !elements.every((element) => region?.contains(element))) region = region.parentElement; - if (!region || region === document.body || region === document.documentElement) return null; - - const boxes = elements.map((element) => element.getBoundingClientRect()).filter((box) => box.width > 0 && box.height > 0); - if (boxes.length === 0) return null; - const left = Math.max(0, Math.min(...boxes.map((box) => box.left)) + window.scrollX - padding); - const top = Math.max(0, Math.min(...boxes.map((box) => box.top)) + window.scrollY - padding); - const right = Math.min(document.documentElement.scrollWidth, Math.max(...boxes.map((box) => box.right)) + window.scrollX + padding); - const bottom = Math.min(document.documentElement.scrollHeight, Math.max(...boxes.map((box) => box.bottom)) + window.scrollY + padding); - return { x: left, y: top, width: right - left, height: bottom - top }; - }, - { targetAttribute: TARGET_ATTRIBUTE, padding: REGION_PADDING } - ); - if (!clip || clip.width < 1 || clip.height < 1) return false; - - await page.screenshot({ path: filePath, clip }); - return true; +function addPadding(region: ScreenshotRegion, imageWidth: number, imageHeight: number, padding = REGION_PADDING): ScreenshotRegion { + const x = Math.max(0, region.x - padding); + const y = Math.max(0, region.y - padding); + const maxX = Math.min(imageWidth, region.x + region.width + padding); + const maxY = Math.min(imageHeight, region.y + region.height + padding); + return { x, y, width: maxX - x, height: maxY - y }; } -async function clearChangeMarkers(page: Page): Promise { +async function removeVisualAnnotations(page: Page): Promise { try { - await page.evaluate((targetAttribute) => { - for (const element of document.querySelectorAll(`[${targetAttribute}]`)) element.removeAttribute(targetAttribute); - }, TARGET_ATTRIBUTE); + await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => { + for (const element of elements) element.remove(); + }); } catch {} } -function getChangedAriaNodes(before: string | null, after: string | null): AriaTarget[] { - const remaining = collectInteractiveNodes(before).map((node) => JSON.stringify(node)); - const changed: AriaTarget[] = []; +async function findOverlayRegion(page: Page, image: PNG, detectUnmarkedOverlay: boolean): Promise { + let box: { x: number; y: number; width: number; height: number } | null = null; + try { + const dialogs = page.locator('[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible'); + if ((await dialogs.count()) > 0) box = await dialogs.last().boundingBox(); + } catch {} - for (const node of collectInteractiveNodes(after)) { - const serialized = JSON.stringify(node); - const match = remaining.indexOf(serialized); - if (match >= 0) { - remaining.splice(match, 1); - continue; - } - if (typeof node.role !== 'string') continue; - changed.push({ role: node.role as AriaTarget['role'], name: typeof node.name === 'string' ? node.name : '' }); + if (!box && detectUnmarkedOverlay) { + try { + box = await findUnmarkedOverlay(page); + } catch {} + } + + try { + const viewport = page.viewportSize(); + if (!box || !viewport) return null; + + const scaleX = image.width / viewport.width; + const scaleY = image.height / viewport.height; + const x = Math.max(0, Math.floor(box.x * scaleX) - REGION_PADDING); + const y = Math.max(0, Math.floor(box.y * scaleY) - REGION_PADDING); + const maxX = Math.min(image.width, Math.ceil((box.x + box.width) * scaleX) + REGION_PADDING); + const maxY = Math.min(image.height, Math.ceil((box.y + box.height) * scaleY) + REGION_PADDING); + if (maxX <= x || maxY <= y) return null; + return { x, y, width: maxX - x, height: maxY - y }; + } catch { + return null; } - return changed; } -interface AriaTarget { - role: Parameters[0]; - name: string; +async function findUnmarkedOverlay(page: Page): Promise<{ x: number; y: number; width: number; height: number } | null> { + return page.evaluate(() => { + const elements = [...document.body.querySelectorAll('*')].map((element) => { + const style = getComputedStyle(element); + const box = element.getBoundingClientRect(); + const zIndex = Number.parseInt(style.zIndex, 10); + return { element, style, box, zIndex, area: box.width * box.height }; + }); + const isVisibleLayer = ({ style, box, zIndex }: (typeof elements)[number]) => { + if (style.visibility === 'hidden' || style.display === 'none' || Number(style.opacity) === 0) return false; + if (box.width <= 0 || box.height <= 0) return false; + if (style.position !== 'fixed' && style.position !== 'absolute') return false; + return Number.isFinite(zIndex); + }; + const backdropZIndex = elements.filter((item) => isVisibleLayer(item) && item.box.width >= window.innerWidth && item.box.height >= window.innerHeight).reduce((highest, item) => Math.max(highest, item.zIndex), Number.NEGATIVE_INFINITY); + if (!Number.isFinite(backdropZIndex)) return null; + + const candidates = elements + .filter((item) => { + if (!isVisibleLayer(item)) return false; + if (item.box.width >= window.innerWidth && item.box.height >= window.innerHeight) return false; + if (item.zIndex < backdropZIndex) return false; + return item.element.matches('button, input, select, textarea, a[href]') || !!item.element.querySelector('button, input, select, textarea, a[href]'); + }) + .sort((left, right) => right.zIndex - left.zIndex || left.area - right.area); + const box = candidates[0]?.box; + if (!box) return null; + return { x: box.x, y: box.y, width: box.width, height: box.height }; + }); } + +export interface ScreenshotRegion { + x: number; + y: number; + width: number; + height: number; +} + +export type InteractionCaptureResult = 'captured' | 'unchanged' | 'failed'; diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index c6beb96..2abb331 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -6,7 +6,7 @@ import type { WebPageState } from '../../../src/state-manager.ts'; import { safeFilename, sanitizeFilename } from '../../../src/utils/strings.ts'; import type { DocStateTransition } from './ai/tools.ts'; import type { DocbotConfig } from './config.ts'; -import { captureInteractionStateScreenshot } from './interaction-screenshots.ts'; +import { captureInteractionAfter, captureInteractionBefore } from './interaction-screenshots.ts'; const DEFAULT_MAX_SECTION_SCREENSHOTS = 8; @@ -61,7 +61,7 @@ export function getScreenshotSections(research: string): ScreenshotSection[] { return sections; } -export async function captureInteractionScreenshot(explorer: Explorer, beforeState: WebPageState, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { +export async function captureInteractionScreenshot(explorer: Explorer, beforeScreenshot: Buffer | null, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { const page = explorer.page; if (!page) { return null; @@ -72,9 +72,9 @@ export async function captureInteractionScreenshot(explorer: Explorer, beforeSta const stateName = sanitizeFilename(transition.targetState?.label || transition.action) || 'state'; const stateId = state.id ? `_${state.id}` : ''; const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}${stateId}`, '.png')); - let captured = await captureInteractionStateScreenshot(page, beforeState, state, filePath); - if (!captured) captured = await captureViewport(page, filePath); - if (!captured) return null; + const result = await captureInteractionAfter(page, beforeScreenshot, filePath, transition.changes?.urlChanged !== true); + if (result === 'unchanged') return null; + if (result === 'failed' && !(await captureViewport(page, filePath))) return null; return { title: transition.targetState?.label || transition.action, @@ -84,6 +84,12 @@ export async function captureInteractionScreenshot(explorer: Explorer, beforeSta }; } +export async function captureBeforeInteraction(explorer: Explorer): Promise { + const page = explorer.page; + if (!page) return null; + return captureInteractionBefore(page); +} + async function captureViewport(page: any, filePath: string): Promise { try { await page.screenshot({ path: filePath }); diff --git a/bun.lock b/bun.lock index 19615e2..2a15937 100644 --- a/bun.lock +++ b/bun.lock @@ -51,7 +51,9 @@ "micromatch": "^4.0.8", "ora-classic": "^5.4.2", "parse5": "^8.0.0", + "pixelmatch": "^7.2.0", "playwright": "^1.60", + "pngjs": "^7.0.0", "react": "^19.1.1", "sambanova-ai-provider": "^1.2.2", "strip-ansi": "^7.1.2", @@ -68,6 +70,7 @@ "@types/debug": "^4.1.12", "@types/jsdom": "^27.0.0", "@types/micromatch": "^4.0.9", + "@types/pngjs": "^6.0.5", "@types/react": "^18.2.0", "@types/yargs": "^17.0.24", "bunosh": "^0.4.0", @@ -1012,6 +1015,8 @@ "@types/pg-pool": ["@types/pg-pool@2.0.6", "", { "dependencies": { "@types/pg": "*" } }, "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ=="], + "@types/pngjs": ["@types/pngjs@6.0.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ=="], + "@types/prompts": ["@types/prompts@2.4.9", "", { "dependencies": { "@types/node": "*", "kleur": "^3.0.3" } }, "sha512-qTxFi6Buiu8+50/+3DGIWLHM6QuWsEKugJnnP6iv2Mc4ncxE4A/OJkjuVOA+5X0X1S/nq5VJRa8Lu+nwcvbrKA=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], @@ -2144,6 +2149,8 @@ "pirates": ["pirates@4.0.7", "", {}, "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA=="], + "pixelmatch": ["pixelmatch@7.2.0", "", { "dependencies": { "pngjs": "^7.0.0" }, "bin": { "pixelmatch": "bin/pixelmatch" } }, "sha512-xhcb4yHu9sM/G7foGzoLtXYcC0zHEaOXXjRKhGup0fw78Nf2Tkiapv4EQyMzrbcmQPsllAI7DbFY2UT7PlI9Pg=="], + "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], "playwright": ["playwright@1.60.0", "", { "dependencies": { "playwright-core": "1.60.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA=="], diff --git a/docs/doc-collection/interactive-mode.md b/docs/doc-collection/interactive-mode.md index 26ec8fb..d3e2fe2 100644 --- a/docs/doc-collection/interactive-mode.md +++ b/docs/doc-collection/interactive-mode.md @@ -78,7 +78,7 @@ Screenshots are on by default (`screenshot: true`) in both modes. For every docu Images land in `output/docs/screenshots/` and are embedded in the page files, each section shot labeled with the CSS selector it was taken from. -Interactive states are captured before the collector restores the original page. A dialog is cropped to its active overlay when semantic dialog markup is available; other changed screen areas receive a viewport screenshot. +Interactive states are captured before the collector restores the original page. DocBot compares viewport screenshots from immediately before and after the action, finds the rectangle containing the changed pixels, adds a 30-pixel margin, and saves that fragment. If the images cannot be compared safely, it saves the current viewport instead. ```ts docs: { @@ -88,8 +88,6 @@ docs: { Set `screenshot: false` to turn captures off entirely. This also disables screenshot-assisted research, which makes the run cheaper and faster but text-only. -Interaction screenshots focus on the visible area changed by an action. DocBot first captures a newly opened dialog, then combines ARIA and DOM differences to find the smallest live container that covers the change. If the change belongs to the whole document or cannot be located safely, it captures the current viewport. - ## Error handling `ignoreErrors` controls page-level crawl failures. `true` keeps the current best-effort behavior and skips every failed page, `false` stops the crawl on the first error, and an array skips only errors whose code, name, or message contains one of the listed strings. diff --git a/package.json b/package.json index 4b9b142..9ad5a30 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,9 @@ "micromatch": "^4.0.8", "ora-classic": "^5.4.2", "parse5": "^8.0.0", + "pixelmatch": "^7.2.0", "playwright": "^1.60", + "pngjs": "^7.0.0", "react": "^19.1.1", "sambanova-ai-provider": "^1.2.2", "strip-ansi": "^7.1.2", @@ -113,6 +115,7 @@ "@types/debug": "^4.1.12", "@types/jsdom": "^27.0.0", "@types/micromatch": "^4.0.9", + "@types/pngjs": "^6.0.5", "@types/react": "^18.2.0", "@types/yargs": "^17.0.24", "bunosh": "^0.4.0", diff --git a/src/utils/html-diff.ts b/src/utils/html-diff.ts index b666a0c..2d90c98 100644 --- a/src/utils/html-diff.ts +++ b/src/utils/html-diff.ts @@ -190,7 +190,7 @@ export function computeHtmlFingerprint(html: string): string[] { /** * Compares two HTML documents and returns differences along with a diff subtree. */ -export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlConfig?: HtmlConfig, options: { includeTextChanges?: boolean } = {}): Promise { +export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlConfig?: HtmlConfig): Promise { const originalDocument = parseDocument(originalHtml, htmlConfig); const modifiedDocument = parseDocument(modifiedHtml, htmlConfig); @@ -203,7 +203,7 @@ export async function htmlDiff(originalHtml: string, modifiedHtml: string, htmlC const similarity = calculateSimilarity(originalLines, modifiedLines); const { added, removed } = findDifferences(originalLines, modifiedLines); - const parts = await buildDiffParts(originalDocument, modifiedDocument, options.includeTextChanges === true); + const parts = await buildDiffParts(originalDocument, modifiedDocument); const structuralAdditions = parts.flatMap((p) => p.added.filter((a) => a.startsWith('ELEMENT:'))); const allAdded = [...added, ...structuralAdditions]; @@ -237,27 +237,9 @@ function parseDocument(html: string, htmlConfig?: HtmlConfig): DocumentNode { function sanitizeDocumentTreeForDiff(document: parse5TreeAdapter.Document): void { // Remove standard non-semantic tags stripElementsByTag(document, new Set([...NON_SEMANTIC_TAGS, 'iframe'])); - stripExplorbotArtifacts(document); pruneDocumentHeadForDiff(document); } -function stripExplorbotArtifacts(node: ParentNodeLike): void { - if (!node.childNodes) return; - - for (let i = node.childNodes.length - 1; i >= 0; i--) { - const child = node.childNodes[i]; - if (!('tagName' in child) || !child.tagName) continue; - - const element = child as ElementNode; - if (element.attrs?.some((attr) => attr.name === 'data-explorbot-annotation')) { - node.childNodes.splice(i, 1); - continue; - } - element.attrs = element.attrs?.filter((attr) => !attr.name.startsWith('data-explorbot-')) ?? []; - stripExplorbotArtifacts(element); - } -} - /** * Prune non-essential elements from document head for diff purposes. * Only keeps title element, removes everything else. @@ -466,7 +448,7 @@ function findStableContainer(topLevelPath: string, originalMap: NodeMap, modifie return { path: 'html[1]/body[1]', selector: 'body' }; } -async function buildDiffParts(originalDocument: DocumentNode, modifiedDocument: DocumentNode, includeTextChanges: boolean): Promise { +async function buildDiffParts(originalDocument: DocumentNode, modifiedDocument: DocumentNode): Promise { const originalMap = collectElementMap(originalDocument); const modifiedMap = collectElementMap(modifiedDocument); @@ -483,7 +465,7 @@ async function buildDiffParts(originalDocument: DocumentNode, modifiedDocument: continue; } - if (attributesDiffer(element, originalElement) || (includeTextChanges && directTextDiffer(element, originalElement))) { + if (attributesDiffer(element, originalElement)) { changedPaths.push(path); } } diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index 416a222..c74f26f 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -6,7 +6,7 @@ import { DocBot } from '../../boat/doc-collector/src/docbot.ts'; import { normalizeAction, renderPageDocumentation, renderSpecIndex } from '../../boat/doc-collector/src/docs-renderer.ts'; import { getDocPageKey, shouldCrawlDocPath } from '../../boat/doc-collector/src/path-filter.ts'; import { extractResearchNavigationTargets } from '../../boat/doc-collector/src/research-navigation.ts'; -import { captureDocumentationScreenshots, captureInteractionScreenshot, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; +import { captureDocumentationScreenshots, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; describe('doc-collector path filter', () => { @@ -435,167 +435,6 @@ describe('doc-collector screenshots', () => { expect(captured.map((item) => item.selector)).toEqual([undefined, '.mainnav-menu']); }); - it('captures the smallest live container covering ARIA changes', async () => { - const calls: Array<{ clip?: { x: number; y: number; width: number; height: number } }> = []; - let evaluations = 0; - const clip = { x: 20, y: 30, width: 400, height: 300 }; - const page = { - async screenshot(options: any) { - calls.push(options); - }, - getByRole() { - return { - async all() { - return [ - { - async evaluate() {}, - }, - ]; - }, - }; - }, - async evaluate() { - evaluations++; - return evaluations === 1 ? clip : undefined; - }, - }; - - const result = await captureInteractionScreenshot( - { page } as any, - { url: '/dashboard', ariaSnapshot: '- button "Old"' }, - { url: '/dashboard', ariaSnapshot: '- button "Old"\n- link "New item"' }, - { action: 'Clicked tab: Details', before: 'before', after: 'after', targetState: { kind: 'section', label: 'Details', url: '/dashboard' }, element: { role: 'tab', name: 'Details', section: 'Tabs', container: '.tabs' } }, - { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } - ); - - expect(result?.kind).toBe('state'); - expect(calls[0].clip).toEqual(clip); - }); - - it('uses the reverse DOM diff to capture a container after content is removed', async () => { - const calls: Array<{ selector?: string }> = []; - const markedSelectors: string[] = []; - const page = { - async screenshot(options: any) { - calls.push(options); - }, - locator(selector: string) { - return { - first() { - return { - async evaluate() { - markedSelectors.push(selector); - }, - async screenshot() { - calls.push({ selector }); - }, - }; - }, - }; - }, - async evaluate() { - return undefined; - }, - }; - - const result = await captureInteractionScreenshot( - { page } as any, - { url: '/search', html: '
One
Two
' }, - { url: '/search', html: '
One
' }, - { action: 'Clicked button: Clear', before: 'before', after: 'after' }, - { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } - ); - - expect(result?.kind).toBe('state'); - expect(markedSelectors.length).toBeGreaterThan(0); - expect(calls).toEqual([{ selector: markedSelectors[0] }]); - }); - - it('falls back to the viewport when changes only share the document root', async () => { - const calls: Array<{ viewport?: boolean }> = []; - const page = { - async screenshot() { - calls.push({ viewport: true }); - }, - getByRole: () => ({ all: async () => [{ evaluate: async () => {} }] }), - async evaluate() { - return false; - }, - }; - - const result = await captureInteractionScreenshot( - { page } as any, - { url: '/dashboard', ariaSnapshot: '- link "Report"' }, - { url: '/reports/1', ariaSnapshot: '- heading "Report"\n- button "Download"' }, - { action: 'Clicked link: Report', before: 'before', after: 'after' }, - { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } - ); - - expect(result?.kind).toBe('state'); - expect(calls).toEqual([{ viewport: true }]); - }); - - it('uses the viewport when the only changed container is the document root', async () => { - const calls: string[] = []; - const page = { - async screenshot() { - calls.push('viewport'); - }, - locator(selector: string) { - return { - async evaluateAll() {}, - first() { - return { - async evaluate() { - return true; - }, - async screenshot() { - calls.push(selector); - }, - }; - }, - }; - }, - }; - - await captureInteractionScreenshot( - { page } as any, - { url: '/products', html: '

Products

Catalog
' }, - { url: '/account', html: '

Account

' }, - { action: 'Clicked link: Account', before: 'before', after: 'after' }, - { pageFilePath: 'output/docs/pages/products.md', screenshotsDir: 'output/docs/screenshots', config: {} } - ); - - expect(calls).toEqual(['viewport']); - }); - - it('captures a newly visible dialog before calculating other changes', async () => { - const calls: string[] = []; - const page = { - locator(selector: string) { - return { - last() { - return { - async screenshot() { - calls.push(selector); - }, - }; - }, - }; - }, - }; - - const result = await captureInteractionScreenshot( - { page } as any, - { url: '/dashboard', ariaSnapshot: '- button "Open"' }, - { url: '/dashboard', ariaSnapshot: '- dialog "Settings"\n - button "Close"' }, - { action: 'Clicked button: Open', before: 'before', after: 'after' }, - { pageFilePath: 'output/docs/pages/dashboard.md', screenshotsDir: 'output/docs/screenshots', config: {} } - ); - - expect(result?.kind).toBe('state'); - expect(calls).toEqual(['[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible']); - }); }); describe('doc-collector scope and signal', () => { @@ -906,6 +745,7 @@ describe('documentarian interactive mode', () => { { url: '/suites', title: 'Suites', h1: 'Suites', ariaSnapshot: '- heading "Suites"\n- dialog "Import tests":\n - heading "Import tests"' }, ]; let stateIndex = 0; + const screenshotLifecycle: string[] = []; const provider = { async generateObject() { return { @@ -923,6 +763,7 @@ describe('documentarian interactive mode', () => { action() { return { async attempt(command: string) { + screenshotLifecycle.push(command.startsWith('I.amOnPage') ? 'restore' : 'click'); stateIndex = command.startsWith('I.amOnPage') ? 0 : 1; return true; }, @@ -936,10 +777,21 @@ describe('documentarian interactive mode', () => { | Element | Type | ARIA | CSS | |------|------|------|------| -| 'Import tests' | button | { role: 'button', text: 'Import tests' } | 'button.import' |` +| 'Import tests' | button | { role: 'button', text: 'Import tests' } | 'button.import' |`, + { + async before() { + screenshotLifecycle.push('before'); + return Buffer.from('before'); + }, + async after(beforeScreenshot) { + screenshotLifecycle.push(`after:${beforeScreenshot?.toString()}`); + return null; + }, + } ); expect(result.interactions?.[0]?.targetState).toEqual({ kind: 'dialog', label: 'Import tests', url: '/suites' }); + expect(screenshotLifecycle).toEqual(['before', 'click', 'after:before', 'restore']); expect(stateIndex).toBe(0); }); diff --git a/tests/unit/html-diff.test.ts b/tests/unit/html-diff.test.ts index 8baf8bd..1f39c05 100644 --- a/tests/unit/html-diff.test.ts +++ b/tests/unit/html-diff.test.ts @@ -6,17 +6,6 @@ function allSubtrees(result: { parts: { subtree: string }[] }): string { } describe('HTML Diff', () => { - it('ignores Explorbot attributes and visual annotation overlays', async () => { - const original = '
'; - const modified = '
e9
'; - - const result = await htmlDiff(original, modified); - - expect(result.parts).toEqual([]); - expect(result.added).toEqual([]); - expect(result.removed).toEqual([]); - }); - it('should detect no changes in identical HTML', async () => { const html1 = ` @@ -205,16 +194,6 @@ describe('HTML Diff', () => { expect(result.removed).toContain('BUTTON:Submit'); }); - it('can locate a text-only change for screenshot regions', async () => { - const original = '

Unavailable

'; - const modified = '

Available now

'; - - const result = await htmlDiff(original, modified, undefined, { includeTextChanges: true }); - - expect(result.parts).toHaveLength(1); - expect(result.parts[0].container).toBe('#card'); - }); - it('should sanitize scripts and non-semantic nodes from diff output', async () => { const original = ` diff --git a/tests/unit/interaction-screenshots.test.ts b/tests/unit/interaction-screenshots.test.ts new file mode 100644 index 0000000..5a59a8f --- /dev/null +++ b/tests/unit/interaction-screenshots.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { PNG } from 'pngjs'; +import { captureInteractionAfter, findChangedRegion } from '../../boat/doc-collector/src/interaction-screenshots.ts'; + +describe('DocBot interaction screenshot diff', () => { + it('returns the changed pixel bounds with 30px padding', () => { + const before = createImage(200, 120); + const after = createImage(200, 120, { x: 80, y: 50, width: 20, height: 10 }); + + expect(findChangedRegion(before, after)).toEqual({ x: 50, y: 20, width: 80, height: 70 }); + }); + + it('clamps padding to the image edges', () => { + const before = createImage(100, 80); + const after = createImage(100, 80, { x: 5, y: 3, width: 10, height: 8 }); + + expect(findChangedRegion(before, after)).toEqual({ x: 0, y: 0, width: 45, height: 41 }); + }); + + it('returns null when no pixels changed', () => { + const image = createImage(50, 40); + expect(findChangedRegion(image, image)).toBeNull(); + }); + + it('returns null for screenshots with different dimensions', () => { + expect(findChangedRegion(createImage(50, 40), createImage(60, 40))).toBeNull(); + }); + + it('writes the cropped changed region from the after screenshot', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-pixel-diff-')); + const filePath = path.join(directory, 'change.png'); + const before = createImage(200, 120); + const after = createImage(200, 120, { x: 80, y: 50, width: 20, height: 10 }); + const page = { + locator: () => ({ evaluateAll: async () => {} }), + screenshot: async () => after, + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath)).toBe('captured'); + const cropped = PNG.sync.read(readFileSync(filePath)); + expect({ width: cropped.width, height: cropped.height }).toEqual({ width: 80, height: 70 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('does not save a viewport when the screenshots are visually unchanged', async () => { + const image = createImage(100, 80); + const page = { + locator: () => ({ evaluateAll: async () => {} }), + screenshot: async () => image, + } as any; + + expect(await captureInteractionAfter(page, image, 'unused.png')).toBe('unchanged'); + }); + + it('does not invoke overlay detection for a local change', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-local-diff-')); + const filePath = path.join(directory, 'local.png'); + const before = createImage(200, 120); + const after = createImage(200, 120, { x: 80, y: 50, width: 20, height: 10 }); + let overlayDetectionCalled = false; + const page = { + locator(selector: string) { + if (selector === '[data-explorbot-annotation]') return { evaluateAll: async () => {} }; + overlayDetectionCalled = true; + throw new Error(`Unexpected overlay locator: ${selector}`); + }, + screenshot: async () => after, + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath, true)).toBe('captured'); + expect(overlayDetectionCalled).toBe(false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('crops a modal with 30px padding instead of including its backdrop', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-modal-diff-')); + const filePath = path.join(directory, 'modal.png'); + const before = createImage(1000, 800); + const after = createImage(1000, 800, { x: 0, y: 0, width: 1000, height: 800 }); + const page = { + locator(selector: string) { + if (selector === '[data-explorbot-annotation]') return { evaluateAll: async () => {} }; + return { count: async () => 1, last: () => ({ boundingBox: async () => ({ x: 350, y: 200, width: 300, height: 400 }) }) }; + }, + viewportSize: () => ({ width: 1000, height: 800 }), + screenshot: async () => after, + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath)).toBe('captured'); + const cropped = PNG.sync.read(readFileSync(filePath)); + expect({ width: cropped.width, height: cropped.height }).toEqual({ width: 360, height: 460 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('uses a compact unmarked overlay for same-page interactions', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-unmarked-overlay-')); + const filePath = path.join(directory, 'overlay.png'); + const before = createImage(1000, 800); + const after = createImage(1000, 800, { x: 0, y: 0, width: 1000, height: 800 }); + const page = { + locator(selector: string) { + if (selector === '[data-explorbot-annotation]') return { evaluateAll: async () => {} }; + return { count: async () => 0, last: () => ({ boundingBox: async () => null }) }; + }, + viewportSize: () => ({ width: 1000, height: 800 }), + screenshot: async () => after, + evaluate: async () => ({ x: 350, y: 200, width: 300, height: 400 }), + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath, true)).toBe('captured'); + const cropped = PNG.sync.read(readFileSync(filePath)); + expect({ width: cropped.width, height: cropped.height }).toEqual({ width: 360, height: 460 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); + +function createImage(width: number, height: number, region?: { x: number; y: number; width: number; height: number }): Buffer { + const image = new PNG({ width, height, colorType: 6 }); + image.data.fill(255); + if (region) { + for (let y = region.y; y < region.y + region.height; y++) { + for (let x = region.x; x < region.x + region.width; x++) { + const offset = (y * width + x) * 4; + image.data[offset] = 0; + image.data[offset + 1] = 0; + image.data[offset + 2] = 0; + } + } + } + return PNG.sync.write(image); +} From 814e8c95802c283d2af4bf477a9c699a527bd5e9 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 23 Jul 2026 16:00:41 +0200 Subject: [PATCH 4/4] Fix --- tests/unit/doc-collector.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index c74f26f..924e5e8 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -434,7 +434,6 @@ describe('doc-collector screenshots', () => { expect(screenshots[1].relativePath).toBe('../screenshots/users_sign_in_navigation.png'); expect(captured.map((item) => item.selector)).toEqual([undefined, '.mainnav-menu']); }); - }); describe('doc-collector scope and signal', () => {