diff --git a/boat/doc-collector/src/ai/documentarian.ts b/boat/doc-collector/src/ai/documentarian.ts index 72555f6..70d77fa 100644 --- a/boat/doc-collector/src/ai/documentarian.ts +++ b/boat/doc-collector/src/ai/documentarian.ts @@ -94,6 +94,9 @@ class Documentarian { if ((interaction.changes?.newElements || 0) > 0) { return true; } + if ((interaction.changes?.removedElements || 0) > 0) { + return true; + } return (interaction.discoveredUrls || []).length > 0; }); } diff --git a/boat/doc-collector/src/ai/tools.ts b/boat/doc-collector/src/ai/tools.ts index 9b1ccc8..658643c 100644 --- a/boat/doc-collector/src/ai/tools.ts +++ b/boat/doc-collector/src/ai/tools.ts @@ -49,7 +49,10 @@ export interface InteractionScreenshot { relativePath: string; } -export type CaptureInteractionState = (state: WebPageState, transition: DocStateTransition) => Promise; +export interface CaptureInteractionState { + before(): Promise; + after(beforeScreenshot: Buffer | null, state: WebPageState, transition: DocStateTransition): Promise; +} const DEFAULT_MAX_PRIMARY_CANDIDATES = 3; const DEFAULT_MAX_INTERACTIONS = 5; @@ -130,6 +133,8 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager return null; } + const beforeScreenshot = await captureState?.before(); + const executed = await attemptInteraction(explorer, candidate); if (!executed) { return null; @@ -151,13 +156,13 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager }); if (captureState && isMeaningfulStateTransition(transition)) { - const screenshot = await captureState(afterState, transition); + const screenshot = await captureState.after(beforeScreenshot ?? null, afterState, transition); if (screenshot) { transition.screenshot = screenshot; } } - if (urlChanged || ariaChanges.newCount > 0) { + if (urlChanged || ariaChanges.newCount > 0 || ariaChanges.removedCount > 0) { await restoreInteractionState(explorer, restoreUrl); } @@ -192,11 +197,18 @@ async function restoreInteractionState(explorer: Explorer, restoreUrl: string, p } function buildTransition(candidate: InteractionCandidate, beforeState: WebPageState, afterState: WebPageState, changes: InteractionChanges): DocStateTransition { + const existingUrls = new Set(collectLinks(beforeState).map((link) => link.url)); const transition: DocStateTransition = { action: describeAction(candidate), before: summarizeInteractiveState(beforeState), after: summarizeInteractiveState(afterState), - discoveredUrls: collectLinks(afterState).map((link) => link.url), + discoveredUrls: [ + ...new Set( + collectLinks(afterState) + .map((link) => link.url) + .filter((url) => !existingUrls.has(url)) + ), + ], newCapabilities: collectDiscoveryNotes(afterState, changes), element: buildInteractionElement(candidate), changes, @@ -239,7 +251,7 @@ function isMeaningfulStateTransition(transition: DocStateTransition): boolean { if (transition.targetUrl || transition.changes?.urlChanged) { return true; } - return (transition.changes?.newElements || 0) > 0; + return (transition.changes?.newElements || 0) > 0 || (transition.changes?.removedElements || 0) > 0; } function buildInteractionElement(candidate: InteractionCandidate): InteractionElement { diff --git a/boat/doc-collector/src/cli.ts b/boat/doc-collector/src/cli.ts index 2fba9f7..3c57677 100644 --- a/boat/doc-collector/src/cli.ts +++ b/boat/doc-collector/src/cli.ts @@ -91,6 +91,7 @@ export function createDocsCommands(name = 'docs'): Command { output: 'docs', screenshot: true, interactive: false, + ignoreErrors: true, collapseDynamicPages: true, scope: 'site', includePaths: [], diff --git a/boat/doc-collector/src/config.ts b/boat/doc-collector/src/config.ts index b92036f..650ab83 100644 --- a/boat/doc-collector/src/config.ts +++ b/boat/doc-collector/src/config.ts @@ -115,6 +115,7 @@ class DocbotConfigParser { output: 'docs', screenshot: true, interactive: false, + ignoreErrors: true, collapseDynamicPages: true, scope: 'site', includePaths: [], @@ -151,6 +152,7 @@ interface DocbotConfig { maxPages?: number; output?: string; screenshot?: boolean; + ignoreErrors?: boolean | string[]; prompt?: string; collapseDynamicPages?: boolean; scope?: 'site' | 'section' | 'subtree'; diff --git a/boat/doc-collector/src/docbot.ts b/boat/doc-collector/src/docbot.ts index 956629a..e394cc4 100644 --- a/boat/doc-collector/src/docbot.ts +++ b/boat/doc-collector/src/docbot.ts @@ -6,11 +6,12 @@ import { normalizeUrl } from '../../../src/state-manager.ts'; import { tag } from '../../../src/utils/logger.ts'; import { sanitizeFilename } from '../../../src/utils/strings.ts'; import { Documentarian, type PageDocumentation } from './ai/documentarian.ts'; +import type { DocStateTransition } from './ai/tools.ts'; import { type DocbotConfig, DocbotConfigParser } from './config.ts'; import { type DocumentedPage, type SkippedPage, renderPageDocumentation, renderSpecIndex } from './docs-renderer.ts'; import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts'; import { extractResearchNavigationTargets } from './research-navigation.ts'; -import { type DocumentationScreenshot, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts'; +import { type DocumentationScreenshot, captureBeforeInteraction, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts'; import { renderMermaidBody } from './state-diagram.ts'; class DocBot { @@ -113,16 +114,18 @@ class DocBot { force: true, }); const pagePath = this.getPageFilePath(state.url); - const documentation = await this.documentarian.document(state, research, async (interactionState, transition) => { - if (!this.shouldUseScreenshots()) { - return null; - } - return captureInteractionScreenshot(this.explorBot.getExplorer(), interactionState, transition, { - pageFilePath: pagePath, - screenshotsDir: this.getScreenshotsDir(), - config: this.config, - }); - }); + const captureState = this.shouldUseScreenshots() + ? { + before: () => captureBeforeInteraction(this.explorBot.getExplorer()), + after: (beforeScreenshot: Buffer | null, interactionState: WebPageState, transition: DocStateTransition) => + captureInteractionScreenshot(this.explorBot.getExplorer(), beforeScreenshot, interactionState, transition, { + pageFilePath: pagePath, + screenshotsDir: this.getScreenshotsDir(), + config: this.config, + }), + } + : undefined; + const documentation = await this.documentarian.document(state, research, captureState); const lowSignalReason = this.getLowSignalReason(documentation, research); if (lowSignalReason) { skipped.push({ @@ -163,6 +166,9 @@ class DocBot { } } catch (error) { const reason = error instanceof Error ? error.message : String(error); + if (!this.shouldIgnoreError(error)) { + throw error; + } tag('warning').log(`Skipping ${target}: ${reason}`); skipped.push({ url: target, @@ -410,6 +416,28 @@ class DocBot { return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`; } + private shouldIgnoreError(error: unknown): boolean { + const ignoreErrors = this.config.docs?.ignoreErrors; + if (ignoreErrors === undefined || ignoreErrors === true) return true; + if (ignoreErrors === false) return false; + + const details = [error instanceof Error ? error.name : '', error instanceof Error ? error.message : String(error)]; + if (typeof error === 'object' && error && 'code' in error) { + details.push(String(error.code)); + } + const normalized = details + .join(' ') + .toLowerCase() + .replaceAll(/[\W_]+/g, ' '); + return ignoreErrors.some((pattern) => { + const normalizedPattern = pattern + .trim() + .toLowerCase() + .replaceAll(/[\W_]+/g, ' '); + return normalizedPattern.length > 0 && normalized.includes(normalizedPattern); + }); + } + private countInteractiveElements(research: string): number { const matches = [...research.matchAll(/\((\d+) elements?\)/g)]; return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0); diff --git a/boat/doc-collector/src/interaction-screenshots.ts b/boat/doc-collector/src/interaction-screenshots.ts new file mode 100644 index 0000000..09684c7 --- /dev/null +++ b/boat/doc-collector/src/interaction-screenshots.ts @@ -0,0 +1,160 @@ +import { writeFileSync } from 'node:fs'; +import pixelmatch from 'pixelmatch'; +import type { Page } from 'playwright'; +import { PNG } from 'pngjs'; + +const REGION_PADDING = 30; +const SCREENSHOT_OPTIONS = { animations: 'disabled', caret: 'hide' } as const; + +export async function captureInteractionBefore(page: Page): Promise { + await removeVisualAnnotations(page); + try { + return await page.screenshot(SCREENSHOT_OPTIONS); + } catch { + return null; + } +} + +export async function captureInteractionAfter(page: Page, beforeScreenshot: Buffer | null, filePath: string, detectUnmarkedOverlay = false): Promise { + if (!beforeScreenshot) return 'failed'; + + await removeVisualAnnotations(page); + try { + const afterScreenshot = await page.screenshot(SCREENSHOT_OPTIONS); + const before = PNG.sync.read(beforeScreenshot); + const after = PNG.sync.read(afterScreenshot); + if (before.width !== after.width || before.height !== after.height) return 'failed'; + const changedPixels = findChangedPixelBounds(before, after); + if (!changedPixels) return 'unchanged'; + const fullViewportChanged = changedPixels.x === 0 && changedPixels.y === 0 && changedPixels.width === after.width && changedPixels.height === after.height; + const changedRegion = addPadding(changedPixels, after.width, after.height); + const overlayRegion = fullViewportChanged ? await findOverlayRegion(page, after, detectUnmarkedOverlay) : null; + saveRegion(after, overlayRegion || changedRegion, filePath); + return 'captured'; + } catch { + return 'failed'; + } +} + +export function findChangedRegion(beforeScreenshot: Buffer, afterScreenshot: Buffer, padding = REGION_PADDING): ScreenshotRegion | null { + const before = PNG.sync.read(beforeScreenshot); + const after = PNG.sync.read(afterScreenshot); + if (before.width !== after.width || before.height !== after.height) return null; + const changedPixels = findChangedPixelBounds(before, after); + return changedPixels ? addPadding(changedPixels, before.width, before.height, padding) : null; +} + +function saveRegion(after: PNG, region: ScreenshotRegion, filePath: string): void { + const cropped = new PNG({ width: region.width, height: region.height }); + PNG.bitblt(after, cropped, region.x, region.y, region.width, region.height, 0, 0); + writeFileSync(filePath, PNG.sync.write(cropped)); +} + +function findChangedPixelBounds(before: PNG, after: PNG): ScreenshotRegion | null { + const diff = Buffer.alloc(before.width * before.height * 4); + const changedPixels = pixelmatch(before.data, after.data, diff, before.width, before.height, { diffMask: true }); + if (changedPixels === 0) return null; + + let left = before.width; + let top = before.height; + let right = 0; + let bottom = 0; + + for (let y = 0; y < before.height; y++) { + for (let x = 0; x < before.width; x++) { + if (diff[(y * before.width + x) * 4 + 3] === 0) continue; + left = Math.min(left, x); + top = Math.min(top, y); + right = Math.max(right, x); + bottom = Math.max(bottom, y); + } + } + + return { x: left, y: top, width: right - left + 1, height: bottom - top + 1 }; +} + +function addPadding(region: ScreenshotRegion, imageWidth: number, imageHeight: number, padding = REGION_PADDING): ScreenshotRegion { + const x = Math.max(0, region.x - padding); + const y = Math.max(0, region.y - padding); + const maxX = Math.min(imageWidth, region.x + region.width + padding); + const maxY = Math.min(imageHeight, region.y + region.height + padding); + return { x, y, width: maxX - x, height: maxY - y }; +} + +async function removeVisualAnnotations(page: Page): Promise { + try { + await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => { + for (const element of elements) element.remove(); + }); + } catch {} +} + +async function findOverlayRegion(page: Page, image: PNG, detectUnmarkedOverlay: boolean): Promise { + let box: { x: number; y: number; width: number; height: number } | null = null; + try { + const dialogs = page.locator('[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible'); + if ((await dialogs.count()) > 0) box = await dialogs.last().boundingBox(); + } catch {} + + if (!box && detectUnmarkedOverlay) { + try { + box = await findUnmarkedOverlay(page); + } catch {} + } + + try { + const viewport = page.viewportSize(); + if (!box || !viewport) return null; + + const scaleX = image.width / viewport.width; + const scaleY = image.height / viewport.height; + const x = Math.max(0, Math.floor(box.x * scaleX) - REGION_PADDING); + const y = Math.max(0, Math.floor(box.y * scaleY) - REGION_PADDING); + const maxX = Math.min(image.width, Math.ceil((box.x + box.width) * scaleX) + REGION_PADDING); + const maxY = Math.min(image.height, Math.ceil((box.y + box.height) * scaleY) + REGION_PADDING); + if (maxX <= x || maxY <= y) return null; + return { x, y, width: maxX - x, height: maxY - y }; + } catch { + return null; + } +} + +async function findUnmarkedOverlay(page: Page): Promise<{ x: number; y: number; width: number; height: number } | null> { + return page.evaluate(() => { + const elements = [...document.body.querySelectorAll('*')].map((element) => { + const style = getComputedStyle(element); + const box = element.getBoundingClientRect(); + const zIndex = Number.parseInt(style.zIndex, 10); + return { element, style, box, zIndex, area: box.width * box.height }; + }); + const isVisibleLayer = ({ style, box, zIndex }: (typeof elements)[number]) => { + if (style.visibility === 'hidden' || style.display === 'none' || Number(style.opacity) === 0) return false; + if (box.width <= 0 || box.height <= 0) return false; + if (style.position !== 'fixed' && style.position !== 'absolute') return false; + return Number.isFinite(zIndex); + }; + const backdropZIndex = elements.filter((item) => isVisibleLayer(item) && item.box.width >= window.innerWidth && item.box.height >= window.innerHeight).reduce((highest, item) => Math.max(highest, item.zIndex), Number.NEGATIVE_INFINITY); + if (!Number.isFinite(backdropZIndex)) return null; + + const candidates = elements + .filter((item) => { + if (!isVisibleLayer(item)) return false; + if (item.box.width >= window.innerWidth && item.box.height >= window.innerHeight) return false; + if (item.zIndex < backdropZIndex) return false; + return item.element.matches('button, input, select, textarea, a[href]') || !!item.element.querySelector('button, input, select, textarea, a[href]'); + }) + .sort((left, right) => right.zIndex - left.zIndex || left.area - right.area); + const box = candidates[0]?.box; + if (!box) return null; + return { x: box.x, y: box.y, width: box.width, height: box.height }; + }); +} + +export interface ScreenshotRegion { + x: number; + y: number; + width: number; + height: number; +} + +export type InteractionCaptureResult = 'captured' | 'unchanged' | 'failed'; diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index 81c5ec8..2abb331 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -3,10 +3,10 @@ import path from 'node:path'; import { parseResearchSections } from '../../../src/ai/researcher/parser.ts'; import type Explorer from '../../../src/explorer.ts'; import type { WebPageState } from '../../../src/state-manager.ts'; -import { detectFocusArea } from '../../../src/utils/aria.ts'; import { safeFilename, sanitizeFilename } from '../../../src/utils/strings.ts'; import type { DocStateTransition } from './ai/tools.ts'; import type { DocbotConfig } from './config.ts'; +import { captureInteractionAfter, captureInteractionBefore } from './interaction-screenshots.ts'; const DEFAULT_MAX_SECTION_SCREENSHOTS = 8; @@ -61,7 +61,7 @@ export function getScreenshotSections(research: string): ScreenshotSection[] { return sections; } -export async function captureInteractionScreenshot(explorer: Explorer, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { +export async function captureInteractionScreenshot(explorer: Explorer, beforeScreenshot: Buffer | null, state: WebPageState, transition: DocStateTransition, options: DocumentationScreenshotOptions): Promise { const page = explorer.page; if (!page) { return null; @@ -70,18 +70,11 @@ export async function captureInteractionScreenshot(explorer: Explorer, state: We mkdirSync(options.screenshotsDir, { recursive: true }); const pageName = sanitizeFilename(state.url || 'page') || 'page'; const stateName = sanitizeFilename(transition.targetState?.label || transition.action) || 'state'; - const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}`, '.png')); - const focus = detectFocusArea(state.ariaSnapshot || null); - - try { - if (focus.detected) { - await page.locator('[role="dialog"], [role="alertdialog"], [aria-modal="true"]').last().screenshot({ path: filePath }); - } else { - await page.screenshot({ path: filePath }); - } - } catch { - return null; - } + const stateId = state.id ? `_${state.id}` : ''; + const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_${stateName}${stateId}`, '.png')); + const result = await captureInteractionAfter(page, beforeScreenshot, filePath, transition.changes?.urlChanged !== true); + if (result === 'unchanged') return null; + if (result === 'failed' && !(await captureViewport(page, filePath))) return null; return { title: transition.targetState?.label || transition.action, @@ -91,6 +84,21 @@ export async function captureInteractionScreenshot(explorer: Explorer, state: We }; } +export async function captureBeforeInteraction(explorer: Explorer): Promise { + const page = explorer.page; + if (!page) return null; + return captureInteractionBefore(page); +} + +async function captureViewport(page: any, filePath: string): Promise { + try { + await page.screenshot({ path: filePath }); + return true; + } catch { + return false; + } +} + async function captureFullPageScreenshot(page: any, pageName: string, options: DocumentationScreenshotOptions): Promise { const filePath = path.join(options.screenshotsDir, safeFilename(`${pageName}_page`, '.png')); try { diff --git a/bun.lock b/bun.lock index 19615e2..2a15937 100644 --- a/bun.lock +++ b/bun.lock @@ -51,7 +51,9 @@ "micromatch": "^4.0.8", "ora-classic": "^5.4.2", "parse5": "^8.0.0", + "pixelmatch": "^7.2.0", "playwright": "^1.60", + "pngjs": "^7.0.0", "react": "^19.1.1", "sambanova-ai-provider": "^1.2.2", "strip-ansi": "^7.1.2", @@ -68,6 +70,7 @@ "@types/debug": "^4.1.12", "@types/jsdom": "^27.0.0", "@types/micromatch": "^4.0.9", + "@types/pngjs": "^6.0.5", "@types/react": "^18.2.0", "@types/yargs": "^17.0.24", "bunosh": "^0.4.0", @@ -1012,6 +1015,8 @@ "@types/pg-pool": ["@types/pg-pool@2.0.6", "", { "dependencies": { "@types/pg": "*" } }, "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ=="], + "@types/pngjs": ["@types/pngjs@6.0.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ=="], + "@types/prompts": ["@types/prompts@2.4.9", "", { "dependencies": { "@types/node": "*", "kleur": "^3.0.3" } }, "sha512-qTxFi6Buiu8+50/+3DGIWLHM6QuWsEKugJnnP6iv2Mc4ncxE4A/OJkjuVOA+5X0X1S/nq5VJRa8Lu+nwcvbrKA=="], "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], @@ -2144,6 +2149,8 @@ "pirates": ["pirates@4.0.7", "", {}, "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA=="], + "pixelmatch": ["pixelmatch@7.2.0", "", { "dependencies": { "pngjs": "^7.0.0" }, "bin": { "pixelmatch": "bin/pixelmatch" } }, "sha512-xhcb4yHu9sM/G7foGzoLtXYcC0zHEaOXXjRKhGup0fw78Nf2Tkiapv4EQyMzrbcmQPsllAI7DbFY2UT7PlI9Pg=="], + "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], "playwright": ["playwright@1.60.0", "", { "dependencies": { "playwright-core": "1.60.0" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA=="], diff --git a/docs/doc-collection/basics.md b/docs/doc-collection/basics.md index c3477a0..37e47ad 100644 --- a/docs/doc-collection/basics.md +++ b/docs/doc-collection/basics.md @@ -39,6 +39,7 @@ export default { maxPages: 100, // how many pages to document output: 'docs', // subfolder inside your output dir screenshot: true, // capture page and section screenshots + ignoreErrors: true, // skip pages that fail instead of stopping the crawl }, }; ``` diff --git a/docs/doc-collection/interactive-mode.md b/docs/doc-collection/interactive-mode.md index 7bf2e67..d3e2fe2 100644 --- a/docs/doc-collection/interactive-mode.md +++ b/docs/doc-collection/interactive-mode.md @@ -78,7 +78,7 @@ Screenshots are on by default (`screenshot: true`) in both modes. For every docu Images land in `output/docs/screenshots/` and are embedded in the page files, each section shot labeled with the CSS selector it was taken from. -Interactive states are captured before the collector restores the original page. A dialog is cropped to its active overlay when semantic dialog markup is available; other changed screen areas receive a viewport screenshot. +Interactive states are captured before the collector restores the original page. DocBot compares viewport screenshots from immediately before and after the action, finds the rectangle containing the changed pixels, adds a 30-pixel margin, and saves that fragment. If the images cannot be compared safely, it saves the current viewport instead. ```ts docs: { @@ -87,3 +87,13 @@ docs: { ``` Set `screenshot: false` to turn captures off entirely. This also disables screenshot-assisted research, which makes the run cheaper and faster but text-only. + +## Error handling + +`ignoreErrors` controls page-level crawl failures. `true` keeps the current best-effort behavior and skips every failed page, `false` stops the crawl on the first error, and an array skips only errors whose code, name, or message contains one of the listed strings. + +```ts +docs: { + ignoreErrors: ['timeout', 'navigation interrupted'], +} +``` diff --git a/package.json b/package.json index 4b9b142..9ad5a30 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,9 @@ "micromatch": "^4.0.8", "ora-classic": "^5.4.2", "parse5": "^8.0.0", + "pixelmatch": "^7.2.0", "playwright": "^1.60", + "pngjs": "^7.0.0", "react": "^19.1.1", "sambanova-ai-provider": "^1.2.2", "strip-ansi": "^7.1.2", @@ -113,6 +115,7 @@ "@types/debug": "^4.1.12", "@types/jsdom": "^27.0.0", "@types/micromatch": "^4.0.9", + "@types/pngjs": "^6.0.5", "@types/react": "^18.2.0", "@types/yargs": "^17.0.24", "bunosh": "^0.4.0", diff --git a/src/ai/researcher/locators.ts b/src/ai/researcher/locators.ts index 520a86f..e9a0ea7 100644 --- a/src/ai/researcher/locators.ts +++ b/src/ai/researcher/locators.ts @@ -228,7 +228,7 @@ export function WithLocators(Base: T) { const eidxList = section.elements.map((el) => el.eidx).filter(Boolean) as string[]; if (eidxList.length < 2) continue; - const ancestor = await this.explorer.runWithBrowserRecovery('recoverContainerFromChildren', () => WebElement.commonAncestor(this.explorer.playwrightHelper.page, eidxList)); + const ancestor = await this.explorer.withPage((page) => WebElement.commonAncestor(page, eidxList)); if (!ancestor) continue; const candidates: string[] = []; diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index caa56ef..924e5e8 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -437,6 +437,24 @@ describe('doc-collector screenshots', () => { }); describe('doc-collector scope and signal', () => { + it('supports all, none, and selected page error ignoring', () => { + const bot = new DocBot(); + + (bot as any).config = { docs: { ignoreErrors: true } }; + expect((bot as any).shouldIgnoreError('Navigation timeout')).toBe(true); + + (bot as any).config = { docs: { ignoreErrors: false } }; + expect((bot as any).shouldIgnoreError('Navigation timeout')).toBe(false); + + (bot as any).config = { docs: { ignoreErrors: ['timeout', 'connection refused'] } }; + expect((bot as any).shouldIgnoreError(new Error('Navigation TIMEOUT after 30s'))).toBe(true); + expect((bot as any).shouldIgnoreError(Object.assign(new Error('Navigation failed'), { code: 'ERR_CONNECTION_REFUSED' }))).toBe(true); + expect((bot as any).shouldIgnoreError(new Error('Page crashed'))).toBe(false); + + (bot as any).config = { docs: { ignoreErrors: [''] } }; + expect((bot as any).shouldIgnoreError(new Error('Page crashed'))).toBe(false); + }); + it('keeps subtree scope around the start page', () => { const bot = new DocBot(); (bot as any).config = { docs: { scope: 'subtree' } }; @@ -726,6 +744,7 @@ describe('documentarian interactive mode', () => { { url: '/suites', title: 'Suites', h1: 'Suites', ariaSnapshot: '- heading "Suites"\n- dialog "Import tests":\n - heading "Import tests"' }, ]; let stateIndex = 0; + const screenshotLifecycle: string[] = []; const provider = { async generateObject() { return { @@ -743,6 +762,7 @@ describe('documentarian interactive mode', () => { action() { return { async attempt(command: string) { + screenshotLifecycle.push(command.startsWith('I.amOnPage') ? 'restore' : 'click'); stateIndex = command.startsWith('I.amOnPage') ? 0 : 1; return true; }, @@ -756,10 +776,21 @@ describe('documentarian interactive mode', () => { | Element | Type | ARIA | CSS | |------|------|------|------| -| 'Import tests' | button | { role: 'button', text: 'Import tests' } | 'button.import' |` +| 'Import tests' | button | { role: 'button', text: 'Import tests' } | 'button.import' |`, + { + async before() { + screenshotLifecycle.push('before'); + return Buffer.from('before'); + }, + async after(beforeScreenshot) { + screenshotLifecycle.push(`after:${beforeScreenshot?.toString()}`); + return null; + }, + } ); expect(result.interactions?.[0]?.targetState).toEqual({ kind: 'dialog', label: 'Import tests', url: '/suites' }); + expect(screenshotLifecycle).toEqual(['before', 'click', 'after:before', 'restore']); expect(stateIndex).toBe(0); }); @@ -837,7 +868,7 @@ describe('documentarian interactive mode', () => { { action: 'Clicked link: Item A', before: '1', after: '2', targetUrl: '/items/a' }, { action: 'Clicked button: Save', before: '1', after: '2', changes: { urlChanged: false, newElements: 2, removedElements: 0 } }, { action: 'Clicked tab: Merged', before: '1', after: '2', discoveredUrls: ['/branches/merged'] }, - { action: 'Clicked button: No change', before: '1', after: '1', changes: { urlChanged: false, newElements: 0, removedElements: 0 } }, + { action: 'Clicked button: No change', before: '1', after: '1', discoveredUrls: [], changes: { urlChanged: false, newElements: 0, removedElements: 0 } }, ]); expect(interactions).toHaveLength(3); diff --git a/tests/unit/interaction-screenshots.test.ts b/tests/unit/interaction-screenshots.test.ts new file mode 100644 index 0000000..5a59a8f --- /dev/null +++ b/tests/unit/interaction-screenshots.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { PNG } from 'pngjs'; +import { captureInteractionAfter, findChangedRegion } from '../../boat/doc-collector/src/interaction-screenshots.ts'; + +describe('DocBot interaction screenshot diff', () => { + it('returns the changed pixel bounds with 30px padding', () => { + const before = createImage(200, 120); + const after = createImage(200, 120, { x: 80, y: 50, width: 20, height: 10 }); + + expect(findChangedRegion(before, after)).toEqual({ x: 50, y: 20, width: 80, height: 70 }); + }); + + it('clamps padding to the image edges', () => { + const before = createImage(100, 80); + const after = createImage(100, 80, { x: 5, y: 3, width: 10, height: 8 }); + + expect(findChangedRegion(before, after)).toEqual({ x: 0, y: 0, width: 45, height: 41 }); + }); + + it('returns null when no pixels changed', () => { + const image = createImage(50, 40); + expect(findChangedRegion(image, image)).toBeNull(); + }); + + it('returns null for screenshots with different dimensions', () => { + expect(findChangedRegion(createImage(50, 40), createImage(60, 40))).toBeNull(); + }); + + it('writes the cropped changed region from the after screenshot', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-pixel-diff-')); + const filePath = path.join(directory, 'change.png'); + const before = createImage(200, 120); + const after = createImage(200, 120, { x: 80, y: 50, width: 20, height: 10 }); + const page = { + locator: () => ({ evaluateAll: async () => {} }), + screenshot: async () => after, + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath)).toBe('captured'); + const cropped = PNG.sync.read(readFileSync(filePath)); + expect({ width: cropped.width, height: cropped.height }).toEqual({ width: 80, height: 70 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('does not save a viewport when the screenshots are visually unchanged', async () => { + const image = createImage(100, 80); + const page = { + locator: () => ({ evaluateAll: async () => {} }), + screenshot: async () => image, + } as any; + + expect(await captureInteractionAfter(page, image, 'unused.png')).toBe('unchanged'); + }); + + it('does not invoke overlay detection for a local change', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-local-diff-')); + const filePath = path.join(directory, 'local.png'); + const before = createImage(200, 120); + const after = createImage(200, 120, { x: 80, y: 50, width: 20, height: 10 }); + let overlayDetectionCalled = false; + const page = { + locator(selector: string) { + if (selector === '[data-explorbot-annotation]') return { evaluateAll: async () => {} }; + overlayDetectionCalled = true; + throw new Error(`Unexpected overlay locator: ${selector}`); + }, + screenshot: async () => after, + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath, true)).toBe('captured'); + expect(overlayDetectionCalled).toBe(false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('crops a modal with 30px padding instead of including its backdrop', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-modal-diff-')); + const filePath = path.join(directory, 'modal.png'); + const before = createImage(1000, 800); + const after = createImage(1000, 800, { x: 0, y: 0, width: 1000, height: 800 }); + const page = { + locator(selector: string) { + if (selector === '[data-explorbot-annotation]') return { evaluateAll: async () => {} }; + return { count: async () => 1, last: () => ({ boundingBox: async () => ({ x: 350, y: 200, width: 300, height: 400 }) }) }; + }, + viewportSize: () => ({ width: 1000, height: 800 }), + screenshot: async () => after, + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath)).toBe('captured'); + const cropped = PNG.sync.read(readFileSync(filePath)); + expect({ width: cropped.width, height: cropped.height }).toEqual({ width: 360, height: 460 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('uses a compact unmarked overlay for same-page interactions', async () => { + const directory = mkdtempSync(path.join(tmpdir(), 'docbot-unmarked-overlay-')); + const filePath = path.join(directory, 'overlay.png'); + const before = createImage(1000, 800); + const after = createImage(1000, 800, { x: 0, y: 0, width: 1000, height: 800 }); + const page = { + locator(selector: string) { + if (selector === '[data-explorbot-annotation]') return { evaluateAll: async () => {} }; + return { count: async () => 0, last: () => ({ boundingBox: async () => null }) }; + }, + viewportSize: () => ({ width: 1000, height: 800 }), + screenshot: async () => after, + evaluate: async () => ({ x: 350, y: 200, width: 300, height: 400 }), + } as any; + + try { + expect(await captureInteractionAfter(page, before, filePath, true)).toBe('captured'); + const cropped = PNG.sync.read(readFileSync(filePath)); + expect({ width: cropped.width, height: cropped.height }).toEqual({ width: 360, height: 460 }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); + +function createImage(width: number, height: number, region?: { x: number; y: number; width: number; height: number }): Buffer { + const image = new PNG({ width, height, colorType: 6 }); + image.data.fill(255); + if (region) { + for (let y = region.y; y < region.y + region.height; y++) { + for (let x = region.x; x < region.x + region.width; x++) { + const offset = (y * width + x) * 4; + image.data[offset] = 0; + image.data[offset + 1] = 0; + image.data[offset + 2] = 0; + } + } + } + return PNG.sync.write(image); +}