Improve DocBot error handling and interaction screenshots

boat/doc-collector/src/ai/documentarian.ts

@@ -94,6 +94,9 @@ class Documentarian {
       if ((interaction.changes?.newElements || 0) > 0) {
         return true;
       }
+      if ((interaction.changes?.removedElements || 0) > 0) {
+        return true;
+      }
       return (interaction.discoveredUrls || []).length > 0;
     });
   }

boat/doc-collector/src/ai/tools.ts

@@ -49,7 +49,10 @@ export interface InteractionScreenshot {
   relativePath: string;
 }
 
-export type CaptureInteractionState = (state: WebPageState, transition: DocStateTransition) => Promise<InteractionScreenshot | null>;
+export interface CaptureInteractionState {
+  before(): Promise<Buffer | null>;
+  after(beforeScreenshot: Buffer | null, state: WebPageState, transition: DocStateTransition): Promise<InteractionScreenshot | null>;
+}
 
 const DEFAULT_MAX_PRIMARY_CANDIDATES = 3;
 const DEFAULT_MAX_INTERACTIONS = 5;
@@ -130,6 +133,8 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager
     return null;
   }
 
+  const beforeScreenshot = await captureState?.before();
+
   const executed = await attemptInteraction(explorer, candidate);
   if (!executed) {
     return null;
@@ -151,13 +156,13 @@ async function executeInteraction(explorer: Explorer, stateManager: StateManager
   });
 
   if (captureState && isMeaningfulStateTransition(transition)) {
-    const screenshot = await captureState(afterState, transition);
+    const screenshot = await captureState.after(beforeScreenshot ?? null, afterState, transition);
     if (screenshot) {
       transition.screenshot = screenshot;
     }
   }
 
-  if (urlChanged || ariaChanges.newCount > 0) {
+  if (urlChanged || ariaChanges.newCount > 0 || ariaChanges.removedCount > 0) {
     await restoreInteractionState(explorer, restoreUrl);
   }
 
@@ -192,11 +197,18 @@ async function restoreInteractionState(explorer: Explorer, restoreUrl: string, p
 }
 
 function buildTransition(candidate: InteractionCandidate, beforeState: WebPageState, afterState: WebPageState, changes: InteractionChanges): DocStateTransition {
+  const existingUrls = new Set(collectLinks(beforeState).map((link) => link.url));
   const transition: DocStateTransition = {
     action: describeAction(candidate),
     before: summarizeInteractiveState(beforeState),
     after: summarizeInteractiveState(afterState),
-    discoveredUrls: collectLinks(afterState).map((link) => link.url),
+    discoveredUrls: [
+      ...new Set(
+        collectLinks(afterState)
+          .map((link) => link.url)
+          .filter((url) => !existingUrls.has(url))
+      ),
+    ],
     newCapabilities: collectDiscoveryNotes(afterState, changes),
     element: buildInteractionElement(candidate),
     changes,
@@ -239,7 +251,7 @@ function isMeaningfulStateTransition(transition: DocStateTransition): boolean {
   if (transition.targetUrl || transition.changes?.urlChanged) {
     return true;
   }
-  return (transition.changes?.newElements || 0) > 0;
+  return (transition.changes?.newElements || 0) > 0 || (transition.changes?.removedElements || 0) > 0;
 }
 
 function buildInteractionElement(candidate: InteractionCandidate): InteractionElement {

boat/doc-collector/src/cli.ts

@@ -91,6 +91,7 @@ export function createDocsCommands(name = 'docs'): Command {
             output: 'docs',
             screenshot: true,
             interactive: false,
+            ignoreErrors: true,
             collapseDynamicPages: true,
             scope: 'site',
             includePaths: [],

boat/doc-collector/src/config.ts

@@ -115,6 +115,7 @@ class DocbotConfigParser {
           output: 'docs',
           screenshot: true,
           interactive: false,
+          ignoreErrors: true,
           collapseDynamicPages: true,
           scope: 'site',
           includePaths: [],
@@ -151,6 +152,7 @@ interface DocbotConfig {
     maxPages?: number;
     output?: string;
     screenshot?: boolean;
+    ignoreErrors?: boolean | string[];
     prompt?: string;
     collapseDynamicPages?: boolean;
     scope?: 'site' | 'section' | 'subtree';

boat/doc-collector/src/docbot.ts

@@ -6,11 +6,12 @@ import { normalizeUrl } from '../../../src/state-manager.ts';
 import { tag } from '../../../src/utils/logger.ts';
 import { sanitizeFilename } from '../../../src/utils/strings.ts';
 import { Documentarian, type PageDocumentation } from './ai/documentarian.ts';
+import type { DocStateTransition } from './ai/tools.ts';
 import { type DocbotConfig, DocbotConfigParser } from './config.ts';
 import { type DocumentedPage, type SkippedPage, renderPageDocumentation, renderSpecIndex } from './docs-renderer.ts';
 import { getDocPageKey, shouldCrawlDocPath } from './path-filter.ts';
 import { extractResearchNavigationTargets } from './research-navigation.ts';
-import { type DocumentationScreenshot, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts';
+import { type DocumentationScreenshot, captureBeforeInteraction, captureDocumentationScreenshots, captureInteractionScreenshot } from './screenshots.ts';
 import { renderMermaidBody } from './state-diagram.ts';
 
 class DocBot {
@@ -113,16 +114,18 @@ class DocBot {
           force: true,
         });
         const pagePath = this.getPageFilePath(state.url);
-        const documentation = await this.documentarian.document(state, research, async (interactionState, transition) => {
-          if (!this.shouldUseScreenshots()) {
-            return null;
-          }
-          return captureInteractionScreenshot(this.explorBot.getExplorer(), interactionState, transition, {
-            pageFilePath: pagePath,
-            screenshotsDir: this.getScreenshotsDir(),
-            config: this.config,
-          });
-        });
+        const captureState = this.shouldUseScreenshots()
+          ? {
+              before: () => captureBeforeInteraction(this.explorBot.getExplorer()),
+              after: (beforeScreenshot: Buffer | null, interactionState: WebPageState, transition: DocStateTransition) =>
+                captureInteractionScreenshot(this.explorBot.getExplorer(), beforeScreenshot, interactionState, transition, {
+                  pageFilePath: pagePath,
+                  screenshotsDir: this.getScreenshotsDir(),
+                  config: this.config,
+                }),
+            }
+          : undefined;
+        const documentation = await this.documentarian.document(state, research, captureState);
         const lowSignalReason = this.getLowSignalReason(documentation, research);
         if (lowSignalReason) {
           skipped.push({
@@ -163,6 +166,9 @@ class DocBot {
         }
       } catch (error) {
         const reason = error instanceof Error ? error.message : String(error);
+        if (!this.shouldIgnoreError(error)) {
+          throw error;
+        }
         tag('warning').log(`Skipping ${target}: ${reason}`);
         skipped.push({
           url: target,
@@ -410,6 +416,28 @@ class DocBot {
     return `low-signal page: only ${documentation.can.length} proven actions and ${interactiveCount} interactive elements`;
   }
 
+  private shouldIgnoreError(error: unknown): boolean {
+    const ignoreErrors = this.config.docs?.ignoreErrors;
+    if (ignoreErrors === undefined || ignoreErrors === true) return true;
+    if (ignoreErrors === false) return false;
+
+    const details = [error instanceof Error ? error.name : '', error instanceof Error ? error.message : String(error)];
+    if (typeof error === 'object' && error && 'code' in error) {
+      details.push(String(error.code));
+    }
+    const normalized = details
+      .join(' ')
+      .toLowerCase()
+      .replaceAll(/[\W_]+/g, ' ');
+    return ignoreErrors.some((pattern) => {
+      const normalizedPattern = pattern
+        .trim()
+        .toLowerCase()
+        .replaceAll(/[\W_]+/g, ' ');
+      return normalizedPattern.length > 0 && normalized.includes(normalizedPattern);
+    });
+  }
+
   private countInteractiveElements(research: string): number {
     const matches = [...research.matchAll(/\((\d+) elements?\)/g)];
     return matches.reduce((sum, match) => sum + Number.parseInt(match[1], 10), 0);

boat/doc-collector/src/interaction-screenshots.ts

@@ -0,0 +1,160 @@
+import { writeFileSync } from 'node:fs';
+import pixelmatch from 'pixelmatch';
+import type { Page } from 'playwright';
+import { PNG } from 'pngjs';
+
+const REGION_PADDING = 30;
+const SCREENSHOT_OPTIONS = { animations: 'disabled', caret: 'hide' } as const;
+
+export async function captureInteractionBefore(page: Page): Promise<Buffer | null> {
+  await removeVisualAnnotations(page);
+  try {
+    return await page.screenshot(SCREENSHOT_OPTIONS);
+  } catch {
+    return null;
+  }
+}
+
+export async function captureInteractionAfter(page: Page, beforeScreenshot: Buffer | null, filePath: string, detectUnmarkedOverlay = false): Promise<InteractionCaptureResult> {
+  if (!beforeScreenshot) return 'failed';
+
+  await removeVisualAnnotations(page);
+  try {
+    const afterScreenshot = await page.screenshot(SCREENSHOT_OPTIONS);
+    const before = PNG.sync.read(beforeScreenshot);
+    const after = PNG.sync.read(afterScreenshot);
+    if (before.width !== after.width || before.height !== after.height) return 'failed';
+    const changedPixels = findChangedPixelBounds(before, after);
+    if (!changedPixels) return 'unchanged';
+    const fullViewportChanged = changedPixels.x === 0 && changedPixels.y === 0 && changedPixels.width === after.width && changedPixels.height === after.height;
+    const changedRegion = addPadding(changedPixels, after.width, after.height);
+    const overlayRegion = fullViewportChanged ? await findOverlayRegion(page, after, detectUnmarkedOverlay) : null;
+    saveRegion(after, overlayRegion || changedRegion, filePath);
+    return 'captured';
+  } catch {
+    return 'failed';
+  }
+}
+
+export function findChangedRegion(beforeScreenshot: Buffer, afterScreenshot: Buffer, padding = REGION_PADDING): ScreenshotRegion | null {
+  const before = PNG.sync.read(beforeScreenshot);
+  const after = PNG.sync.read(afterScreenshot);
+  if (before.width !== after.width || before.height !== after.height) return null;
+  const changedPixels = findChangedPixelBounds(before, after);
+  return changedPixels ? addPadding(changedPixels, before.width, before.height, padding) : null;
+}
+
+function saveRegion(after: PNG, region: ScreenshotRegion, filePath: string): void {
+  const cropped = new PNG({ width: region.width, height: region.height });
+  PNG.bitblt(after, cropped, region.x, region.y, region.width, region.height, 0, 0);
+  writeFileSync(filePath, PNG.sync.write(cropped));
+}
+
+function findChangedPixelBounds(before: PNG, after: PNG): ScreenshotRegion | null {
+  const diff = Buffer.alloc(before.width * before.height * 4);
+  const changedPixels = pixelmatch(before.data, after.data, diff, before.width, before.height, { diffMask: true });
+  if (changedPixels === 0) return null;
+
+  let left = before.width;
+  let top = before.height;
+  let right = 0;
+  let bottom = 0;
+
+  for (let y = 0; y < before.height; y++) {
+    for (let x = 0; x < before.width; x++) {
+      if (diff[(y * before.width + x) * 4 + 3] === 0) continue;
+      left = Math.min(left, x);
+      top = Math.min(top, y);
+      right = Math.max(right, x);
+      bottom = Math.max(bottom, y);
+    }
+  }
+
+  return { x: left, y: top, width: right - left + 1, height: bottom - top + 1 };
+}
+
+function addPadding(region: ScreenshotRegion, imageWidth: number, imageHeight: number, padding = REGION_PADDING): ScreenshotRegion {
+  const x = Math.max(0, region.x - padding);
+  const y = Math.max(0, region.y - padding);
+  const maxX = Math.min(imageWidth, region.x + region.width + padding);
+  const maxY = Math.min(imageHeight, region.y + region.height + padding);
+  return { x, y, width: maxX - x, height: maxY - y };
+}
+
+async function removeVisualAnnotations(page: Page): Promise<void> {
+  try {
+    await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => {
+      for (const element of elements) element.remove();
+    });
+  } catch {}
+}
+
+async function findOverlayRegion(page: Page, image: PNG, detectUnmarkedOverlay: boolean): Promise<ScreenshotRegion | null> {
+  let box: { x: number; y: number; width: number; height: number } | null = null;
+  try {
+    const dialogs = page.locator('[role="dialog"]:visible, [role="alertdialog"]:visible, [aria-modal="true"]:visible');
+    if ((await dialogs.count()) > 0) box = await dialogs.last().boundingBox();
+  } catch {}
+
+  if (!box && detectUnmarkedOverlay) {
+    try {
+      box = await findUnmarkedOverlay(page);
+    } catch {}
+  }
+
+  try {
+    const viewport = page.viewportSize();
+    if (!box || !viewport) return null;
+
+    const scaleX = image.width / viewport.width;
+    const scaleY = image.height / viewport.height;
+    const x = Math.max(0, Math.floor(box.x * scaleX) - REGION_PADDING);
+    const y = Math.max(0, Math.floor(box.y * scaleY) - REGION_PADDING);
+    const maxX = Math.min(image.width, Math.ceil((box.x + box.width) * scaleX) + REGION_PADDING);
+    const maxY = Math.min(image.height, Math.ceil((box.y + box.height) * scaleY) + REGION_PADDING);
+    if (maxX <= x || maxY <= y) return null;
+    return { x, y, width: maxX - x, height: maxY - y };
+  } catch {
+    return null;
+  }
+}
+
+async function findUnmarkedOverlay(page: Page): Promise<{ x: number; y: number; width: number; height: number } | null> {
+  return page.evaluate(() => {
+    const elements = [...document.body.querySelectorAll('*')].map((element) => {
+      const style = getComputedStyle(element);
+      const box = element.getBoundingClientRect();
+      const zIndex = Number.parseInt(style.zIndex, 10);
+      return { element, style, box, zIndex, area: box.width * box.height };
+    });
+    const isVisibleLayer = ({ style, box, zIndex }: (typeof elements)[number]) => {
+      if (style.visibility === 'hidden' || style.display === 'none' || Number(style.opacity) === 0) return false;
+      if (box.width <= 0 || box.height <= 0) return false;
+      if (style.position !== 'fixed' && style.position !== 'absolute') return false;
+      return Number.isFinite(zIndex);
+    };
+    const backdropZIndex = elements.filter((item) => isVisibleLayer(item) && item.box.width >= window.innerWidth && item.box.height >= window.innerHeight).reduce((highest, item) => Math.max(highest, item.zIndex), Number.NEGATIVE_INFINITY);
+    if (!Number.isFinite(backdropZIndex)) return null;
+
+    const candidates = elements
+      .filter((item) => {
+        if (!isVisibleLayer(item)) return false;
+        if (item.box.width >= window.innerWidth && item.box.height >= window.innerHeight) return false;
+        if (item.zIndex < backdropZIndex) return false;
+        return item.element.matches('button, input, select, textarea, a[href]') || !!item.element.querySelector('button, input, select, textarea, a[href]');
+      })
+      .sort((left, right) => right.zIndex - left.zIndex || left.area - right.area);
+    const box = candidates[0]?.box;
+    if (!box) return null;
+    return { x: box.x, y: box.y, width: box.width, height: box.height };
+  });
+}
+
+export interface ScreenshotRegion {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export type InteractionCaptureResult = 'captured' | 'unchanged' | 'failed';