diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 2266c51..2893f71 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -63,6 +63,7 @@ import { BlueskyAccountError, connectBlueskyAccount, budgetStatus, + crawlDedupeKey, startContactImport, importContactChunk, finishContactImport, @@ -3668,6 +3669,11 @@ async function approveRecommendation( url: target, ...(recommendation.campaign_id ? { campaignId: recommendation.campaign_id } : {}), }, + // The same key `POST /prospects/by-url` uses, so approving research + // for twenty people at one company reads that company's site once. + // Without it a page naming fifty-seven people asks for that page + // fifty-seven times, and every read finds the same fifty-seven. + dedupeKey: crawlDedupeKey(target), }); research = { queued: queued.queued, url: target }; } else { diff --git a/packages/pipeline/src/auto-approve.test.ts b/packages/pipeline/src/auto-approve.test.ts index eece804..5cae43e 100644 --- a/packages/pipeline/src/auto-approve.test.ts +++ b/packages/pipeline/src/auto-approve.test.ts @@ -292,6 +292,76 @@ describe('autoApproveInternal', () => { expect(result.queuedCrawls).toBe(0); }); + test('reads one company site once, however many people it names', async () => { + // The failure that got past the per-person cooldown and into production. + // A crawl of accenture.com returns 57 people; each becomes a research + // card; each card queued another crawl of the same page, which returned + // the same 57. Prod reached 226 pending crawls of one URL and 81 of + // another before it was stopped. The cooldown could not catch it because + // every one of those people was new. + seeded = await seedDatabase('auto-one-crawl'); + await withDomain(seeded.db); + + for (let index = 0; index < 8; index += 1) { + const personId = `per_same_co_${index}`; + await seeded.db.execute({ + sql: `INSERT INTO people (id, display_name, status, identity_confidence, + current_company_id, created_at, updated_at) + VALUES (?, ?, 'qualified', 0.9, 'co_auto', ?, ?)`, + args: [personId, `Colleague ${index}`, now(), now()], + }); + await seeded.db.execute({ + sql: `INSERT INTO recommendations (id, workspace_id, campaign_id, person_id, action, + network, priority, reason, policy_status, policy_version, expected_goal, status, + created_at) + VALUES (?, ?, ?, ?, 'refresh_research', 'website', 50, 'because', + 'allow_with_approval', '2026-08-11', 'qualify', 'pending', ?)`, + args: [newId('recommendation'), SEED.workspaceId, SEED.campaignId, personId, now()], + }); + } + + const result = await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId }); + + // Every card clears — the queue is the point. + expect(result.approved).toBeGreaterThanOrEqual(8); + + // But the site is read once. This is the assertion that matters. + const jobs = await queryOne<{ n: number }>( + seeded.db, + `SELECT count(*) AS n FROM jobs WHERE kind = 'crawl_site' AND status = 'pending'`, + ); + expect(Number(jobs?.n)).toBe(1); + expect(result.queuedCrawls).toBe(1); + }); + + test('the key frees itself once the crawl finishes', async () => { + // Suppressing duplicates must not suppress future work: the index covers + // only pending and running jobs for exactly this reason. + seeded = await seedDatabase('auto-crawl-key-frees'); + await withDomain(seeded.db); + await card(seeded.db, 'refresh_research'); + + expect( + (await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId })).queuedCrawls, + ).toBe(1); + + await seeded.db.execute({ + sql: `UPDATE jobs SET status = 'done' WHERE kind = 'crawl_site'`, + args: [], + }); + // Age the research so the per-person cooldown does not mask the result. + await seeded.db.execute({ + sql: `UPDATE actions SET created_at = ? WHERE kind = 'refresh_research'`, + args: [new Date(Date.now() - 48 * 3_600_000).toISOString()], + }); + + await card(seeded.db, 'refresh_research'); + + expect( + (await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId })).queuedCrawls, + ).toBe(1); + }); + test('honours the limit so one workspace cannot hold the tick', async () => { seeded = await seedDatabase('auto-limit'); diff --git a/packages/pipeline/src/auto-approve.ts b/packages/pipeline/src/auto-approve.ts index 297d0a9..06ef3cb 100644 --- a/packages/pipeline/src/auto-approve.ts +++ b/packages/pipeline/src/auto-approve.ts @@ -226,13 +226,25 @@ export async function autoApproveInternal( ); if (site?.domain) { + const url = normaliseDomain(site.domain); + const queued = await enqueue(db, { workspaceId: input.workspaceId, kind: 'crawl_site', payload: { - url: normaliseDomain(site.domain), + url, ...(row.campaign_id ? { campaignId: row.campaign_id } : {}), }, + // One crawl per site at a time. Without this a research card is a + // crawl *per person*, and a page that names many people asks for the + // same page many times: production queued accenture.com 226 times, + // toptal.com 81, from a single pass. The site is read once and every + // person on it is served by that read. + // + // The index behind this covers only pending and running jobs, so the + // key frees itself the moment the crawl finishes — this suppresses + // duplicates, never future work. + dedupeKey: crawlDedupeKey(url), }); if (queued.queued) queuedCrawls += 1; @@ -309,6 +321,29 @@ function normaliseDomain(domain: string): string { return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`; } +/** + * The key that collapses many requests for one site into a single crawl. + * + * Deliberately the same string `POST /prospects/by-url` has always used — + * `crawl:`, without `www.` — because two paths deduping under different + * keys do not deduplicate against each other, which is the entire point. That + * route had this from the start; the approval paths never passed a key at all, + * which is why one of them could queue accenture.com 226 times while the other + * could not queue it twice. + * + * Keyed on the host rather than the full URL so that `example.com` and + * `https://www.example.com/` are one crawl, which is what they fetch. + */ +export function crawlDedupeKey(url: string): string { + try { + return `crawl:${new URL(url).hostname.replace(/^www\./, '')}`; + } catch { + // Unparseable is still deduplicable against itself, and the crawl job will + // fail on its own terms rather than here. + return `crawl:${url.toLowerCase()}`; + } +} + /** Every workspace with internal cards waiting, for the worker to sweep. */ export async function workspacesWithInternalBacklog(db: Client): Promise { const placeholders = AUTO_APPROVED.map(() => '?').join(', '); diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index d352323..54498d6 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -87,6 +87,7 @@ export { autoApproveInternal, workspacesWithInternalBacklog, AUTO_APPROVE_ACTOR, + crawlDedupeKey, type AutoApproveResult, } from './auto-approve'; export {