From b8bda91e76efe9021e0d6b56f97d16977ea0ac82 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 16:16:03 +0000 Subject: [PATCH] Read a company's site once, however many people it names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runaway behind #61 and #62. A crawl of accenture.com returns 57 people; each becomes a `refresh_research` card; each card queued another crawl of the same page, which returned the same 57. Production reached 226 pending crawls of that one URL and 81 of toptal.com before it was stopped. The per-person cooldown in #62 could not catch this, and the samples said so plainly: cooldowns stayed at zero while pending crawls climbed 227 to 426. Every one of those 57 people was new, so none of them had any research history to be cooled down against. The cooldown was the right mechanism for the wrong axis — it bounds how often one person is re-researched, and the amplification is per *site*. `enqueue` has supported a dedupe key since 0007, and the index behind it deliberately covers only pending and running jobs so a key frees itself when the job finishes. `POST /prospects/by-url` has passed one from the start. Neither approval path ever did, which is why one route could not queue a host twice and another could queue it 226 times. Both approval paths now pass the same key that route uses — `crawl:` without `www.` — because two paths deduping under different keys do not deduplicate against each other, which is the whole point. Keyed on the host rather than the URL, so `example.com` and `https://www.example.com/` are the one crawl they actually are. The cards still all clear; only the duplicate crawls collapse. Emptying the queue was never the problem. Co-Authored-By: Claude Opus 5 (1M context) --- apps/api/src/app.ts | 6 ++ packages/pipeline/src/auto-approve.test.ts | 70 ++++++++++++++++++++++ packages/pipeline/src/auto-approve.ts | 37 +++++++++++- packages/pipeline/src/index.ts | 1 + 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 2266c51..2893f71 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -63,6 +63,7 @@ import { BlueskyAccountError, connectBlueskyAccount, budgetStatus, + crawlDedupeKey, startContactImport, importContactChunk, finishContactImport, @@ -3668,6 +3669,11 @@ async function approveRecommendation( url: target, ...(recommendation.campaign_id ? { campaignId: recommendation.campaign_id } : {}), }, + // The same key `POST /prospects/by-url` uses, so approving research + // for twenty people at one company reads that company's site once. + // Without it a page naming fifty-seven people asks for that page + // fifty-seven times, and every read finds the same fifty-seven. + dedupeKey: crawlDedupeKey(target), }); research = { queued: queued.queued, url: target }; } else { diff --git a/packages/pipeline/src/auto-approve.test.ts b/packages/pipeline/src/auto-approve.test.ts index eece804..5cae43e 100644 --- a/packages/pipeline/src/auto-approve.test.ts +++ b/packages/pipeline/src/auto-approve.test.ts @@ -292,6 +292,76 @@ describe('autoApproveInternal', () => { expect(result.queuedCrawls).toBe(0); }); + test('reads one company site once, however many people it names', async () => { + // The failure that got past the per-person cooldown and into production. + // A crawl of accenture.com returns 57 people; each becomes a research + // card; each card queued another crawl of the same page, which returned + // the same 57. Prod reached 226 pending crawls of one URL and 81 of + // another before it was stopped. The cooldown could not catch it because + // every one of those people was new. + seeded = await seedDatabase('auto-one-crawl'); + await withDomain(seeded.db); + + for (let index = 0; index < 8; index += 1) { + const personId = `per_same_co_${index}`; + await seeded.db.execute({ + sql: `INSERT INTO people (id, display_name, status, identity_confidence, + current_company_id, created_at, updated_at) + VALUES (?, ?, 'qualified', 0.9, 'co_auto', ?, ?)`, + args: [personId, `Colleague ${index}`, now(), now()], + }); + await seeded.db.execute({ + sql: `INSERT INTO recommendations (id, workspace_id, campaign_id, person_id, action, + network, priority, reason, policy_status, policy_version, expected_goal, status, + created_at) + VALUES (?, ?, ?, ?, 'refresh_research', 'website', 50, 'because', + 'allow_with_approval', '2026-08-11', 'qualify', 'pending', ?)`, + args: [newId('recommendation'), SEED.workspaceId, SEED.campaignId, personId, now()], + }); + } + + const result = await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId }); + + // Every card clears — the queue is the point. + expect(result.approved).toBeGreaterThanOrEqual(8); + + // But the site is read once. This is the assertion that matters. + const jobs = await queryOne<{ n: number }>( + seeded.db, + `SELECT count(*) AS n FROM jobs WHERE kind = 'crawl_site' AND status = 'pending'`, + ); + expect(Number(jobs?.n)).toBe(1); + expect(result.queuedCrawls).toBe(1); + }); + + test('the key frees itself once the crawl finishes', async () => { + // Suppressing duplicates must not suppress future work: the index covers + // only pending and running jobs for exactly this reason. + seeded = await seedDatabase('auto-crawl-key-frees'); + await withDomain(seeded.db); + await card(seeded.db, 'refresh_research'); + + expect( + (await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId })).queuedCrawls, + ).toBe(1); + + await seeded.db.execute({ + sql: `UPDATE jobs SET status = 'done' WHERE kind = 'crawl_site'`, + args: [], + }); + // Age the research so the per-person cooldown does not mask the result. + await seeded.db.execute({ + sql: `UPDATE actions SET created_at = ? WHERE kind = 'refresh_research'`, + args: [new Date(Date.now() - 48 * 3_600_000).toISOString()], + }); + + await card(seeded.db, 'refresh_research'); + + expect( + (await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId })).queuedCrawls, + ).toBe(1); + }); + test('honours the limit so one workspace cannot hold the tick', async () => { seeded = await seedDatabase('auto-limit'); diff --git a/packages/pipeline/src/auto-approve.ts b/packages/pipeline/src/auto-approve.ts index 297d0a9..06ef3cb 100644 --- a/packages/pipeline/src/auto-approve.ts +++ b/packages/pipeline/src/auto-approve.ts @@ -226,13 +226,25 @@ export async function autoApproveInternal( ); if (site?.domain) { + const url = normaliseDomain(site.domain); + const queued = await enqueue(db, { workspaceId: input.workspaceId, kind: 'crawl_site', payload: { - url: normaliseDomain(site.domain), + url, ...(row.campaign_id ? { campaignId: row.campaign_id } : {}), }, + // One crawl per site at a time. Without this a research card is a + // crawl *per person*, and a page that names many people asks for the + // same page many times: production queued accenture.com 226 times, + // toptal.com 81, from a single pass. The site is read once and every + // person on it is served by that read. + // + // The index behind this covers only pending and running jobs, so the + // key frees itself the moment the crawl finishes — this suppresses + // duplicates, never future work. + dedupeKey: crawlDedupeKey(url), }); if (queued.queued) queuedCrawls += 1; @@ -309,6 +321,29 @@ function normaliseDomain(domain: string): string { return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`; } +/** + * The key that collapses many requests for one site into a single crawl. + * + * Deliberately the same string `POST /prospects/by-url` has always used — + * `crawl:`, without `www.` — because two paths deduping under different + * keys do not deduplicate against each other, which is the entire point. That + * route had this from the start; the approval paths never passed a key at all, + * which is why one of them could queue accenture.com 226 times while the other + * could not queue it twice. + * + * Keyed on the host rather than the full URL so that `example.com` and + * `https://www.example.com/` are one crawl, which is what they fetch. + */ +export function crawlDedupeKey(url: string): string { + try { + return `crawl:${new URL(url).hostname.replace(/^www\./, '')}`; + } catch { + // Unparseable is still deduplicable against itself, and the crawl job will + // fail on its own terms rather than here. + return `crawl:${url.toLowerCase()}`; + } +} + /** Every workspace with internal cards waiting, for the worker to sweep. */ export async function workspacesWithInternalBacklog(db: Client): Promise { const placeholders = AUTO_APPROVED.map(() => '?').join(', '); diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index d352323..54498d6 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -87,6 +87,7 @@ export { autoApproveInternal, workspacesWithInternalBacklog, AUTO_APPROVE_ACTOR, + crawlDedupeKey, type AutoApproveResult, } from './auto-approve'; export {