Read a company's site once, however many people it names

apps/api/src/app.ts

@@ -63,6 +63,7 @@ import {
   BlueskyAccountError,
   connectBlueskyAccount,
   budgetStatus,
+  crawlDedupeKey,
   startContactImport,
   importContactChunk,
   finishContactImport,
@@ -3668,6 +3669,11 @@ async function approveRecommendation(
           url: target,
           ...(recommendation.campaign_id ? { campaignId: recommendation.campaign_id } : {}),
         },
+        // The same key `POST /prospects/by-url` uses, so approving research
+        // for twenty people at one company reads that company's site once.
+        // Without it a page naming fifty-seven people asks for that page
+        // fifty-seven times, and every read finds the same fifty-seven.
+        dedupeKey: crawlDedupeKey(target),
       });
       research = { queued: queued.queued, url: target };
     } else {

packages/pipeline/src/auto-approve.test.ts

@@ -292,6 +292,76 @@ describe('autoApproveInternal', () => {
     expect(result.queuedCrawls).toBe(0);
   });
 
+  test('reads one company site once, however many people it names', async () => {
+    // The failure that got past the per-person cooldown and into production.
+    // A crawl of accenture.com returns 57 people; each becomes a research
+    // card; each card queued another crawl of the same page, which returned
+    // the same 57. Prod reached 226 pending crawls of one URL and 81 of
+    // another before it was stopped. The cooldown could not catch it because
+    // every one of those people was new.
+    seeded = await seedDatabase('auto-one-crawl');
+    await withDomain(seeded.db);
+
+    for (let index = 0; index < 8; index += 1) {
+      const personId = `per_same_co_${index}`;
+      await seeded.db.execute({
+        sql: `INSERT INTO people (id, display_name, status, identity_confidence,
+              current_company_id, created_at, updated_at)
+              VALUES (?, ?, 'qualified', 0.9, 'co_auto', ?, ?)`,
+        args: [personId, `Colleague ${index}`, now(), now()],
+      });
+      await seeded.db.execute({
+        sql: `INSERT INTO recommendations (id, workspace_id, campaign_id, person_id, action,
+              network, priority, reason, policy_status, policy_version, expected_goal, status,
+              created_at)
+              VALUES (?, ?, ?, ?, 'refresh_research', 'website', 50, 'because',
+              'allow_with_approval', '2026-08-11', 'qualify', 'pending', ?)`,
+        args: [newId('recommendation'), SEED.workspaceId, SEED.campaignId, personId, now()],
+      });
+    }
+
+    const result = await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId });
+
+    // Every card clears — the queue is the point.
+    expect(result.approved).toBeGreaterThanOrEqual(8);
+
+    // But the site is read once. This is the assertion that matters.
+    const jobs = await queryOne<{ n: number }>(
+      seeded.db,
+      `SELECT count(*) AS n FROM jobs WHERE kind = 'crawl_site' AND status = 'pending'`,
+    );
+    expect(Number(jobs?.n)).toBe(1);
+    expect(result.queuedCrawls).toBe(1);
+  });
+
+  test('the key frees itself once the crawl finishes', async () => {
+    // Suppressing duplicates must not suppress future work: the index covers
+    // only pending and running jobs for exactly this reason.
+    seeded = await seedDatabase('auto-crawl-key-frees');
+    await withDomain(seeded.db);
+    await card(seeded.db, 'refresh_research');
+
+    expect(
+      (await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId })).queuedCrawls,
+    ).toBe(1);
+
+    await seeded.db.execute({
+      sql: `UPDATE jobs SET status = 'done' WHERE kind = 'crawl_site'`,
+      args: [],
+    });
+    // Age the research so the per-person cooldown does not mask the result.
+    await seeded.db.execute({
+      sql: `UPDATE actions SET created_at = ? WHERE kind = 'refresh_research'`,
+      args: [new Date(Date.now() - 48 * 3_600_000).toISOString()],
+    });
+
+    await card(seeded.db, 'refresh_research');
+
+    expect(
+      (await autoApproveInternal(seeded.db, { workspaceId: SEED.workspaceId })).queuedCrawls,
+    ).toBe(1);
+  });
+
   test('honours the limit so one workspace cannot hold the tick', async () => {
     seeded = await seedDatabase('auto-limit');
 

packages/pipeline/src/auto-approve.ts

@@ -226,13 +226,25 @@ export async function autoApproveInternal(
       );
 
       if (site?.domain) {
+        const url = normaliseDomain(site.domain);
+
         const queued = await enqueue(db, {
           workspaceId: input.workspaceId,
           kind: 'crawl_site',
           payload: {
-            url: normaliseDomain(site.domain),
+            url,
             ...(row.campaign_id ? { campaignId: row.campaign_id } : {}),
           },
+          // One crawl per site at a time. Without this a research card is a
+          // crawl *per person*, and a page that names many people asks for the
+          // same page many times: production queued accenture.com 226 times,
+          // toptal.com 81, from a single pass. The site is read once and every
+          // person on it is served by that read.
+          //
+          // The index behind this covers only pending and running jobs, so the
+          // key frees itself the moment the crawl finishes — this suppresses
+          // duplicates, never future work.
+          dedupeKey: crawlDedupeKey(url),
         });
 
         if (queued.queued) queuedCrawls += 1;
@@ -309,6 +321,29 @@ function normaliseDomain(domain: string): string {
   return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
 }
 
+/**
+ * The key that collapses many requests for one site into a single crawl.
+ *
+ * Deliberately the same string `POST /prospects/by-url` has always used —
+ * `crawl:<host>`, without `www.` — because two paths deduping under different
+ * keys do not deduplicate against each other, which is the entire point. That
+ * route had this from the start; the approval paths never passed a key at all,
+ * which is why one of them could queue accenture.com 226 times while the other
+ * could not queue it twice.
+ *
+ * Keyed on the host rather than the full URL so that `example.com` and
+ * `https://www.example.com/` are one crawl, which is what they fetch.
+ */
+export function crawlDedupeKey(url: string): string {
+  try {
+    return `crawl:${new URL(url).hostname.replace(/^www\./, '')}`;
+  } catch {
+    // Unparseable is still deduplicable against itself, and the crawl job will
+    // fail on its own terms rather than here.
+    return `crawl:${url.toLowerCase()}`;
+  }
+}
+
 /** Every workspace with internal cards waiting, for the worker to sweep. */
 export async function workspacesWithInternalBacklog(db: Client): Promise<string[]> {
   const placeholders = AUTO_APPROVED.map(() => '?').join(', ');

packages/pipeline/src/index.ts

@@ -87,6 +87,7 @@ export {
   autoApproveInternal,
   workspacesWithInternalBacklog,
   AUTO_APPROVE_ACTOR,
+  crawlDedupeKey,
   type AutoApproveResult,
 } from './auto-approve';
 export {