From 702fd5211f23afedb9d3a9fc62bcb66935555db1 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:30:42 +0000 Subject: [PATCH 1/2] Charge AI training crawlers for access (@profullstack/x402-gateway) Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider, Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales page at /crawl, and a paid pass opens the site for a day. People, search engines and retrieval crawlers pass through untouched. robots.txt is now generated from the same lists. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- .env.example | 6 ++++++ apps/web/app/robots.txt/route.ts | 9 +++++++++ apps/web/lib/crawl-gateway.ts | 27 +++++++++++++++++++++++++++ apps/web/package.json | 1 + apps/web/proxy.ts | 20 ++++++++++++++++++++ bun.lock | 3 +++ 6 files changed, 66 insertions(+) create mode 100644 apps/web/app/robots.txt/route.ts create mode 100644 apps/web/lib/crawl-gateway.ts create mode 100644 apps/web/proxy.ts diff --git a/.env.example b/.env.example index aedd08f..92645a3 100644 --- a/.env.example +++ b/.env.example @@ -154,3 +154,9 @@ REDIS_URL= # ------------------------------------------------------------------- worker WORKER_TICK_MS=60000 + +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over +# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives +# the USDC. Unset = crawlers still get 402, nothing sold. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/web/app/robots.txt/route.ts b/apps/web/app/robots.txt/route.ts new file mode 100644 index 0000000..e3ce9df --- /dev/null +++ b/apps/web/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from "@profullstack/x402-gateway/next"; +import { gateway } from "../../lib/crawl-gateway"; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ["/api/"], +}); diff --git a/apps/web/lib/crawl-gateway.ts b/apps/web/lib/crawl-gateway.ts new file mode 100644 index 0000000..fb1e676 --- /dev/null +++ b/apps/web/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://outreachgraph.com", + siteName: "OutreachGraph", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:support@outreachgraph.com", +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/package.json b/apps/web/package.json index 2c63005..39912ab 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,6 +12,7 @@ "dependencies": { "@outreachgraph/contracts": "workspace:*", "@outreachgraph/domain": "workspace:*", + "@profullstack/x402-gateway": "^0.1.0", "next": "16.3.0", "react": "19.2.8", "react-dom": "19.2.8" diff --git a/apps/web/proxy.ts b/apps/web/proxy.ts new file mode 100644 index 0000000..63662c2 --- /dev/null +++ b/apps/web/proxy.ts @@ -0,0 +1,20 @@ +import { gate } from "./lib/crawl-gateway"; +import { NextResponse, type NextRequest } from "next/server"; + +export async function proxy(request: NextRequest) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + + return NextResponse.next(); +} + +export const config = { + // Everything but Next's own assets and static files. API routes stay + // covered on purpose: a training crawler hitting the API gets 402 too. + matcher: [ + "/((?!_next/static|_next/image|favicon.ico|.*\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff|woff2|ttf|otf|mp3|mp4|webmanifest)$).*)", + ], +}; diff --git a/bun.lock b/bun.lock index 551bb5b..46f72cd 100644 --- a/bun.lock +++ b/bun.lock @@ -78,6 +78,7 @@ "dependencies": { "@outreachgraph/contracts": "workspace:*", "@outreachgraph/domain": "workspace:*", + "@profullstack/x402-gateway": "^0.1.0", "next": "16.3.0", "react": "19.2.8", "react-dom": "19.2.8", @@ -387,6 +388,8 @@ "@pinojs/redact": ["@pinojs/redact@0.4.0", "", {}, "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.1.0", "", {}, "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], "@tailwindcss/node": ["@tailwindcss/node@4.3.3", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.24.1", "jiti": "^2.7.0", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.3" } }, "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg=="], From 45a7c14ab02dd99b580bcf8a1cf8a0bfeeb335bd Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 21:40:01 +0000 Subject: [PATCH 2/2] Format Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2 --- apps/web/app/robots.txt/route.ts | 6 +++--- apps/web/lib/crawl-gateway.ts | 14 +++++++------- apps/web/proxy.ts | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/web/app/robots.txt/route.ts b/apps/web/app/robots.txt/route.ts index e3ce9df..36bcddb 100644 --- a/apps/web/app/robots.txt/route.ts +++ b/apps/web/app/robots.txt/route.ts @@ -1,9 +1,9 @@ -import { robotsRoute } from "@profullstack/x402-gateway/next"; -import { gateway } from "../../lib/crawl-gateway"; +import { robotsRoute } from '@profullstack/x402-gateway/next'; +import { gateway } from '../../lib/crawl-gateway'; // Generated from the same crawler lists the gateway enforces: training // crawlers are refused everywhere but /crawl (where they can buy a pass), // retrieval crawlers are named as welcome, everyone else gets the rules below. export const GET = robotsRoute(gateway, { - disallow: ["/api/"], + disallow: ['/api/'], }); diff --git a/apps/web/lib/crawl-gateway.ts b/apps/web/lib/crawl-gateway.ts index fb1e676..3b6dc8e 100644 --- a/apps/web/lib/crawl-gateway.ts +++ b/apps/web/lib/crawl-gateway.ts @@ -1,5 +1,5 @@ -import { createGateway } from "@profullstack/x402-gateway"; -import { x402Proxy } from "@profullstack/x402-gateway/next"; +import { createGateway } from '@profullstack/x402-gateway'; +import { x402Proxy } from '@profullstack/x402-gateway/next'; /** * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, @@ -16,11 +16,11 @@ import { x402Proxy } from "@profullstack/x402-gateway/next"; const env = (name: string) => process.env[name]; export const gateway = createGateway({ - siteUrl: env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://outreachgraph.com", - siteName: "OutreachGraph", - coinpay: { apiKey: env("COINPAY_X402_KEY") }, - payTo: env("CRAWL_PAY_TO"), - contact: "mailto:support@outreachgraph.com", + siteUrl: env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://outreachgraph.com', + siteName: 'OutreachGraph', + coinpay: { apiKey: env('COINPAY_X402_KEY') }, + payTo: env('CRAWL_PAY_TO'), + contact: 'mailto:support@outreachgraph.com', }); /** Resolves to a Response for a refused crawler, or undefined to carry on. */ diff --git a/apps/web/proxy.ts b/apps/web/proxy.ts index 63662c2..ce07af6 100644 --- a/apps/web/proxy.ts +++ b/apps/web/proxy.ts @@ -1,5 +1,5 @@ -import { gate } from "./lib/crawl-gateway"; -import { NextResponse, type NextRequest } from "next/server"; +import { gate } from './lib/crawl-gateway'; +import { NextResponse, type NextRequest } from 'next/server'; export async function proxy(request: NextRequest) { // Crawl gateway first: AI training crawlers get 402 Payment Required (or the @@ -15,6 +15,6 @@ export const config = { // Everything but Next's own assets and static files. API routes stay // covered on purpose: a training crawler hitting the API gets 402 too. matcher: [ - "/((?!_next/static|_next/image|favicon.ico|.*\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff|woff2|ttf|otf|mp3|mp4|webmanifest)$).*)", + '/((?!_next/static|_next/image|favicon.ico|.*\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff|woff2|ttf|otf|mp3|mp4|webmanifest)$).*)', ], };