diff --git a/.env.example b/.env.example index aedd08f..92645a3 100644 --- a/.env.example +++ b/.env.example @@ -154,3 +154,9 @@ REDIS_URL= # ------------------------------------------------------------------- worker WORKER_TICK_MS=60000 + +# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over +# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives +# the USDC. Unset = crawlers still get 402, nothing sold. +COINPAY_X402_KEY= +CRAWL_PAY_TO= diff --git a/apps/web/app/robots.txt/route.ts b/apps/web/app/robots.txt/route.ts new file mode 100644 index 0000000..36bcddb --- /dev/null +++ b/apps/web/app/robots.txt/route.ts @@ -0,0 +1,9 @@ +import { robotsRoute } from '@profullstack/x402-gateway/next'; +import { gateway } from '../../lib/crawl-gateway'; + +// Generated from the same crawler lists the gateway enforces: training +// crawlers are refused everywhere but /crawl (where they can buy a pass), +// retrieval crawlers are named as welcome, everyone else gets the rules below. +export const GET = robotsRoute(gateway, { + disallow: ['/api/'], +}); diff --git a/apps/web/lib/crawl-gateway.ts b/apps/web/lib/crawl-gateway.ts new file mode 100644 index 0000000..3b6dc8e --- /dev/null +++ b/apps/web/lib/crawl-gateway.ts @@ -0,0 +1,27 @@ +import { createGateway } from '@profullstack/x402-gateway'; +import { x402Proxy } from '@profullstack/x402-gateway/next'; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, + * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over + * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval + * crawlers behind AI search pass through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. Without + * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training + * crawlers with 402, just with an empty offer. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://outreachgraph.com', + siteName: 'OutreachGraph', + coinpay: { apiKey: env('COINPAY_X402_KEY') }, + payTo: env('CRAWL_PAY_TO'), + contact: 'mailto:support@outreachgraph.com', +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/package.json b/apps/web/package.json index 2c63005..39912ab 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,6 +12,7 @@ "dependencies": { "@outreachgraph/contracts": "workspace:*", "@outreachgraph/domain": "workspace:*", + "@profullstack/x402-gateway": "^0.1.0", "next": "16.3.0", "react": "19.2.8", "react-dom": "19.2.8" diff --git a/apps/web/proxy.ts b/apps/web/proxy.ts new file mode 100644 index 0000000..ce07af6 --- /dev/null +++ b/apps/web/proxy.ts @@ -0,0 +1,20 @@ +import { gate } from './lib/crawl-gateway'; +import { NextResponse, type NextRequest } from 'next/server'; + +export async function proxy(request: NextRequest) { + // Crawl gateway first: AI training crawlers get 402 Payment Required (or the + // sales page at /crawl) unless they present a paid pass. People, Googlebot + // and retrieval crawlers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + + return NextResponse.next(); +} + +export const config = { + // Everything but Next's own assets and static files. API routes stay + // covered on purpose: a training crawler hitting the API gets 402 too. + matcher: [ + '/((?!_next/static|_next/image|favicon.ico|.*\.(?:svg|png|jpg|jpeg|gif|webp|ico|css|js|map|woff|woff2|ttf|otf|mp3|mp4|webmanifest)$).*)', + ], +}; diff --git a/bun.lock b/bun.lock index 551bb5b..46f72cd 100644 --- a/bun.lock +++ b/bun.lock @@ -78,6 +78,7 @@ "dependencies": { "@outreachgraph/contracts": "workspace:*", "@outreachgraph/domain": "workspace:*", + "@profullstack/x402-gateway": "^0.1.0", "next": "16.3.0", "react": "19.2.8", "react-dom": "19.2.8", @@ -387,6 +388,8 @@ "@pinojs/redact": ["@pinojs/redact@0.4.0", "", {}, "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.1.0", "", {}, "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], "@tailwindcss/node": ["@tailwindcss/node@4.3.3", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.24.1", "jiti": "^2.7.0", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.3" } }, "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg=="],