Charge AI training crawlers for access (@profullstack/x402-gateway)

Training crawlers (GPTBot, ClaudeBot, CCBot, meta-externalagent, Bytespider,
Applebot-Extended) get 402 Payment Required with an x402 offer, or the sales
page at /crawl, and a paid pass opens the site for a day. People, search
engines and retrieval crawlers pass through untouched. robots.txt is now
generated from the same lists.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YafYxayh7Gqe5MWNNQMev2

Author: ralyodio

.env.example

@@ -159,4 +159,9 @@ VAPID_PRIVATE_KEY=your-vapid-private-key
 
 APPLE_ID=your-developer-email
 APPLE_TEAM_ID=your-apple-team-id
-APPLE_APP_SPECIFIC_PASSWORD=your-app-specific-password
+APPLE_APP_SPECIFIC_PASSWORD=your-app-specific-password
+# Crawl gateway (@profullstack/x402-gateway): AI training crawlers pay $1/day over
+# x402. A SCOPED CoinPay key (payments:create) and the EVM address that receives
+# the USDC. Unset = crawlers still get 402, nothing sold.
+COINPAY_X402_KEY=
+CRAWL_PAY_TO=

apps/web/package.json

@@ -19,6 +19,7 @@
     "@profullstack/autoblog": "github:profullstack/autoblog#75e54af",
     "@profullstack/emailer": "^1.0.3",
     "@profullstack/stack": "^0.1.3",
+    "@profullstack/x402-gateway": "^0.1.0",
     "@serwist/next": "^9.5.0",
     "@supabase/ssr": "^0.5.0",
     "@supabase/supabase-js": "^2.47.0",

apps/web/public/robots.txt

@@ -0,0 +1,9 @@
+import { robotsRoute } from '@profullstack/x402-gateway/next';
+import { gateway } from '@/lib/crawl-gateway';
+
+// Generated from the same crawler lists the gateway enforces: training
+// crawlers are refused everywhere but /crawl (where they can buy a pass),
+// retrieval crawlers are named as welcome, everyone else gets the rules below.
+export const GET = robotsRoute(gateway, {
+  disallow: ['/api/'],
+});

apps/web/src/app/robots.txt/route.ts

@@ -0,0 +1,27 @@
+import { createGateway } from '@profullstack/x402-gateway';
+import { x402Proxy } from '@profullstack/x402-gateway/next';
+
+/**
+ * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot,
+ * meta-externalagent, Bytespider, Applebot-Extended, ...) by the day over
+ * x402, settled by CoinPay in USDC. People, Googlebot and the retrieval
+ * crawlers behind AI search pass through untouched.
+ *
+ * Runs inside the middleware, so nothing here may import Node-only modules.
+ * The env is read through a non-literal key on purpose: Next inlines
+ * `process.env.NAME` at build time, and these are runtime secrets. Without
+ * COINPAY_X402_KEY and CRAWL_PAY_TO the gateway still answers training
+ * crawlers with 402, just with an empty offer.
+ */
+const env = (name: string) => process.env[name];
+
+export const gateway = createGateway({
+  siteUrl: env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://pairux.com',
+  siteName: 'PairUX',
+  coinpay: { apiKey: env('COINPAY_X402_KEY') },
+  payTo: env('CRAWL_PAY_TO'),
+  contact: 'mailto:support@pairux.com',
+});
+
+/** Resolves to a Response for a refused crawler, or undefined to carry on. */
+export const gate = x402Proxy(gateway);

apps/web/src/lib/crawl-gateway.ts

@@ -1,3 +1,4 @@
+import { gate } from '@/lib/crawl-gateway';
 import { NextResponse, type NextRequest } from 'next/server';
 import { updateSession } from '@/lib/supabase/middleware';
 import { CORS_HEADERS } from '@/lib/cors';
@@ -43,6 +44,12 @@ function buildCsp(nonce: string, embeddable: boolean): string {
 }
 
 export async function middleware(request: NextRequest) {
+  // Crawl gateway first: AI training crawlers get 402 Payment Required (or the
+  // sales page at /crawl) unless they present a paid pass. People, Googlebot
+  // and retrieval crawlers fall through to everything below.
+  const answer = await gate(request);
+  if (answer) return answer;
+
   const { pathname } = request.nextUrl;
 
   const isEmbed = pathname === '/embed' || pathname.startsWith('/embed/');