feat: x402 crawl gateway for training crawlers

package.json

@@ -66,6 +66,7 @@
     "@profullstack/pairux-embed": "^0.1.0",
     "@profullstack/stack": "^0.1.3",
     "@profullstack/text-type-detection": "^1.0.0",
+    "@profullstack/x402-gateway": "^0.11.0",
     "@simplewebauthn/browser": "^14.0.0",
     "@simplewebauthn/server": "^14.0.3",
     "@supabase/ssr": "^0.7.0",

src/lib/server/crawl-gateway.js

@@ -0,0 +1,28 @@
+import { createGateway } from '@profullstack/x402-gateway';
+import { x402Proxy } from '@profullstack/x402-gateway/next';
+
+/**
+ * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, ...)
+ * by the day over x402, settled by CoinPay in USDC, with a CoinPay checkout
+ * for people at /crawl. Browsers, Googlebot and AI chat/search fetchers pass
+ * through untouched.
+ *
+ * Runs inside the middleware, so nothing here may import Node-only modules.
+ * The env is read through a non-literal key on purpose: Next inlines
+ * `process.env.NAME` at build time, and these are runtime secrets.
+ */
+const env = (name) => process.env[name];
+
+export const gateway = createGateway({
+  siteUrl: (env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://qrypt.chat').replace(/\/+$/, ''),
+  siteName: 'QryptChat',
+  coinpay: { apiKey: env('COINPAY_X402_KEY') },
+  payTo: env('CRAWL_PAY_TO'),
+  contact: 'mailto:anthony@profullstack.com',
+  // The API (health checks, auth hooks, webhooks, chat), the realtime socket
+  // and the crawl manifests are never charged.
+  openPaths: ['/api/', '/ws', '/_next/', '/robots.txt', '/sitemap.xml', '/llms.txt', '/.well-known/', '/manifest.json', '/sw.js'],
+});
+
+/** Resolves to a Response for a refused crawler, or undefined to carry on. */
+export const gate = x402Proxy(gateway);

src/middleware.js

@@ -1,5 +1,6 @@
 import { NextResponse } from 'next/server';
 import { getClientIp, RateLimiter } from '@/lib/server/rate-limiter.js';
+import { gate } from '@/lib/server/crawl-gateway.js';
 
 const authRateLimiter = new RateLimiter({ maxRequests: 10, windowMs: 60 * 1000 });
 const webhookRateLimiter = new RateLimiter({ maxRequests: 100, windowMs: 60 * 1000 });
@@ -13,7 +14,7 @@ function rateLimitedResponse() {
   );
 }
 
-export function middleware(request) {
+export async function middleware(request) {
   const { pathname, host } = request.nextUrl;
 
   // WWW redirect
@@ -23,6 +24,12 @@ export function middleware(request) {
     return NextResponse.redirect(url, 301);
   }
 
+  // Crawl gateway: AI training crawlers get 402 Payment Required (or the pass
+  // page at /crawl) unless they present a paid pass. People, Googlebot and AI
+  // chat fetchers fall through to everything below.
+  const answer = await gate(request);
+  if (answer) return answer;
+
   // Health checks (status.profullstack.com) are never rate limited.
   if (pathname === '/api/health') return NextResponse.next();
 

tests/crawl-gateway.test.js

@@ -0,0 +1,40 @@
+import { beforeAll, describe, expect, it } from 'vitest';
+
+const GPTBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)';
+const HUMAN = {
+  'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+  accept: 'text/html,application/xhtml+xml',
+  'accept-language': 'en-US',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+  'sec-fetch-dest': 'document',
+  'x-real-ip': '198.51.100.7',
+};
+
+let gateway;
+beforeAll(async () => {
+  process.env.COINPAY_X402_KEY = 'cp_live_test';
+  process.env.CRAWL_PAY_TO = '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5';
+  ({ gateway } = await import('../src/lib/server/crawl-gateway.js'));
+});
+
+describe('crawl gateway', () => {
+  it('answers a training crawler 402 with PAYMENT-REQUIRED', async () => {
+    const res = await gateway.handle(new Request('https://qrypt.chat/', { headers: { 'user-agent': GPTBOT, 'x-real-ip': '203.0.113.9' } }));
+    expect(res?.status).toBe(402);
+    expect(res?.headers.get('payment-required')).toBeTruthy();
+  });
+
+  it('offers a $1 day pass at /crawl', async () => {
+    const res = await gateway.handle(new Request('https://qrypt.chat/crawl', { headers: { accept: 'application/json' } }));
+    expect(res?.status).toBe(402);
+    expect(res?.headers.get('payment-required')).toBeTruthy();
+    const body = await res.json();
+    expect(String(body.accepts[0].amount)).toBe('1000000');
+  });
+
+  it('lets browsers and the API through', async () => {
+    expect(await gateway.handle(new Request('https://qrypt.chat/', { headers: HUMAN }))).toBeNull();
+    expect(await gateway.handle(new Request('https://qrypt.chat/api/health', { headers: { 'user-agent': GPTBOT } }))).toBeNull();
+  });
+});