diff --git a/bun.lock b/bun.lock index d0d9ce21..91d83fe4 100644 --- a/bun.lock +++ b/bun.lock @@ -9,11 +9,12 @@ "@noble/hashes": "^2.0.0", "@profullstack/autoblog": "github:profullstack/autoblog#75e54af77cbcf61dbd90fb3ed529c1b2dad1ed6f", "@profullstack/encrypt": "^0.1.0", - "@profullstack/footer": "0.2.5", + "@profullstack/footer": "latest", "@profullstack/openprofile": "^0.2.0", "@profullstack/pairux-embed": "^0.1.0", "@profullstack/stack": "^0.1.3", "@profullstack/text-type-detection": "^1.0.0", + "@profullstack/x402-gateway": "^0.11.0", "@simplewebauthn/browser": "^14.0.0", "@simplewebauthn/server": "^14.0.3", "@supabase/ssr": "^0.7.0", @@ -421,6 +422,8 @@ "@profullstack/hqtui": ["@profullstack/hqtui@0.10.2", "", { "bin": { "hqtui": "bin/hqtui.mjs" } }, "sha512-rflKGzBWdX8xXsQgoMq9L0QhK4DP6a0+vqQu0Uq29XoB6OT4zM2QhMXXcIw2FgyoUyF43nf4dDFB0CauMSvwRg=="], + "@profullstack/keys": ["@profullstack/keys@0.1.0", "", {}, "sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w=="], + "@profullstack/openprofile": ["@profullstack/openprofile@0.2.0", "", {}, "sha512-cdFsBbfRXurvy5KqHDNcKXxurK2s/YlfVgnILFRpVl2hq3860rExqDee8IfsFGxw5cPdqGv2fxUxtek1Xy2U5Q=="], "@profullstack/pairux-embed": ["@profullstack/pairux-embed@0.1.0", "", { "dependencies": { "livekit-client": "^2.17.0" } }, "sha512-dGRx0kW6Y0awBOy9nbWrqAEcZNJEHrBOfxG/RikmNHuGyE+BaP0Qn1ZFis8dDWyQdEJ6VsTIH6xJbu+Gx5ITqg=="], @@ -431,6 +434,8 @@ "@profullstack/text-type-detection": ["@profullstack/text-type-detection@1.0.0", "", {}, "sha512-3tG0Icz2gwLfwbOwdYxgCTTcjCyAAhEnU82ReIrbOQiN8f3R84WrPLWfE4Pqg/ce57VFVPpNkSmd5doaXKV+dw=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.11.0", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q=="], + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-rc.3", "", {}, "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q=="], "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.52.4", "", { "os": "android", "cpu": "arm" }, "sha512-BTm2qKNnWIQ5auf4deoetINJm2JzvihvGb9R6K/ETwKLql/Bb3Eg2H1FBp1gUb4YGbydMA3jcmQTR73q7J+GAA=="], diff --git a/package.json b/package.json index 573741e0..25ca3ecc 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@profullstack/pairux-embed": "^0.1.0", "@profullstack/stack": "^0.1.3", "@profullstack/text-type-detection": "^1.0.0", + "@profullstack/x402-gateway": "^0.11.0", "@simplewebauthn/browser": "^14.0.0", "@simplewebauthn/server": "^14.0.3", "@supabase/ssr": "^0.7.0", diff --git a/src/lib/server/crawl-gateway.js b/src/lib/server/crawl-gateway.js new file mode 100644 index 00000000..7ddfc445 --- /dev/null +++ b/src/lib/server/crawl-gateway.js @@ -0,0 +1,28 @@ +import { createGateway } from '@profullstack/x402-gateway'; +import { x402Proxy } from '@profullstack/x402-gateway/next'; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, ...) + * by the day over x402, settled by CoinPay in USDC, with a CoinPay checkout + * for people at /crawl. Browsers, Googlebot and AI chat/search fetchers pass + * through untouched. + * + * Runs inside the middleware, so nothing here may import Node-only modules. + * The env is read through a non-literal key on purpose: Next inlines + * `process.env.NAME` at build time, and these are runtime secrets. + */ +const env = (name) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: (env('SITE_URL') || env('NEXT_PUBLIC_SITE_URL') || 'https://qrypt.chat').replace(/\/+$/, ''), + siteName: 'QryptChat', + coinpay: { apiKey: env('COINPAY_X402_KEY') }, + payTo: env('CRAWL_PAY_TO'), + contact: 'mailto:anthony@profullstack.com', + // The API (health checks, auth hooks, webhooks, chat), the realtime socket + // and the crawl manifests are never charged. + openPaths: ['/api/', '/ws', '/_next/', '/robots.txt', '/sitemap.xml', '/llms.txt', '/.well-known/', '/manifest.json', '/sw.js'], +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/src/middleware.js b/src/middleware.js index 7efc23a1..c6013595 100644 --- a/src/middleware.js +++ b/src/middleware.js @@ -1,5 +1,6 @@ import { NextResponse } from 'next/server'; import { getClientIp, RateLimiter } from '@/lib/server/rate-limiter.js'; +import { gate } from '@/lib/server/crawl-gateway.js'; const authRateLimiter = new RateLimiter({ maxRequests: 10, windowMs: 60 * 1000 }); const webhookRateLimiter = new RateLimiter({ maxRequests: 100, windowMs: 60 * 1000 }); @@ -13,7 +14,7 @@ function rateLimitedResponse() { ); } -export function middleware(request) { +export async function middleware(request) { const { pathname, host } = request.nextUrl; // WWW redirect @@ -23,6 +24,12 @@ export function middleware(request) { return NextResponse.redirect(url, 301); } + // Crawl gateway: AI training crawlers get 402 Payment Required (or the pass + // page at /crawl) unless they present a paid pass. People, Googlebot and AI + // chat fetchers fall through to everything below. + const answer = await gate(request); + if (answer) return answer; + // Health checks (status.profullstack.com) are never rate limited. if (pathname === '/api/health') return NextResponse.next(); diff --git a/tests/crawl-gateway.test.js b/tests/crawl-gateway.test.js new file mode 100644 index 00000000..8c965852 --- /dev/null +++ b/tests/crawl-gateway.test.js @@ -0,0 +1,40 @@ +import { beforeAll, describe, expect, it } from 'vitest'; + +const GPTBOT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; +const HUMAN = { + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + accept: 'text/html,application/xhtml+xml', + 'accept-language': 'en-US', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-dest': 'document', + 'x-real-ip': '198.51.100.7', +}; + +let gateway; +beforeAll(async () => { + process.env.COINPAY_X402_KEY = 'cp_live_test'; + process.env.CRAWL_PAY_TO = '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5'; + ({ gateway } = await import('../src/lib/server/crawl-gateway.js')); +}); + +describe('crawl gateway', () => { + it('answers a training crawler 402 with PAYMENT-REQUIRED', async () => { + const res = await gateway.handle(new Request('https://qrypt.chat/', { headers: { 'user-agent': GPTBOT, 'x-real-ip': '203.0.113.9' } })); + expect(res?.status).toBe(402); + expect(res?.headers.get('payment-required')).toBeTruthy(); + }); + + it('offers a $1 day pass at /crawl', async () => { + const res = await gateway.handle(new Request('https://qrypt.chat/crawl', { headers: { accept: 'application/json' } })); + expect(res?.status).toBe(402); + expect(res?.headers.get('payment-required')).toBeTruthy(); + const body = await res.json(); + expect(String(body.accepts[0].amount)).toBe('1000000'); + }); + + it('lets browsers and the API through', async () => { + expect(await gateway.handle(new Request('https://qrypt.chat/', { headers: HUMAN }))).toBeNull(); + expect(await gateway.handle(new Request('https://qrypt.chat/api/health', { headers: { 'user-agent': GPTBOT } }))).toBeNull(); + }); +});