From 81d8c5fde4f2f1fb13197f3bb5fb5c419f988d27 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 4 Nov 2026 20:10:47 +0000 Subject: [PATCH 1/5] feat(graphile): remove multipart uploads and graphile-llm metering Uploads go through the presigned path only: the Upload scalar, the *Upload mutation inputs, the graphql-upload middleware and the streaming resolver are gone, and image/upload/attachment columns are plain columns on the mutation inputs. graphile-llm no longer carries a metering plugin; the embedder has one result shape. --- .github/workflows/run-tests.yaml | 2 +- CLAUDE.md | 2 +- graphile/graphile-llm/package.json | 1 - .../src/__tests__/preset-schema.test.ts | 124 ++ graphile/graphile-llm/src/config-cache.ts | 216 ---- graphile/graphile-llm/src/embedder.ts | 7 +- graphile/graphile-llm/src/index.ts | 16 - graphile/graphile-llm/src/metering.ts | 431 ------- .../src/plugins/llm-module-plugin.ts | 14 +- .../src/plugins/metering-plugin.ts | 231 ---- .../src/plugins/text-mutation-plugin.ts | 4 +- .../src/plugins/text-search-plugin.ts | 3 +- graphile/graphile-llm/src/preset.ts | 41 +- graphile/graphile-llm/src/types.ts | 58 +- .../__tests__/dedup-liveness.test.ts | 81 -- .../__tests__/file-projection.test.ts | 36 + .../__tests__/managed-upload.test.ts | 668 ----------- .../src/file-projection.ts | 48 + .../src/index.ts | 9 +- .../src/managed-upload.ts | 418 ------- .../src/plugin.ts | 2 +- .../src/types.ts | 2 +- .../__tests__/upload-resolver.test.ts | 371 ------ graphile/graphile-settings/package.json | 3 - .../src/presets/constructive-preset.ts | 14 - .../graphile-settings/src/upload-resolver.ts | 277 ----- graphile/graphile-upload-plugin/CHANGELOG.md | 222 ---- graphile/graphile-upload-plugin/README.md | 15 - .../__tests__/plugin.test.ts | 1049 ----------------- .../__tests__/scalar.test.ts | 133 --- .../__tests__/types.test.ts | 147 --- .../graphile-upload-plugin/jest.config.js | 18 - graphile/graphile-upload-plugin/package.json | 52 - graphile/graphile-upload-plugin/src/index.ts | 39 - graphile/graphile-upload-plugin/src/plugin.ts | 372 ------ graphile/graphile-upload-plugin/src/preset.ts | 46 - graphile/graphile-upload-plugin/src/types.ts | 104 -- .../graphile-upload-plugin/tsconfig.esm.json | 8 - graphile/graphile-upload-plugin/tsconfig.json | 8 - graphql/dev-server/README.md | 4 +- graphql/dev-server/src/preset.ts | 7 +- .../seed/upload-columns/schema.sql | 26 + .../upload-columns.integration.test.ts | 137 +++ graphql/server/README.md | 2 - graphql/server/package.json | 2 - .../__tests__/request-protection.test.ts | 14 - .../server/src/middleware/multipart-bridge.ts | 32 - .../src/middleware/request-protection.ts | 11 +- graphql/server/src/server.ts | 9 - .../src/loaders/database-settings.ts | 3 - packages/express-context/src/types.ts | 1 - pnpm-lock.yaml | 239 ---- 52 files changed, 399 insertions(+), 5380 deletions(-) create mode 100644 graphile/graphile-llm/src/__tests__/preset-schema.test.ts delete mode 100644 graphile/graphile-llm/src/config-cache.ts delete mode 100644 graphile/graphile-llm/src/metering.ts delete mode 100644 graphile/graphile-llm/src/plugins/metering-plugin.ts create mode 100644 graphile/graphile-presigned-url-plugin/__tests__/file-projection.test.ts delete mode 100644 graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/file-projection.ts delete mode 100644 graphile/graphile-presigned-url-plugin/src/managed-upload.ts delete mode 100644 graphile/graphile-settings/__tests__/upload-resolver.test.ts delete mode 100644 graphile/graphile-settings/src/upload-resolver.ts delete mode 100644 graphile/graphile-upload-plugin/CHANGELOG.md delete mode 100644 graphile/graphile-upload-plugin/README.md delete mode 100644 graphile/graphile-upload-plugin/__tests__/plugin.test.ts delete mode 100644 graphile/graphile-upload-plugin/__tests__/scalar.test.ts delete mode 100644 graphile/graphile-upload-plugin/__tests__/types.test.ts delete mode 100644 graphile/graphile-upload-plugin/jest.config.js delete mode 100644 graphile/graphile-upload-plugin/package.json delete mode 100644 graphile/graphile-upload-plugin/src/index.ts delete mode 100644 graphile/graphile-upload-plugin/src/plugin.ts delete mode 100644 graphile/graphile-upload-plugin/src/preset.ts delete mode 100644 graphile/graphile-upload-plugin/src/types.ts delete mode 100644 graphile/graphile-upload-plugin/tsconfig.esm.json delete mode 100644 graphile/graphile-upload-plugin/tsconfig.json create mode 100644 graphql/server-test/__fixtures__/seed/upload-columns/schema.sql create mode 100644 graphql/server-test/__tests__/upload-columns.integration.test.ts delete mode 100644 graphql/server/src/middleware/multipart-bridge.ts diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 93da3d912f..b1162ea995 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -113,7 +113,7 @@ jobs: - batch: graphql packages: 'graphql/query graphql/codegen' - batch: graphile-unit - packages: 'graphile/graphile-plugin-utils graphile/graphile-cache graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin graphile/graphile-storage-registry' + packages: 'graphile/graphile-plugin-utils graphile/graphile-cache graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-storage-registry' - batch: agentic packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/permissions agentic/chat agentic/cli agentic/db-tools agentic/pi agentic/dsh agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/metering agentic/agent-conversation agentic/pi-host agentic/run-log-client agentic/run-log-gate agentic/machine-protocol agentic/machine-runner agentic/agent-cli' - batch: pgpm-unit diff --git a/CLAUDE.md b/CLAUDE.md index 1fc9350389..d0281a1b62 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,7 +84,7 @@ React Query Hooks or Prisma-like ORM Client **`graphql/server`** - Express + PostGraphile. Supports multi-endpoint routing via subdomain/host detection (schema builder with app-public/auth/admin sub-endpoints). Uses `LISTEN/NOTIFY` for schema cache invalidation. -**`graphile/graphile-settings`** - Centralizes all PostGraphile plugin config: connection filters, full-text search, PostGIS, i18n, meta-schema, many-to-many, search, upload plugin. Single `getGraphileSettings(opts)` entry point. +**`graphile/graphile-settings`** - Centralizes all PostGraphile plugin config: connection filters, full-text search, PostGIS, i18n, meta-schema, many-to-many, search, presigned uploads. Single `getGraphileSettings(opts)` entry point. **`packages/query-builder`** - Fluent SQL query builder for SELECT/INSERT/UPDATE/DELETE with JOINs, WHERE, GROUP BY. Schema-qualified tables. diff --git a/graphile/graphile-llm/package.json b/graphile/graphile-llm/package.json index 72460b18a1..1cd9668766 100644 --- a/graphile/graphile-llm/package.json +++ b/graphile/graphile-llm/package.json @@ -30,7 +30,6 @@ }, "dependencies": { "@agentic-kit/ollama": "workspace:*", - "@constructive-io/express-context": "workspace:^", "@constructive-io/llm-env": "workspace:^", "graphile-cache": "workspace:^" }, diff --git a/graphile/graphile-llm/src/__tests__/preset-schema.test.ts b/graphile/graphile-llm/src/__tests__/preset-schema.test.ts new file mode 100644 index 0000000000..6065c646b9 --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/preset-schema.test.ts @@ -0,0 +1,124 @@ +import { ConnectionFilterPreset } from 'graphile-connection-filter'; +import { createPgvectorAdapter } from 'graphile-search/adapters/pgvector'; +import { VectorCodecPlugin } from 'graphile-search/codecs/vector-codec'; +import { createUnifiedSearchPlugin } from 'graphile-search/plugin'; +import type { GraphQLResponse } from 'graphile-test'; +import { getConnections, seed } from 'graphile-test'; +import { join } from 'path'; +import type { PgTestClient } from 'pgsql-test'; + +import { GraphileLlmPreset } from '../preset'; + +// The preset as production loads it (graphile-settings calls `GraphileLlmPreset()` +// and the dev stack adds an embedder), built into a real schema with the +// plugins it enriches, and queried through grafast. + +type QueryFn = ( + query: string, + variables?: Record +) => Promise>; + +const OLLAMA_URL = 'http://localhost:11434'; + +async function ollamaReachable(): Promise { + try { + const res = await fetch(`${OLLAMA_URL}/api/tags`); + return res.ok; + } catch { + return false; + } +} + +describe('GraphileLlmPreset builds a real schema', () => { + let db: PgTestClient; + let teardown: () => Promise; + let query: QueryFn; + + beforeAll(async () => { + const connections = await getConnections( + { + schemas: ['llm_test'], + preset: { + extends: [ + ConnectionFilterPreset(), + GraphileLlmPreset({ + defaultEmbedder: { provider: 'ollama', model: 'nomic-embed-text', baseUrl: OLLAMA_URL }, + enableRag: true + }) + ], + plugins: [ + VectorCodecPlugin, + createUnifiedSearchPlugin({ adapters: [createPgvectorAdapter()] }) + ] + }, + useRoot: true, + authRole: 'postgres' + }, + [seed.sqlfile([join(__dirname, './setup.sql')])] + ); + db = connections.db; + teardown = connections.teardown; + query = connections.query; + }); + + afterAll(async () => { + if (teardown) await teardown(); + }); + + beforeEach(async () => { + await db.beforeEach(); + }); + + afterEach(async () => { + await db.afterEach(); + }); + + it('has exactly the module, text search, text mutation and RAG plugins', () => { + const names = GraphileLlmPreset({ enableRag: true }).plugins!.map((p) => p.name); + expect(names).toEqual(['LlmModulePlugin', 'LlmTextSearchPlugin', 'LlmTextMutationPlugin', 'LlmRagPlugin']); + }); + + it('takes no metering option', () => { + // @ts-expect-error there is no metering plugin to turn on + expect(GraphileLlmPreset({ metering: true }).plugins!.map((p) => p.name)).toEqual([ + 'LlmModulePlugin', + 'LlmTextSearchPlugin', + 'LlmTextMutationPlugin' + ]); + }); + + it('exposes the text search field, the text companion field and embedText', async () => { + const result = await query<{ + nearby: { inputFields: Array<{ name: string }> }; + input: { inputFields: Array<{ name: string }> }; + root: { fields: Array<{ name: string }> }; + }>(` + query { + nearby: __type(name: "VectorNearbyInput") { inputFields { name } } + input: __type(name: "ArticleInput") { inputFields { name } } + root: __type(name: "Query") { fields { name } } + } + `); + + expect(result.errors).toBeUndefined(); + const names = (fields: Array<{ name: string }>) => fields.map((f) => f.name); + expect(names(result.data!.nearby.inputFields)).toContain('text'); + expect(names(result.data!.input.inputFields)).toContain('embeddingText'); + expect(names(result.data!.root.fields)).toContain('embedText'); + }); + + it('embedText reads the embedder result through the one embedder shape', async () => { + if (!(await ollamaReachable())) { + console.log('Ollama not reachable; skipping the live embed'); + return; + } + + const result = await query<{ embedText: { vector: number[]; dimensions: number } }>(` + query { embedText(text: "warehouse lease") { vector dimensions } } + `); + + expect(result.errors).toBeUndefined(); + expect(result.data!.embedText.dimensions).toBe(768); + expect(result.data!.embedText.vector).toHaveLength(768); + }); +}); diff --git a/graphile/graphile-llm/src/config-cache.ts b/graphile/graphile-llm/src/config-cache.ts deleted file mode 100644 index b71b6cb053..0000000000 --- a/graphile/graphile-llm/src/config-cache.ts +++ /dev/null @@ -1,216 +0,0 @@ -/** - * config-cache — Per-database LLM billing configuration cache - * - * Caches resolved billing function names per database_id. - * Uses an LRU cache with TTL so config changes propagate within a bounded window - * without requiring a server restart. - * - * Resolution flow: - * Billing config from `metaschema_modules_public.billing_module` - * (schema name + function names for record_usage, check_billing_quota) - * - * All queries run through the Graphile `withPgClient` callback, which gives us - * a client connected to the tenant database with proper role settings. - * - * The LLM module config (provider, model, etc.) is already resolved by the - * LlmModulePlugin at schema-build time. This cache handles the runtime-only - * billing piece. - */ - -import { ModuleConfigCache } from 'graphile-cache'; - -// ─── Types ────────────────────────────────────────────────────────────────── - -/** - * Generic pg client interface matching what Graphile's withPgClient provides. - * Avoids a hard dependency on the `pg` package. - */ -export interface PgClient { - query(sql: string, values?: unknown[]): Promise<{ rows: Record[] }>; -} - -/** - * Billing function metadata resolved from the billing_module metaschema table. - */ -export interface BillingConfig { - /** Private schema containing the billing functions */ - privateSchema: string; - /** Name of the record_usage function */ - recordUsageFunction: string; - /** Name of the check_billing_quota function */ - checkBillingQuotaFunction: string; - /** Public schema containing meters table */ - publicSchema: string; -} - -/** - * Inference log table metadata resolved from the inference_log_module. - */ -export interface InferenceLogConfig { - /** Schema containing the usage_log_inference table */ - schema: string; - /** Name of the inference log table */ - tableName: string; -} - -/** - * Per-database cached configuration for the LLM billing integration. - */ -export interface LlmBillingCacheEntry { - /** Billing function references (null if billing_module not provisioned) */ - billing: BillingConfig | null; - /** Inference log table references (null if inference_log_module not provisioned) */ - inferenceLog: InferenceLogConfig | null; -} - -// ─── SQL Queries ──────────────────────────────────────────────────────────── - -/** - * Check if the billing_module table exists before querying it. - * This prevents hard errors on databases that don't have the billing - * module provisioned (the metaschema_modules_public schema or the - * billing_module table might not exist at all). - */ -const BILLING_MODULE_SQL = ` - SELECT - s.schema_name AS public_schema, - ps.schema_name AS private_schema, - bm.record_usage_function - FROM metaschema_modules_public.billing_module bm - JOIN metaschema_public.schema s ON bm.schema_id = s.id - JOIN metaschema_public.schema ps ON bm.private_schema_id = ps.id - WHERE bm.database_id = $1 - LIMIT 1 -`; - -/** - * Resolve the inference log module's schema and table name. - */ -const INFERENCE_LOG_MODULE_SQL = ` - SELECT - s.schema_name AS schema, - ilm.inference_log_table_name AS table_name - FROM metaschema_modules_public.inference_log_module ilm - JOIN metaschema_public.schema s ON ilm.schema_id = s.id - WHERE ilm.database_id = $1 - LIMIT 1 -`; -// ─── Cache ────────────────────────────────────────────────────────────────── - -const billingCache = new ModuleConfigCache({ - name: 'billing-config', - ttlMs: 5 * 60 * 1000, // 5 minutes - max: 50 -}); - -// ─── Resolution Functions ─────────────────────────────────────────────────── - -/** - * SQL to check if a schema exists. Used as a guard before querying - * metaschema tables that may not be provisioned. - */ -const SCHEMA_EXISTS_SQL = ` - SELECT 1 FROM information_schema.schemata WHERE schema_name = $1 LIMIT 1 -`; - -/** - * The module table is absent on a database that never provisioned it; only - * that undefined_table probe result means "not configured". Anything else - * (permissions, connectivity, bad SQL) is a real failure and propagates. - */ -const isModuleTableMissing = (err: unknown): boolean => - (err as { code?: string } | null)?.code === '42P01'; - -async function resolveInferenceLogConfig( - pgClient: PgClient, - databaseId: string -): Promise { - try { - const schemaCheck = await pgClient.query(SCHEMA_EXISTS_SQL, ['metaschema_modules_public']); - if (schemaCheck.rows.length === 0) return null; - - const result = await pgClient.query(INFERENCE_LOG_MODULE_SQL, [databaseId]); - const row = result.rows[0]; - if (!row?.schema || !row?.table_name) return null; - - return { - schema: row.schema as string, - tableName: row.table_name as string - }; - } catch (err) { - if (isModuleTableMissing(err)) return null; - throw err; - } -} - -async function resolveBillingConfig( - pgClient: PgClient, - databaseId: string -): Promise { - try { - // Guard: check if the metaschema_modules_public schema exists. - // If the database doesn't have the billing module provisioned, - // this schema (or the billing_module table) won't exist. - const schemaCheck = await pgClient.query(SCHEMA_EXISTS_SQL, ['metaschema_modules_public']); - if (schemaCheck.rows.length === 0) return null; - - const result = await pgClient.query(BILLING_MODULE_SQL, [databaseId]); - const row = result.rows[0]; - if (!row?.record_usage_function) return null; - - return { - publicSchema: row.public_schema as string, - privateSchema: row.private_schema as string, - recordUsageFunction: row.record_usage_function as string, - // The check_billing_quota function name follows the inflection pattern - checkBillingQuotaFunction: 'check_billing_quota' - }; - } catch (err) { - if (isModuleTableMissing(err)) return null; - throw err; - } -} - -// ─── Public API ───────────────────────────────────────────────────────────── - -/** - * Resolve billing config for a database. - * Results are cached per database_id with a 5-minute TTL. - * - * @param pgClient - A client connected to the tenant database (from withPgClient) - * @param databaseId - The database UUID - */ -export async function getLlmBillingConfig( - pgClient: PgClient, - databaseId: string -): Promise { - const cached = billingCache.get(databaseId); - if (cached) return cached; - - const [billing, inferenceLog] = await Promise.all([ - resolveBillingConfig(pgClient, databaseId), - resolveInferenceLogConfig(pgClient, databaseId) - ]); - - const entry: LlmBillingCacheEntry = { billing, inferenceLog }; - billingCache.set(databaseId, entry); - return entry; -} - -/** - * Invalidate the cached config for a specific database (or all). - */ -export function invalidateLlmBillingConfig(databaseId?: string): void { - if (databaseId) { - billingCache.delete(databaseId); - } else { - billingCache.clear(); - } -} - -/** - * Get cache stats for diagnostics. - */ -export function getLlmBillingCacheStats(): { size: number; max: number } { - return { size: billingCache.size, max: 50 }; -} diff --git a/graphile/graphile-llm/src/embedder.ts b/graphile/graphile-llm/src/embedder.ts index 13612b1a76..5918a0659b 100644 --- a/graphile/graphile-llm/src/embedder.ts +++ b/graphile/graphile-llm/src/embedder.ts @@ -7,8 +7,8 @@ * Per-request model/baseUrl overrides are read from AsyncLocalStorage * (`llmConfigStore`). embed-inputs.ts sets these overrides from * the database's `llm_module` config via `ctx.useLlm()`. This means - * the same embedder function — and its metering wrapper — handle every - * request, regardless of which tenant's model config applies. + * the same embedder function handles every request, regardless of which + * tenant's model config applies. */ import { AsyncLocalStorage } from 'node:async_hooks'; @@ -35,8 +35,7 @@ export interface LlmConfigOverrides { * AsyncLocalStorage for per-request LLM config overrides. * * Set by the plan-time embedding step in embed-inputs.ts (from ctx.useLlm()), - * read by the embedder at call time. The metering wrapper sits between - * them and is completely unaware — it just wraps the same function. + * read by the embedder at call time. */ export const llmConfigStore = new AsyncLocalStorage(); diff --git a/graphile/graphile-llm/src/index.ts b/graphile/graphile-llm/src/index.ts index efefa7aed6..aa287f89d2 100644 --- a/graphile/graphile-llm/src/index.ts +++ b/graphile/graphile-llm/src/index.ts @@ -47,9 +47,6 @@ export { createLlmTextSearchPlugin, embedTextInWhere } from './plugins/text-sear export type { EmbedRule } from './embed-inputs'; export { embedField, planTimeEmbedding } from './embed-inputs'; -// Metering plugin (opt-in billing integration) -export { createLlmMeteringPlugin } from './plugins/metering-plugin'; - // Agent discovery (queries agent_chat_module config table at runtime) export type { AgentDiscovery,AgentTableInfo } from './plugins/agent-discovery-plugin'; export { clearAgentDiscoveryCache,getAgentDiscovery } from './plugins/agent-discovery-plugin'; @@ -70,18 +67,6 @@ export { buildChatCompleterFromModule } from './chat'; -// Metering utilities (for custom integration) -export type { InferenceLogEntry,MeteringContext, MeteringOptions, MeterResult, WithPgClient } from './metering'; -export { logInferenceUsage, meteredChat, meteredEmbed, QuotaExceededError } from './metering'; - -// Config cache (for custom integration) -export type { BillingConfig, InferenceLogConfig, LlmBillingCacheEntry, PgClient } from './config-cache'; -export { - getLlmBillingCacheStats, - getLlmBillingConfig, - invalidateLlmBillingConfig -} from './config-cache'; - // Types export type { ChatConfig, @@ -96,6 +81,5 @@ export type { GraphileLlmOptions, LlmModuleData, LlmUsage, - MeteringConfig, RagDefaults } from './types'; diff --git a/graphile/graphile-llm/src/metering.ts b/graphile/graphile-llm/src/metering.ts deleted file mode 100644 index 0053538a3e..0000000000 --- a/graphile/graphile-llm/src/metering.ts +++ /dev/null @@ -1,431 +0,0 @@ -/** - * metering — Billing-aware wrappers for embedder and chat functions - * - * Wraps EmbedderFunction and ChatFunction with: - * 1. Pre-check: `check_billing_quota(meter_slug, entity_id, estimated_amount)` - * 2. Execute the underlying function - * 3. Post-record: `record_usage(meter_slug, entity_id, actual_amount)` - * - * When the quota check fails, the wrapper returns null (graceful degradation) - * instead of throwing, so the search pipeline can fall back to text-only. - * - * Token counts: - * - Chat: real provider counts via ChatResult.usage (from OllamaAdapter.stream()) - * - Embedding: real provider counts via EmbeddingResult.promptTokens (from /api/embed) - * - * The billing functions live in the tenant database and are called via the - * Graphile `withPgClient` callback. Function locations (schema, names) are - * resolved from `billing_module` metaschema and cached by `config-cache.ts`. - */ - -import type { BillingClient } from '@constructive-io/express-context'; -import { createBillingClient } from '@constructive-io/express-context'; - -import type { BillingConfig, InferenceLogConfig, PgClient } from './config-cache'; -import type { ChatFunction, ChatMessage, ChatOptions, EmbedderFunction } from './types'; - -// ─── Types ────────────────────────────────────────────────────────────────── - -/** - * Callback matching Graphile's withPgClient signature. - * Acquires a pg client, calls the callback, then releases the client. - */ -export type WithPgClient = ( - pgSettings: Record, - callback: (pgClient: PgClient) => Promise, -) => Promise; - -export interface MeteringContext { - /** Callback to acquire a tenant database client */ - withPgClient: WithPgClient; - /** pgSettings from the GraphQL context (for role/claims) */ - pgSettings: Record; - /** Billing function references from the billing_module */ - billing: BillingConfig; - /** Entity ID to meter against (from JWT claims) */ - entityId: string; - /** Per-request correlation ID (from request.id pgSetting) */ - requestId: string | null; - /** Database UUID from JWT claims */ - databaseId: string; - /** Actor (user) ID from JWT claims */ - actorId: string | null; - /** Inference log table config (null if inference_log_module not provisioned) */ - inferenceLog: InferenceLogConfig | null; -} - -export interface MeteringOptions { - /** Meter slug for embedding operations (default: model name from build config) */ - embeddingMeterSlug?: string; - /** Meter slug for chat completion operations (default: model name from build config) */ - chatMeterSlug?: string; - /** Whether to skip metering entirely (e.g. for local dev). Default: false */ - skipMetering?: boolean; - /** Embedding model name (for inference log) */ - embeddingModel?: string; - /** Chat model name (for inference log) */ - chatModel?: string; - /** Provider name (for inference log) */ - provider?: string; -} - -export interface MeterResult { - /** The result from the underlying function, or null if quota exceeded */ - result: T | null; - /** Whether the call was metered */ - metered: boolean; - /** Whether the call was skipped due to quota limits */ - quotaExceeded: boolean; - /** Latency of the underlying function call in ms */ - latencyMs: number; -} - -// ─── Billing Client Adapter ───────────────────────────────────────────────── - -/** - * Create a BillingClient from a MeteringContext by adapting Graphile's 2-arg - * withPgClient(pgSettings, fn) to the express-context 1-arg withPgClient(fn). - */ -function billingClientFromCtx(ctx: MeteringContext): BillingClient { - const adaptedWithPgClient = (fn: (client: any) => Promise): Promise => - ctx.withPgClient(ctx.pgSettings, fn as any) as unknown as Promise; - - return createBillingClient( - adaptedWithPgClient, - ctx.entityId, - ctx.billing, - ctx.inferenceLog - ); -} - -// ─── Inference Usage Log ──────────────────────────────────────────────────── - -export interface InferenceLogEntry { - databaseId: string; - entityId: string; - actorId: string | null; - model: string; - provider: string | null; - service: 'llm' | 'embedding' | 'tts' | 'stt' | 'ocr' | 'image_gen' | 'search' | 'compute'; - operation: string; - inputTokens: number; - outputTokens: number; - totalTokens: number; - cacheReadTokens: number | null; - cacheWriteTokens: number | null; - latencyMs: number; - ragEnabled: boolean; - chunksRetrieved: number | null; - embeddingModel: string | null; - embeddingLatencyMs: number | null; - status: 'success' | 'quota_exceeded' | 'provider_error' | 'timeout'; - errorType: string | null; - rawUsage: Record | null; -} - -/** - * Write a row to the usage_log_inference table. - * Delegates to the shared BillingClient from express-context. - */ -export async function logInferenceUsage( - ctx: MeteringContext, - entry: InferenceLogEntry -): Promise { - const client = billingClientFromCtx(ctx); - return client.logInference({ - entityId: entry.entityId, - actorId: entry.actorId, - model: entry.model, - provider: entry.provider, - service: entry.service, - operation: entry.operation, - inputTokens: entry.inputTokens, - outputTokens: entry.outputTokens, - totalTokens: entry.totalTokens, - latencyMs: entry.latencyMs, - status: entry.status, - cacheReadTokens: entry.cacheReadTokens, - cacheWriteTokens: entry.cacheWriteTokens, - ragEnabled: entry.ragEnabled, - chunksRetrieved: entry.chunksRetrieved, - embeddingModel: entry.embeddingModel, - embeddingLatencyMs: entry.embeddingLatencyMs, - errorType: entry.errorType, - rawUsage: entry.rawUsage - }); -} - -// ─── Metered Embedder ─────────────────────────────────────────────────────── - -/** - * Wrap an embedder with billing quota check + usage recording. - * - * The returned MeterResult contains `quotaExceeded: true` when the pre-check - * fails, enabling the caller to fall back to text-only search. - */ -export async function meteredEmbed( - embedder: EmbedderFunction, - text: string, - ctx: MeteringContext | null, - options: MeteringOptions = {} -): Promise> { - const startTime = Date.now(); - - // No billing context → just embed without metering - if (!ctx) { - const { embedding } = await embedder(text); - return { - result: embedding, - metered: false, - quotaExceeded: false, - latencyMs: Date.now() - startTime - }; - } - - const meterSlug = options.embeddingMeterSlug; - if (!meterSlug) { - const { embedding } = await embedder(text); - return { - result: embedding, - metered: false, - quotaExceeded: false, - latencyMs: Date.now() - startTime - }; - } - - if (options.skipMetering) { - const { embedding } = await embedder(text); - return { - result: embedding, - metered: false, - quotaExceeded: false, - latencyMs: Date.now() - startTime - }; - } - - // Pre-check: can this entity afford this call? - const billingClient = billingClientFromCtx(ctx); - const allowed = await billingClient.checkQuota(meterSlug); - - if (!allowed) { - logInferenceUsage(ctx, { - databaseId: ctx.databaseId, - entityId: ctx.entityId, - actorId: ctx.actorId, - model: options.embeddingModel ?? meterSlug, - provider: options.provider ?? null, - service: 'embedding', - operation: 'create', - inputTokens: 0, - outputTokens: 0, - totalTokens: 0, - cacheReadTokens: null, - cacheWriteTokens: null, - latencyMs: Date.now() - startTime, - ragEnabled: false, - chunksRetrieved: null, - embeddingModel: options.embeddingModel ?? null, - embeddingLatencyMs: null, - status: 'quota_exceeded', - errorType: null, - rawUsage: null - }).catch(() => {}); - - return { - result: null, - metered: true, - quotaExceeded: true, - latencyMs: Date.now() - startTime - }; - } - - // Execute embedding — real token count from provider via EmbeddingResult - const { embedding, promptTokens } = await embedder(text); - const latencyMs = Date.now() - startTime; - - billingClient.recordUsage(meterSlug, promptTokens, { - request_id: ctx.requestId, - input_chars: text.length, - prompt_tokens: promptTokens, - dims: embedding.length, - latency_ms: latencyMs - }).catch(() => {}); - - // Log to inference usage table - logInferenceUsage(ctx, { - databaseId: ctx.databaseId, - entityId: ctx.entityId, - actorId: ctx.actorId, - model: options.embeddingModel ?? meterSlug, - provider: options.provider ?? null, - service: 'embedding', - operation: 'create', - inputTokens: promptTokens, - outputTokens: 0, - totalTokens: promptTokens, - cacheReadTokens: null, - cacheWriteTokens: null, - latencyMs, - ragEnabled: false, - chunksRetrieved: null, - embeddingModel: options.embeddingModel ?? null, - embeddingLatencyMs: latencyMs, - status: 'success', - errorType: null, - rawUsage: { prompt_tokens: promptTokens } - }).catch(() => {}); - - return { - result: embedding, - metered: true, - quotaExceeded: false, - latencyMs - }; -} - -// ─── Metered Chat ─────────────────────────────────────────────────────────── - -/** - * Wrap a chat completion call with billing quota check + usage recording. - */ -export async function meteredChat( - chat: ChatFunction, - messages: ChatMessage[], - ctx: MeteringContext | null, - chatOptions?: ChatOptions, - meteringOptions: MeteringOptions = {} -): Promise> { - const startTime = Date.now(); - - if (!ctx) { - const chatResult = await chat(messages, chatOptions); - return { - result: chatResult.content, - metered: false, - quotaExceeded: false, - latencyMs: Date.now() - startTime - }; - } - - const meterSlug = meteringOptions.chatMeterSlug; - if (!meterSlug) { - const chatResult = await chat(messages, chatOptions); - return { - result: chatResult.content, - metered: false, - quotaExceeded: false, - latencyMs: Date.now() - startTime - }; - } - - if (meteringOptions.skipMetering) { - const chatResult = await chat(messages, chatOptions); - return { - result: chatResult.content, - metered: false, - quotaExceeded: false, - latencyMs: Date.now() - startTime - }; - } - - // Pre-check: can this entity afford this call? - const billingClient = billingClientFromCtx(ctx); - const allowed = await billingClient.checkQuota(meterSlug); - - if (!allowed) { - const estimatedInputTokens = Math.ceil(messages.reduce((sum, m) => sum + m.content.length, 0) / 4); - logInferenceUsage(ctx, { - databaseId: ctx.databaseId, - entityId: ctx.entityId, - actorId: ctx.actorId, - model: meteringOptions.chatModel ?? meterSlug, - provider: meteringOptions.provider ?? null, - service: 'llm', - operation: 'chat', - inputTokens: estimatedInputTokens, - outputTokens: 0, - totalTokens: estimatedInputTokens, - cacheReadTokens: null, - cacheWriteTokens: null, - latencyMs: Date.now() - startTime, - ragEnabled: false, - chunksRetrieved: null, - embeddingModel: null, - embeddingLatencyMs: null, - status: 'quota_exceeded', - errorType: null, - rawUsage: null - }).catch(() => {}); - - return { - result: null, - metered: true, - quotaExceeded: true, - latencyMs: Date.now() - startTime - }; - } - - // Execute chat completion — returns real token usage from provider - const chatResult = await chat(messages, chatOptions); - const latencyMs = Date.now() - startTime; - const usage = chatResult.usage; - - billingClient.recordUsage(meterSlug, usage.totalTokens, { - request_id: ctx.requestId, - input_tokens: usage.input, - output_tokens: usage.output, - cache_read_tokens: usage.cacheRead, - cache_write_tokens: usage.cacheWrite, - messages_count: messages.length, - latency_ms: latencyMs - }).catch(() => {}); - - // Log to inference usage table with real provider token counts - logInferenceUsage(ctx, { - databaseId: ctx.databaseId, - entityId: ctx.entityId, - actorId: ctx.actorId, - model: meteringOptions.chatModel ?? meterSlug, - provider: meteringOptions.provider ?? null, - service: 'llm', - operation: 'chat', - inputTokens: usage.input, - outputTokens: usage.output, - totalTokens: usage.totalTokens, - cacheReadTokens: usage.cacheRead || null, - cacheWriteTokens: usage.cacheWrite || null, - latencyMs, - ragEnabled: false, - chunksRetrieved: null, - embeddingModel: null, - embeddingLatencyMs: null, - status: 'success', - errorType: null, - rawUsage: { reasoning: usage.reasoning } - }).catch(() => {}); - - return { - result: chatResult.content, - metered: true, - quotaExceeded: false, - latencyMs - }; -} - -// ─── Error Types ──────────────────────────────────────────────────────────── - -export class QuotaExceededError extends Error { - public readonly code = 'QUOTA_EXCEEDED'; - public readonly meterSlug: string; - public readonly entityId: string; - - constructor(meterSlug: string, entityId: string) { - super( - `LLM quota exceeded for meter '${meterSlug}' on entity '${entityId}'. ` + - 'Upgrade your plan or wait for the next billing period.' - ); - this.name = 'QuotaExceededError'; - this.meterSlug = meterSlug; - this.entityId = entityId; - } -} diff --git a/graphile/graphile-llm/src/plugins/llm-module-plugin.ts b/graphile/graphile-llm/src/plugins/llm-module-plugin.ts index cbec7c1b39..39c534da02 100644 --- a/graphile/graphile-llm/src/plugins/llm-module-plugin.ts +++ b/graphile/graphile-llm/src/plugins/llm-module-plugin.ts @@ -17,13 +17,9 @@ * * Per-database model/baseUrl overrides (from `llm_module` via `ctx.useLlm()`) * are applied at request time via `llmConfigStore` (AsyncLocalStorage) in - * the plan-time embedding step (embed-inputs.ts). The same embedder function — and - * its metering wrapper — handles every request; only the model name and - * base URL parameters change per-tenant. - * - * This plugin is intentionally pure — no billing or metering logic. - * The optional LlmMeteringPlugin wraps the embedder with billing integration - * if loaded (it runs after this plugin and before the consumer plugins). + * the plan-time embedding step (embed-inputs.ts). The same embedder function + * handles every request; only the model name and base URL parameters change + * per-tenant. */ import type { GraphileConfig } from 'graphile-config'; @@ -42,9 +38,9 @@ declare global { llmEmbedder: EmbedderFunction | null; /** The resolved chat completion function, or null if not configured */ llmChatCompleter: ChatFunction | null; - /** The embedding model name (used as billing meter slug) */ + /** The embedding model name */ llmEmbeddingModel: string | null; - /** The chat model name (used as billing meter slug) */ + /** The chat model name */ llmChatModel: string | null; } } diff --git a/graphile/graphile-llm/src/plugins/metering-plugin.ts b/graphile/graphile-llm/src/plugins/metering-plugin.ts deleted file mode 100644 index 754f4aabfd..0000000000 --- a/graphile/graphile-llm/src/plugins/metering-plugin.ts +++ /dev/null @@ -1,231 +0,0 @@ -/** - * LlmMeteringPlugin - * - * Opt-in billing integration for graphile-llm. Completely separate from the - * pure LLM plugins (text-search, text-mutation, rag). - * - * **How it works:** - * 1. At schema build time, replaces `build.llmEmbedder` with a metered wrapper - * that has the same `(text: string) => Promise` signature - * 2. At request time, wraps every root query/mutation resolver to set up a - * request-scoped MeteringContext via AsyncLocalStorage - * 3. When the embedder is called (by any plugin), the wrapper checks - * AsyncLocalStorage for a metering context and if found, calls - * check_billing_quota before and record_usage after - * 4. If quota is exceeded, the wrapper returns null — the calling plugin sees - * null and handles it (search falls back to text-only, mutations throw) - * - * The pure plugins never import metering, config-cache, or billing types. - * They call the embedder and handle null results — that's it. - * - * **Entity ID resolution:** - * The billing `entity_id` is resolved via a configurable callback. - * Default: reads `jwt.claims.user_id` from pgSettings. Override via - * `metering.resolveEntityId` in GraphileLlmPreset options. - * - * **Graceful behavior:** - * - billing_module not provisioned → embedder passes through unmetered - * - entity_id not available → embedder passes through unmetered - * - check_billing_quota throws → call is allowed (billing is opt-in) - * - record_usage throws → call succeeds, recording silently skipped - * - quota exceeded → embedder returns null - */ - -import { AsyncLocalStorage } from 'node:async_hooks'; - -import type { GraphileConfig } from 'graphile-config'; - -import type { PgClient } from '../config-cache'; -import { getLlmBillingConfig } from '../config-cache'; -import type { MeteringContext, MeteringOptions, WithPgClient } from '../metering'; -import { meteredEmbed } from '../metering'; -import type { EmbedderFunction, MeteringConfig } from '../types'; - -// ─── TypeScript Augmentation ──────────────────────────────────────────────── - -declare global { - namespace GraphileConfig { - interface Plugins { - LlmMeteringPlugin: true; - } - } -} - -// ─── Request-scoped context via AsyncLocalStorage ─────────────────────────── - -const meteringStore = new AsyncLocalStorage(); - -// ─── Helpers ──────────────────────────────────────────────────────────────── - -function defaultResolveEntityId(pgSettings: Record): string | null { - return pgSettings['jwt.claims.user_id'] ?? null; -} - -async function buildMeteringContext( - graphqlContext: any, - resolveEntityId: (pgSettings: Record) => string | null -): Promise { - const pgSettings: Record = graphqlContext?.pgSettings ?? {}; - const entityId = resolveEntityId(pgSettings); - const databaseId = pgSettings['jwt.claims.database_id'] ?? null; - const requestId = pgSettings['request.id'] ?? null; - const actorId = pgSettings['jwt.claims.user_id'] ?? null; - if (!entityId || !databaseId) return null; - - const withPgClient: WithPgClient | undefined = graphqlContext?.withPgClient; - if (!withPgClient) return null; - - let billingConfig = null; - let inferenceLogConfig = null; - try { - await withPgClient(pgSettings, async (pgClient: PgClient) => { - const entry = await getLlmBillingConfig(pgClient, databaseId); - billingConfig = entry.billing; - inferenceLogConfig = entry.inferenceLog; - }); - } catch { - return null; - } - - if (!billingConfig) return null; - - return { - withPgClient, - pgSettings, - billing: billingConfig, - entityId, - requestId, - databaseId, - actorId, - inferenceLog: inferenceLogConfig - }; -} - -/** - * Wrap an embedder with metering that reads context from AsyncLocalStorage. - * The returned function has the same signature as the original embedder, - * so downstream plugins are unaware of billing. - * - * When no metering context is in scope, the original embedder is called directly. - * When quota is exceeded, returns null instead of a vector. - */ -function wrapEmbedderWithMetering( - embedder: EmbedderFunction, - meteringOptions: MeteringOptions -): (text: string) => Promise { - return async (text: string): Promise => { - const ctx = meteringStore.getStore(); - - if (!ctx) { - // No metering context in scope — call original embedder directly - const startTime = Date.now(); - const { embedding } = await embedder(text); - const latencyMs = Date.now() - startTime; - console.log(`[graphile-llm] Embed (unmetered): dims=${embedding?.length ?? 0}, latency=${latencyMs}ms`); - return embedding; - } - - const result = await meteredEmbed(embedder, text, ctx, meteringOptions); - - if (result.quotaExceeded) { - return null; - } - - return result.result; - }; -} - -// ─── Plugin ───────────────────────────────────────────────────────────────── - -export function createLlmMeteringPlugin( - meteringConfig: MeteringConfig = {} -): GraphileConfig.Plugin { - const { - embeddingMeterSlug: configEmbeddingSlug, - chatMeterSlug: configChatSlug, - skipMetering, - resolveEntityId = defaultResolveEntityId - } = meteringConfig; - - return { - name: 'LlmMeteringPlugin', - version: '0.2.0', - description: - 'Wraps LLM embedder/chat with billing quota checks and usage recording', - after: ['LlmModulePlugin'], - before: ['LlmTextSearchPlugin', 'LlmTextMutationPlugin', 'LlmRagPlugin'], - - schema: { - hooks: { - build(build) { - const originalEmbedder: EmbedderFunction | null = (build as any).llmEmbedder; - - if (!originalEmbedder) { - console.log('[graphile-llm] Metering plugin loaded but no embedder configured — skipping'); - return build; - } - - // Meter slug = model name by default (three-level waterfall: model → inference → universal) - const embeddingModel: string | null = (build as any).llmEmbeddingModel; - const chatModel: string | null = (build as any).llmChatModel; - const embeddingSlug = configEmbeddingSlug ?? embeddingModel ?? undefined; - const chatSlug = configChatSlug ?? chatModel ?? undefined; - - if (embeddingSlug) { - console.log(`[graphile-llm] Metering enabled — embedding meter: ${embeddingSlug}`); - } else { - console.log('[graphile-llm] Metering enabled but no embedding model name — usage will not be metered'); - } - - const meteringOptions: MeteringOptions = { - embeddingMeterSlug: embeddingSlug, - chatMeterSlug: chatSlug, - skipMetering, - embeddingModel: embeddingModel ?? undefined, - chatModel: chatModel ?? undefined - }; - - // Replace the embedder with a metered version. - // Same signature except it can return null (quota exceeded). - const meteredEmbedder = wrapEmbedderWithMetering(originalEmbedder, meteringOptions); - - return build.extend(build, { - llmEmbedder: meteredEmbedder as unknown as EmbedderFunction - }, 'LlmMeteringPlugin replacing llmEmbedder with metered version'); - }, - - /** - * Wrap every root query/mutation resolver to establish the - * request-scoped metering context via AsyncLocalStorage. - */ - GraphQLObjectType_fields_field(field, build, context) { - const { - scope: { isRootQuery, isRootMutation } - } = context as any; - - if (!isRootQuery && !isRootMutation) return field; - - // Only wrap if we actually replaced the embedder - if (!(build as any).llmEmbedder) return field; - - const defaultResolver = (obj: any) => obj[(context as any).scope.fieldName]; - const { resolve: oldResolve = defaultResolver, ...rest } = field; - - return { - ...rest, - async resolve(source: any, args: any, graphqlContext: any, info: any) { - // Build the metering context for this request - const ctx = await buildMeteringContext(graphqlContext, resolveEntityId); - - // Run the original resolver within the AsyncLocalStorage scope - // so any embedder calls made by downstream plugins pick up the ctx - return meteringStore.run(ctx, () => { - return oldResolve(source, args, graphqlContext, info); - }); - } - }; - } - } - } - }; -} diff --git a/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts b/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts index 7643b38138..d5847645a6 100644 --- a/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts +++ b/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts @@ -9,8 +9,8 @@ * Example: * mutation { createArticle(input: { embeddingText: "Machine learning concepts" }) } * - * If the embedder returns null (e.g. quota exceeded when the metering plugin - * is loaded), the mutation throws an error — unlike search, mutations cannot + * If the embedder returns null, the mutation throws an error — unlike + * search, mutations cannot * silently skip writing a vector the user asked for. * * This is the mutation counterpart to LlmTextSearchPlugin (which handles diff --git a/graphile/graphile-llm/src/plugins/text-search-plugin.ts b/graphile/graphile-llm/src/plugins/text-search-plugin.ts index f86fa4fb04..a6a7e998c2 100644 --- a/graphile/graphile-llm/src/plugins/text-search-plugin.ts +++ b/graphile/graphile-llm/src/plugins/text-search-plugin.ts @@ -20,8 +20,7 @@ * If no embedder is configured, `text` on `VectorNearbyInput` is an error and * `unifiedSearch` runs on the text adapters only. * - * If the embedder returns null (e.g. quota exceeded when the metering - * plugin is loaded), behavior depends on `onQuotaExceeded`: + * If the embedder returns null, behavior depends on `onQuotaExceeded`: * - `'degrade'` (default): drops the text and continues without the vector * filter (text-only search for `unifiedSearch`). * - `'throw'`: throws an error. diff --git a/graphile/graphile-llm/src/preset.ts b/graphile/graphile-llm/src/preset.ts index c5a36b0be1..27e07c6cb0 100644 --- a/graphile/graphile-llm/src/preset.ts +++ b/graphile/graphile-llm/src/preset.ts @@ -8,7 +8,6 @@ * - Resolves an embedder from configuration (llm_module, env vars, or preset options) * - Adds a `text: String` field to `VectorNearbyInput` for text-based vector search * - Adds `{column}Text: String` companion fields on mutation inputs for vector columns - * - Optionally enables billing/metering via the LlmMeteringPlugin * * Included in ConstructivePreset when `enableLlm` is true (the default). * Can also be used standalone by adding it directly to your preset's `extends` array. @@ -43,34 +42,13 @@ * }; * ``` * - * @example With billing metering (opt-in, meter slug = model name by default): + * @example Fail instead of degrading when the embedder is unavailable: * ```typescript * GraphileLlmPreset({ - * defaultEmbedder: { provider: 'openai', model: 'text-embedding-3-small' }, - * metering: true, - * // → embedding calls metered under 'text-embedding-3-small' meter slug - * // → three-level waterfall: text-embedding-3-small → inference pool → universal - * }) - * ``` - * - * @example With strict quota enforcement (fail if embedding unavailable): - * ```typescript - * GraphileLlmPreset({ - * defaultEmbedder: { provider: 'openai', model: 'text-embedding-3-small' }, - * metering: true, + * defaultEmbedder: { provider: 'ollama', model: 'nomic-embed-text' }, * onQuotaExceeded: 'throw', - * // → if billing quota is exceeded, queries with unifiedSearch throw an error - * // → without this, queries silently fall back to text-only search - * }) - * ``` - * - * @example With custom entity_id resolution (bill per-database): - * ```typescript - * GraphileLlmPreset({ - * defaultEmbedder: { provider: 'openai', model: 'text-embedding-3-small' }, - * metering: { - * resolveEntityId: (pgSettings) => pgSettings['jwt.claims.database_id'], - * }, + * // → a unifiedSearch whose text cannot be embedded throws + * // → without this, it falls back to text-only search * }) * ``` */ @@ -78,7 +56,6 @@ import type { GraphileConfig } from 'graphile-config'; import { createLlmModulePlugin } from './plugins/llm-module-plugin'; -import { createLlmMeteringPlugin } from './plugins/metering-plugin'; import { createLlmRagPlugin } from './plugins/rag-plugin'; import { createLlmTextMutationPlugin } from './plugins/text-mutation-plugin'; import { createLlmTextSearchPlugin } from './plugins/text-search-plugin'; @@ -98,21 +75,13 @@ export function GraphileLlmPreset( enableTextMutations = true, enableRag = false, onQuotaExceeded, - ragDefaults, - metering + ragDefaults } = options; const plugins: GraphileConfig.Plugin[] = [ createLlmModulePlugin(options) ]; - // Metering is opt-in: only loaded when metering is truthy - // (true, or a MeteringConfig object) - if (metering) { - const meteringConfig = metering === true ? {} : metering; - plugins.push(createLlmMeteringPlugin(meteringConfig)); - } - if (enableTextSearch) { plugins.push(createLlmTextSearchPlugin({ onQuotaExceeded })); } diff --git a/graphile/graphile-llm/src/types.ts b/graphile/graphile-llm/src/types.ts index c60d8e6bfe..d260644d9e 100644 --- a/graphile/graphile-llm/src/types.ts +++ b/graphile/graphile-llm/src/types.ts @@ -37,7 +37,6 @@ export interface EmbedderConfig { /** * Token usage metadata returned by LLM providers. - * Maps to the billing schema's inference_log columns. */ export interface LlmUsage { /** Prompt / input tokens consumed */ @@ -186,47 +185,6 @@ export interface ChunkTableInfo { contentField: string; } -// ─── Metering Types ───────────────────────────────────────────────────────── - -/** - * Configuration for billing/metering integration. - * When provided, embedding and chat calls are wrapped with quota checks - * and usage recording via the billing_module functions. - */ -export interface MeteringConfig { - /** - * Meter slug for embedding operations. - * Must match a slug in the billing_module meters table. - * - * @default the embedding model name (e.g. 'text-embedding-3-small') - * — meter slug = model name, so each model has its own meter - * in the three-level waterfall (per-model → inference pool → universal). - */ - embeddingMeterSlug?: string; - - /** - * Meter slug for chat completion operations. - * - * @default the chat model name (e.g. 'gpt-4o-mini') - */ - chatMeterSlug?: string; - - /** - * Disable metering entirely (e.g. for local dev). - * When true, billing functions are never called. - * @default false - */ - skipMetering?: boolean; - - /** - * Resolve the billing entity_id from pgSettings. - * The entity_id identifies who gets billed (user, org, etc.). - * - * @default reads jwt.claims.user_id - */ - resolveEntityId?: (pgSettings: Record) => string | null; -} - // ─── Plugin Options ───────────────────────────────────────────────────────── /** @@ -254,7 +212,7 @@ export interface GraphileLlmOptions { enableTextSearch?: boolean; /** - * Controls behavior when the embedder returns null (e.g. billing quota exceeded). + * Controls behavior when the embedder returns null (provider unavailable or embedding failed). * * - `'degrade'` (default): Skip the vector search silently and fall back to * text-only adapters (tsvector, BM25, trgm). If no text adapters are @@ -287,18 +245,4 @@ export interface GraphileLlmOptions { * Individual queries can override these values. */ ragDefaults?: RagDefaults; - - /** - * Billing/metering configuration (opt-in). - * When truthy, loads the LlmMeteringPlugin which wraps the embedder - * with billing quota checks + usage recording. - * - * Set to `true` to enable metering with defaults (entity_id from jwt.claims.user_id). - * Provide a MeteringConfig object for fine-grained control (custom entity_id, meter slugs). - * Set to `false` or omit to disable metering entirely. - * - * @default undefined (metering disabled) - */ - metering?: boolean | MeteringConfig; - } diff --git a/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts index 63f1d2fafc..8a956c3132 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/dedup-liveness.test.ts @@ -22,7 +22,6 @@ jest.mock('../src/s3-signer', () => ({ import { HeadObjectCommand } from '@aws-sdk/client-s3'; -import { finalizeStagedUpload } from '../src/managed-upload'; import { processSingleFile } from '../src/plugin'; import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from '../src/types'; @@ -162,83 +161,3 @@ describe('content-addressed upload whose hash matches a custom-key row', () => { expect(queries.find((q) => /^DELETE/.test(q.text))?.values).toEqual([OLD_FILE_ID]); }); }); - -describe('streamed upload landing on a never-completed row', () => { - const staged = { - stagingKey: '.staging/tmp-1', - contentHash: HASH, - contentType: 'text/html', - size: 16, - filename: SITE_KEY, - }; - - function db({ status = 'requested', deleteAffects = true } = {}) { - const queries: Array<{ text: string; values?: unknown[] }> = []; - const client = { - async query(opts: { text: string; values?: unknown[] }) { - queries.push(opts); - if (/set_config/.test(opts.text)) return { rows: [] }; - if (/SELECT id, key, mime_type/.test(opts.text)) { - return { rows: [{ id: OLD_FILE_ID, key: HASH, mime_type: 'text/html', size: 16, filename: null as string | null, status }] }; - } - if (/^DELETE FROM storage_public\.app_files/.test(opts.text)) { - return { rows: deleteAffects ? [{ id: OLD_FILE_ID }] : [] }; - } - if (/record_file/.test(opts.text)) return { rows: [{ id: NEW_FILE_ID }] }; - throw new Error(`unexpected query: ${opts.text}`); - }, - withTransaction: (cb: any) => cb(client), - }; - return { withPgClient: (_settings: any, cb: any) => cb(client), queries }; - } - - const target = (s3: S3Config): any => ({ - databaseId: 'db', - storageConfig: storageConfig(), - bucket, - physicalName: 'site-bucket', - s3, - binding: null, - }); - - it('promotes the staged bytes as a fresh file instead of absorbing them into the requested row', async () => { - const s3 = fakeS3(); - const fake = db(); - - const { projection, deduplicated } = await finalizeStagedUpload({ - target: target(s3), withPgClient: fake.withPgClient, pgSettings: null, staged, - }); - - expect(deduplicated).toBe(false); - expect(projection.id).toBe(NEW_FILE_ID); - expect(heads(s3)).toEqual([]); - expect(fake.queries.find((q) => /^DELETE/.test(q.text))?.values).toEqual([OLD_FILE_ID]); - }); - - it('fails instead of promoting when the caller cannot delete the requested row', async () => { - const s3 = fakeS3(); - const fake = db({ deleteAffects: false }); - - await expect( - finalizeStagedUpload({ target: target(s3), withPgClient: fake.withPgClient, pgSettings: null, staged }), - ).rejects.toThrow(/FILE_NOT_REPLACEABLE/); - - expect(fake.queries.some((q) => /record_file/.test(q.text))).toBe(false); - const sent = (s3.client.send as jest.Mock).mock.calls.map(([command]) => command); - expect(sent.map((command) => command.constructor.name)).toEqual(['DeleteObjectCommand']); - expect(sent[0].input.Key).toBe(staged.stagingKey); - }); - - it('deduplicates against an uploaded row', async () => { - const s3 = fakeS3(); - const fake = db({ status: 'uploaded' }); - - const { projection, deduplicated } = await finalizeStagedUpload({ - target: target(s3), withPgClient: fake.withPgClient, pgSettings: null, staged, - }); - - expect(deduplicated).toBe(true); - expect(projection.id).toBe(OLD_FILE_ID); - expect(fake.queries.some((q) => /DELETE|record_file/.test(q.text))).toBe(false); - }); -}); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/file-projection.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/file-projection.test.ts new file mode 100644 index 0000000000..9eac6038c9 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/file-projection.test.ts @@ -0,0 +1,36 @@ +import { buildFileProjection } from '../src/file-projection'; +import type { S3Config } from '../src/types'; + +const BUCKET_ID = '33333333-3333-3333-3333-333333333333'; +const FILE_ID = '44444444-4444-4444-4444-444444444444'; + +describe('buildFileProjection', () => { + it('names the files row in `id` so a document reference is countable', () => { + const projection = buildFileProjection( + { id: FILE_ID, key: 'abc123', bucketId: BUCKET_ID, mime: 'image/png', size: 42, filename: 'hero.png' }, + { is_public: true }, + { publicUrlPrefix: 'https://cdn.example.com/' } as S3Config, + ); + + expect(projection).toEqual({ + id: FILE_ID, + key: 'abc123', + bucket_id: BUCKET_ID, + mime: 'image/png', + size: 42, + filename: 'hero.png', + url: 'https://cdn.example.com/abc123', + }); + }); + + it('omits `url` for a private bucket rather than storing an expiring one', () => { + const projection = buildFileProjection( + { id: FILE_ID, key: 'abc123', bucketId: BUCKET_ID, mime: 'image/png', size: 42 }, + { is_public: false }, + { publicUrlPrefix: 'https://cdn.example.com' } as S3Config, + ); + + expect(projection.url).toBeUndefined(); + expect(projection.id).toBe(FILE_ID); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts deleted file mode 100644 index 4ab9403265..0000000000 --- a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts +++ /dev/null @@ -1,668 +0,0 @@ -/** - * The managed upload lifecycle: bucket resolution, bucket rules, and the - * staged-object → files row + projection promotion. - * - * The database is faked at the query boundary (each test answers only the - * statements its path issues) so these cover the decisions this module makes — - * which bucket, whether to dedup, what the document ends up containing — - * without a server or S3. - */ - -const mockS3Send = jest.fn(); -jest.mock('@constructive-io/s3-utils', () => ({ - createS3Client: jest.fn(() => ({ send: mockS3Send })), -})); - -import { clearFileRefFieldCache } from '../src/file-ref-registry'; -import { clearBucketCache, clearStorageModuleCache } from '../src/storage-module-cache'; -import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from '../src/types'; - -const DATABASE_ID = '00000000-0000-0000-0000-0000000000db'; -const APP_MODULE_ID = '11111111-1111-1111-1111-111111111111'; -const OTHER_MODULE_ID = '22222222-2222-2222-2222-222222222222'; -const BUCKET_ID = '33333333-3333-3333-3333-333333333333'; -const FILE_ID = '44444444-4444-4444-4444-444444444444'; -const FIELD = { schemaName: 'app_public', tableName: 'posts', columnName: 'image' }; - -interface QueryHandler { - match: RegExp; - rows: (values: unknown[]) => unknown[]; -} - -interface FakeDb { - withPgClient: any; - queries: Array<{ text: string; values?: unknown[] }>; -} - -/** - * A withPgClient whose statements are answered by the first matching handler. - * - * Unmatched statements throw rather than returning zero rows: a resolver that - * silently proceeds on an unanswered query is exactly the bug these tests exist - * to catch. - */ -function fakeDb(handlers: QueryHandler[]): FakeDb { - const queries: Array<{ text: string; values?: unknown[] }> = []; - const client = { - async query(opts: { text: string; values?: unknown[] }) { - queries.push(opts); - const handler = handlers.find((h) => h.match.test(opts.text)); - if (!handler) throw new Error(`unexpected query: ${opts.text}`); - return { rows: handler.rows(opts.values ?? []) }; - }, - withTransaction: (cb: any) => cb(client), - }; - return { - withPgClient: (_settings: any, cb: any) => cb(client), - queries, - }; -} - -const SET_CONFIG: QueryHandler = { match: /set_config/, rows: () => [] }; - -function storageModuleRow(overrides: Record = {}): Record { - return { - id: APP_MODULE_ID, - scope: 'app', - entity_table_id: null, - buckets_schema: 'storage_public', - buckets_table: 'app_buckets', - files_schema: 'storage_public', - files_table: 'app_files', - private_schema: 'storage_private', - endpoint: 'http://localhost:9000', - public_url_prefix: 'https://cdn.example.com', - provider: 'minio', - region: 'us-east-1', - connection_overrides: [], - allowed_origins: null, - upload_url_expiry_seconds: null, - download_url_expiry_seconds: null, - default_max_file_size: 1000, - max_filename_length: null, - cache_ttl_seconds: null, - max_bulk_files: null, - max_bulk_total_size: null, - has_path_shares: false, - has_versioning: false, - has_content_hash: false, - entity_schema: null, - entity_table: null, - ...overrides, - }; -} - -function bucketRow(overrides: Record = {}): Record { - return { - id: BUCKET_ID, - key: 'default-public', - type: 'public', - is_public: true, - owner_id: null, - allowed_mime_types: null, - max_file_size: null, - allow_custom_keys: false, - physical_name: 'myapp-default-public-db', - ...overrides, - }; -} - -const STORAGE_MODULES: QueryHandler = { - match: /function_resolution\.storage_modules\(\)/, - rows: () => [storageModuleRow()], -}; - -const NO_REGISTRY_ROW: QueryHandler = { - match: /file_ref_field/, - rows: () => [], -}; - -function options(): PresignedUrlPluginOptions { - return { credentials: { accessKeyId: 'test', secretAccessKey: 'test' } }; -} - -function storageConfig(): StorageModuleConfig { - return { - id: APP_MODULE_ID, - scope: 'app', - bucketsQualifiedName: 'storage_public.app_buckets', - filesQualifiedName: 'storage_public.app_files', - filesTableName: 'app_files', - recorderQualifiedName: 'storage_private.app_files_record_file', - bucketPhysicalNameQualifiedName: 'storage_private.app_buckets_physical_name', - defaultMaxFileSize: 1000, - hasPathShares: false, - allowedOrigins: null, - } as unknown as StorageModuleConfig; -} - -// Every lookup on this path is cached for the life of the process; each test -// states its own database, so the caches start empty. -beforeEach(() => { - clearStorageModuleCache(); - clearBucketCache(); - clearFileRefFieldCache(); -}); - -describe('buildFileProjection', () => { - it('names the files row in `id` so a document reference is countable', async () => { - const { buildFileProjection } = await import('../src/managed-upload'); - - const projection = buildFileProjection( - { id: FILE_ID, key: 'abc123', bucketId: BUCKET_ID, mime: 'image/png', size: 42, filename: 'hero.png' }, - { is_public: true }, - { publicUrlPrefix: 'https://cdn.example.com/' } as S3Config, - ); - - expect(projection).toEqual({ - id: FILE_ID, - key: 'abc123', - bucket_id: BUCKET_ID, - mime: 'image/png', - size: 42, - filename: 'hero.png', - url: 'https://cdn.example.com/abc123', - }); - }); - - it('omits `url` for a private bucket rather than storing an expiring one', async () => { - const { buildFileProjection } = await import('../src/managed-upload'); - - const projection = buildFileProjection( - { id: FILE_ID, key: 'abc123', bucketId: BUCKET_ID, mime: 'image/png', size: 42 }, - { is_public: false }, - { publicUrlPrefix: 'https://cdn.example.com' } as S3Config, - ); - - expect(projection.url).toBeUndefined(); - expect(projection.id).toBe(FILE_ID); - }); -}); - -describe('resolveManagedUploadTarget', () => { - it('resolves the default tag for an unregistered column, never an env bucket', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - NO_REGISTRY_ROW, - STORAGE_MODULES, - { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'default-public', bucket_type: 'public', physical_name: 'myapp-default-public-db' }] }, - { match: /FROM storage_public\.app_buckets/, rows: () => [bucketRow()] }, - ]); - - const target = await resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: { 'jwt.claims.database_id': DATABASE_ID }, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }); - - expect(target.binding).toBeNull(); - expect(target.physicalName).toBe('myapp-default-public-db'); - expect(target.s3.bucket).toBe('myapp-default-public-db'); - expect(target.s3.endpoint).toBe('http://localhost:9000'); - expect(target.s3.region).toBe('us-east-1'); - - const resolveCall = db.queries.find((q) => /resolve_default_bucket/.test(q.text)); - // scope, entity, public_access, and no explicit key: the reserved default tag. - expect(resolveCall?.values).toEqual(['app', null, true, null]); - }); - - it('passes a registered field\'s declared bucket key and publicness through', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { - match: /file_ref_field/, - rows: () => [{ - id: 'ref-1', - storage_module_id: APP_MODULE_ID, - bucket_key: 'avatars', - bucket_tags: null, - is_public: false, - enforce_fk: true, - }], - }, - STORAGE_MODULES, - { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'avatars', bucket_type: 'private', physical_name: 'myapp-avatars-db' }] }, - { match: /FROM storage_public\.app_buckets/, rows: () => [bucketRow({ key: 'avatars', type: 'private', is_public: false, physical_name: 'myapp-avatars-db' })] }, - ]); - - const target = await resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }); - - const resolveCall = db.queries.find((q) => /resolve_default_bucket/.test(q.text)); - expect(resolveCall?.values).toEqual(['app', null, false, 'avatars']); - expect(target.bucket.key).toBe('avatars'); - }); - - it('refuses a path-keyed bucket, whose keys are chosen by its publisher', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - // A static site's bucket: public, custom keys allowed, addressed by path. - const db = fakeDb([ - SET_CONFIG, - NO_REGISTRY_ROW, - STORAGE_MODULES, - { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'site', bucket_type: 'public', physical_name: 'myapp-site-db' }] }, - { match: /FROM storage_public\.app_buckets/, rows: () => [bucketRow({ key: 'site', allow_custom_keys: true, physical_name: 'myapp-site-db' })] }, - ]); - - await expect( - resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }), - ).rejects.toThrow('BUCKET_PATH_KEYED'); - }); - - it('rejects an unreconciled bucket without calling S3 or provisioning', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - mockS3Send.mockClear(); - const db = fakeDb([ - SET_CONFIG, - NO_REGISTRY_ROW, - STORAGE_MODULES, - { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'default-public', bucket_type: 'public', physical_name: null }] }, - { match: /SELECT id, key, type/, rows: () => [bucketRow({ physical_name: null })] }, - ]); - - await expect(resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - })).rejects.toThrow('STORAGE_BUCKET_NOT_RECONCILED'); - - expect(mockS3Send).not.toHaveBeenCalled(); - expect(db.queries.some((q) => /UPDATE/.test(q.text))).toBe(false); - }); - - it('raises when the database has no storage module to default to', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - NO_REGISTRY_ROW, - { match: /function_resolution\.storage_modules\(\)/, rows: () => [] }, - ]); - - await expect( - resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }), - ).rejects.toThrow('STORAGE_MODULE_NOT_FOUND'); - }); - - // The fallback for an unregistered column selects the database's single - // global plane by shape (no entity key), not by the name its scope happens to - // carry: a database-scope plane registers as 'database', and matching 'app' - // sent every one of them to STORAGE_MODULE_NOT_FOUND. - it('defaults an unregistered column to a database-scope plane', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - NO_REGISTRY_ROW, - { - match: /function_resolution\.storage_modules\(\)/, - rows: () => [storageModuleRow({ - scope: 'database', - buckets_table: 'buckets', - files_table: 'files', - })], - }, - { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'default-public', bucket_type: 'public', physical_name: 'myapp-default-public-db' }] }, - { match: /FROM storage_public\.buckets/, rows: () => [bucketRow()] }, - ]); - - const target = await resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }); - - expect(target.storageConfig.scope).toBe('database'); - expect(target.physicalName).toBe('myapp-default-public-db'); - const resolveCall = db.queries.find((q) => /resolve_default_bucket/.test(q.text)); - expect(resolveCall?.values).toEqual(['database', null, true, null]); - }); - - it('refuses to guess between two global planes for an unregistered column', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - NO_REGISTRY_ROW, - { - match: /function_resolution\.storage_modules\(\)/, - rows: () => [ - storageModuleRow({ scope: 'database', buckets_table: 'buckets', files_table: 'files' }), - storageModuleRow({ id: OTHER_MODULE_ID, scope: 'platform', buckets_table: 'platform_buckets', files_table: 'platform_files' }), - ], - }, - ]); - - await expect( - resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }), - ).rejects.toThrow('STORAGE_MODULE_AMBIGUOUS'); - }); - - it('raises when the registry names a module the database does not have', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { - match: /file_ref_field/, - rows: () => [{ id: 'ref-1', storage_module_id: OTHER_MODULE_ID, bucket_key: null, bucket_tags: null, is_public: null, enforce_fk: false }], - }, - STORAGE_MODULES, - ]); - - await expect( - resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }), - ).rejects.toThrow(OTHER_MODULE_ID); - }); - - it('refuses an entity-scoped module, whose bucket depends on an owner row', async () => { - const { resolveManagedUploadTarget } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { - match: /file_ref_field/, - rows: () => [{ id: 'ref-1', storage_module_id: OTHER_MODULE_ID, bucket_key: null, bucket_tags: null, is_public: null, enforce_fk: false }], - }, - { - match: /function_resolution\.storage_modules\(\)/, - rows: () => [storageModuleRow({ - id: OTHER_MODULE_ID, - scope: 'data_room', - entity_table_id: 'et-1', - entity_schema: 'app_public', - entity_table: 'data_rooms', - buckets_table: 'data_room_buckets', - files_table: 'data_room_files', - })], - }, - ]); - - await expect( - resolveManagedUploadTarget({ - options: options(), - withPgClient: db.withPgClient, - pgSettings: null, - databaseId: DATABASE_ID, - field: FIELD, - defaultPublicAccess: true, - }), - ).rejects.toThrow('STORAGE_SCOPE_UNSUPPORTED'); - }); -}); - -describe('assertUploadAllowedByBucket', () => { - function target(bucket: Partial): any { - return { bucket: { ...bucketRow(), ...bucket }, storageConfig: storageConfig() }; - } - - it('enforces the bucket mime allowlist on the streaming transport too', async () => { - const { assertUploadAllowedByBucket } = await import('../src/managed-upload'); - expect(() => - assertUploadAllowedByBucket(target({ allowed_mime_types: ['image/*'] }), 'application/pdf', 10), - ).toThrow('CONTENT_TYPE_NOT_ALLOWED'); - expect(() => - assertUploadAllowedByBucket(target({ allowed_mime_types: ['image/*'] }), 'image/png', 10), - ).not.toThrow(); - }); - - it('enforces the size cap, falling back to the module default', async () => { - const { assertUploadAllowedByBucket } = await import('../src/managed-upload'); - expect(() => assertUploadAllowedByBucket(target({ max_file_size: 5 }), 'image/png', 6)).toThrow('FILE_TOO_LARGE'); - expect(() => assertUploadAllowedByBucket(target({ max_file_size: null }), 'image/png', 1001)).toThrow('FILE_TOO_LARGE'); - expect(() => assertUploadAllowedByBucket(target({}), 'image/png', 0)).toThrow('INVALID_FILE_SIZE'); - }); -}); - -describe('finalizeStagedUpload', () => { - const s3 = { - client: { send: jest.fn().mockResolvedValue({}) }, - bucket: 'myapp-default-public-db', - region: 'us-east-1', - publicUrlPrefix: 'https://cdn.example.com', - } as unknown as S3Config; - - const target: any = { - databaseId: DATABASE_ID, - storageConfig: storageConfig(), - bucket: bucketRow(), - physicalName: 'myapp-default-public-db', - s3, - binding: null, - }; - - const staged = { - stagingKey: '.staging/tmp-1', - contentHash: 'a'.repeat(64), - contentType: 'image/png', - size: 16, - filename: 'hero.png', - }; - - beforeEach(() => { - (s3.client.send as jest.Mock).mockClear(); - }); - - it('promotes the staged object to its content key and inserts the files row', async () => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { match: /SELECT id, key, mime_type/, rows: () => [] }, - { match: /SELECT id FROM storage_private\.app_files_record_file/, rows: () => [{ id: FILE_ID }] }, - ]); - - const { projection, deduplicated } = await finalizeStagedUpload({ - target, withPgClient: db.withPgClient, pgSettings: null, staged, - }); - - expect(deduplicated).toBe(false); - expect(projection).toEqual({ - id: FILE_ID, - key: staged.contentHash, - bucket_id: BUCKET_ID, - mime: 'image/png', - size: 16, - filename: 'hero.png', - url: `https://cdn.example.com/${staged.contentHash}`, - }); - - const recorder = db.queries.find((q) => /record_file/.test(q.text)); - expect(recorder?.text).toContain('bucket_id :='); - expect(recorder?.text).toContain('upload :='); - expect(recorder?.text).not.toContain('owner_id'); - expect(recorder?.text).not.toContain('is_public'); - expect(recorder?.values).toEqual([ - BUCKET_ID, staged.contentHash, staged.contentHash, 'image/png', 16, 'hero.png', null, - ]); - // Copy to the content key, then drop the staged object. - expect(s3.client.send).toHaveBeenCalledTimes(2); - }); - - it('reuses the existing row and drops the staged bytes on a hash collision', async () => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { - match: /SELECT id, key, mime_type/, - rows: () => [{ id: FILE_ID, key: staged.contentHash, mime_type: 'image/png', size: 16, filename: 'original.png', status: 'uploaded' }], - }, - ]); - - const { projection, deduplicated } = await finalizeStagedUpload({ - target, withPgClient: db.withPgClient, pgSettings: null, staged, - }); - - expect(deduplicated).toBe(true); - expect(projection.id).toBe(FILE_ID); - expect(projection.filename).toBe('original.png'); - expect(db.queries.some((q) => /INSERT/.test(q.text))).toBe(false); - // The row's status is the evidence the bytes exist, so only the staged - // object is deleted; nothing is copied or probed in S3. - const sent = (s3.client.send as jest.Mock).mock.calls.map(([command]) => command.constructor.name); - expect(sent).toEqual(['DeleteObjectCommand']); - }); - - // A row is a claim on bytes rather than proof of them, so only a confirmed - // row may absorb an upload. - describe('row liveness', () => { - const existingRow = (status: string) => ({ - id: FILE_ID, - key: staged.contentHash, - mime_type: 'image/png', - size: 16, - filename: 'original.png', - status, - }); - - it.each(['uploaded', 'processed'])('deduplicates against a %s row', async (status) => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { match: /SELECT id, key, mime_type/, rows: () => [existingRow(status)] }, - ]); - - const { projection, deduplicated } = await finalizeStagedUpload({ - target, withPgClient: db.withPgClient, pgSettings: null, staged, - }); - - expect(deduplicated).toBe(true); - expect(projection.id).toBe(FILE_ID); - expect(db.queries.some((q) => /INSERT|DELETE/.test(q.text))).toBe(false); - }); - - it.each(['requested', 'rejected', 'expired'])( - 'drops the %s row and uploads afresh instead of reporting a dedup hit', - async (status) => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const NEW_FILE_ID = '55555555-5555-5555-5555-555555555555'; - const db = fakeDb([ - SET_CONFIG, - { match: /SELECT id, key, mime_type/, rows: () => [existingRow(status)] }, - { match: /DELETE FROM storage_public\.app_files/, rows: () => [{ id: FILE_ID }] }, - { match: /SELECT id FROM storage_private\.app_files_record_file/, rows: () => [{ id: NEW_FILE_ID }] }, - ]); - - const { projection, deduplicated } = await finalizeStagedUpload({ - target, withPgClient: db.withPgClient, pgSettings: null, staged, - }); - - expect(deduplicated).toBe(false); - // The caller is handed the row that actually names the promoted bytes. - expect(projection.id).toBe(NEW_FILE_ID); - const del = db.queries.find((q) => /DELETE/.test(q.text)); - expect(del?.values).toEqual([FILE_ID]); - // Promote to the content key, then drop the staged object. - expect(s3.client.send).toHaveBeenCalledTimes(2); - }, - ); - - it('reads the status column alongside the row', async () => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { match: /SELECT id, key, mime_type/, rows: () => [existingRow('uploaded')] }, - ]); - await finalizeStagedUpload({ - target, withPgClient: db.withPgClient, pgSettings: null, staged, - }); - expect(db.queries.find((q) => /SELECT id, key/.test(q.text))!.text).toContain('status'); - }); - }); - - it('abandons both keys when the files row cannot be inserted', async () => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { match: /SELECT id, key, mime_type/, rows: () => [] }, - { match: /SELECT id FROM storage_private\.app_files_record_file/, rows: () => { throw new Error('insert boom'); } }, - ]); - - await expect( - finalizeStagedUpload({ target, withPgClient: db.withPgClient, pgSettings: null, staged }), - ).rejects.toThrow('insert boom'); - - // Copy + delete promoted + delete staged: bytes no row names are bytes GC - // can never reach, so they do not outlive the failed call. - expect(s3.client.send).toHaveBeenCalledTimes(3); - }); - - it('fails loudly when the storage module has no generated recorder', async () => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([ - SET_CONFIG, - { match: /SELECT id, key, mime_type/, rows: () => [] }, - ]); - - await expect( - finalizeStagedUpload({ - target: { - ...target, - storageConfig: { ...storageConfig(), recorderQualifiedName: null }, - }, - withPgClient: db.withPgClient, - pgSettings: null, - staged, - }), - ).rejects.toThrow( - `STORAGE_RECORDER_MISSING: storage module ${APP_MODULE_ID} (app_files)`, - ); - expect(db.queries.some((q) => /INSERT INTO/.test(q.text))).toBe(false); - }); - - it('rejects bytes the bucket does not allow before writing anything', async () => { - const { finalizeStagedUpload } = await import('../src/managed-upload'); - const db = fakeDb([SET_CONFIG]); - - await expect( - finalizeStagedUpload({ - target: { ...target, bucket: { ...bucketRow(), allowed_mime_types: ['image/jpeg'] } }, - withPgClient: db.withPgClient, - pgSettings: null, - staged, - }), - ).rejects.toThrow('CONTENT_TYPE_NOT_ALLOWED'); - - expect(db.queries).toHaveLength(0); - expect(s3.client.send).not.toHaveBeenCalled(); - }); -}); diff --git a/graphile/graphile-presigned-url-plugin/src/file-projection.ts b/graphile/graphile-presigned-url-plugin/src/file-projection.ts new file mode 100644 index 0000000000..4bfbdd67b6 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/file-projection.ts @@ -0,0 +1,48 @@ +/** + * The projection document a managed `image`/`upload` column stores. + * + * A presigned upload creates the files row; the column then carries this + * document, whose `id` names that row. Nothing here bakes a presigned URL into + * a row: `url` is populated only for a public bucket, where it is a stable CDN + * address rather than a credential with an expiry. + */ + +import type { FileProjection, S3Config } from './types'; + +/** + * `id` is the files row — the load-bearing field: it is what makes the column a + * projection rather than a second, unmanaged copy of the truth, and it is what + * storage GC counts before collecting an object. + * + * `url` is set only for public buckets. Prefer `id` plus the files row's + * `downloadUrl`, which is late-bound and works for private buckets too. + */ +export type { FileProjection } from './types'; + +/** + * Build the projection document for a files row. + * + * A public bucket has a stable address, so `url` is a real, durable value there. + * A private bucket has no such address — only presigned, expiring ones — so the + * field is omitted rather than filled with a URL that dies in an hour. + */ +export function buildFileProjection( + file: { id: string; key: string; bucketId: string; mime: string; size: number; filename?: string | null }, + bucket: { is_public: boolean }, + s3: S3Config, +): FileProjection { + const projection: FileProjection = { + id: file.id, + key: file.key, + bucket_id: file.bucketId, + mime: file.mime, + size: file.size, + }; + if (file.filename) projection.filename = file.filename; + + if (bucket.is_public && s3.publicUrlPrefix) { + projection.url = `${s3.publicUrlPrefix.replace(/\/$/, '')}/${file.key}`; + } + + return projection; +} diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index b5aa36980f..3ed4236ec7 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -35,14 +35,7 @@ export { resolveDefaultBucket } from './default-bucket'; export { createDownloadUrlPlugin } from './download-url-field'; export type { FileRefFieldBinding } from './file-ref-registry'; export { clearFileRefFieldCache, FileRefFieldNotRegisteredError, getFileRefFieldBinding } from './file-ref-registry'; -export { - assertUploadAllowedByBucket, - buildFileProjection, - type FileProjection, - finalizeStagedUpload, - type ManagedUploadTarget, - resolveManagedUploadTarget, -} from './managed-upload'; +export { buildFileProjection, type FileProjection } from './file-projection'; export { assertBucketReconciled, resolveS3ForDatabase, diff --git a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts deleted file mode 100644 index 424367d456..0000000000 --- a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts +++ /dev/null @@ -1,418 +0,0 @@ -/** - * The managed upload lifecycle, shared by both transports. - * - * Multipart-through-GraphQL and presigned two-step are transports for one - * lifecycle, not two data models: either way an object gets a files row, a - * server-chosen key, a tenant-resolved bucket, and a projection document that - * carries the files row's id. The only difference is who moves the bytes. - * - * This module owns the parts that are the same: - * * `resolveManagedUploadTarget` — from a document column to a concrete - * (storage module, bucket, physical bucket, S3 config). - * * `finalizeStagedUpload` — from bytes already in S3 under a staging key to a - * files row and a projection document, deduplicating on content hash. - * * `buildFileProjection` — the document shape the column stores. - * - * Nothing here bakes a presigned URL into a row: `url` is populated only for a - * public bucket, where it is a stable CDN address rather than a credential with - * an expiry. - */ - -import { Logger } from '@pgpmjs/logger'; -import { type WithPgClient, withRequestPgClient } from 'graphile-plugin-utils'; - -import { resolveDefaultBucket } from './default-bucket'; -import { isLiveFileRow } from './file-lifecycle'; -import { fileNotReplaceable } from './file-not-replaceable'; -import { type FileRefFieldBinding, getFileRefFieldBinding } from './file-ref-registry'; -import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket'; -import { copyS3Object, deleteS3Object } from './s3-signer'; -import { recordManagedFile } from './storage-file-recorder'; -import { getBucketConfig, loadAllStorageModules } from './storage-module-cache'; -import type { BucketConfig, FileProjection, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; - -const log = new Logger('graphile-presigned-url:managed-upload'); - -/** - * The document a managed `image`/`upload` column stores. - * - * `id` is the files row — the load-bearing field: it is what makes the column a - * projection rather than a second, unmanaged copy of the truth, and it is what - * storage GC counts before collecting an object. - * - * `url` is retained for existing readers of the pre-managed shape and is set - * only for public buckets. Prefer `id` plus the files row's `downloadUrl`, which - * is late-bound and works for private buckets too. - */ -export type { FileProjection } from './types'; - -/** - * Build the projection document for a files row. - * - * A public bucket has a stable address, so `url` is a real, durable value there. - * A private bucket has no such address — only presigned, expiring ones — so the - * field is omitted rather than filled with a URL that dies in an hour. - */ -export function buildFileProjection( - file: { id: string; key: string; bucketId: string; mime: string; size: number; filename?: string | null }, - bucket: { is_public: boolean }, - s3: S3Config, -): FileProjection { - const projection: FileProjection = { - id: file.id, - key: file.key, - bucket_id: file.bucketId, - mime: file.mime, - size: file.size, - }; - if (file.filename) projection.filename = file.filename; - - if (bucket.is_public && s3.publicUrlPrefix) { - projection.url = `${s3.publicUrlPrefix.replace(/\/$/, '')}/${file.key}`; - } - - return projection; -} - -/** - * Everything a managed upload needs before bytes move. - */ -export interface ManagedUploadTarget { - databaseId: string; - storageConfig: StorageModuleConfig; - bucket: BucketConfig; - physicalName: string; - s3: S3Config; - /** The registry row, or null when the column predates registration. */ - binding: FileRefFieldBinding | null; -} - -/** - * Resolve where a write to a document column lands. - * - * Two routes, one rule — the bucket is always resolved inside the tenant: - * * a registered column names its storage module, and either a logical bucket - * key or the reserved default tag for its declared publicness; - * * an unregistered column (a bare `image`/`upload` on a database provisioned - * before the registry) falls back to the database's single global storage - * plane and the same reserved default tag. That is a *tenant* default, not - * an environment one. - * - * The fallback is by shape, not by scope name: a global plane is one with no - * entity key (`entity_table_id IS NULL`), which is what a database-wide plane is - * whether it registered as 'database', 'platform', or 'app'. Matching the name - * instead sent every tenant whose plane registers as 'database' — i.e. every - * database-scope plane — to STORAGE_MODULE_NOT_FOUND. - * - * A database with no global plane raises, and so does one with several: there is - * nowhere unambiguously tenant-owned to put the bytes, and the deployment's - * configured bucket is not an answer. - */ -export async function resolveManagedUploadTarget(args: { - options: PresignedUrlPluginOptions; - withPgClient: WithPgClient; - pgSettings: Record | null; - databaseId: string; - field: { schemaName: string; tableName: string; columnName: string }; - /** Publicness to use when the column is unregistered. */ - defaultPublicAccess: boolean; -}): Promise { - const { options, withPgClient, pgSettings, databaseId, field, defaultPublicAccess } = args; - - // Registry and module registration are read as the request's own role, - // through the claim-scoped function_resolution SECURITY DEFINER readers. - const binding = await withRequestPgClient(withPgClient, pgSettings, async (pgClient): Promise => { - try { - return await getFileRefFieldBinding(pgClient, databaseId, field); - } catch (err: any) { - // An unregistered column is a legitimate state (it predates the registry) - // and falls back to the tenant's global plane below. Any other - // failure — a broken connection, a missing registry table — is not. - if (err?.name === 'FileRefFieldNotRegisteredError') return null; - throw err; - } - }); - - const allConfigs = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => - loadAllStorageModules(pgClient, databaseId), - ); - - const globalConfigs = allConfigs.filter((c) => c.entityTableId === null); - - if (!binding && globalConfigs.length > 1) { - // Several global planes and no registry row to disambiguate: picking one is - // picking a tenant's bucket at random. Name them and refuse. - throw new Error( - `STORAGE_MODULE_AMBIGUOUS: ${field.schemaName}.${field.tableName}.${field.columnName} is an ` + - `unregistered upload column and database ${databaseId} has ${globalConfigs.length} global ` + - `storage planes (scopes: ${globalConfigs.map((c) => c.scope).join(', ')}); register the column ` + - 'so it names the plane it writes to', - ); - } - - const storageConfig = binding - ? allConfigs.find((c) => c.id === binding.storageModuleId) - : globalConfigs[0]; - - if (!storageConfig) { - throw new Error( - binding - ? `STORAGE_MODULE_NOT_FOUND: file_ref_field ${binding.id} names storage module ` + - `${binding.storageModuleId}, which database ${databaseId} does not have` - : `STORAGE_MODULE_NOT_FOUND: ${field.schemaName}.${field.tableName}.${field.columnName} is an ` + - `unregistered upload column and database ${databaseId} has no global storage plane to ` + - 'default to; there is no environment bucket to fall back to', - ); - } - - if (storageConfig.entityTableId !== null) { - // An entity-keyed module resolves its bucket per owning row, and a - // multipart column write does not carry one. Refuse rather than write a - // tenant's file into whichever bucket happened to resolve. - throw new Error( - `STORAGE_SCOPE_UNSUPPORTED: ${field.schemaName}.${field.tableName}.${field.columnName} binds to ` + - `'${storageConfig.scope}'-scoped storage, which resolves its bucket per owner row. ` + - 'Use the presigned upload mutation, which takes an ownerId.', - ); - } - - const publicAccess = binding?.isPublic ?? defaultPublicAccess; - - // Bucket resolution and the bucket read run under the request role: what the - // caller may store into is exactly what RLS lets them see. - const coordinate = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => - resolveDefaultBucket( - pgClient, - storageConfig.scope, - null, - publicAccess, - binding?.bucketKey ?? null, - ), - ); - - const bucket = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => - getBucketConfig(pgClient, storageConfig, databaseId, coordinate.resolvedKey), - ); - if (!bucket) { - throw new Error( - `BUCKET_NOT_FOUND: bucket "${coordinate.resolvedKey}" resolved for ` + - `${field.schemaName}.${field.tableName}.${field.columnName} is not readable`, - ); - } - - if (bucket.allow_custom_keys) { - // A path-keyed bucket (e.g. a static site's) is addressed by the keys its - // publisher chose; this lane can only mint content-hash keys, which would - // pollute it with unreachable objects. Path-keyed uploads go through the - // presigned lane, which accepts an explicit `key`. - throw new Error( - `BUCKET_PATH_KEYED: bucket "${bucket.key}" allows custom keys and is addressed by path; ` + - 'the multipart upload lane only writes content-addressed keys. Use the presigned upload ' + - 'mutation with an explicit key.', - ); - } - - const physicalName = assertBucketReconciled(bucket, databaseId); - - return { - databaseId, - storageConfig, - bucket, - physicalName, - s3: resolveS3ForDatabase(options, storageConfig, physicalName), - binding, - }; -} - -/** - * Validate an upload against the resolved bucket's rules. - * - * The same rules the presigned lane enforces — a transport must not be a way - * around a bucket's mime allowlist or size cap. - */ -export function assertUploadAllowedByBucket( - target: ManagedUploadTarget, - contentType: string, - size: number, -): void { - const { bucket, storageConfig } = target; - - if (bucket.allowed_mime_types && bucket.allowed_mime_types.length > 0) { - const isAllowed = bucket.allowed_mime_types.some((pattern) => { - if (pattern === '*/*') return true; - if (pattern.endsWith('/*')) return contentType.startsWith(pattern.slice(0, -1)); - return contentType === pattern; - }); - if (!isAllowed) { - throw new Error(`CONTENT_TYPE_NOT_ALLOWED: ${contentType} not in bucket allowed types`); - } - } - - const maxSize = bucket.max_file_size ?? storageConfig.defaultMaxFileSize; - if (size > maxSize) { - throw new Error(`FILE_TOO_LARGE: ${size} bytes exceeds the ${maxSize} byte limit`); - } - if (size <= 0) { - throw new Error('INVALID_FILE_SIZE: an upload must carry at least one byte'); - } -} - -/** Delete an object we are abandoning, without masking the failure in progress. */ -async function bestEffortDelete(s3: S3Config, key: string): Promise { - try { - await deleteS3Object(s3, key); - } catch (err) { - log.warn(`Failed to clean up abandoned object ${key}: ${err}`); - } -} - -/** - * Turn bytes already staged in S3 into a files row and a projection document. - * - * The content hash is only known once the stream has been read, so a streaming - * transport writes to a staging key first and promotes here: - * - * * hash already present in this bucket → drop the staged object, reuse the - * existing files row. Dedup is a property of the object, so it holds no - * matter which transport wrote it first. - * * otherwise → server-side copy to the content-addressed key, drop the staged - * object, insert the files row. - * - * The row is inserted *after* the bytes land, so the confirm-upload job the - * insert trigger enqueues finds the object and completes the - * `requested → uploaded` transition without any extra wiring here. - * - * Every failure path leaves S3 as it found it. Bytes written by this call and - * not reachable through a files row would be invisible to storage GC, which - * collects objects by walking rows — so an object is only left behind once the - * row naming it exists. - */ -export async function finalizeStagedUpload(args: { - target: ManagedUploadTarget; - withPgClient: WithPgClient; - pgSettings: Record | null; - staged: { - stagingKey: string; - contentHash: string; - contentType: string; - size: number; - filename?: string | null; - }; -}): Promise<{ projection: FileProjection; deduplicated: boolean }> { - const { target, withPgClient, pgSettings, staged } = args; - const { storageConfig, bucket, s3 } = target; - - assertUploadAllowedByBucket(target, staged.contentType, staged.size); - - const finalKey = staged.contentHash; - - const existing = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - const result = await pgClient.query({ - text: `SELECT id, key, mime_type, size, filename, status - FROM ${storageConfig.filesQualifiedName} - WHERE content_hash = $1 AND bucket_id = $2 - LIMIT 1`, - values: [staged.contentHash, bucket.id], - }); - return result.rows[0] as - | { - id: string; - key: string; - mime_type: string; - size: number; - filename: string | null; - status: string; - } - | undefined; - }); - - // Only a row that already stands for stored bytes may absorb this upload. One - // that never received them is dropped, and the staged object is promoted as a - // fresh file below — which is also what keeps the insert possible, since the - // final key is the content hash and (bucket_id, key) is unique. The GC job the - // delete enqueues re-takes the reference count when it runs, by which point - // the replacement row exists, so it no-ops. - if (existing && !isLiveFileRow(existing)) { - log.info( - `Restarting upload of hash ${staged.contentHash}: file ${existing.id} has no bytes behind it` - ); - // The DELETE runs under the request role, so RLS decides whether this caller - // may drop the row. A row it can see but not delete stays, and the upload - // fails here rather than colliding on (bucket_id, key) below. - const deleted = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - const result = await pgClient.query({ - text: `DELETE FROM ${storageConfig.filesQualifiedName} WHERE id = $1 RETURNING id`, - values: [existing.id], - }); - return result.rows.length; - }); - if (deleted === 0) { - // Nothing names the staged object, so dropping it cannot cost anyone bytes. - await bestEffortDelete(s3, staged.stagingKey); - throw fileNotReplaceable(existing.id, existing.key); - } - } else if (existing) { - log.info(`Dedup hit: file ${existing.id} already carries hash ${staged.contentHash}`); - await deleteS3Object(s3, staged.stagingKey); - - return { - projection: buildFileProjection( - { - id: existing.id, - key: existing.key, - bucketId: bucket.id, - mime: existing.mime_type, - size: Number(existing.size), - filename: existing.filename, - }, - bucket, - s3, - ), - deduplicated: true, - }; - } - - await copyS3Object(s3, staged.stagingKey, finalKey, staged.contentType); - - let fileId: string; - try { - fileId = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { - return recordManagedFile(pgClient, storageConfig, { - bucketId: bucket.id, - key: finalKey, - contentHash: staged.contentHash, - mimeType: staged.contentType, - size: staged.size, - filename: staged.filename, - }); - }); - } catch (err) { - // No row names either key, so both are unreachable to GC. Dropping the - // promoted copy is safe precisely because the dedup probe above found no row - // on this hash: nothing else in this bucket is entitled to those bytes. - // Cleanup must never replace the failure that caused it. - await bestEffortDelete(s3, finalKey); - await bestEffortDelete(s3, staged.stagingKey); - throw err; - } - - await deleteS3Object(s3, staged.stagingKey); - - log.info(`Managed upload created file ${fileId} at ${bucket.key}/${finalKey}`); - - return { - projection: buildFileProjection( - { - id: fileId, - key: finalKey, - bucketId: bucket.id, - mime: staged.contentType, - size: staged.size, - filename: staged.filename, - }, - bucket, - s3, - ), - deduplicated: false, - }; -} diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index fa1573ce69..1fc46f86d1 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -32,7 +32,7 @@ import { validateCustomKey } from './custom-key'; import { resolveDefaultBucket } from './default-bucket'; import { isLiveFileRow } from './file-lifecycle'; import { fileNotReplaceable } from './file-not-replaceable'; -import { buildFileProjection, type FileProjection } from './managed-upload'; +import { buildFileProjection, type FileProjection } from './file-projection'; import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket'; import { deleteS3Object,generatePresignedPutUrl } from './s3-signer'; import { recordManagedFile } from './storage-file-recorder'; diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index d6c3c11bb8..07c8599457 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -169,7 +169,7 @@ export interface RequestUploadUrlPayload { } /** - * The document a managed `image`/`upload` column stores. See `./managed-upload`. + * The document a managed `image`/`upload` column stores. See `./file-projection`. */ export interface FileProjection { id: string; diff --git a/graphile/graphile-settings/__tests__/upload-resolver.test.ts b/graphile/graphile-settings/__tests__/upload-resolver.test.ts deleted file mode 100644 index 240d52457f..0000000000 --- a/graphile/graphile-settings/__tests__/upload-resolver.test.ts +++ /dev/null @@ -1,371 +0,0 @@ -/** - * The multipart upload lane, end to end minus the network. - * - * S3 is mocked at the streamer/client boundary and the database at the query - * boundary, so these assert what the lane decides: that the bytes go to the - * tenant's resolved bucket rather than an environment one, that a files row is - * created, and that the value stored in the column names that row while keeping - * the `url` its existing readers depend on. - */ - -import { Readable } from 'stream'; - -const DATABASE_ID = '00000000-0000-0000-0000-0000000000db'; -const MODULE_ID = '11111111-1111-1111-1111-111111111111'; -const BUCKET_ID = '33333333-3333-3333-3333-333333333333'; -const FILE_ID = '44444444-4444-4444-4444-444444444444'; - -const FIELD = { schemaName: 'app_public', tableName: 'posts', columnName: 'image' }; - -interface QueryHandler { - match: RegExp; - rows: () => unknown[]; -} - -function storageModuleRow(): Record { - return { - id: MODULE_ID, - scope: 'app', - entity_table_id: null, - buckets_schema: 'storage_public', - buckets_table: 'app_buckets', - files_schema: 'storage_public', - files_table: 'app_files', - private_schema: 'storage_private', - endpoint: 'http://localhost:9000', - public_url_prefix: 'https://cdn.example.com', - provider: 'minio', - region: 'us-east-1', - allowed_origins: null, - upload_url_expiry_seconds: null, - download_url_expiry_seconds: null, - default_max_file_size: 1048576, - max_filename_length: null, - cache_ttl_seconds: null, - max_bulk_files: null, - max_bulk_total_size: null, - has_path_shares: false, - has_versioning: false, - entity_schema: null, - entity_table: null, - }; -} - -function bucketRow(isPublic: boolean): Record { - return { - id: BUCKET_ID, - key: isPublic ? 'default-public' : 'default', - type: isPublic ? 'public' : 'private', - is_public: isPublic, - owner_id: null, - allowed_mime_types: null, - max_file_size: null, - allow_custom_keys: false, - physical_name: 'myapp-default-public-db', - }; -} - -/** A withPgClient answering exactly the statements this lane issues. */ -function fakeContext(opts: { isPublic?: boolean; existingFile?: boolean; failInsert?: boolean } = {}) { - const isPublic = opts.isPublic ?? true; - const queries: Array<{ text: string; values?: unknown[] }> = []; - const handlers: QueryHandler[] = [ - { match: /set_config/, rows: () => [] }, - { match: /current_database_id/, rows: () => [{ id: DATABASE_ID }] }, - { match: /file_ref_field/, rows: () => [] }, - { match: /function_resolution\.storage_modules\(\)/, rows: () => [storageModuleRow()] }, - { - match: /resolve_default_bucket/, - rows: () => [{ - bucket_id: BUCKET_ID, - resolved_key: isPublic ? 'default-public' : 'default', - bucket_type: isPublic ? 'public' : 'private', - physical_name: 'myapp-default-public-db', - }], - }, - { match: /SELECT id, key, type/, rows: () => [bucketRow(isPublic)] }, - { - match: /SELECT id, key, mime_type/, - rows: () => opts.existingFile - ? [{ id: FILE_ID, key: 'existing-key', mime_type: 'image/png', size: 16, filename: 'first.png', status: 'uploaded' }] - : [], - }, - { - match: /app_files_record_file/, - rows: () => { - if (opts.failInsert) throw new Error('insert boom'); - return [{ id: FILE_ID }]; - }, - }, - ]; - - const client = { - async query(o: { text: string; values?: unknown[] }) { - queries.push(o); - const handler = handlers.find((h) => h.match.test(o.text)); - if (!handler) throw new Error(`unexpected query: ${o.text}`); - return { rows: handler.rows() }; - }, - withTransaction: (cb: any) => cb(client), - }; - - return { - context: { - withPgClient: (_settings: any, cb: any) => cb(client), - pgSettings: { 'jwt.claims.database_id': DATABASE_ID }, - }, - queries, - }; -} - -async function loadUploadResolverModule(opts: { detectedContentType: string }) { - jest.resetModules(); - - const mockDetectContentType = jest.fn().mockResolvedValue({ - stream: Readable.from([Buffer.alloc(16)]), - magic: { type: opts.detectedContentType, charset: 'binary' }, - contentType: opts.detectedContentType, - }); - - const mockUploadWithContentType = jest.fn().mockImplementation(async ({ readStream }: any) => { - // Drain the stream so the hashing pass-through sees every byte, exactly as - // a real multipart upload to S3 would. - for await (const _chunk of readStream) { /* consumed */ } - return { - upload: { Location: 'https://cdn.example.com/uploaded-file' }, - contentType: opts.detectedContentType, - }; - }); - - jest.doMock('@constructive-io/graphql-env', () => ({ - getEnvOptions: jest.fn(() => ({ - storage: { accessKeyId: 'test', secretAccessKey: 'test' }, - })), - })); - - jest.doMock('@constructive-io/s3-streamer', () => ({ - __esModule: true, - default: jest.fn().mockImplementation(() => ({ - uploadWithContentType: mockUploadWithContentType, - detectContentType: mockDetectContentType, - })), - })); - - const s3Send = jest.fn().mockResolvedValue({}); - jest.doMock('@constructive-io/s3-utils', () => ({ - createS3Client: jest.fn(() => ({ send: s3Send })), - })); - - const mod = await import('../src/upload-resolver'); - const { clearBucketCache, clearStorageModuleCache, clearFileRefFieldCache } = - await import('graphile-presigned-url-plugin'); - clearBucketCache(); - clearStorageModuleCache(); - clearFileRefFieldCache(); - - return { ...mod, mockDetectContentType, mockUploadWithContentType, s3Send }; -} - -function definitionFor(defs: any[], name: string) { - const def = defs.find((d) => 'name' in d && d.name === name); - if (!def) throw new Error(`Missing ${name} upload field definition`); - return def; -} - -function makeFakeUpload(filename: string) { - return { - filename, - createReadStream: jest.fn(() => Readable.from([Buffer.alloc(16)])), - }; -} - -describe('multipart upload resolver', () => { - it('rejects a disallowed MIME before anything is written', async () => { - const { constructiveUploadFieldDefinitions, mockDetectContentType, mockUploadWithContentType } = - await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); - const { context } = fakeContext(); - - await expect( - definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( - makeFakeUpload('document.pdf') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, - ), - ).rejects.toThrow('UPLOAD_MIMETYPE'); - - expect(mockDetectContentType).toHaveBeenCalledTimes(1); - expect(mockUploadWithContentType).not.toHaveBeenCalled(); - }); - - it('stores a projection that names the files row, keeping url/filename/mime', async () => { - const { constructiveUploadFieldDefinitions, mockUploadWithContentType } = - await loadUploadResolverModule({ detectedContentType: 'image/png' }); - const { context, queries } = fakeContext(); - - const result: any = await definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( - makeFakeUpload('photo.png') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, - ); - - // The load-bearing new field: GC counts document references by files-row id. - expect(result.id).toBe(FILE_ID); - expect(result.bucket_id).toBe(BUCKET_ID); - expect(result.size).toBe(16); - // The content hash of 16 zero bytes — the key is derived from the bytes, not - // from a random string. - expect(result.key).toBe('374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb'); - // Retained for existing readers of the pre-managed shape. - expect(result.filename).toBe('photo.png'); - expect(result.mime).toBe('image/png'); - expect(result.url).toBe('https://cdn.example.com/374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb'); - - // The bytes went to the tenant's bucket, and to a staging key first. - const call = mockUploadWithContentType.mock.calls[0][0]; - expect(call.bucket).toBe('myapp-default-public-db'); - expect(call.key).toMatch(/^\.staging\//); - - const recorder = queries.find((q) => /app_files_record_file/.test(q.text)); - expect(recorder).toBeDefined(); - expect(recorder?.text).not.toContain('INSERT INTO'); - }); - - it('deduplicates against an existing row rather than inserting a second one', async () => { - const { constructiveUploadFieldDefinitions } = - await loadUploadResolverModule({ detectedContentType: 'image/png' }); - const { context, queries } = fakeContext({ existingFile: true }); - - const result: any = await definitionFor(constructiveUploadFieldDefinitions, 'upload').resolve( - makeFakeUpload('photo.png') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'upload', field: FIELD } }, - ); - - expect(result.id).toBe(FILE_ID); - expect(result.key).toBe('existing-key'); - expect(queries.some((q) => /INSERT/.test(q.text))).toBe(false); - }); - - it('returns a bare URL for an attachment column, whose type cannot hold a document', async () => { - const { constructiveUploadFieldDefinitions } = - await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); - const { context } = fakeContext(); - - const result = await definitionFor(constructiveUploadFieldDefinitions, 'attachment').resolve( - makeFakeUpload('doc.pdf') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'attachment', field: { ...FIELD, columnName: 'doc' } } }, - ); - - expect(typeof result).toBe('string'); - expect(result).toMatch(/^https:\/\/cdn\.example\.com\//); - }); - - it('refuses to put an expiring URL in an attachment column on a private bucket', async () => { - const { constructiveUploadFieldDefinitions } = - await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); - const { context } = fakeContext({ isPublic: false }); - - await expect( - definitionFor(constructiveUploadFieldDefinitions, 'attachment').resolve( - makeFakeUpload('doc.pdf') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'attachment', field: { ...FIELD, columnName: 'doc' } } }, - ), - ).rejects.toThrow('ATTACHMENT_BUCKET_NOT_PUBLIC'); - }); - - it('leaves nothing in S3 when the files row cannot be created', async () => { - const { constructiveUploadFieldDefinitions, s3Send } = - await loadUploadResolverModule({ detectedContentType: 'image/png' }); - // The insert is the last step, so it fails with the bytes already staged. - const { context } = fakeContext({ failInsert: true }); - - await expect( - definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( - makeFakeUpload('photo.png') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, - ), - ).rejects.toThrow('insert boom'); - - // Copy to the content key, then both the promoted and the staged object are - // removed: no files row names either, so GC could never reach them. - expect(s3Send).toHaveBeenCalledTimes(3); - }); - - it('refuses to upload when the plugin cannot say which column is being written', async () => { - const { constructiveUploadFieldDefinitions } = - await loadUploadResolverModule({ detectedContentType: 'image/png' }); - const { context } = fakeContext(); - - await expect( - definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( - makeFakeUpload('photo.png') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'image' } }, - ), - ).rejects.toThrow('UPLOAD_FIELD_UNKNOWN'); - }); - - it('rejects an extension that disagrees with the bytes, before anything is written', async () => { - // The html-as-jpg attack: a browser fetching this as an image would execute - // the script in it. - const { constructiveUploadFieldDefinitions, mockUploadWithContentType } = - await loadUploadResolverModule({ detectedContentType: 'text/html' }); - const { context, queries } = fakeContext(); - - await expect( - definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( - makeFakeUpload('avatar.jpg') as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, - ), - ).rejects.toThrow('UPLOAD_TYPE_MISMATCH'); - - expect(mockUploadWithContentType).not.toHaveBeenCalled(); - expect(queries.some((q) => /INSERT/.test(q.text))).toBe(false); - }); - - it('rejects a declared MIME type that disagrees with the bytes', async () => { - const { constructiveUploadFieldDefinitions, mockUploadWithContentType } = - await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); - const { context } = fakeContext(); - - await expect( - definitionFor(constructiveUploadFieldDefinitions, 'upload').resolve( - { ...makeFakeUpload('report.pdf'), mimetype: 'image/png' } as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'upload', field: FIELD } }, - ), - ).rejects.toThrow('UPLOAD_TYPE_MISMATCH'); - - expect(mockUploadWithContentType).not.toHaveBeenCalled(); - }); - - it('accepts a text file whose extension the bytes cannot confirm in detail', async () => { - // Leading bytes can tell text from binary, not CSV from plain text; treating - // that as a mismatch would reject every legitimate text upload. - const { constructiveUploadFieldDefinitions } = - await loadUploadResolverModule({ detectedContentType: 'text/plain' }); - const { context } = fakeContext(); - - const result: any = await definitionFor(constructiveUploadFieldDefinitions, 'upload').resolve( - { ...makeFakeUpload('rows.csv'), mimetype: 'text/csv' } as any, - {}, - context, - { uploadPlugin: { tags: {}, type: 'upload', field: FIELD } }, - ); - - expect(result.id).toBe(FILE_ID); - }); -}); diff --git a/graphile/graphile-settings/package.json b/graphile/graphile-settings/package.json index 37bf089066..350c4529c6 100644 --- a/graphile/graphile-settings/package.json +++ b/graphile/graphile-settings/package.json @@ -33,7 +33,6 @@ "@constructive-io/bucket-provisioner": "workspace:^", "@constructive-io/graphql-env": "workspace:^", "@constructive-io/graphql-types": "workspace:^", - "@constructive-io/s3-streamer": "workspace:^", "@constructive-io/upload-names": "workspace:^", "@dataplan/json": "1.0.1", "@dataplan/pg": "1.1.1", @@ -61,12 +60,10 @@ "graphile-presigned-url-plugin": "workspace:^", "graphile-realtime-subscriptions": "workspace:^", "graphile-search": "workspace:^", - "graphile-upload-plugin": "workspace:^", "graphile-utils": "5.0.3", "graphql": "16.13.0", "inflekt": "^0.8.1", "lru-cache": "^11.2.7", - "mime-bytes": "workspace:^", "pg": "^8.21.0", "pg-query-context": "workspace:^", "pg-sql2": "5.0.1", diff --git a/graphile/graphile-settings/src/presets/constructive-preset.ts b/graphile/graphile-settings/src/presets/constructive-preset.ts index f99327d7af..05b1a07c56 100644 --- a/graphile/graphile-settings/src/presets/constructive-preset.ts +++ b/graphile/graphile-settings/src/presets/constructive-preset.ts @@ -11,7 +11,6 @@ import { createPostgisOperatorFactory,GraphilePostgisPreset } from 'graphile-pos import { PresignedUrlPreset } from 'graphile-presigned-url-plugin'; import { RealtimeSubscriptionsPreset } from 'graphile-realtime-subscriptions'; import { createMatchesOperatorFactory, createTrgmOperatorFactories,UnifiedSearchPreset } from 'graphile-search'; -import { UploadPreset } from 'graphile-upload-plugin'; import { ConflictDetectorPreset, @@ -26,7 +25,6 @@ import { RequiredInputPreset } from '../plugins'; import { getStorageCredentials } from '../presigned-url-resolver'; -import { constructiveUploadFieldDefinitions } from '../upload-resolver'; /** * Feature flags that control which optional Graphile plugins are included @@ -41,7 +39,6 @@ export interface ConstructivePresetOptions { enableAggregates?: boolean; enablePostgis?: boolean; enableSearch?: boolean; - enableDirectUploads?: boolean; enablePresignedUploads?: boolean; enableManyToMany?: boolean; enableConnectionFilter?: boolean; @@ -75,7 +72,6 @@ const DEFAULTS: Required = { enableAggregates: false, enablePostgis: true, enableSearch: true, - enableDirectUploads: true, enablePresignedUploads: true, enableManyToMany: true, enableConnectionFilter: true, @@ -113,7 +109,6 @@ const DEFAULTS: Required = { * - enableSearch -> UnifiedSearchPreset (tsvector, BM25, pg_trgm, pgvector) * - enablePostgis -> GraphilePostgisPreset * - enableLtree -> GraphileLtreePreset - * - enableDirectUploads -> UploadPreset * - enablePresignedUploads -> PresignedUrlPreset, BucketProvisionerPreset * - enableAggregates -> PgAggregatesPreset (off by default) * - enableRealtime -> RealtimeSubscriptionsPreset (off by default) @@ -187,15 +182,6 @@ export function createConstructivePreset( presets.push(GraphileLtreePreset); } - if (opts.enableDirectUploads) { - presets.push( - UploadPreset({ - uploadFieldDefinitions: constructiveUploadFieldDefinitions, - maxFileSize: 10 * 1024 * 1024 // 10MB - }) - ); - } - if (opts.enablePresignedUploads) { presets.push( PresignedUrlPreset({ diff --git a/graphile/graphile-settings/src/upload-resolver.ts b/graphile/graphile-settings/src/upload-resolver.ts deleted file mode 100644 index 3574549353..0000000000 --- a/graphile/graphile-settings/src/upload-resolver.ts +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Upload resolver for the Constructive upload plugin (multipart `Upload` scalar). - * - * This is the streaming transport into the *managed* storage lane: bytes arrive - * on the mutation, and the file they carry gets the same treatment a presigned - * upload gets — a bucket resolved inside the tenant, a content-addressed key, a - * files row, and a projection document naming that row. - * - * It used to be a second storage model: stream to `BUCKET_NAME` under a random - * key, hand back a URL, record nothing. Objects written that way belonged to no - * database, could not be deduplicated, listed, or access-controlled, and storage - * GC could not see that a document still pointed at them. Nothing in this path - * comes from the environment but the object-store credentials: the bucket and - * its connection (endpoint/provider/region) are the tenant's storage module. - * - * Compatibility: `image`/`upload` columns still receive `url` alongside the new - * `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep - * working while they migrate to `id` + the files row's late-bound `downloadUrl`. - * - * ENV VARS: STORAGE_ACCESS_KEY_ID, STORAGE_SECRET_ACCESS_KEY (required). - */ - -import Streamer from '@constructive-io/s3-streamer'; -import { createHash, randomUUID } from 'crypto'; -import { - finalizeStagedUpload, - type PresignedUrlPluginOptions, - resolveManagedUploadTarget, - withRequestPgClient, -} from 'graphile-presigned-url-plugin'; -import type { - FileUpload, - UploadFieldDefinition, - UploadFieldIdentity, - UploadPluginInfo, -} from 'graphile-upload-plugin'; -import { checkTypeAgreement } from 'mime-bytes'; -import { Transform } from 'stream'; - -import { getStorageCredentials } from './presigned-url-resolver'; - -const DEFAULT_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/svg+xml']; - -let managedOptions: PresignedUrlPluginOptions | null = null; - -function getManagedOptions(): PresignedUrlPluginOptions { - if (!managedOptions) { - managedOptions = { - credentials: getStorageCredentials, - }; - } - return managedOptions; -} - -/** A staging key: transient, and never what the object ends up under. */ -function stagingKey(): string { - return `.staging/${randomUUID()}`; -} - -async function resolveDatabaseId(pgClient: any): Promise { - const result = await pgClient.query({ text: `SELECT jwt_private.current_database_id() AS id` }); - return result.rows[0]?.id ?? null; -} - -/** - * Which default bucket an *unregistered* column resolves to. - * - * Columns written by the pre-managed resolver held a directly-embedded URL, so - * their readers assume a publicly addressable object; resolving them to the - * private default would break every page rendering one. A registered column - * states its own intent and this is not consulted. - */ -const LEGACY_DEFAULT_PUBLIC_ACCESS = true; - -/** The mime allowlist for a column, from its smart tags or its type. */ -function allowedMimeTypes(tags: Record | undefined, typ: string | undefined): string[] { - const VALID_MIME = /^[a-z]+\/[a-z0-9][a-z0-9!#$&\-.^_+]*$/i; - if (tags?.mime) { - return String(tags.mime) - .trim() - .split(',') - .map((a: string) => a.trim()) - .filter((m: string) => VALID_MIME.test(m)); - } - return typ === 'image' ? DEFAULT_IMAGE_MIME_TYPES : []; -} - -/** - * Hash and measure bytes as they stream past, without buffering them. - * - * The final key is the content hash, which is only known once the last byte has - * gone by — so the object is staged first and promoted after. Nothing is held in - * memory: a 2GB upload streams through this the same as a 2KB one. - */ -function hashingPassThrough(): Transform & { digest: () => string; bytes: () => number } { - const hash = createHash('sha256'); - let bytes = 0; - const stream = new Transform({ - transform(chunk, _encoding, callback) { - hash.update(chunk); - bytes += chunk.length; - callback(null, chunk); - }, - }); - return Object.assign(stream, { - digest: () => hash.digest('hex'), - bytes: () => bytes, - }); -} - -/** - * Stream an upload into managed storage and return the value the column stores. - * - * Shape by column type hint: - * * `image` / `upload` (jsonb domains) → the projection document, including a - * compatibility `url` for public buckets. - * * `attachment` (text domain) → the object's public URL. A text column cannot - * hold a projection, so the files row is still authoritative but the column - * itself carries no id; the row keeps the object alive. A private bucket - * raises rather than storing an expiring presigned URL in a column. - */ -async function uploadResolver( - upload: FileUpload, - _args: unknown, - context: any, - info: { uploadPlugin: UploadPluginInfo }, -): Promise { - const { tags, type, field } = info.uploadPlugin; - const typ = type || tags?.type; - - const withPgClient = context?.withPgClient; - const pgSettings = context?.pgSettings ?? null; - if (!withPgClient) { - throw new Error( - 'UPLOAD_NO_PG_CLIENT: a managed upload resolves its bucket in the database, so the ' + - 'GraphQL context must carry withPgClient', - ); - } - if (!field) { - throw new Error( - 'UPLOAD_FIELD_UNKNOWN: the upload plugin did not report which column is being written, ' + - 'so the storage module and bucket backing it cannot be resolved', - ); - } - - const databaseId = await withRequestPgClient(withPgClient, pgSettings, (pgClient: any) => - resolveDatabaseId(pgClient), - ); - if (!databaseId) throw new Error('DATABASE_NOT_FOUND'); - - const target = await resolveManagedUploadTarget({ - options: getManagedOptions(), - withPgClient, - pgSettings, - databaseId, - field: field as UploadFieldIdentity, - defaultPublicAccess: LEGACY_DEFAULT_PUBLIC_ACCESS, - }); - - if (typ === 'attachment' && !target.bucket.is_public) { - throw new Error( - 'ATTACHMENT_BUCKET_NOT_PUBLIC: an attachment column stores a plain URL, and the resolved ' + - `bucket "${target.bucket.key}" is private, whose only URLs expire. Use an upload column, ` + - 'which stores the file id and resolves a fresh download URL on read.', - ); - } - - const s3 = new Streamer({ client: target.s3.client, defaultBucket: target.physicalName }); - const { filename } = upload; - - // Validate before persisting: content type comes from the leading bytes, not - // from the client's claim about them. - const detected = await s3.detectContentType({ - readStream: upload.createReadStream(), - filename, - }); - - // The three claims must agree, and disagreement is rejection rather than - // relabelling: an `.jpg` carrying HTML is served to a browser as an image and - // executed as a script. Compared against `magic.type` — what the bytes say — - // not `contentType`, which the detector may have refined using the very - // extension under suspicion. - const agreement = checkTypeAgreement({ - filename, - declaredMime: upload.mimetype, - detectedMime: detected.magic?.type, - }); - if (!agreement.ok) { - detected.stream.destroy(); - throw new Error( - `UPLOAD_TYPE_MISMATCH: ${agreement.violation.message}. The upload was rejected rather than ` + - 'stored under a type it is not.', - ); - } - - const allowed = allowedMimeTypes(tags, typ); - if (allowed.length && !allowed.includes(detected.contentType)) { - detected.stream.destroy(); - throw new Error('UPLOAD_MIMETYPE'); - } - - const staged = stagingKey(); - const hashing = hashingPassThrough(); - const uploadResult = await s3.uploadWithContentType({ - readStream: detected.stream.pipe(hashing), - contentType: detected.contentType, - magic: detected.magic, - key: staged, - bucket: target.physicalName, - }); - - // Owns the staged key from here: it either promotes it into a files row or - // removes it, so a failed upload leaves nothing behind in S3. - const { projection } = await finalizeStagedUpload({ - target, - withPgClient, - pgSettings, - staged: { - stagingKey: staged, - contentHash: hashing.digest(), - contentType: uploadResult.contentType, - size: hashing.bytes(), - filename, - }, - }); - - switch (typ) { - case 'image': - case 'upload': - // `filename` and `mime` were in the pre-managed shape and stay in it; - // `url` is populated for public buckets and deprecated in favour of `id`. - return { ...projection, filename, mime: uploadResult.contentType }; - case 'attachment': - default: - if (!projection.url) { - throw new Error( - `ATTACHMENT_NO_PUBLIC_URL: bucket "${target.bucket.key}" has no public URL prefix ` + - 'configured, so there is no durable URL to store in a text column', - ); - } - return projection.url; - } -} - -/** - * Upload field definitions for Constructive's three upload domain types. - * - * These match columns whose PostgreSQL type is one of the domains defined - * in constructive-db/pgpm-modules/types/: - * - * - `image` (public schema) — jsonb domain for images with versions - * - `upload` (public schema) — jsonb domain for generic file uploads - * - `attachment` (public schema) — text domain for simple URL attachments - * - * These domain types are part of the platform's core type system, deployed - * to every application database. They rarely change, so this config is stable. - */ -export const constructiveUploadFieldDefinitions: UploadFieldDefinition[] = [ - { - name: 'image', - namespaceName: 'public', - type: 'image', - resolve: uploadResolver, - }, - { - name: 'upload', - namespaceName: 'public', - type: 'upload', - resolve: uploadResolver, - }, - { - name: 'attachment', - namespaceName: 'public', - type: 'attachment', - resolve: uploadResolver, - }, -]; diff --git a/graphile/graphile-upload-plugin/CHANGELOG.md b/graphile/graphile-upload-plugin/CHANGELOG.md deleted file mode 100644 index 9f819ff149..0000000000 --- a/graphile/graphile-upload-plugin/CHANGELOG.md +++ /dev/null @@ -1,222 +0,0 @@ -# Change Log - -All notable changes to this project will be documented in this file. -See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. - -# [2.28.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.27.0...graphile-upload-plugin@2.28.0) (2026-10-14) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.27.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.26.1...graphile-upload-plugin@2.27.0) (2026-09-17) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.26.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.26.0...graphile-upload-plugin@2.26.1) (2026-09-01) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.26.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.25.0...graphile-upload-plugin@2.26.0) (2026-08-18) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.25.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.24.2...graphile-upload-plugin@2.25.0) (2026-08-08) - -### Features - -- **storage:** route uploads through managed files rows and tenant-resolved buckets ([a965f3a](https://github.com/constructive-io/constructive/commit/a965f3abe1f8e72d58ad042054fb98e28c4f4a19)), closes [constructive-planning#1476](https://github.com/constructive-planning/issues/1476) - -## [2.24.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.24.1...graphile-upload-plugin@2.24.2) (2026-08-05) - -### Bug Fixes - -- **deps:** widen graphile peer ranges and resolve connection-filter from the workspace ([a5f5b63](https://github.com/constructive-io/constructive/commit/a5f5b63580f91bdfc669a00cfe7f401ff3336d3e)) - -## [2.24.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.24.0...graphile-upload-plugin@2.24.1) (2026-08-04) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.24.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.23.3...graphile-upload-plugin@2.24.0) (2026-08-02) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.23.3](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.23.2...graphile-upload-plugin@2.23.3) (2026-08-01) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.23.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.23.1...graphile-upload-plugin@2.23.2) (2026-07-31) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.23.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.23.0...graphile-upload-plugin@2.23.1) (2026-07-31) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.23.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.22.1...graphile-upload-plugin@2.23.0) (2026-07-31) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.22.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.22.0...graphile-upload-plugin@2.22.1) (2026-07-30) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.22.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.21.0...graphile-upload-plugin@2.22.0) (2026-07-29) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.21.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.20.0...graphile-upload-plugin@2.21.0) (2026-07-29) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.20.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.19.0...graphile-upload-plugin@2.20.0) (2026-07-28) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.19.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.18.1...graphile-upload-plugin@2.19.0) (2026-07-28) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.18.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.18.0...graphile-upload-plugin@2.18.1) (2026-07-28) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.18.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.17.0...graphile-upload-plugin@2.18.0) (2026-07-27) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.17.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.16.0...graphile-upload-plugin@2.17.0) (2026-07-27) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.16.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.15.2...graphile-upload-plugin@2.16.0) (2026-07-27) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.15.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.15.1...graphile-upload-plugin@2.15.2) (2026-07-26) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.15.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.15.0...graphile-upload-plugin@2.15.1) (2026-07-26) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.15.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.14.0...graphile-upload-plugin@2.15.0) (2026-07-26) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.14.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.13.1...graphile-upload-plugin@2.14.0) (2026-07-12) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.13.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.13.0...graphile-upload-plugin@2.13.1) (2026-07-11) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.13.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.12.2...graphile-upload-plugin@2.13.0) (2026-06-28) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.12.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.12.1...graphile-upload-plugin@2.12.2) (2026-06-18) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.12.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.12.0...graphile-upload-plugin@2.12.1) (2026-06-17) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.12.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.11.2...graphile-upload-plugin@2.12.0) (2026-05-30) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.11.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.11.1...graphile-upload-plugin@2.11.2) (2026-05-29) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.11.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.11.0...graphile-upload-plugin@2.11.1) (2026-05-21) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.11.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.10.1...graphile-upload-plugin@2.11.0) (2026-05-14) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.10.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.10.0...graphile-upload-plugin@2.10.1) (2026-05-11) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.10.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.9.0...graphile-upload-plugin@2.10.0) (2026-05-11) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.9.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.8.0...graphile-upload-plugin@2.9.0) (2026-05-08) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.8.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.7.0...graphile-upload-plugin@2.8.0) (2026-05-08) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.7.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.6.0...graphile-upload-plugin@2.7.0) (2026-05-06) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.6.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.5.2...graphile-upload-plugin@2.6.0) (2026-04-18) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.5.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.5.1...graphile-upload-plugin@2.5.2) (2026-03-27) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.5.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.5.0...graphile-upload-plugin@2.5.1) (2026-03-26) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.5.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.4.4...graphile-upload-plugin@2.5.0) (2026-03-26) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.4.4](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.4.3...graphile-upload-plugin@2.4.4) (2026-03-15) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.4.3](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.4.2...graphile-upload-plugin@2.4.3) (2026-03-15) - -### Bug Fixes - -- align all graphile ecosystem deps to latest compatible pinned RC versions ([d3f7761](https://github.com/constructive-io/constructive/commit/d3f7761182e0470cf992028155d1a5fa00cf9211)) - -## [2.4.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.4.1...graphile-upload-plugin@2.4.2) (2026-03-12) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.4.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.3.0...graphile-upload-plugin@2.4.1) (2026-03-12) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.4.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.3.0...graphile-upload-plugin@2.4.0) (2026-03-12) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.3.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.2.2...graphile-upload-plugin@2.3.0) (2026-03-12) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.2.2](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.2.1...graphile-upload-plugin@2.2.2) (2026-03-04) - -**Note:** Version bump only for package graphile-upload-plugin - -## [2.2.1](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.2.0...graphile-upload-plugin@2.2.1) (2026-02-25) - -### Bug Fixes - -- pin all Graphile ecosystem RC versions and sync inconsistencies ([8f553c1](https://github.com/constructive-io/constructive/commit/8f553c10d0bdc868bc1ac34136a6dfdab76f61b5)) - -# [2.2.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@2.1.0...graphile-upload-plugin@2.2.0) (2026-02-24) - -**Note:** Version bump only for package graphile-upload-plugin - -# [2.1.0](https://github.com/constructive-io/constructive/compare/graphile-upload-plugin@1.1.1...graphile-upload-plugin@2.1.0) (2026-02-24) - -### Features - -- **upload:** add graphile upload plugin package ([bc22681](https://github.com/constructive-io/constructive/commit/bc22681361da94516f2b063a4b6d0a212b96ff4e)) diff --git a/graphile/graphile-upload-plugin/README.md b/graphile/graphile-upload-plugin/README.md deleted file mode 100644 index 91775f61bc..0000000000 --- a/graphile/graphile-upload-plugin/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# graphile-upload-plugin - -

- -

- -

- - - - - -

- -File upload support for PostGraphile v5. diff --git a/graphile/graphile-upload-plugin/__tests__/plugin.test.ts b/graphile/graphile-upload-plugin/__tests__/plugin.test.ts deleted file mode 100644 index 5b47f2a41c..0000000000 --- a/graphile/graphile-upload-plugin/__tests__/plugin.test.ts +++ /dev/null @@ -1,1049 +0,0 @@ -import { EventEmitter } from 'events'; -import { GraphQLError,GraphQLScalarType } from 'graphql'; -import { Readable } from 'stream'; - -import { createUploadPlugin } from '../src/plugin'; -import type { UploadFieldDefinition } from '../src/types'; - -/** - * Tests for the upload plugin's field generation and mutation wrapping logic. - * Tests the internal `relevantUploadType` matching, `GraphQLInputObjectType_fields` - * hook, and `GraphQLObjectType_fields_field` hook. - */ - -// Shared mock resolver -const mockResolve = jest.fn(async (upload: any) => `url:${upload.filename}`); - -// Helper to create a mock readable stream -function createMockStream() { - const emitter = new EventEmitter(); - (emitter as any).destroyed = false; - (emitter as any).destroy = jest.fn((_err?: Error) => { - (emitter as any).destroyed = true; - }); - return emitter as any; -} - -// Helper to create the plugin and extract hooks -function getHooks(defs: UploadFieldDefinition[] = []) { - const plugin = createUploadPlugin({ uploadFieldDefinitions: defs }); - return plugin.schema!.hooks as any; -} - -// Helper to create a mock build object -function createMockBuild(registeredTypes: Map = new Map()) { - const types = registeredTypes; - return { - graphql: { GraphQLScalarType, GraphQLError }, - registerScalarType( - name: string, - _ext: any, - factory: () => any, - _reason: string - ) { - types.set(name, factory()); - }, - getTypeByName(name: string) { - return types.get(name); - }, - inflection: { - attribute({ attributeName }: { codec: any; attributeName: string }) { - // Simple camelCase inflection mock - return attributeName; - } - }, - extend(base: any, extra: any, _reason?: string) { - return { ...base, ...extra }; - } - }; -} - -describe('relevantUploadType matching', () => { - const typeDef: UploadFieldDefinition = { - name: 'text', - namespaceName: 'pg_catalog', - type: 'Upload', - resolve: mockResolve - }; - - const tagDef: UploadFieldDefinition = { - tag: 'upload', - resolve: mockResolve - }; - - it('should match by PG type name/namespace via codec extensions', () => { - const hooks = getHooks([typeDef]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const attribute = { - codec: { - name: 'text', - extensions: { pg: { name: 'text', schemaName: 'pg_catalog' } } - } - }; - - // Test through GraphQLInputObjectType_fields hook - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { avatar_url: attribute } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).toHaveProperty('avatar_urlUpload'); - }); - - it('should match by codec name as fallback', () => { - const hooks = getHooks([typeDef]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const attribute = { - codec: { name: 'text' } - // No extensions.pg - }; - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { avatar_url: attribute } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).toHaveProperty('avatar_urlUpload'); - }); - - it('should match by smart tag', () => { - const hooks = getHooks([tagDef]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const attribute = { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - }; - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { photo: attribute } - }, - isPgPatch: true, - isPgBaseInput: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).toHaveProperty('photoUpload'); - }); - - it('should not match when tag is absent', () => { - const hooks = getHooks([tagDef]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const attribute = { - codec: { name: 'text' }, - extensions: { tags: {} } - }; - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { photo: attribute } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).not.toHaveProperty('photoUpload'); - }); - - it('should throw when definitions are ambiguous', () => { - const hooks = getHooks([ - { name: 'text', namespaceName: 'pg_catalog', type: 'Upload', resolve: mockResolve }, - { tag: 'upload', resolve: mockResolve } - ]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - // Attribute that matches BOTH definitions - const attribute = { - codec: { - name: 'text', - extensions: { pg: { name: 'text', schemaName: 'pg_catalog' } } - }, - extensions: { tags: { upload: true } } - }; - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { photo: attribute } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - expect(() => - hooks.GraphQLInputObjectType_fields({}, build, context) - ).toThrow('Upload field definitions are ambiguous'); - }); -}); - -describe('GraphQLInputObjectType_fields hook', () => { - const def: UploadFieldDefinition = { - tag: 'upload', - resolve: mockResolve - }; - - it('should skip non-codec input types', () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const context = { - scope: { - pgCodec: null as any, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - } - }; - - const fields = { existingField: {} }; - const result = hooks.GraphQLInputObjectType_fields(fields, build, context); - expect(result).toBe(fields); - }); - - it('should skip codecs without attributes', () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const context = { - scope: { - pgCodec: { name: 'MyType' } as any, // no attributes - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - } - }; - - const fields = { existingField: {} }; - const result = hooks.GraphQLInputObjectType_fields(fields, build, context); - expect(result).toBe(fields); - }); - - it('should skip non-mutation input types', () => { - const hooks = getHooks([def]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - }, - isPgPatch: false, - isPgBaseInput: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const fields = {}; - const result = hooks.GraphQLInputObjectType_fields(fields, build, context); - expect(result).toBe(fields); - }); - - it('should add upload fields for isPgPatch scope', () => { - const hooks = getHooks([def]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - }, - isPgPatch: true, - isPgBaseInput: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).toHaveProperty('photoUpload'); - }); - - it('should add upload fields for isMutationInput scope', () => { - const hooks = getHooks([def]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - }, - isPgPatch: false, - isPgBaseInput: false, - isMutationInput: true - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).toHaveProperty('photoUpload'); - }); - - it('should skip when Upload type is not registered', () => { - const hooks = getHooks([def]); - // Don't call init, so Upload type is not registered - const build = createMockBuild(); - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).not.toHaveProperty('photoUpload'); - }); - - it('should throw on duplicate upload field names', () => { - const hooks = getHooks([ - { tag: 'upload', resolve: mockResolve }, - { tag: 'upload2', resolve: mockResolve } - ]); - const types = new Map(); - const build = createMockBuild(types); - // Override inflection to produce same field name for different attributes - build.inflection.attribute = () => 'photo'; - hooks.init({}, build); - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - photo1: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - }, - photo2: { - codec: { name: 'text' }, - extensions: { tags: { upload2: true } } - } - } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - expect(() => - hooks.GraphQLInputObjectType_fields({}, build, context) - ).toThrow(/Two columns produce the same upload field name/); - }); - - it('should handle multiple upload fields on the same codec', () => { - const hooks = getHooks([def]); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - avatar: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - }, - cover_photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - }, - name: { - codec: { name: 'text' }, - extensions: { tags: {} } - } - } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).toHaveProperty('avatarUpload'); - expect(result).toHaveProperty('cover_photoUpload'); - expect(result).not.toHaveProperty('nameUpload'); - }); -}); - -describe('GraphQLObjectType_fields_field hook (mutation resolver wrapping)', () => { - beforeEach(() => { - mockResolve.mockClear(); - }); - - const def: UploadFieldDefinition = { - tag: 'upload', - resolve: mockResolve, - type: 'image' - }; - - it('should skip non-mutation fields', () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const field = { resolve: jest.fn() }; - const context = { - scope: { - isRootMutation: false, - fieldName: 'createPost', - pgCodec: { name: 'Post', attributes: {} } - } - }; - - const result = hooks.GraphQLObjectType_fields_field(field, build, context); - expect(result).toBe(field); - }); - - it('should skip when pgCodec is missing', () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const field = { resolve: jest.fn() }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: null as any - } - }; - - const result = hooks.GraphQLObjectType_fields_field(field, build, context); - expect(result).toBe(field); - }); - - it('should skip when no upload fields match', () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const field = { resolve: jest.fn() }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - title: { - codec: { name: 'text' }, - extensions: { tags: {} } - } - } - } - } - }; - - const result = hooks.GraphQLObjectType_fields_field(field, build, context); - expect(result).toBe(field); - }); - - it('should wrap resolve when upload fields match', () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve, type: 'Post' }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const result = hooks.GraphQLObjectType_fields_field(field, build, context); - expect(result.resolve).not.toBe(oldResolve); - expect(result.type).toBe('Post'); // Spread rest preserved - }); - - it('should resolve upload promises and call custom resolver', async () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const wrappedField = hooks.GraphQLObjectType_fields_field( - field, - build, - context - ); - - const mockStream = createMockStream(); - const fakeUpload = { filename: 'photo.jpg', createReadStream: jest.fn().mockReturnValue(mockStream) }; - const args = { - input: { - photoUpload: Promise.resolve(fakeUpload), - title: 'My Post' - } - }; - const ctx = { user: { id: '1' } }; - const info = { fieldName: 'createPost' }; - - await wrappedField.resolve({}, args, ctx, info); - - // Upload resolver should have been called with the upload - expect(mockResolve).toHaveBeenCalledTimes(1); - expect(mockResolve).toHaveBeenCalledWith( - fakeUpload, - args, - ctx, - expect.objectContaining({ - uploadPlugin: { - tags: { upload: true }, - type: 'image' - } - }) - ); - - // The original column value should be set - expect((args.input as any).photo).toBe('url:photo.jpg'); - expect((args.input as any).photoUpload).toBeUndefined(); - - // The old resolver should have been called - expect(oldResolve).toHaveBeenCalledTimes(1); - }); - - it('should recursively resolve nested upload promises', async () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const wrappedField = hooks.GraphQLObjectType_fields_field( - field, - build, - context - ); - - const mockStream = createMockStream(); - const fakeUpload = { filename: 'deep.png', createReadStream: jest.fn().mockReturnValue(mockStream) }; - const args = { - input: { - nested: { - photoUpload: Promise.resolve(fakeUpload) - } - } - }; - - await wrappedField.resolve({}, args, {}, {}); - - // Should have found and resolved the nested promise - expect(mockResolve).toHaveBeenCalledTimes(1); - }); - - it('should use default resolver when no resolve is provided on field', async () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - // Field without a resolve function - const field = { type: 'Post' }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const wrappedField = hooks.GraphQLObjectType_fields_field( - field, - build, - context - ); - - const source = { createPost: { id: '1', title: 'Test' } }; - const args = { input: { title: 'Test' } }; - - const result = await wrappedField.resolve(source, args, {}, {}); - // Default resolver: (obj) => obj[fieldName] - expect(result).toEqual({ id: '1', title: 'Test' }); - }); - - it('should skip non-Promise values during recursive walk', async () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const wrappedField = hooks.GraphQLObjectType_fields_field( - field, - build, - context - ); - - const args = { - input: { - title: 'Normal string', - count: 42, - active: true, - meta: null as any - } - }; - - await wrappedField.resolve({}, args, {}, {}); - - // No uploads to resolve - expect(mockResolve).not.toHaveBeenCalled(); - expect(oldResolve).toHaveBeenCalledTimes(1); - }); - - it('should handle upload promises that are not in uploadResolversByFieldName', async () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const wrappedField = hooks.GraphQLObjectType_fields_field( - field, - build, - context - ); - - // Promise with a key that doesn't match any upload field - const args = { - input: { - unknownUpload: Promise.resolve({ filename: 'nope.txt' }) - } - }; - - await wrappedField.resolve({}, args, {}, {}); - - // Should not call the upload resolver since key doesn't match - expect(mockResolve).not.toHaveBeenCalled(); - expect(oldResolve).toHaveBeenCalledTimes(1); - }); - - it('should resolve thenable uploads (not just Promise instances)', async () => { - const hooks = getHooks([def]); - const build = createMockBuild(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - const context = { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - - const wrappedField = hooks.GraphQLObjectType_fields_field( - field, - build, - context - ); - - const mockStream = createMockStream(); - const fakeUpload = { - filename: 'thenable.jpg', - createReadStream: jest.fn().mockReturnValue(mockStream) - }; - const args = { - input: { - photoUpload: { - then: (resolve: (value: unknown) => void) => resolve(fakeUpload) - } - } - }; - - await wrappedField.resolve({}, args, {}, {}); - - expect(mockResolve).toHaveBeenCalledTimes(1); - expect((args.input as any).photo).toBe('url:thenable.jpg'); - expect((args.input as any).photoUpload).toBeUndefined(); - }); -}); - -describe('createUploadPlugin', () => { - it('should return a valid plugin object', () => { - const plugin = createUploadPlugin(); - - expect(plugin.name).toBe('UploadPlugin'); - expect(plugin.version).toBe('2.0.0'); - expect(plugin.description).toBe('File upload support for PostGraphile v5'); - expect(plugin.after).toEqual([ - 'PgAttributesPlugin', - 'PgMutationCreatePlugin', - 'PgMutationUpdateDeletePlugin' - ]); - expect(plugin.schema).toBeDefined(); - expect(plugin.schema!.hooks).toBeDefined(); - }); - - it('should default to empty uploadFieldDefinitions', () => { - const hooks = getHooks(); - const types = new Map(); - const build = createMockBuild(types); - hooks.init({}, build); - - // No definitions, so no upload fields should be created - const context = { - scope: { - pgCodec: { - name: 'MyTable', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: {} } - } - } - }, - isPgBaseInput: true, - isPgPatch: false, - isMutationInput: false - }, - fieldWithHooks: (_scope: any, config: any) => config - }; - - const result = hooks.GraphQLInputObjectType_fields({}, build, context); - expect(result).not.toHaveProperty('photoUpload'); - }); -}); - -describe('maxFileSize option', () => { - function getUploadHooks(defs: UploadFieldDefinition[], opts: { maxFileSize?: number } = {}) { - const plugin = createUploadPlugin({ uploadFieldDefinitions: defs, ...opts }); - return plugin.schema!.hooks as any; - } - - function createUploadContext() { - return { - scope: { - isRootMutation: true, - fieldName: 'createPost', - pgCodec: { - name: 'Post', - attributes: { - photo: { - codec: { name: 'text' }, - extensions: { tags: { upload: true } } - } - } - } - } - }; - } - - async function consumeReadable(stream: Readable): Promise { - let total = 0; - for await (const chunk of stream) { - total += Buffer.isBuffer(chunk) - ? chunk.length - : Buffer.byteLength(String(chunk)); - } - return total; - } - - it('should reject files exceeding maxFileSize with a controlled error', async () => { - const def: UploadFieldDefinition = { - tag: 'upload', resolve: jest.fn(async (upload: any) => { - const stream = upload.createReadStream(); - await consumeReadable(stream); - return 'resolved-url'; - }), - type: 'image' - }; - - const hooks = getUploadHooks([def], { maxFileSize: 100 }); - const build = createMockBuild(); - const context = createUploadContext(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - - const wrappedField = hooks.GraphQLObjectType_fields_field(field, build, context); - - const fakeUpload = { - filename: 'large.jpg', - createReadStream: jest.fn().mockReturnValue(Readable.from([Buffer.alloc(150)])) - }; - const args = { input: { photoUpload: Promise.resolve(fakeUpload) } }; - - await expect(wrappedField.resolve({}, args, {}, {})).rejects.toThrow( - 'File exceeds maximum size of 100 bytes' - ); - expect(oldResolve).not.toHaveBeenCalled(); - }); - - it('should allow files within maxFileSize', async () => { - const def: UploadFieldDefinition = { - tag: 'upload', resolve: jest.fn(async (upload: any) => { - const stream = upload.createReadStream(); - await consumeReadable(stream); - return 'resolved-url'; - }), - type: 'image' - }; - - const hooks = getUploadHooks([def], { maxFileSize: 1000 }); - const build = createMockBuild(); - const context = createUploadContext(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - - const wrappedField = hooks.GraphQLObjectType_fields_field(field, build, context); - - const fakeUpload = { - filename: 'small.jpg', - createReadStream: jest.fn().mockReturnValue(Readable.from([Buffer.alloc(50)])) - }; - const args = { input: { photoUpload: Promise.resolve(fakeUpload) } }; - - await wrappedField.resolve({}, args, {}, {}); - expect(oldResolve).toHaveBeenCalledTimes(1); - expect((args.input as any).photo).toBe('resolved-url'); - }); - - it('should enforce maxFileSize on the stream consumed by resolver', async () => { - const def: UploadFieldDefinition = { - tag: 'upload', resolve: jest.fn(async (upload: any) => { - const stream = upload.createReadStream(); - await consumeReadable(stream); - return 'resolved-url'; - }), - type: 'image' - }; - - const hooks = getUploadHooks([def], { maxFileSize: 100 }); - const build = createMockBuild(); - const context = createUploadContext(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - - const wrappedField = hooks.GraphQLObjectType_fields_field(field, build, context); - - const fakeUpload = { - filename: 'oversize.jpg', - createReadStream: jest.fn().mockReturnValue(Readable.from([Buffer.alloc(120)])) - }; - const args = { input: { photoUpload: Promise.resolve(fakeUpload) } }; - - await expect(wrappedField.resolve({}, args, {}, {})).rejects.toThrow( - 'File exceeds maximum size of 100 bytes' - ); - // If plugin pre-read the stream before resolver, this would be 2 calls. - expect(fakeUpload.createReadStream).toHaveBeenCalledTimes(1); - }); - - it('should enforce maxFileSize across multiple createReadStream calls', async () => { - const def: UploadFieldDefinition = { - tag: 'upload', resolve: jest.fn(async (upload: any) => { - const stream1 = upload.createReadStream(); - await consumeReadable(stream1); - - const stream2 = upload.createReadStream(); - await consumeReadable(stream2); - - return 'resolved-url'; - }), - type: 'image' - }; - - const hooks = getUploadHooks([def], { maxFileSize: 100 }); - const build = createMockBuild(); - const context = createUploadContext(); - - const oldResolve = jest.fn().mockResolvedValue({ id: '1' }); - const field = { resolve: oldResolve }; - - const wrappedField = hooks.GraphQLObjectType_fields_field(field, build, context); - - const fakeUpload = { - filename: 'photo.jpg', - createReadStream: jest.fn() - .mockReturnValueOnce(Readable.from([Buffer.alloc(10)])) - .mockReturnValueOnce(Readable.from([Buffer.alloc(150)])) - }; - const args = { input: { photoUpload: Promise.resolve(fakeUpload) } }; - - await expect(wrappedField.resolve({}, args, {}, {})).rejects.toThrow( - 'File exceeds maximum size of 100 bytes' - ); - expect(fakeUpload.createReadStream).toHaveBeenCalledTimes(2); - expect(oldResolve).not.toHaveBeenCalled(); - }); -}); - -describe('UploadPlugin export', () => { - it('should export UploadPlugin as alias for createUploadPlugin', async () => { - const { UploadPlugin, createUploadPlugin } = await import('../src/plugin'); - expect(UploadPlugin).toBe(createUploadPlugin); - }); -}); diff --git a/graphile/graphile-upload-plugin/__tests__/scalar.test.ts b/graphile/graphile-upload-plugin/__tests__/scalar.test.ts deleted file mode 100644 index 1c58bede85..0000000000 --- a/graphile/graphile-upload-plugin/__tests__/scalar.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { GraphQLError,GraphQLScalarType } from 'graphql'; - -import { createUploadPlugin } from '../src/plugin'; - -/** - * Tests for the Upload scalar type created by the plugin. - * Verifies parity with the v4 Upload scalar behavior. - */ - -function getUploadScalar(): GraphQLScalarType { - const plugin = createUploadPlugin(); - const hooks = plugin.schema!.hooks as any; - - // Simulate the build object that PostGraphile v5 provides - let registeredScalar: GraphQLScalarType | undefined; - const mockBuild = { - graphql: { GraphQLScalarType, GraphQLError }, - registerScalarType( - name: string, - _extensions: any, - factory: () => GraphQLScalarType, - _reason: string - ) { - registeredScalar = factory(); - } - }; - - hooks.init({}, mockBuild); - - if (!registeredScalar) { - throw new Error('Upload scalar was not registered'); - } - return registeredScalar; -} - -describe('Upload scalar', () => { - let scalar: GraphQLScalarType; - - beforeAll(() => { - scalar = getUploadScalar(); - }); - - it('should have the correct name and description', () => { - expect(scalar.name).toBe('Upload'); - expect(scalar.description).toBe( - 'The `Upload` scalar type represents a file upload.' - ); - }); - - describe('parseValue', () => { - it('should return the promise from a valid upload value', () => { - const fakePromise = Promise.resolve({ filename: 'test.txt' }); - const value = { promise: fakePromise }; - - const result = scalar.parseValue(value); - expect(result).toBe(fakePromise); - }); - - it('should accept any thenable promise property', () => { - const thenableLike = { then: jest.fn() }; - const value = { promise: thenableLike }; - - const result = scalar.parseValue(value); - expect(result).toBe(thenableLike); - }); - - it('should throw GraphQLError for null value', () => { - expect(() => scalar.parseValue(null)).toThrow(GraphQLError); - expect(() => scalar.parseValue(null)).toThrow('Upload value invalid.'); - }); - - it('should throw GraphQLError for undefined value', () => { - expect(() => scalar.parseValue(undefined)).toThrow(GraphQLError); - expect(() => scalar.parseValue(undefined)).toThrow( - 'Upload value invalid.' - ); - }); - - it('should throw GraphQLError for plain object without promise', () => { - expect(() => scalar.parseValue({ foo: 'bar' })).toThrow(GraphQLError); - expect(() => scalar.parseValue({ foo: 'bar' })).toThrow( - 'Upload value invalid.' - ); - }); - - it('should throw GraphQLError when promise is not thenable', () => { - expect(() => scalar.parseValue({ promise: 'not-a-promise' })).toThrow( - GraphQLError - ); - expect(() => scalar.parseValue({ promise: 42 })).toThrow( - 'Upload value invalid.' - ); - }); - - it('should throw GraphQLError for string value', () => { - expect(() => scalar.parseValue('test')).toThrow(GraphQLError); - }); - - it('should throw GraphQLError for number value', () => { - expect(() => scalar.parseValue(123)).toThrow(GraphQLError); - }); - }); - - describe('parseLiteral', () => { - it('should always throw GraphQLError', () => { - const fakeAst = { kind: 'StringValue', value: 'test' }; - expect(() => scalar.parseLiteral(fakeAst as any, {})).toThrow( - GraphQLError - ); - expect(() => scalar.parseLiteral(fakeAst as any, {})).toThrow( - 'Upload literal unsupported.' - ); - }); - }); - - describe('serialize', () => { - it('should always throw GraphQLError', () => { - expect(() => scalar.serialize('anything')).toThrow(GraphQLError); - expect(() => scalar.serialize('anything')).toThrow( - 'Upload serialization unsupported.' - ); - }); - - it('should throw even for null/undefined values', () => { - expect(() => scalar.serialize(null)).toThrow( - 'Upload serialization unsupported.' - ); - expect(() => scalar.serialize(undefined)).toThrow( - 'Upload serialization unsupported.' - ); - }); - }); -}); diff --git a/graphile/graphile-upload-plugin/__tests__/types.test.ts b/graphile/graphile-upload-plugin/__tests__/types.test.ts deleted file mode 100644 index 96977d3658..0000000000 --- a/graphile/graphile-upload-plugin/__tests__/types.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -import type { - FileUpload, - UploadFieldDefinition, - UploadPluginInfo, - UploadPluginOptions, - UploadResolver} from '../src/types'; - -/** - * Tests for the UploadFieldDefinition type interface. - * Verifies both type-name based and smart-tag based forms are supported. - */ - -describe('UploadFieldDefinition', () => { - const mockResolver: UploadResolver = async ( - upload: FileUpload, - _args: any, - _context: any, - _info: { uploadPlugin: UploadPluginInfo } - ) => { - return upload.filename; - }; - - describe('type-name based definition', () => { - it('should accept name, namespaceName, type, and resolve', () => { - const def: UploadFieldDefinition = { - name: 'text', - namespaceName: 'pg_catalog', - type: 'Upload', - resolve: mockResolver - }; - - expect(def.name).toBe('text'); - expect(def.namespaceName).toBe('pg_catalog'); - expect(def.type).toBe('Upload'); - expect(def.resolve).toBe(mockResolver); - expect(def.tag).toBeUndefined(); - }); - - it('should require all mandatory fields', () => { - const def: UploadFieldDefinition = { - name: 'jsonb', - namespaceName: 'pg_catalog', - type: 'JSON', - resolve: mockResolver - }; - - expect(def).toBeDefined(); - }); - }); - - describe('smart-tag based definition', () => { - it('should accept tag and resolve', () => { - const def: UploadFieldDefinition = { - tag: 'upload', - resolve: mockResolver - }; - - expect(def.tag).toBe('upload'); - expect(def.resolve).toBe(mockResolver); - expect(def.name).toBeUndefined(); - expect(def.namespaceName).toBeUndefined(); - }); - - it('should accept optional type field', () => { - const def: UploadFieldDefinition = { - tag: 'imageUpload', - resolve: mockResolver, - type: 'image' - }; - - expect(def.tag).toBe('imageUpload'); - expect(def.type).toBe('image'); - }); - }); - - describe('UploadResolver', () => { - it('should receive FileUpload with expected properties', async () => { - const mockUpload: FileUpload = { - filename: 'test.png', - mimetype: 'image/png', - encoding: '7bit', - createReadStream: jest.fn() - }; - - const info = { - uploadPlugin: { - tags: { upload: true }, - type: 'image' - } - }; - - const result = await mockResolver(mockUpload, {}, {}, info); - expect(result).toBe('test.png'); - }); - - it('should work with minimal FileUpload (only filename and createReadStream)', async () => { - const mockUpload: FileUpload = { - filename: 'doc.pdf', - createReadStream: jest.fn() - }; - - const info = { - uploadPlugin: { - tags: {} - } - }; - - const result = await mockResolver(mockUpload, {}, {}, info); - expect(result).toBe('doc.pdf'); - }); - }); - - describe('UploadPluginOptions', () => { - it('should accept empty options', () => { - const opts: UploadPluginOptions = {}; - expect(opts.uploadFieldDefinitions).toBeUndefined(); - expect(opts.maxFileSize).toBeUndefined(); - }); - - it('should accept an array of mixed field definitions', () => { - const opts: UploadPluginOptions = { - uploadFieldDefinitions: [ - { - name: 'text', - namespaceName: 'pg_catalog', - type: 'Upload', - resolve: mockResolver - }, - { - tag: 'upload', - resolve: mockResolver - } - ] - }; - - expect(opts.uploadFieldDefinitions).toHaveLength(2); - }); - - it('should accept maxFileSize option', () => { - const opts: UploadPluginOptions = { - maxFileSize: 5 * 1024 * 1024 // 5 MB - }; - - expect(opts.maxFileSize).toBe(5 * 1024 * 1024); - }); - }); -}); diff --git a/graphile/graphile-upload-plugin/jest.config.js b/graphile/graphile-upload-plugin/jest.config.js deleted file mode 100644 index eecd073352..0000000000 --- a/graphile/graphile-upload-plugin/jest.config.js +++ /dev/null @@ -1,18 +0,0 @@ -/** @type {import('ts-jest').JestConfigWithTsJest} */ -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - transform: { - '^.+\\.tsx?$': [ - 'ts-jest', - { - babelConfig: false, - tsconfig: 'tsconfig.json' - } - ] - }, - transformIgnorePatterns: [`/node_modules/*`], - testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$', - moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], - modulePathIgnorePatterns: ['dist/*'] -}; diff --git a/graphile/graphile-upload-plugin/package.json b/graphile/graphile-upload-plugin/package.json deleted file mode 100644 index 9035a3eddb..0000000000 --- a/graphile/graphile-upload-plugin/package.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "name": "graphile-upload-plugin", - "version": "2.28.0", - "description": "File upload support for PostGraphile v5", - "author": "Constructive ", - "homepage": "https://github.com/constructive-io/constructive", - "license": "MIT", - "main": "index.js", - "module": "esm/index.js", - "types": "index.d.ts", - "scripts": { - "clean": "makage clean", - "prepack": "npm run build", - "build": "makage build", - "build:dev": "makage build --dev", - "lint": "eslint . --fix", - "test": "jest" - }, - "publishConfig": { - "access": "public", - "directory": "dist" - }, - "repository": { - "type": "git", - "url": "https://github.com/constructive-io/constructive" - }, - "keywords": [ - "postgraphile", - "graphile", - "constructive", - "plugin", - "postgres", - "graphql", - "upload", - "file-upload", - "s3" - ], - "bugs": { - "url": "https://github.com/constructive-io/constructive/issues" - }, - "peerDependencies": { - "graphile-build": "^5.1.1", - "graphile-build-pg": "^5.1.3", - "graphile-config": "^1.1.0", - "graphql": "^16.9.0", - "postgraphile": "^5.1.3" - }, - "devDependencies": { - "@types/node": "^22.19.11", - "makage": "^0.8.0" - } -} diff --git a/graphile/graphile-upload-plugin/src/index.ts b/graphile/graphile-upload-plugin/src/index.ts deleted file mode 100644 index d555a29aba..0000000000 --- a/graphile/graphile-upload-plugin/src/index.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * PostGraphile v5 Upload Plugin - * - * Provides file upload capabilities for PostGraphile v5 mutations. - * - * @example - * ```typescript - * import { UploadPlugin, UploadPreset } from 'graphile-upload-plugin'; - * - * // Option 1: Use the preset (recommended) - * const preset = { - * extends: [ - * UploadPreset({ - * uploadFieldDefinitions: [ - * { tag: 'upload', resolve: myUploadResolver }, - * ], - * }), - * ], - * }; - * - * // Option 2: Use the plugin directly - * const plugin = UploadPlugin({ - * uploadFieldDefinitions: [ - * { tag: 'upload', resolve: myUploadResolver }, - * ], - * }); - * ``` - */ - -export { createUploadPlugin,UploadPlugin } from './plugin'; -export { UploadPreset } from './preset'; -export type { - FileUpload, - UploadFieldDefinition, - UploadFieldIdentity, - UploadPluginInfo, - UploadPluginOptions, - UploadResolver -} from './types'; diff --git a/graphile/graphile-upload-plugin/src/plugin.ts b/graphile/graphile-upload-plugin/src/plugin.ts deleted file mode 100644 index 1a157f50e9..0000000000 --- a/graphile/graphile-upload-plugin/src/plugin.ts +++ /dev/null @@ -1,372 +0,0 @@ -/** - * PostGraphile v5 Upload Plugin - * - * Adds file upload support to PostGraphile v5 mutations. For columns that match - * the configured upload field definitions (by PG type name/namespace or by smart - * tag), this plugin: - * - * 1. Registers a GraphQL `Upload` scalar type - * 2. Adds `*Upload` input fields on mutation input types - * 3. Wraps mutation field resolvers to process file uploads before the mutation - * executes, calling the user-supplied resolver for each upload - * - * In v5, the `GraphQLObjectType_fields_field` hook wraps the `resolve` function - * (which still exists on mutation fields alongside `plan`) to intercept uploads - * at the HTTP layer before the plan executes. - * - * COMPATIBILITY NOTE: - * This plugin uses v4-style resolver wrapping via GraphQLObjectType_fields_field hook. - * grafserv v5 supports this through its backwards-compatibility layer. - * This plugin requires grafserv's resolver support to be enabled (default in v5 RC). - * It will NOT work in a pure grafast plan-only execution context. - */ - -import 'graphile-build'; -import 'graphile-build-pg'; - -import type { GraphileConfig } from 'graphile-config'; -import { isInputType } from 'graphql'; -import { Readable, Transform } from 'stream'; - -import type { UploadFieldDefinition, UploadFieldIdentity, UploadPluginOptions } from './types'; - -/** - * Read the mutated column's PG identity off the codec. - * - * Returns null when the codec is not table-backed (no `extensions.pg`), so a - * resolver that needs the identity can refuse rather than guess. - */ -function uploadFieldIdentity( - pgCodec: any, - attributeName: string, -): UploadFieldIdentity | null { - const pgExt = pgCodec?.extensions?.pg; - const schemaName = pgExt?.schemaName; - const tableName = pgExt?.name; - if (!schemaName || !tableName) return null; - return { schemaName, tableName, columnName: attributeName }; -} - -/** - * Determines whether a codec attribute matches an upload field definition. - * Returns the matching definition or undefined. - */ -function relevantUploadType( - attribute: any, - uploadFieldDefinitions: UploadFieldDefinition[] -): UploadFieldDefinition | undefined { - const types = uploadFieldDefinitions.filter(({ name, namespaceName, tag }) => { - if (name && namespaceName) { - // Type-name based matching: check the attribute's codec PG extension metadata - const pgExt = attribute.codec?.extensions?.pg; - if (pgExt && pgExt.name === name && pgExt.schemaName === namespaceName) { - return true; - } - // Fallback: check codec name directly - if (attribute.codec?.name === name) { - return true; - } - } - if (tag) { - // Smart-tag based matching: check if the attribute has the specified tag - const tags = attribute.extensions?.tags; - if (tags && tags[tag]) { - return true; - } - } - return false; - }); - - if (types.length === 1) { - return types[0]; - } else if (types.length > 1) { - throw new Error('Upload field definitions are ambiguous'); - } - return undefined; -} - -function createSizeLimitStream( - source: Readable, - maxFileSize: number -): Readable { - let bytesRead = 0; - - const limiter = new Transform({ - transform(chunk, _encoding, callback) { - const chunkSize = Buffer.isBuffer(chunk) - ? chunk.length - : Buffer.byteLength(String(chunk)); - bytesRead += chunkSize; - - if (bytesRead > maxFileSize) { - callback(new Error(`File exceeds maximum size of ${maxFileSize} bytes`)); - return; - } - - callback(null, chunk); - } - }); - - // Ensure source stream errors are always forwarded to the consumer stream. - source.on('error', (error: Error) => { - limiter.destroy(error); - }); - - limiter.on('error', (error) => { - if (!source.destroyed) { - source.destroy(error as Error); - } - }); - - source.pipe(limiter); - return limiter; -} - -function isThenable(value: unknown): value is PromiseLike { - return ( - value !== null && - (typeof value === 'object' || typeof value === 'function') && - typeof (value as PromiseLike).then === 'function' - ); -} - -function wrapUploadForMaxFileSize(upload: any, maxFileSize?: number): any { - if (!maxFileSize) { - return upload; - } - - return { - ...upload, - createReadStream() { - const source = upload.createReadStream(); - return createSizeLimitStream(source, maxFileSize); - } - }; -} - -/** - * Creates the Upload plugin with the given options. - */ -export function createUploadPlugin( - options: UploadPluginOptions = {} -): GraphileConfig.Plugin { - const { uploadFieldDefinitions = [], maxFileSize } = options; - - return { - name: 'UploadPlugin', - version: '2.0.0', - description: 'File upload support for PostGraphile v5', - after: ['PgAttributesPlugin', 'PgMutationCreatePlugin', 'PgMutationUpdateDeletePlugin'], - - schema: { - hooks: { - // Register the Upload scalar type - init(_, build) { - const { - graphql: { GraphQLScalarType, GraphQLError } - } = build; - - const GraphQLUpload = new GraphQLScalarType({ - name: 'Upload', - description: 'The `Upload` scalar type represents a file upload.', - parseValue(value: any) { - const maybe = value; - if ( - maybe && - maybe.promise && - typeof maybe.promise.then === 'function' - ) { - return maybe.promise; - } - throw new GraphQLError('Upload value invalid.'); - }, - parseLiteral(_ast: any) { - throw new GraphQLError('Upload literal unsupported.'); - }, - serialize() { - throw new GraphQLError('Upload serialization unsupported.'); - } - }); - - build.registerScalarType( - GraphQLUpload.name, - {}, - () => GraphQLUpload, - 'UploadPlugin registering Upload scalar' - ); - - return _; - }, - - // Add *Upload input fields alongside matching columns on mutation input types - GraphQLInputObjectType_fields(fields, build, context) { - const { - scope: { pgCodec, isPgPatch, isPgBaseInput, isMutationInput } - } = context; - - // Only process row-based input types (create inputs, patch inputs, base inputs) - if (!pgCodec || !pgCodec.attributes) { - return fields; - } - - // Must be a mutation-related input type - if (!isPgPatch && !isPgBaseInput && !isMutationInput) { - return fields; - } - - // getTypeByName is untyped beyond GraphQLNamedType, and the field - // below needs an input type — narrow rather than assert, so a - // non-input 'Upload' registered by another plugin skips instead of - // building an invalid schema. - const UploadType = build.getTypeByName('Upload'); - if (!UploadType || !isInputType(UploadType)) { - return fields; - } - - let newFields = fields; - - for (const [attributeName, attribute] of Object.entries( - pgCodec.attributes as Record - )) { - const matchedDef = relevantUploadType(attribute, uploadFieldDefinitions); - if (!matchedDef) continue; - - // Generate the upload field name: columnName + 'Upload' - const baseFieldName = build.inflection.attribute({ - codec: pgCodec as any, - attributeName - }); - const uploadFieldName = baseFieldName + 'Upload'; - - if (newFields[uploadFieldName]) { - throw new Error( - `Two columns produce the same upload field name '${uploadFieldName}' ` + - `on codec '${pgCodec.name}'; one of them is '${attributeName}'` - ); - } - - newFields = build.extend( - newFields, - { - [uploadFieldName]: context.fieldWithHooks( - { fieldName: uploadFieldName, isPgUploadField: true } as any, - { - description: attribute.description - ? `Upload for ${attribute.description}` - : `File upload for the \`${baseFieldName}\` field.`, - type: UploadType - } - ) - }, - `UploadPlugin adding upload field '${uploadFieldName}' for ` + - `attribute '${attributeName}' on '${pgCodec.name}'` - ); - } - - return newFields; - }, - - // Wrap mutation field resolvers to process file uploads - GraphQLObjectType_fields_field(field, build, context) { - const { - scope: { isRootMutation, fieldName, pgCodec } - } = context; - - if (!isRootMutation || !pgCodec || !pgCodec.attributes) { - return field; - } - - // Build the mapping of upload field names to their resolvers - const uploadResolversByFieldName: Record = {}; - const tags: Record = {}; - const types: Record = {}; - const originals: Record = {}; - const identities: Record = {}; - - for (const [attributeName, attribute] of Object.entries( - pgCodec.attributes as Record - )) { - const matchedDef = relevantUploadType(attribute, uploadFieldDefinitions); - if (!matchedDef) continue; - - const baseFieldName = build.inflection.attribute({ - codec: pgCodec as any, - attributeName - }); - const uploadFieldName = baseFieldName + 'Upload'; - - uploadResolversByFieldName[uploadFieldName] = matchedDef.resolve; - tags[uploadFieldName] = attribute.extensions?.tags || {}; - types[uploadFieldName] = matchedDef.type || ''; - originals[uploadFieldName] = baseFieldName; - identities[uploadFieldName] = uploadFieldIdentity(pgCodec, attributeName); - } - - // If no upload fields match this mutation's codec, skip wrapping - if (Object.keys(uploadResolversByFieldName).length === 0) { - return field; - } - - const defaultResolver = (obj: any) => obj[fieldName]; - const { resolve: oldResolve = defaultResolver, ...rest } = field; - - return { - ...rest, - async resolve(source: any, args: any, context: any, info: any) { - // Recursively walk args to find and resolve upload thenables. - async function resolvePromises(obj: any): Promise { - const pending: Promise[] = []; - - for (const key of Object.keys(obj)) { - const value = obj[key]; - if (isThenable(value)) { - const resolver = uploadResolversByFieldName[key]; - if (!resolver) continue; - - pending.push((async () => { - const upload = await value; - const uploadForResolver = wrapUploadForMaxFileSize(upload, maxFileSize); - - obj[originals[key]] = await resolver( - uploadForResolver, - args, - context, - { - ...info, - uploadPlugin: { - tags: tags[key], - type: types[key], - ...(identities[key] ? { field: identities[key] } : {}) - } - } - ); - - // Remove consumed `*Upload` key so downstream resolvers only - // see the materialized column value. - delete obj[key]; - })()); - continue; - } - - if (value !== null && typeof value === 'object') { - pending.push(resolvePromises(value)); - } - } - - if (pending.length > 0) { - await Promise.all(pending); - } - } - - await resolvePromises(args); - return oldResolve(source, args, context, info); - } - }; - } - } - } - }; -} - -export const UploadPlugin = createUploadPlugin; -export default UploadPlugin; diff --git a/graphile/graphile-upload-plugin/src/preset.ts b/graphile/graphile-upload-plugin/src/preset.ts deleted file mode 100644 index fa6f2bed85..0000000000 --- a/graphile/graphile-upload-plugin/src/preset.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * PostGraphile v5 Upload Preset - * - * Provides a convenient preset for including upload support in PostGraphile. - */ - -import type { GraphileConfig } from 'graphile-config'; - -import { createUploadPlugin } from './plugin'; -import type { UploadPluginOptions } from './types'; - -/** - * Creates a preset that includes the upload plugin with the given options. - * - * @example - * ```typescript - * import { UploadPreset } from 'graphile-upload-plugin'; - * - * const preset = { - * extends: [ - * UploadPreset({ - * uploadFieldDefinitions: [ - * { - * tag: 'upload', - * resolve: async (upload, args, context, info) => { - * // Process the upload and return the value to store in the column - * const stream = upload.createReadStream(); - * const url = await uploadToStorage(stream, upload.filename); - * return url; - * }, - * }, - * ], - * }), - * ], - * }; - * ``` - */ -export function UploadPreset( - options: UploadPluginOptions = {} -): GraphileConfig.Preset { - return { - plugins: [createUploadPlugin(options)] - }; -} - -export default UploadPreset; diff --git a/graphile/graphile-upload-plugin/src/types.ts b/graphile/graphile-upload-plugin/src/types.ts deleted file mode 100644 index b31f5b99d6..0000000000 --- a/graphile/graphile-upload-plugin/src/types.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { Readable } from 'stream'; - -/** - * Represents a file upload received from the client. - */ -export interface FileUpload { - filename: string; - mimetype?: string; - encoding?: string; - createReadStream: () => Readable; -} - -/** - * Identifies the column an upload is being written into. - * - * A resolver that persists through managed storage needs the column's identity, - * not just its type: which bucket the bytes land in is a property of the field - * declaration, recorded per (table, column), so a resolver given only - * `{tags, type}` can do no better than a server-global bucket. - */ -export interface UploadFieldIdentity { - /** PostgreSQL schema of the table being mutated (e.g. 'app_public') */ - schemaName: string; - /** PostgreSQL table name being mutated (e.g. 'products') */ - tableName: string; - /** PostgreSQL column name receiving the upload (e.g. 'photo') */ - columnName: string; -} - -/** - * Additional metadata passed to the upload resolver via the info parameter. - */ -export interface UploadPluginInfo { - tags: Record; - type?: string; - /** - * The column being written. Absent only when the codec carries no PG - * identity, which a managed resolver must treat as an error rather than - * falling back to a default bucket. - */ - field?: UploadFieldIdentity; -} - -/** - * A function that processes an uploaded file and returns the value to store - * in the database column (e.g., a URL string or a JSON object). - * - * Called with the upload stream. The filename comes from user input - * and MUST be sanitized by the implementation before use in file paths - * or storage keys (e.g., strip path separators, limit length, validate characters). - */ -export type UploadResolver = ( - upload: FileUpload, - args: any, - context: any, - info: { uploadPlugin: UploadPluginInfo } -) => Promise; - -/** - * Defines which columns should receive upload handling. - * - * Two forms are supported: - * 1. Type-name based: match columns by their PG type name and namespace. - * 2. Smart-tag based: match columns that have a specific smart tag. - */ -export type UploadFieldDefinition = - | { - /** PG type name (e.g., 'text') */ - name: string; - /** PG schema name (e.g., 'pg_catalog') */ - namespaceName: string; - /** The GraphQL type name to override to (e.g., 'Upload') */ - type: string; - /** The resolver function to process the upload */ - resolve: UploadResolver; - tag?: never; - } - | { - /** Smart tag name to match (e.g., '@upload') */ - tag: string; - /** The resolver function to process the upload */ - resolve: UploadResolver; - name?: never; - namespaceName?: never; - /** Optional type hint for the resolver */ - type?: string; - }; - -/** - * Plugin options for the upload plugin. - */ -export interface UploadPluginOptions { - /** - * Array of upload field definitions that configure which columns - * should get upload handling. - */ - uploadFieldDefinitions?: UploadFieldDefinition[]; - - /** - * Maximum file size in bytes. If set, streams exceeding this are rejected. - * Default: no limit (rely on graphql-upload middleware). - */ - maxFileSize?: number; -} diff --git a/graphile/graphile-upload-plugin/tsconfig.esm.json b/graphile/graphile-upload-plugin/tsconfig.esm.json deleted file mode 100644 index 0843fea973..0000000000 --- a/graphile/graphile-upload-plugin/tsconfig.esm.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "moduleResolution": "bundler", - "outDir": "dist/esm", - "module": "ESNext" - } -} diff --git a/graphile/graphile-upload-plugin/tsconfig.json b/graphile/graphile-upload-plugin/tsconfig.json deleted file mode 100644 index 9c8a7d7c10..0000000000 --- a/graphile/graphile-upload-plugin/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "outDir": "dist", - "rootDir": "src" - }, - "include": ["src/**/*"] -} diff --git a/graphql/dev-server/README.md b/graphql/dev-server/README.md index 2c076984eb..f4eabb7a53 100644 --- a/graphql/dev-server/README.md +++ b/graphql/dev-server/README.md @@ -39,7 +39,7 @@ await server.stop(); ## What it does -- Builds a Constructive-style PostGraphile v5 preset via `createConstructivePreset()`, so all the standard presets (connection filters, search, PostGIS, ltree, direct uploads, meta-schema, inflection, type mappings, etc.) are baked in automatically. +- Builds a Constructive-style PostGraphile v5 preset via `createConstructivePreset()`, so all the standard presets (connection filters, search, PostGIS, ltree, meta-schema, inflection, type mappings, etc.) are baked in automatically. - Serves the schemas in `api.exposedSchemas` from a single database. - Runs every request as one fixed role (`api.roleName` → `api.anonRole` → `pg.user`). There is no per-request auth and no routing. - Exposes `/healthz`, `/graphql`, and `/graphiql`. @@ -51,7 +51,7 @@ The following plugin groups resolve per-tenant configuration from a `database_id - **Presigned uploads / bucket provisioner** (`enablePresignedUploads: false`) — resolve per-tenant storage config keyed by `database_id`. - **LLM** (`enableLlm: false`) — resolves per-tenant billing / inference-log config keyed by `database_id`. -Direct uploads stay enabled: they stream to a fixed, env-configured bucket and need no `database_id`. +The dev server therefore has no upload path; uploads are presigned and resolve the tenant's storage module. ## Configuration diff --git a/graphql/dev-server/src/preset.ts b/graphql/dev-server/src/preset.ts index c47cbbe8e0..114482b368 100644 --- a/graphql/dev-server/src/preset.ts +++ b/graphql/dev-server/src/preset.ts @@ -16,10 +16,9 @@ export interface DevPresetInput { * configured schemas with a fixed role and no scoped-routing plane. Only * plugins that do NOT depend on a tenant database id are enabled. The presigned * upload / bucket-provisioner plugins resolve per-tenant storage config from a - * `database_id`, and the LLM plugin resolves per-tenant billing / inference-log - * config from a `database_id`, so all three are excluded here. Direct uploads - * stream to a fixed env-configured bucket and need no database id, so they stay. - * Nothing enabled here reads `jwt.claims.database_id`. + * `database_id`, and the LLM plugin resolves per-tenant model config from a + * `database_id`, so all three are excluded here. Nothing enabled here reads + * `jwt.claims.database_id`. */ export const buildDevPreset = ({ pool, schemas, role }: DevPresetInput): GraphileConfig.Preset => ({ extends: [ diff --git a/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql b/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql new file mode 100644 index 0000000000..a276a2f91a --- /dev/null +++ b/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql @@ -0,0 +1,26 @@ +-- A table with the column types the managed storage lane projects into +-- (`image`, `upload`, `attachment`). + +CREATE SCHEMA IF NOT EXISTS upload_columns; + +GRANT USAGE ON SCHEMA upload_columns TO administrator, authenticated, anonymous; + +-- The platform's types module declares these; a suite without it gets the bare shape. +DO $$ BEGIN + IF to_regtype('public.image') IS NULL THEN CREATE DOMAIN public.image AS jsonb; END IF; + IF to_regtype('public.upload') IS NULL THEN CREATE DOMAIN public.upload AS jsonb; END IF; + IF to_regtype('public.attachment') IS NULL THEN CREATE DOMAIN public.attachment AS text; END IF; +END $$; + +CREATE TABLE upload_columns.products ( + id serial PRIMARY KEY, + title text NOT NULL, + photo public.image, + manual public.upload, + brochure public.attachment +); + +GRANT SELECT, INSERT, UPDATE, DELETE ON upload_columns.products TO administrator, authenticated, anonymous; +GRANT USAGE, SELECT ON SEQUENCE upload_columns.products_id_seq TO administrator, authenticated, anonymous; + +INSERT INTO upload_columns.products (title) VALUES ('kettle'); diff --git a/graphql/server-test/__tests__/upload-columns.integration.test.ts b/graphql/server-test/__tests__/upload-columns.integration.test.ts new file mode 100644 index 0000000000..8d37131743 --- /dev/null +++ b/graphql/server-test/__tests__/upload-columns.integration.test.ts @@ -0,0 +1,137 @@ +/** + * Integration tests — `image` / `upload` / `attachment` columns through the + * routed server. + * + * A file reaches a column through the presigned path: the upload mutation + * creates the files row, the client PUTs the bytes, and an ordinary update + * writes the projection document. The mutation inputs therefore carry only the + * column itself; there is no second, streaming input on the mutation and no + * multipart body on /graphql. + * + * pnpm test -- --testPathPattern=upload-columns + */ + +import path from 'path'; +import type supertest from 'supertest'; + +import { getConnections, seed } from '../src'; + +jest.setTimeout(120000); + +const sharedSeedRoot = path.join(__dirname, '..', '..', '..', '__fixtures__', 'seed'); +const localSeedRoot = path.join(__dirname, '..', '__fixtures__', 'seed'); +const pgpmWorkspace = path.join(sharedSeedRoot, '..', '..'); + +const databaseId = '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9'; +const schema = 'upload_columns'; +const metaSchemas = [ + 'catalog_private', + 'routing_public', + 'apps_public', + 'metaschema_public', + 'metaschema_modules_public' +]; + +const INPUT_FIELDS = ` + query InputFields($name: String!) { + __type(name: $name) { + inputFields { + name + type { name ofType { name } } + } + } + } +`; + +const UPDATE_PRODUCT = ` + mutation UpdateProduct($input: UpdateProductInput!) { + updateProduct(input: $input) { + product { id title photo manual brochure } + } + } +`; + +const fieldNames = (res: supertest.Response): string[] => + res.body.data.__type.inputFields.map((f: { name: string }) => f.name); + +describe('image / upload / attachment columns', () => { + let request: supertest.Agent; + let teardown: () => Promise; + + const post = () => + request.post('/graphql').set('X-Database-Id', databaseId).set('X-Schemata', schema); + + beforeAll(async () => { + ({ request, teardown } = await getConnections( + { + schemas: [schema], + authRole: 'anonymous', + server: { useRouting: true, api: { isPublic: false, metaSchemas } } + }, + [ + seed.pgpm(pgpmWorkspace), + seed.sqlfile([ + path.join(sharedSeedRoot, 'scoped', 'test-data.sql'), + path.join(localSeedRoot, 'upload-columns', 'schema.sql') + ]) + ] + )); + }); + + afterAll(async () => { + if (teardown) await teardown(); + }); + + it('builds a schema for a table whose columns use every storage document type', async () => { + const res = await post().send({ query: INPUT_FIELDS, variables: { name: 'ProductPatch' } }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + expect(fieldNames(res)).toEqual(['id', 'title', 'photo', 'manual', 'brochure']); + }); + + it('emits only the column on the create input, with no streaming companion', async () => { + const res = await post().send({ query: INPUT_FIELDS, variables: { name: 'ProductInput' } }); + + expect(res.body.errors).toBeUndefined(); + expect(fieldNames(res)).toEqual(['id', 'title', 'photo', 'manual', 'brochure']); + }); + + it('does not register an Upload scalar', async () => { + const res = await post().send({ query: `{ __type(name: "Upload") { name } }` }); + + expect(res.body.errors).toBeUndefined(); + expect(res.body.data.__type).toBeNull(); + }); + + it('writes the projection document through the ordinary update', async () => { + const photo = { + id: '0f0f0f0f-0000-4000-8000-000000000001', + key: 'a'.repeat(64), + bucket_id: '0f0f0f0f-0000-4000-8000-000000000002', + mime: 'image/png', + size: 16 + }; + const res = await post().send({ + query: UPDATE_PRODUCT, + variables: { input: { id: 1, productPatch: { photo } } } + }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + expect(res.body.data.updateProduct.product.photo).toEqual(photo); + }); + + it('refuses a multipart body on /graphql', async () => { + const operations = JSON.stringify({ + query: UPDATE_PRODUCT, + variables: { input: { id: 1, productPatch: { photo: null } } } + }); + const res = await post() + .field('operations', operations) + .field('map', JSON.stringify({ '0': ['variables.input.productPatch.photo'] })) + .attach('0', Buffer.from('not a png'), { filename: 'kettle.png', contentType: 'image/png' }); + + expect(res.status).toBe(415); + }); +}); diff --git a/graphql/server/README.md b/graphql/server/README.md index 19d7a3fdde..3e05e521b2 100644 --- a/graphql/server/README.md +++ b/graphql/server/README.md @@ -70,7 +70,6 @@ Runs an Express server that wires CORS, uploads, domain parsing, auth, and PostG - Automatic GraphQL API generation from PostgreSQL schemas - RLS-aware authentication and per-request `pgSettings` - Meta-schema routing by domain + subdomain -- File uploads via `graphql-upload` - GraphiQL and health check endpoints - Schema cache flush via `/flush` or database notifications - Opt-in observability for memory, DB activity, and Graphile build debugging @@ -101,7 +100,6 @@ For the operational workflow, sampler output, and heap snapshot usage, see [docs - `GET /healthz` -> health check - `GET /graphiql` -> GraphiQL UI - `GET /graphql` / `POST /graphql` -> GraphQL endpoint -- `POST /graphql` (multipart) -> file uploads - `POST /flush` -> clears cached Graphile schema for the current API; mounted only when `API_FLUSH_TOKEN` is set, and requires `Authorization: Bearer $API_FLUSH_TOKEN`. Without the variable the route is not served (404) — operators deploying a schema-cache flush must set it on the server and on every caller. The `LISTEN/NOTIFY` invalidation path is unaffected. - `GET /debug/memory` -> memory/process/Graphile debug snapshot when observability is enabled - `GET /debug/db` -> PostgreSQL activity/locks/pool debug snapshot when observability is enabled diff --git a/graphql/server/package.json b/graphql/server/package.json index d563d49aa8..5c5dec79ff 100644 --- a/graphql/server/package.json +++ b/graphql/server/package.json @@ -76,7 +76,6 @@ "graphile-settings": "workspace:^", "graphile-utils": "5.0.3", "graphql": "16.13.0", - "graphql-upload": "^13.0.0", "lru-cache": "^11.2.7", "pg": "^8.21.0", "pg-cache": "workspace:^", @@ -90,7 +89,6 @@ "@types/cookie-parser": "^1.4.10", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", - "@types/graphql-upload": "^8.0.12", "@types/pg": "^8.20.4", "@types/supertest": "^7.2.1", "graphile-test": "workspace:*", diff --git a/graphql/server/src/middleware/__tests__/request-protection.test.ts b/graphql/server/src/middleware/__tests__/request-protection.test.ts index 24c6ecadd9..46c564aa2f 100644 --- a/graphql/server/src/middleware/__tests__/request-protection.test.ts +++ b/graphql/server/src/middleware/__tests__/request-protection.test.ts @@ -108,20 +108,6 @@ describe('createRequestProtectionMiddleware', () => { expect(next).toHaveBeenCalled(); }); - it('exempts multipart uploads, whose bodies are files bounded by the upload limits', async () => { - const req = fakeReq( - { - 'content-length': '50000000', - 'content-type': 'multipart/form-data; boundary=----x' - }, - jest.fn().mockResolvedValue(protection({ maxRequestBytes: 2_000 })) - ); - - const { next } = await run(req); - - expect(next).toHaveBeenCalled(); - }); - it('does not reject a request that declared no length', async () => { const req = fakeReq({}, jest.fn().mockResolvedValue(protection({ maxRequestBytes: 2_000 }))); diff --git a/graphql/server/src/middleware/multipart-bridge.ts b/graphql/server/src/middleware/multipart-bridge.ts deleted file mode 100644 index 26e39c35bd..0000000000 --- a/graphql/server/src/middleware/multipart-bridge.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { RequestHandler } from 'express'; - -/** - * Bridge middleware between graphql-upload and grafserv (PostGraphile v5). - * - * graphql-upload's `graphqlUploadExpress()` parses multipart/form-data requests - * and sets `req.body` to a standard GraphQL operation object, but leaves the - * original Content-Type header unchanged. grafserv rejects any Content-Type it - * doesn't recognize (415 Unsupported Media Type) because its allowlist only - * includes `application/json` and `application/graphql`. - * - * This middleware rewrites the Content-Type header to `application/json` after - * graphql-upload has finished parsing, so grafserv accepts the request. - * - * This is the standard bridge pattern for grafserv + graphql-upload. - * See: https://github.com/graphql/graphql-http/discussions/36 - * - * Must be registered immediately after `graphqlUploadExpress()` in the - * middleware chain — it relies on graphql-upload having already parsed the - * multipart body into `req.body`. - * - * Security note: multipart/form-data is a CORS "simple request" content type, - * which means browsers send it without a preflight OPTIONS check. The CSRF risk - * is mitigated by the CORS middleware that runs before this middleware in the - * Express pipeline. - */ -export const multipartBridge: RequestHandler = (req, _res, next) => { - if (req.body != null && req.headers['content-type']?.startsWith('multipart/form-data')) { - req.headers['content-type'] = 'application/json'; - } - next(); -}; diff --git a/graphql/server/src/middleware/request-protection.ts b/graphql/server/src/middleware/request-protection.ts index f96fa8706e..72e9dae463 100644 --- a/graphql/server/src/middleware/request-protection.ts +++ b/graphql/server/src/middleware/request-protection.ts @@ -22,14 +22,6 @@ const resolveProtection = async (req: Request): Promise => { return resolved ?? DEFAULT_REQUEST_PROTECTION; }; -/** - * Multipart requests carry file bodies, which are streamed by the upload - * plugin and bounded by the upload limits rather than by the GraphQL request - * size: applying a JSON-sized cap to them would reject every upload. - */ -const isMultipart = (req: Request): boolean => - (req.get('content-type') ?? '').toLowerCase().startsWith('multipart/form-data'); - /** * Express middleware that resolves request protection and enforces the one * bound that has to be checked before the body is read. @@ -60,8 +52,7 @@ export const createRequestProtectionMiddleware = (): RequestHandler => { const declaredLength = Number(req.get('content-length') ?? ''); if ( Number.isFinite(declaredLength) && - declaredLength > protection.maxRequestBytes && - !isMultipart(req) + declaredLength > protection.maxRequestBytes ) { recordRefusal(req, 'request_too_large'); respondWithGraphQLError( diff --git a/graphql/server/src/server.ts b/graphql/server/src/server.ts index 8e922ad33a..ca935be3d5 100644 --- a/graphql/server/src/server.ts +++ b/graphql/server/src/server.ts @@ -10,7 +10,6 @@ import { PgpmOptions } from '@pgpmjs/types'; import cookieParser from 'cookie-parser'; import express, { Express, NextFunction, Request, RequestHandler, Response } from 'express'; import { closeAllCaches,graphileCache } from 'graphile-cache'; -import graphqlUpload from 'graphql-upload'; import type { Server as HttpServer } from 'http'; import { Pool, PoolClient } from 'pg'; import { getPgPool } from 'pg-cache'; @@ -38,7 +37,6 @@ import { favicon } from './middleware/favicon'; import { createFlushMiddleware, flushService } from './middleware/flush'; import { createFnRouter } from './middleware/fn'; import { graphile } from './middleware/graphile'; -import { multipartBridge } from './middleware/multipart-bridge'; import { createDebugDatabaseMiddleware } from './middleware/observability/debug-db'; import { debugMemory } from './middleware/observability/debug-memory'; import { localObservabilityOnly } from './middleware/observability/guard'; @@ -154,13 +152,6 @@ class Server { app.use(poweredBy('constructive')); app.use(cookieParser()); app.use(cors(fallbackOrigin)); - app.use('/graphql', graphqlUpload.graphqlUploadExpress({ - maxFileSize: 10 * 1024 * 1024, // 10 MB - maxFiles: 10 - })); - - // Rewrite Content-Type after graphql-upload so grafserv accepts the request - app.use('/graphql', multipartBridge); app.use(parseDomains() as RequestHandler); app.use(clientIpMiddleware); app.use(requestIdMiddleware()); diff --git a/packages/express-context/src/loaders/database-settings.ts b/packages/express-context/src/loaders/database-settings.ts index e2b294afb2..9233c0e05b 100644 --- a/packages/express-context/src/loaders/database-settings.ts +++ b/packages/express-context/src/loaders/database-settings.ts @@ -19,7 +19,6 @@ const databaseSettingsSql = ` enable_aggregates AS resolved_enable_aggregates, enable_postgis AS resolved_enable_postgis, enable_search AS resolved_enable_search, - enable_direct_uploads AS resolved_enable_direct_uploads, enable_presigned_uploads AS resolved_enable_presigned_uploads, enable_many_to_many AS resolved_enable_many_to_many, enable_connection_filter AS resolved_enable_connection_filter, @@ -37,7 +36,6 @@ interface DatabaseSettingsRow { resolved_enable_aggregates: boolean; resolved_enable_postgis: boolean; resolved_enable_search: boolean; - resolved_enable_direct_uploads: boolean; resolved_enable_presigned_uploads: boolean; resolved_enable_many_to_many: boolean; resolved_enable_connection_filter: boolean; @@ -67,7 +65,6 @@ export const databaseSettingsLoader: ModuleLoader = createModu enableAggregates: row.resolved_enable_aggregates, enablePostgis: row.resolved_enable_postgis, enableSearch: row.resolved_enable_search, - enableDirectUploads: row.resolved_enable_direct_uploads, enablePresignedUploads: row.resolved_enable_presigned_uploads, enableManyToMany: row.resolved_enable_many_to_many, enableConnectionFilter: row.resolved_enable_connection_filter, diff --git a/packages/express-context/src/types.ts b/packages/express-context/src/types.ts index a75b28019f..18a363fc16 100644 --- a/packages/express-context/src/types.ts +++ b/packages/express-context/src/types.ts @@ -18,7 +18,6 @@ export interface DatabaseSettings { enableAggregates: boolean; enablePostgis: boolean; enableSearch: boolean; - enableDirectUploads: boolean; enablePresignedUploads: boolean; enableManyToMany: boolean; enableConnectionFilter: boolean; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4307eb718e..84a481cce5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -859,9 +859,6 @@ importers: '@agentic-kit/ollama': specifier: workspace:* version: link:../../agentic/ollama/dist - '@constructive-io/express-context': - specifier: workspace:^ - version: link:../../packages/express-context/dist '@constructive-io/llm-env': specifier: workspace:^ version: link:../../packages/llm-env/dist @@ -1449,9 +1446,6 @@ importers: '@constructive-io/graphql-types': specifier: workspace:^ version: link:../../graphql/types/dist - '@constructive-io/s3-streamer': - specifier: workspace:^ - version: link:../../uploads/s3-streamer/dist '@constructive-io/upload-names': specifier: workspace:^ version: link:../../uploads/upload-names/dist @@ -1533,9 +1527,6 @@ importers: graphile-search: specifier: workspace:^ version: link:../graphile-search/dist - graphile-upload-plugin: - specifier: workspace:^ - version: link:../graphile-upload-plugin/dist graphile-utils: specifier: 5.0.3 version: 5.0.3(@dataplan/pg@1.1.1(@dataplan/json@1.0.1(grafast@1.1.2(graphql@16.13.0)))(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.1.2(graphql@16.13.0))(graphile-build-pg@5.1.3(@dataplan/pg@1.1.1(@dataplan/json@1.0.1(grafast@1.1.2(graphql@16.13.0)))(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.1.2(graphql@16.13.0))(graphile-build@5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1))(graphile-build@5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(tamedevil@0.1.1) @@ -1548,9 +1539,6 @@ importers: lru-cache: specifier: ^11.2.7 version: 11.2.7 - mime-bytes: - specifier: workspace:^ - version: link:../../uploads/mime-bytes/dist pg: specifier: ^8.21.0 version: 8.21.0 @@ -1685,32 +1673,6 @@ importers: version: 0.8.0 publishDirectory: dist - graphile/graphile-upload-plugin: - dependencies: - graphile-build: - specifier: 5.1.1 - version: 5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0) - graphile-build-pg: - specifier: 5.1.3 - version: 5.1.3(@dataplan/pg@1.1.1(@dataplan/json@1.0.1(grafast@1.1.2(graphql@16.13.0)))(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0))(grafast@1.1.2(graphql@16.13.0))(graphile-build@5.1.1(grafast@1.1.2(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0))(graphile-config@1.1.0)(graphql@16.13.0)(pg-sql2@5.0.1)(pg@8.21.0)(tamedevil@0.1.1) - graphile-config: - specifier: 1.1.0 - version: 1.1.0 - graphql: - specifier: 16.13.0 - version: 16.13.0 - postgraphile: - specifier: 5.1.4 - version: 5.1.4(51dc2e186be12dc7266f70b2b43fe3f8) - devDependencies: - '@types/node': - specifier: ^22.19.11 - version: 22.19.11 - makage: - specifier: ^0.8.0 - version: 0.8.0 - publishDirectory: dist - graphql/codegen: dependencies: '@0no-co/graphql.web': @@ -2284,9 +2246,6 @@ importers: graphql: specifier: 16.13.0 version: 16.13.0 - graphql-upload: - specifier: ^13.0.0 - version: 13.0.0(graphql@16.13.0) lru-cache: specifier: ^11.2.7 version: 11.2.7 @@ -2321,9 +2280,6 @@ importers: '@types/express': specifier: ^5.0.6 version: 5.0.6 - '@types/graphql-upload': - specifier: ^8.0.12 - version: 8.0.12 '@types/pg': specifier: ^8.20.4 version: 8.20.4 @@ -8123,12 +8079,6 @@ packages: integrity: sha512-6+dKdh9q/I8xDBnKQKddCBKaWBWLmJ97HTiSbAXVpL7LEgDfOkKF98UVCaZ5KJrtdN5Wa5ndXUiqD3XR9XGqWQ==, } - '@types/accepts@1.3.7': - resolution: - { - integrity: sha512-Pay9fq2lM2wXPWbteBsRAGiWH2hig4ZE2asK+mm7kUzlxRTfL961rj89I6zV/E3PcIkDqyuBEcMxFT7rccugeQ==, - } - '@types/aria-query@5.0.4': resolution: { @@ -8171,12 +8121,6 @@ packages: integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==, } - '@types/content-disposition@0.5.9': - resolution: - { - integrity: sha512-8uYXI3Gw35MhiVYhG3s295oihrxRyytcRHjSjqnqZVDDy/xcGBRny7+Xj1Wgfhv5QzRtN2hB2dVRBUX9XW3UcQ==, - } - '@types/cookie-parser@1.4.10': resolution: { @@ -8191,12 +8135,6 @@ packages: integrity: sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==, } - '@types/cookies@0.9.2': - resolution: - { - integrity: sha512-1AvkDdZM2dbyFybL4fxpuNCaWyv//0AwsuUk2DWeXyM1/5ZKm6W3z6mQi24RZ4l2ucY+bkSHzbDVpySqPGuV8A==, - } - '@types/cors@2.8.19': resolution: { @@ -8233,18 +8171,6 @@ packages: integrity: sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==, } - '@types/graphql-upload@8.0.12': - resolution: - { - integrity: sha512-M0ZPZqNUzKNB16q5woEzgG/Q8DjICV80K7JvDSRnDmDFfrRdfFX/n6PbmqAN7gCzECcHVnw1gk6N4Cg0FwxCqA==, - } - - '@types/http-assert@1.5.6': - resolution: - { - integrity: sha512-TTEwmtjgVbYAzZYWyeHPrrtWnfVkm8tQkP8P21uQifPgMRgjrow3XDEYqucuC8SKZJT7pUnhU/JymvjggxO9vw==, - } - '@types/http-errors@2.0.5': resolution: { @@ -8299,24 +8225,6 @@ packages: integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==, } - '@types/keygrip@1.0.6': - resolution: - { - integrity: sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==, - } - - '@types/koa-compose@3.2.9': - resolution: - { - integrity: sha512-BroAZ9FTvPiCy0Pi8tjD1OfJ7bgU1gQf0eR6e1Vm+JJATy9eKOG3hQMFtMciMawiSOVnLMdmUOC46s7HBhSTsA==, - } - - '@types/koa@3.0.1': - resolution: - { - integrity: sha512-VkB6WJUQSe0zBpR+Q7/YIUESGp5wPHcaXr0xueU5W0EOUWtlSbblsl+Kl31lyRQ63nIILh0e/7gXjQ09JXJIHw==, - } - '@types/methods@1.1.4': resolution: { @@ -9272,13 +9180,6 @@ packages: integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==, } - busboy@0.3.1: - resolution: - { - integrity: sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==, - } - engines: { node: '>=4.5.0' } - byte-size@8.1.1: resolution: { @@ -9921,13 +9822,6 @@ packages: } engines: { node: '>=0.4.0' } - depd@1.1.2: - resolution: - { - integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==, - } - engines: { node: '>= 0.6' } - depd@2.0.0: resolution: { @@ -9974,13 +9868,6 @@ packages: integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==, } - dicer@0.3.0: - resolution: - { - integrity: sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==, - } - engines: { node: '>=4.5.0' } - diff-sequences@29.6.3: resolution: { @@ -10772,20 +10659,6 @@ packages: integrity: sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==, } - fs-capacitor@6.2.0: - resolution: - { - integrity: sha512-nKcE1UduoSKX27NSZlg879LdQc94OtbOsEmKMN2MBNudXREvijRKx2GEBsTMTfws+BrbkJoEuynbGSVRSpauvw==, - } - engines: { node: '>=10' } - - fs-capacitor@8.0.0: - resolution: - { - integrity: sha512-+Lk6iSKajdGw+7XYxUkwIzreJ2G1JFlYOdnKJv5PzwFLVsoJYBpCuS7WPIUSNT1IbQaEWT1nhYU63Ud03DyzLA==, - } - engines: { node: ^14.17.0 || >=16.0.0 } - fs-constants@1.0.0: resolution: { @@ -11226,15 +11099,6 @@ packages: peerDependencies: graphql: 16.13.0 - graphql-upload@13.0.0: - resolution: - { - integrity: sha512-YKhx8m/uOtKu4Y1UzBFJhbBGJTlk7k4CydlUUiNrtxnwZv0WigbRHP+DVhRNKt7u7DXOtcKZeYJlGtnMXvreXA==, - } - engines: { node: ^12.22.0 || ^14.17.0 || >= 16.0.0 } - peerDependencies: - graphql: 16.13.0 - graphql-ws@6.0.8: resolution: { @@ -11383,13 +11247,6 @@ packages: integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==, } - http-errors@1.8.1: - resolution: - { - integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==, - } - engines: { node: '>= 0.6' } - http-errors@2.0.1: resolution: { @@ -13285,13 +13142,6 @@ packages: } engines: { node: '>= 0.4' } - object-path@0.11.8: - resolution: - { - integrity: sha512-YJjNZrlXJFM42wTBn6zgOJVar9KFJvzx6sTWDte8sWZF//cnjl0BxHNpfZx+ZffXX63A9q0b1zsFiBX4g4X5KA==, - } - engines: { node: '>= 10.12.0' } - on-finished@2.4.1: resolution: { @@ -14922,13 +14772,6 @@ packages: integrity: sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==, } - statuses@1.5.0: - resolution: - { - integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==, - } - engines: { node: '>= 0.6' } - statuses@2.0.2: resolution: { @@ -14942,13 +14785,6 @@ packages: integrity: sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==, } - streamsearch@0.1.2: - resolution: - { - integrity: sha512-jos8u++JKm0ARcSUTAZXOVC0mSox7Bhn6sBgty73P1f3JGf7yG2clTbBNHUdde/kdvP2FESam+vM6l8jBrNxHA==, - } - engines: { node: '>=0.8.0' } - strfy-js@3.2.2: resolution: { @@ -19271,10 +19107,6 @@ snapshots: '@types/accept-language-parser@1.5.8': {} - '@types/accepts@1.3.7': - dependencies: - '@types/node': 22.19.19 - '@types/aria-query@5.0.4': {} '@types/babel__core@7.20.5': @@ -19307,21 +19139,12 @@ snapshots: dependencies: '@types/node': 22.19.19 - '@types/content-disposition@0.5.9': {} - '@types/cookie-parser@1.4.10(@types/express@5.0.6)': dependencies: '@types/express': 5.0.6 '@types/cookiejar@2.1.5': {} - '@types/cookies@0.9.2': - dependencies: - '@types/connect': 3.4.38 - '@types/express': 5.0.6 - '@types/keygrip': 1.0.6 - '@types/node': 22.19.19 - '@types/cors@2.8.19': dependencies: '@types/node': 22.19.19 @@ -19347,15 +19170,6 @@ snapshots: '@types/geojson@7946.0.16': {} - '@types/graphql-upload@8.0.12': - dependencies: - '@types/express': 5.0.6 - '@types/koa': 3.0.1 - fs-capacitor: 8.0.0 - graphql: 16.13.0 - - '@types/http-assert@1.5.6': {} - '@types/http-errors@2.0.5': {} '@types/istanbul-lib-coverage@2.0.6': {} @@ -19388,23 +19202,6 @@ snapshots: '@types/json-schema@7.0.15': {} - '@types/keygrip@1.0.6': {} - - '@types/koa-compose@3.2.9': - dependencies: - '@types/koa': 3.0.1 - - '@types/koa@3.0.1': - dependencies: - '@types/accepts': 1.3.7 - '@types/content-disposition': 0.5.9 - '@types/cookies': 0.9.2 - '@types/http-assert': 1.5.6 - '@types/http-errors': 2.0.5 - '@types/keygrip': 1.0.6 - '@types/koa-compose': 3.2.9 - '@types/node': 22.19.19 - '@types/methods@1.1.4': {} '@types/minimatch@3.0.5': {} @@ -19975,10 +19772,6 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 - busboy@0.3.1: - dependencies: - dicer: 0.3.0 - byte-size@8.1.1: {} bytes@3.1.2: {} @@ -20312,8 +20105,6 @@ snapshots: delayed-stream@1.0.0: {} - depd@1.1.2: {} - depd@2.0.0: {} deprecation@2.3.1: {} @@ -20331,10 +20122,6 @@ snapshots: asap: 2.0.6 wrappy: 1.0.2 - dicer@0.3.0: - dependencies: - streamsearch: 0.1.2 - diff-sequences@29.6.3: {} diff@4.0.2: {} @@ -20827,10 +20614,6 @@ snapshots: dependencies: js-yaml: 3.14.2 - fs-capacitor@6.2.0: {} - - fs-capacitor@8.0.0: {} - fs-constants@1.0.0: {} fs-extra@11.3.3: @@ -21369,14 +21152,6 @@ snapshots: graphql: 16.13.0 tslib: 2.8.1 - graphql-upload@13.0.0(graphql@16.13.0): - dependencies: - busboy: 0.3.1 - fs-capacitor: 6.2.0 - graphql: 16.13.0 - http-errors: 1.8.1 - object-path: 0.11.8 - graphql-ws@6.0.8(graphql@16.13.0)(ws@8.20.1): dependencies: graphql: 16.13.0 @@ -21448,14 +21223,6 @@ snapshots: http-cache-semantics@4.2.0: {} - http-errors@1.8.1: - dependencies: - depd: 1.1.2 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 1.5.0 - toidentifier: 1.0.1 - http-errors@2.0.1: dependencies: depd: 2.0.0 @@ -22932,8 +22699,6 @@ snapshots: object-inspect@1.13.4: {} - object-path@0.11.8: {} - on-finished@2.4.1: dependencies: ee-first: 1.1.1 @@ -24244,8 +24009,6 @@ snapshots: '@stablelib/base64': 1.0.1 fast-sha256: 1.3.0 - statuses@1.5.0: {} - statuses@2.0.2: {} stream-browserify@3.0.0: @@ -24253,8 +24016,6 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - streamsearch@0.1.2: {} - strfy-js@3.2.2: dependencies: minimatch: 10.2.6 From bb3d50ebc81baede217379d926ba805a1de27e40 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 4 Nov 2026 20:43:09 +0000 Subject: [PATCH 2/5] chore(graphile): drop file-ref registry, stale wording; test presigned row projection --- .../src/plugins/text-search-plugin.ts | 10 +- .../src/file-ref-registry.ts | 148 ------------------ .../src/index.ts | 2 - .../upload-columns.integration.test.ts | 59 +++++-- scripts/check-test-coverage.cjs | 2 +- 5 files changed, 52 insertions(+), 169 deletions(-) delete mode 100644 graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts diff --git a/graphile/graphile-llm/src/plugins/text-search-plugin.ts b/graphile/graphile-llm/src/plugins/text-search-plugin.ts index a6a7e998c2..b6845af941 100644 --- a/graphile/graphile-llm/src/plugins/text-search-plugin.ts +++ b/graphile/graphile-llm/src/plugins/text-search-plugin.ts @@ -44,7 +44,7 @@ declare global { * Recursively walk a `where` argument object and embed any VectorNearbyInput * values that have `text` instead of `vector`. * - * If the embedder returns null (e.g. quota exceeded), the text field is + * If the embedder returns null (provider unavailable), the text field is * removed so the pgvector filter is skipped — graceful text-only fallback. * * @deprecated The plugins no longer call this: under Grafast a resolver runs @@ -74,11 +74,11 @@ export async function embedTextInWhere( if (vector === null) { if (onQuotaExceeded === 'throw' || !hasTextAdapters) { throw new Error( - 'unifiedSearch: embedding failed (quota exceeded or provider unavailable). ' + + 'unifiedSearch: embedding failed (provider unavailable). ' + (!hasTextAdapters ? 'No text search adapters available for fallback. ' : 'onQuotaExceeded is set to \'throw\'. ') + - 'Upgrade your plan or adjust onQuotaExceeded to \'degrade\' for text-only fallback.' + 'Set onQuotaExceeded to \'degrade\' for text-only fallback.' ); } return; @@ -106,8 +106,8 @@ export async function embedTextInWhere( if (vector === null) { if (onQuotaExceeded === 'throw') { throw new Error( - 'VectorNearbyInput: embedding failed (quota exceeded or provider unavailable). ' + - 'Upgrade your plan or adjust onQuotaExceeded to \'degrade\' for graceful fallback.' + 'VectorNearbyInput: embedding failed (provider unavailable). ' + + 'Set onQuotaExceeded to \'degrade\' to fall back to the other filters.' ); } delete value.text; diff --git a/graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts b/graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts deleted file mode 100644 index dd563ff63a..0000000000 --- a/graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts +++ /dev/null @@ -1,148 +0,0 @@ -/** - * The `file_ref_field` registry: which storage module and bucket a managed - * document column writes into. - * - * An `image`/`upload` column is a projection of a files row, and the decision of - * *where* those bytes live is a property of the field declaration, not of the - * request. The registry records that intent per (table, column) — a storage - * module plus either a logical bucket key, a tag selector, or nothing at all - * (meaning the reserved default tag for the declared publicness). - * - * This module answers one question — "what does a write to this column bind - * to?" — and answers it loudly: an unregistered column raises rather than - * falling back to a server-global bucket, because a silent fallback is how the - * unmanaged lane produced objects no tenant owned. - */ - -import { Logger } from '@pgpmjs/logger'; -import { LRUCache } from 'lru-cache'; - -const log = new Logger('graphile-presigned-url:file-ref-registry'); - -const FIVE_MINUTES_MS = 1000 * 60 * 5; -const ONE_HOUR_MS = 1000 * 60 * 60; - -/** - * A field's recorded storage intent. - * - * `bucketKey` and `bucketTags` are mutually exclusive by table constraint, and - * both may be absent — resolution then uses the reserved default tag for - * `isPublic`. Nothing here is a physical bucket name or id: the concrete bucket - * is resolved per written row, inside the tenant. - */ -export interface FileRefFieldBinding { - id: string; - storageModuleId: string; - bucketKey: string | null; - bucketTags: string[] | null; - isPublic: boolean | null; - enforceFk: boolean; -} - -/** - * Resolve the registry row for a document column. - * - * Joined through metaschema rather than keyed by name, because the registry - * records field *ids*: the physical (schema, table, column) triple is what the - * GraphQL layer knows, and metaschema is the only thing that maps one to the - * other. - */ -const FILE_REF_FIELD_QUERY = ` - SELECT * FROM function_resolution.storage_file_ref_field($1, $2, $3) -`; - -interface FileRefFieldRow { - id: string; - storage_module_id: string; - bucket_key: string | null; - bucket_tags: string[] | null; - is_public: boolean | null; - enforce_fk: boolean; -} - -/** - * LRU cache of field bindings. - * - * A binding is schema, not data: it changes only when a database is - * re-provisioned, so it caches on the same terms as the storage module config - * next to it. Misses are never cached — an unregistered column is a hard error - * every time it is written, not a remembered "no". - */ -const bindingCache = new LRUCache({ - max: 500, - ttl: process.env.NODE_ENV === 'development' ? FIVE_MINUTES_MS : ONE_HOUR_MS, - updateAgeOnGet: true, -}); - -export class FileRefFieldNotRegisteredError extends Error { - constructor( - public readonly databaseId: string, - public readonly schemaName: string, - public readonly tableName: string, - public readonly columnName: string, - ) { - super( - `FILE_REF_FIELD_NOT_REGISTERED: ${schemaName}.${tableName}.${columnName} ` + - `is not a registered file-reference field in database ${databaseId}. ` + - 'A managed upload needs the declared storage module and bucket intent; ' + - 'there is no server-global bucket to fall back to.', - ); - this.name = 'FileRefFieldNotRegisteredError'; - } -} - -/** - * Look up the storage binding for a document column, or throw. - * - * The read runs on the request's own lane, through the claim-scoped - * function_resolution.storage_file_ref_field(): the database is the request's - * jwt.claims.database_id, never an argument. - */ -export async function getFileRefFieldBinding( - pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, - databaseId: string, - field: { schemaName: string; tableName: string; columnName: string }, -): Promise { - const cacheKey = `file-ref:${databaseId}:${field.schemaName}.${field.tableName}.${field.columnName}`; - const cached = bindingCache.get(cacheKey); - if (cached) return cached; - - const result = await pgClient.query({ - text: FILE_REF_FIELD_QUERY, - values: [field.schemaName, field.tableName, field.columnName], - }); - - if (result.rows.length === 0) { - throw new FileRefFieldNotRegisteredError( - databaseId, - field.schemaName, - field.tableName, - field.columnName, - ); - } - - const row = result.rows[0] as FileRefFieldRow; - const binding: FileRefFieldBinding = { - id: row.id, - storageModuleId: row.storage_module_id, - bucketKey: row.bucket_key, - bucketTags: row.bucket_tags, - isPublic: row.is_public, - enforceFk: row.enforce_fk, - }; - - bindingCache.set(cacheKey, binding); - log.debug( - `Bound ${field.schemaName}.${field.tableName}.${field.columnName} to storage module ` + - `${binding.storageModuleId} (bucket_key=${binding.bucketKey ?? ''})`, - ); - - return binding; -} - -/** - * Drop cached bindings. Used by tests and after a re-provision. - */ -export function clearFileRefFieldCache(): void { - bindingCache.clear(); -} diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index 3ed4236ec7..4bf8039ecd 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -33,8 +33,6 @@ export { validateCustomKey } from './custom-key'; export type { ResolvedBucketCoordinate } from './default-bucket'; export { resolveDefaultBucket } from './default-bucket'; export { createDownloadUrlPlugin } from './download-url-field'; -export type { FileRefFieldBinding } from './file-ref-registry'; -export { clearFileRefFieldCache, FileRefFieldNotRegisteredError, getFileRefFieldBinding } from './file-ref-registry'; export { buildFileProjection, type FileProjection } from './file-projection'; export { assertBucketReconciled, diff --git a/graphql/server-test/__tests__/upload-columns.integration.test.ts b/graphql/server-test/__tests__/upload-columns.integration.test.ts index 8d37131743..688923cd5e 100644 --- a/graphql/server-test/__tests__/upload-columns.integration.test.ts +++ b/graphql/server-test/__tests__/upload-columns.integration.test.ts @@ -11,6 +11,7 @@ * pnpm test -- --testPathPattern=upload-columns */ +import { hashContent } from '@constructive-io/upload-client'; import path from 'path'; import type supertest from 'supertest'; @@ -22,7 +23,9 @@ const sharedSeedRoot = path.join(__dirname, '..', '..', '..', '__fixtures__', 's const localSeedRoot = path.join(__dirname, '..', '__fixtures__', 'seed'); const pgpmWorkspace = path.join(sharedSeedRoot, '..', '..'); +// Alice's database: `simple-seed-storage` gives it a storage module and buckets. const databaseId = '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9'; +const storageSchema = 'simple-storage-public'; const schema = 'upload_columns'; const metaSchemas = [ 'catalog_private', @@ -43,6 +46,15 @@ const INPUT_FIELDS = ` } `; +const UPLOAD_APP_FILE = ` + mutation UploadAppFile($input: UploadAppFileInput!) { + uploadAppFile(input: $input) { + fileId + key + } + } +`; + const UPDATE_PRODUCT = ` mutation UpdateProduct($input: UpdateProductInput!) { updateProduct(input: $input) { @@ -56,22 +68,27 @@ const fieldNames = (res: supertest.Response): string[] => describe('image / upload / attachment columns', () => { let request: supertest.Agent; + let pg: { query: (text: string, values?: unknown[]) => Promise<{ rows: any[] }> }; let teardown: () => Promise; const post = () => - request.post('/graphql').set('X-Database-Id', databaseId).set('X-Schemata', schema); + request + .post('/graphql') + .set('X-Database-Id', databaseId) + .set('X-Schemata', [storageSchema, schema].join(',')); beforeAll(async () => { - ({ request, teardown } = await getConnections( + ({ request, pg, teardown } = await getConnections( { - schemas: [schema], + schemas: [storageSchema, schema], authRole: 'anonymous', server: { useRouting: true, api: { isPublic: false, metaSchemas } } }, [ seed.pgpm(pgpmWorkspace), seed.sqlfile([ - path.join(sharedSeedRoot, 'scoped', 'test-data.sql'), + path.join(localSeedRoot, 'simple-seed-storage', 'schema.sql'), + path.join(localSeedRoot, 'simple-seed-storage', 'test-data.sql'), path.join(localSeedRoot, 'upload-columns', 'schema.sql') ]) ] @@ -104,14 +121,30 @@ describe('image / upload / attachment columns', () => { expect(res.body.data.__type).toBeNull(); }); - it('writes the projection document through the ordinary update', async () => { - const photo = { - id: '0f0f0f0f-0000-4000-8000-000000000001', - key: 'a'.repeat(64), - bucket_id: '0f0f0f0f-0000-4000-8000-000000000002', - mime: 'image/png', - size: 16 - }; + it('writes the projection of a presigned files row through the ordinary update', async () => { + const content = 'kettle photo bytes'; + const contentHash = await hashContent(content); + const uploaded = await post().send({ + query: UPLOAD_APP_FILE, + variables: { + input: { + bucketKey: 'public', + contentHash, + contentType: 'image/png', + size: Buffer.byteLength(content), + filename: 'kettle.png' + } + } + }); + expect(uploaded.body.errors).toBeUndefined(); + const { fileId, key } = uploaded.body.data.uploadAppFile; + const { rows } = await pg.query( + `SELECT bucket_id, mime_type, size FROM "${storageSchema}".app_files WHERE id = $1`, + [fileId] + ); + expect(rows).toHaveLength(1); + + const photo = { id: fileId, key, bucket_id: rows[0].bucket_id, mime: rows[0].mime_type, size: Number(rows[0].size) }; const res = await post().send({ query: UPDATE_PRODUCT, variables: { input: { id: 1, productPatch: { photo } } } @@ -129,7 +162,7 @@ describe('image / upload / attachment columns', () => { }); const res = await post() .field('operations', operations) - .field('map', JSON.stringify({ '0': ['variables.input.productPatch.photo'] })) + .field('map', JSON.stringify({ 0: ['variables.input.productPatch.photo'] })) .attach('0', Buffer.from('not a png'), { filename: 'kettle.png', contentType: 'image/png' }); expect(res.status).toBe(415); diff --git a/scripts/check-test-coverage.cjs b/scripts/check-test-coverage.cjs index 81707a55cb..7188807534 100644 --- a/scripts/check-test-coverage.cjs +++ b/scripts/check-test-coverage.cjs @@ -6,7 +6,7 @@ * lists of package paths. Adding a package to the workspace does not add it to * a batch, and nothing complained: 35 packages / 89 test files had accumulated * in no job at all, so genuinely broken code (a type error in - * graphile-upload-plugin, stale mocks in graphile-realtime-subscriptions) sat + * a graphile plugin, stale mocks in graphile-realtime-subscriptions) sat * green for as long as nobody ran them locally. * * Omission is now countable. A package with tests must either be claimed by a From 054d1df7ee970efa8e12d022012b124ceb614f94 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 4 Nov 2026 21:02:06 +0000 Subject: [PATCH 3/5] chore(graphile-llm): one embedder shape after plan-time embedding; provider wording --- .../src/__tests__/embed-inputs.test.ts | 18 +++++++--------- graphile/graphile-llm/src/embed-inputs.ts | 21 +++++++------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts b/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts index a1d218f852..f6b0575548 100644 --- a/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts +++ b/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts @@ -36,13 +36,12 @@ const VECTORS: Record = { }; const vectorFor = (text: string) => VECTORS[text] ?? [1, 1, 1]; -type EmbedderShape = 'result' | 'array' | 'null' | 'empty'; +type EmbedderShape = 'result' | 'null' | 'empty'; let shape: EmbedderShape = 'result'; const calls: Array<{ text: string; overrides: unknown }> = []; -// `result` is the provider shape ({ embedding }), `array` is the metering -// wrapper's shape, `null` is the metering wrapper refusing (quota exceeded), -// `empty` is a provider answering with no vector. +// `result` is the provider shape ({ embedding }); `null` and `empty` are a +// provider answering with no vector. function fakeEmbedderPlugin(configured: boolean): GraphileConfig.Plugin { return { name: 'FakeEmbedderPlugin', @@ -55,9 +54,8 @@ function fakeEmbedderPlugin(configured: boolean): GraphileConfig.Plugin { ? async (text: string) => { calls.push({ text, overrides: llmConfigStore.getStore() }); if (shape === 'null') return null; - if (shape === 'empty') return []; - const embedding = vectorFor(text); - return shape === 'array' ? embedding : { embedding, promptTokens: 1 }; + if (shape === 'empty') return { embedding: [] }; + return { embedding: vectorFor(text), promptTokens: 1 }; } : null; return build; @@ -194,8 +192,8 @@ describe('plan-time embedding (onQuotaExceeded: degrade)', () => { expect(errors).toEqual([[], [], []]); }); - describe.each(['result', 'array'])('embedder returning the %s shape', (embedderShape) => { - beforeEach(() => { shape = embedderShape; }); + describe('embedder returning the provider shape', () => { + beforeEach(() => { shape = 'result'; }); it('ranks a text search by the embedded vector and embeds once', async () => { const result = await h.query(BY_TEXT, { t: 'revenue' }); @@ -331,7 +329,7 @@ describe('plan-time embedding (onQuotaExceeded: degrade)', () => { shape = 'null'; const result = await h.query(CREATE_FROM_TEXT, { name: 'quota.txt', t: 'rent' }); - expect(result.errors?.[0]?.message).toMatch(/EMBED_QUOTA_EXCEEDED: Cannot embed embeddingText/); + expect(result.errors?.[0]?.message).toMatch(/EMBED_FAILED: Cannot embed embeddingText/); const rows = await h.pg.query("SELECT 1 FROM llm_embed_inputs.docs WHERE name = 'quota.txt'"); expect(rows.rows).toHaveLength(0); }); diff --git a/graphile/graphile-llm/src/embed-inputs.ts b/graphile/graphile-llm/src/embed-inputs.ts index 1d9b35dfdc..d1f2632619 100644 --- a/graphile/graphile-llm/src/embed-inputs.ts +++ b/graphile/graphile-llm/src/embed-inputs.ts @@ -37,16 +37,13 @@ export function embedField(rule: EmbedRule, spec: T): T { return { ...spec, extensions: { ...(spec as { extensions?: object }).extensions, [EMBED_EXT]: rule } }; } -/** - * `build.llmEmbedder` resolves to `{ embedding }` from the providers and to - * `number[] | null` once the metering plugin has wrapped it. - */ +/** `build.llmEmbedder` resolves to `{ embedding }`; an empty or missing vector is a failed embed. */ export function toVectorEmbedder(raw: unknown): VectorEmbedder | null { if (typeof raw !== 'function') return null; return async (text) => { const result = await raw(text); - const vector = Array.isArray(result) ? result : result?.embedding; - return Array.isArray(vector) && vector.length > 0 ? vector : null; + const vector = result?.embedding; + return isNonEmptyVector(vector) ? vector : null; }; } @@ -184,8 +181,8 @@ export function createEmbedIndex(graphql: any) { next[rule.into] = vector; } else if (rule.onQuotaExceeded === 'throw') { throw new Error( - `${typeName}: embedding failed (quota exceeded or provider unavailable). ` + - 'Upgrade your plan or adjust onQuotaExceeded to \'degrade\' for graceful fallback.' + `${typeName}: embedding failed (provider unavailable). ` + + 'Set onQuotaExceeded to \'degrade\' to fall back to the other filters.' ); } return; @@ -198,9 +195,8 @@ export function createEmbedIndex(graphql: any) { next[fieldName] = { __text: text, __vector: vector }; } else if (rule.onQuotaExceeded === 'throw') { throw new Error( - `${fieldName}: embedding failed (quota exceeded or provider unavailable). ` + - 'onQuotaExceeded is set to \'throw\'. ' + - 'Upgrade your plan or adjust onQuotaExceeded to \'degrade\' for text-only fallback.' + `${fieldName}: embedding failed (provider unavailable). ` + + 'onQuotaExceeded is set to \'throw\'; set it to \'degrade\' for text-only fallback.' ); } return; @@ -233,8 +229,7 @@ export function createEmbedIndex(graphql: any) { const vector = await embed(text); if (!vector) { throw new Error( - `EMBED_QUOTA_EXCEEDED: Cannot embed ${fieldName} — embedding quota exceeded. ` + - 'Upgrade your plan or wait for the next billing period.' + `EMBED_FAILED: Cannot embed ${fieldName} — the embedding provider returned no vector.` ); } next[rule.into] = vector; From 3f419e0fbae274a2c21496d78ad62fdf2537b933 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 4 Nov 2026 21:12:02 +0000 Subject: [PATCH 4/5] test(graphile-llm): type the empty embedding fixture --- graphile/graphile-llm/src/__tests__/embed-inputs.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts b/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts index f6b0575548..024b9f6cd7 100644 --- a/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts +++ b/graphile/graphile-llm/src/__tests__/embed-inputs.test.ts @@ -54,7 +54,7 @@ function fakeEmbedderPlugin(configured: boolean): GraphileConfig.Plugin { ? async (text: string) => { calls.push({ text, overrides: llmConfigStore.getStore() }); if (shape === 'null') return null; - if (shape === 'empty') return { embedding: [] }; + if (shape === 'empty') return { embedding: [] as number[] }; return { embedding: vectorFor(text), promptTokens: 1 }; } : null; From 33bed39be502dfbdeb0419a744ff1cf33830e252 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 5 Nov 2026 00:37:59 +0000 Subject: [PATCH 5/5] test(server-test): upload the presigned object before writing its projection --- .../upload-columns.integration.test.ts | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/graphql/server-test/__tests__/upload-columns.integration.test.ts b/graphql/server-test/__tests__/upload-columns.integration.test.ts index 688923cd5e..0f4ce9cf3b 100644 --- a/graphql/server-test/__tests__/upload-columns.integration.test.ts +++ b/graphql/server-test/__tests__/upload-columns.integration.test.ts @@ -11,7 +11,8 @@ * pnpm test -- --testPathPattern=upload-columns */ -import { hashContent } from '@constructive-io/upload-client'; +import { createS3Bucket, createS3Client } from '@constructive-io/s3-utils'; +import { hashContent, putToPresignedUrl } from '@constructive-io/upload-client'; import path from 'path'; import type supertest from 'supertest'; @@ -51,6 +52,7 @@ const UPLOAD_APP_FILE = ` uploadAppFile(input: $input) { fileId key + uploadUrl } } `; @@ -122,37 +124,54 @@ describe('image / upload / attachment columns', () => { }); it('writes the projection of a presigned files row through the ordinary update', async () => { - const content = 'kettle photo bytes'; + const content = 'kettle manual, plain text'; const contentHash = await hashContent(content); + const { rows: buckets } = await pg.query( + `SELECT physical_name FROM "${storageSchema}".app_buckets WHERE key = 'public'` + ); + const s3 = createS3Client({ + provider: 'minio', + region: 'us-east-1', + endpoint: 'http://localhost:9000', + accessKeyId: 'constructive', + secretAccessKey: 'constructive-dev-secret' + }); + expect((await createS3Bucket(s3, buckets[0].physical_name, { provider: 'minio' })).success).toBe(true); + const uploaded = await post().send({ query: UPLOAD_APP_FILE, variables: { input: { bucketKey: 'public', contentHash, - contentType: 'image/png', + contentType: 'text/plain', size: Buffer.byteLength(content), - filename: 'kettle.png' + filename: 'kettle.txt' } } }); expect(uploaded.body.errors).toBeUndefined(); - const { fileId, key } = uploaded.body.data.uploadAppFile; + const { fileId, key, uploadUrl } = uploaded.body.data.uploadAppFile; + expect((await putToPresignedUrl(uploadUrl, content, 'text/plain')).ok).toBe(true); + // The confirm worker is not part of this fixture; it is what moves a row + // from `requested` to `uploaded` once it has inspected the object. + await pg.query(`UPDATE "${storageSchema}".app_files SET status = 'uploaded' WHERE id = $1`, [fileId]); + const { rows } = await pg.query( - `SELECT bucket_id, mime_type, size FROM "${storageSchema}".app_files WHERE id = $1`, + `SELECT bucket_id, mime_type, size, status FROM "${storageSchema}".app_files WHERE id = $1`, [fileId] ); - expect(rows).toHaveLength(1); + expect(rows[0].status).toBe('uploaded'); + const projection = { id: fileId, key, bucket_id: rows[0].bucket_id, mime: rows[0].mime_type, size: Number(rows[0].size) }; - const photo = { id: fileId, key, bucket_id: rows[0].bucket_id, mime: rows[0].mime_type, size: Number(rows[0].size) }; const res = await post().send({ query: UPDATE_PRODUCT, - variables: { input: { id: 1, productPatch: { photo } } } + variables: { input: { id: 1, productPatch: { photo: { ...projection, caption: 'mine' } } } } }); expect(res.status).toBe(200); expect(res.body.errors).toBeUndefined(); - expect(res.body.data.updateProduct.product.photo).toEqual(photo); + expect(res.body.data.updateProduct.product.photo).toMatchObject({ ...projection, caption: 'mine' }); }); it('refuses a multipart body on /graphql', async () => {