F21: bound and serialize Graphile builds with watchdog and drain

.github/workflows/run-tests.yaml

@@ -113,7 +113,7 @@ jobs:
           - batch: graphql
             packages: 'graphql/query graphql/codegen'
           - batch: graphile-unit
-            packages: 'graphile/graphile-plugin-utils graphile/graphile-cache graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin graphile/graphile-storage-registry'
+            packages: 'graphile/graphile-plugin-utils graphile/graphile-realtime-subscriptions graphile/graphile-sql-expression-validator graphile/graphile-upload-plugin graphile/graphile-storage-registry'
           - batch: agentic
             packages: 'agentic/protocol agentic/agentic-kit agentic/agent agentic/harness agentic/permissions agentic/chat agentic/cli agentic/db-tools agentic/pi agentic/dsh agentic/react agentic/agentic-server agentic/anthropic agentic/openai agentic/ollama agentic/run-log agentic/metering agentic/agent-conversation agentic/pi-host agentic/run-log-client agentic/run-log-gate agentic/machine-protocol agentic/machine-runner agentic/agent-cli'
           - batch: pgpm-unit
@@ -246,7 +246,7 @@ jobs:
           - batch: pg-graphql
             packages: 'graphile/graphile-search graphile/graphile-function-source graphile/graphile-ltree graphile/graphile-bulk-mutations graphile/graphile-function-bindings graphile/graphile-history graphile/graphile-meta graphile/graphile-schema graphql/orm-test graphql/test graphql/playwright-test'
           - batch: pg-graphile-extras
-            packages: 'graphile/graphile-i18n graphile/graphile-pg-aggregates graphile/graphile-query graphile/graphile-realtime-test'
+            packages: 'graphile/graphile-cache graphile/graphile-i18n graphile/graphile-pg-aggregates graphile/graphile-query graphile/graphile-realtime-test'
 
     env:
       PGHOST: localhost

graphile/graphile-cache/README.md

@@ -37,6 +37,99 @@ When you import this package, it automatically registers a cleanup callback with
 
 ## Usage
 
+### Exact build identity
+
+The GraphQL server and Explorer cache by `createGraphileBuildCacheKey(domain,
+snapshot)`. A key includes the effective build inputs, the exact pool reference,
+and the configuration owner. Server and Explorer use separate domains. Plain
+data is copied with `snapshotGraphileBuildValue` before it is used for both the
+fingerprint and the preset; opaque objects and functions retain reference
+identity. Keys are HMAC fingerprints with a process-local secret and must not be
+persisted or compared across processes.
+
+Entries carry `serviceKey`, `databaseId` (when known), and `poolKey` separately
+from the fingerprint. Use `clearGraphileEntriesForService`,
+`clearGraphileEntriesForDatabase`, or `clearGraphileEntriesForPool` to invalidate
+all build variants for that owner. Pool cleanup matches `poolKey` exactly.
+
+The server consumes the API, physical pool, service key, and transaction settings
+from `req.constructive`. Request settings remain request-scoped and are not
+captured in the shared schema. Build refusals retain their canonical error code
+and HTTP status; unknown failures return a generic 500 response with the original
+cause kept internally.
+
+Bulk disposal drains report all failures for generations scheduled before the
+call. Work scheduled later belongs to the next drain. An observed bulk failure
+is acknowledged for reporting, while its exact-entry release promise remains
+rejected; acknowledging an error never establishes successful resource release.
+
+### Resident admission
+
+Server and Explorer builds use `buildAdmittedGraphileInstance(metadata, factory)`.
+It reserves capacity before the factory allocates services, waits for an evicted
+entry's public release interfaces, and publishes only after the ready result passes its heap check.
+Upstream background UNLISTEN/client return is outside that public release boundary.
+The count includes reservations, residents, and disposal in progress. A disposal
+failure blocks further admission instead of treating uncertain resources as free.
+This process-wide fence requires a process restart. Disposal counters track pending
+work and can reach zero after a rejected release; neither emptying the cache nor
+reconfiguring limits proves that the failed release reclaimed its resources.
+The public release interfaces provide no verified retry-completion signal, so
+automatically reopening admission would abandon the capacity guarantee.
+
+`configureGraphileAdmission` accepts `max`, `ttl`, `heapMaxBytes`, and
+`buildReserveBytes`. The server exposes these as `graphile.cache` options and
+`GRAPHILE_CACHE_MAX`, `GRAPHILE_CACHE_TTL_MS`, `GRAPHILE_CACHE_HEAP_MAX_BYTES`, and
+`GRAPHILE_CACHE_BUILD_RESERVE_BYTES`. `graphql-env` owns environment parsing and
+defaults; runtime options override environment and file configuration. This
+package reads no environment variables. Defaults retain the 50-entry ceiling,
+use 85% of the V8 heap limit as a watermark, and reserve 64 MiB per pending build.
+The first owner can replace the fallback limits, including raising the resident
+ceiling above 50. Later owners use the strictest limits. TTL changes require an
+empty cache; lowering capacity below owned residents/reservations is rejected.
+TTL expiry remains lazy, with expired entries reclaimed before admission. This is a conservative
+process-heap admission check, not an exact per-instance memory measurement or an
+RSS limit; garbage collection may delay admission after eviction. Raw
+`graphileCache.set` is a low-level API and does not reserve capacity.
+
+### Exact-key build flights
+
+Server and Explorer use the shared `graphileBuildFlights.getOrCreate` owner.
+It registers a flight synchronously, shares its promise for an exact build key,
+and publishes through resident admission only after readiness. A failed flight
+is shared with its current callers; a later request may start a new attempt.
+There is no implicit retry by joined callers.
+
+Preparation scopes fence asynchronous metadata lookup before the exact key is
+known. Service/database/pool invalidation, key deletion, and cache clear fence
+old work before removing residents. Late results are disposed instead of
+published, and cannot erase a replacement flight. Closing caches rejects callers
+promptly while draining the underlying work and its cleanup before pool close.
+`reopenGraphileBuilds` only reopens a fully drained registry.
+
+### Global build coordination
+
+Every unique admitted build runs through one process-wide coordinator. It uses
+one active permit and a bounded FIFO queue (default 16); same-key callers join
+the existing flight without consuming another queue slot. The permit covers
+capacity admission and eviction waits, preset allocation, readiness, publication,
+and failed-generation cleanup. Overflow refuses work before admission.
+
+`graphile.build` / `configureGraphileBuilds` configure `queueMax`, `watchdogMs`
+(default 300000), and `shutdownTimeoutMs` (default 30000). Corresponding environment
+variables are `GRAPHILE_BUILD_QUEUE_MAX`, `GRAPHILE_BUILD_WATCHDOG_MS`, and
+`GRAPHILE_BUILD_SHUTDOWN_TIMEOUT_MS`. Multiple owners retain the strictest limits;
+active work cannot have its watchdog policy changed.
+
+The watchdog fences publication before rejecting active and queued callers. It
+cannot cancel JavaScript work: the permit and any reservation remain owned until
+the actual task and cleanup settle. Watchdog, drain timeout, or failed cleanup
+permanently makes the coordinator unavailable; process restart is required.
+`closeGraphileBuilds` returns false if its single deadline expires, and
+`closeAllCaches` throws without closing pools in that case. Server cache shutdown
+fences builds before waiting for HTTP requests that may be awaiting those builds.
+This is build drain; full request and WebSocket retirement is a separate concern.
+
 ### Basic Usage
 
 ```typescript

graphile/graphile-cache/__tests__/graphile-cache.test.ts

@@ -5,6 +5,7 @@ import type { GraphileCacheEntry } from '../src';
 import {
   cacheGraphileInstance,
   closeAllCaches,
+  disposeUncachedEntry,
   getCacheStats,
   graphileCache,
   GraphileCacheBudgetError} from '../src';
@@ -50,12 +51,16 @@ describe('graphile cache heap budget', () => {
     const small = entry('small', 2 * MiB);
     await cacheGraphileInstance('small', small.entry);
     const huge = entry('huge', 11 * MiB);
+    const releasePresetServices = jest.fn(async () => undefined);
+    huge.entry.releasePresetServices = releasePresetServices;
 
     await expect(cacheGraphileInstance('huge', huge.entry)).rejects.toThrow(
       GraphileCacheBudgetError
     );
 
+    await disposeUncachedEntry(huge.entry);
     expect(huge.release).toHaveBeenCalledTimes(1);
+    expect(releasePresetServices).toHaveBeenCalledTimes(1);
     expect(graphileCache.has('huge')).toBe(false);
     expect(graphileCache.has('small')).toBe(true);
   });

graphile/graphile-cache/jest.config.js

@@ -7,7 +7,8 @@ module.exports = {
       'ts-jest',
       {
         babelConfig: false,
-        tsconfig: 'tsconfig.json',
+        // ts-jest requires isolated transformation for NodeNext package exports.
+        tsconfig: { isolatedModules: true },
       },
     ],
   },

graphile/graphile-cache/package.json

@@ -30,6 +30,7 @@
   },
   "dependencies": {
     "12factor-env": "workspace:^",
+    "@constructive-io/errors": "workspace:^",
     "@pgpmjs/logger": "workspace:^",
     "express": "^5.2.1",
     "grafserv": "1.0.1",
@@ -42,6 +43,7 @@
     "@types/express": "^5.0.6",
     "makage": "^0.8.0",
     "nodemon": "^3.1.14",
+    "pgsql-test": "workspace:^",
     "ts-node": "^10.9.2"
   },
   "keywords": [

graphile/graphile-cache/src/__tests__/admission-watchdog.test.ts

@@ -0,0 +1,56 @@
+import {
+  clearGraphileCache,
+  closeGraphileBuilds,
+  configureGraphileAdmission,
+  configureGraphileBuilds,
+  getCacheStats,
+  getGraphileBuildStats,
+  graphileBuildFlights,
+  type GraphileCacheEntry
+} from '../graphile-cache';
+
+const metadata = (cacheKey: string) => ({ cacheKey, serviceKey: cacheKey, poolKey: 'pool' });
+const entry = (cacheKey: string, release: () => Promise<void>): GraphileCacheEntry => ({
+  cacheKey, heapBytes: 1024 * 1024, createdAt: Date.now(), pgl: { release }, serv: {}, handler: {}, httpServer: { listening: false }
+} as unknown as GraphileCacheEntry);
+
+it('bounds and watches an eviction wait before a new factory can allocate', async () => {
+  jest.useFakeTimers();
+  configureGraphileAdmission({ max: 1, buildReserveBytes: 0 });
+  configureGraphileBuilds({ queueMax: 1, watchdogMs: 10 });
+  let finishRelease!: () => void;
+  let releaseStarted!: () => void;
+  const releasing = new Promise<void>((resolve) => { releaseStarted = resolve; });
+  const release = new Promise<void>((resolve) => { finishRelease = resolve; });
+  const resident = entry('resident', async () => { releaseStarted(); await release; });
+  const create = jest.fn(async () => entry('replacement', async (): Promise<void> => undefined));
+  const queuedCreate = jest.fn(async () => entry('queued', async (): Promise<void> => undefined));
+  try {
+    await graphileBuildFlights.getOrCreate(metadata('resident'), async () => resident);
+    const replacing = graphileBuildFlights.getOrCreate(metadata('replacement'), create);
+    const replacementFailure = expect(replacing).rejects.toMatchObject({ code: 'SCHEMA_BUILD_STUCK' });
+    await releasing;
+    const queued = graphileBuildFlights.getOrCreate(metadata('queued'), queuedCreate);
+    const queueFailure = expect(queued).rejects.toMatchObject({ code: 'SCHEMA_BUILD_STUCK' });
+    const overflow = jest.fn();
+    await expect(graphileBuildFlights.getOrCreate(metadata('overflow'), overflow))
+      .rejects.toMatchObject({ code: 'SCHEMA_BUILD_QUEUE_FULL' });
+    expect(getGraphileBuildStats()).toMatchObject({ active: 1, queued: 1 });
+    expect(getCacheStats()).toMatchObject({ reserved: 0, size: 0, disposing: 1 });
+    expect(create).not.toHaveBeenCalled();
+    expect(overflow).not.toHaveBeenCalled();
+
+    await jest.advanceTimersByTimeAsync(10);
+    await Promise.all([replacementFailure, queueFailure]);
+    expect(getGraphileBuildStats()).toMatchObject({ active: 1, queued: 0, state: 'stuck' });
+    finishRelease();
+    expect(await closeGraphileBuilds(100)).toBe(true);
+    expect(create).not.toHaveBeenCalled();
+    expect(queuedCreate).not.toHaveBeenCalled();
+    expect(getCacheStats()).toMatchObject({ reserved: 0, disposing: 0, size: 0 });
+  } finally {
+    finishRelease();
+    await clearGraphileCache();
+    jest.useRealTimers();
+  }
+});

graphile/graphile-cache/src/__tests__/admission.test.ts

@@ -0,0 +1,86 @@
+import { GraphileAdmission } from '../admission';
+
+const deferred = () => {
+  let resolve!: () => void;
+  const promise = new Promise<void>((r) => { resolve = r; });
+  return { promise, resolve };
+};
+
+describe('resident admission', () => {
+  it('lets the first owner replace fallback count and heap limits', () => {
+    const admission = new GraphileAdmission({ occupied: () => 0, evict: async () => false }, 10, () => 10, 100);
+    admission.configure({ max: 20, heapMaxBytes: 200, buildReserveBytes: 10 });
+    expect(admission.stats).toMatchObject({ max: 20, heapMaxBytes: 200 });
+    admission.configure({ max: 30, heapMaxBytes: 300 });
+    expect(admission.stats).toMatchObject({ max: 20, heapMaxBytes: 200 });
+  });
+
+  it('does not reuse capacity until eviction completes, including concurrent admissions', async () => {
+    const disposal = deferred();
+    let occupied = 1;
+    const evict = jest.fn(async () => {
+      if (!occupied) return false;
+      await disposal.promise;
+      occupied--;
+      return true;
+    });
+    const admission = new GraphileAdmission({ occupied: () => occupied, evict }, 1, () => 10, 100);
+    admission.configure({ buildReserveBytes: 10 });
+    let admitted = false;
+    const first = admission.reserve().then((reservation) => { admitted = true; return reservation; });
+    const second = admission.reserve();
+    const refusal = expect(second).rejects.toMatchObject({ code: 'SCHEMA_CAPACITY_EXHAUSTED' });
+    await Promise.resolve();
+    expect(admitted).toBe(false);
+    disposal.resolve();
+    const reservation = await first;
+    await refusal;
+    expect(admission.stats.reserved).toBe(1);
+    reservation.release();
+    reservation.release();
+    expect(admission.stats.reserved).toBe(0);
+  });
+
+  it('reserves independent build headroom before admitting concurrent builders', async () => {
+    const admission = new GraphileAdmission({ occupied: () => 0, evict: async () => false }, 10, () => 60, 100);
+    admission.configure({ buildReserveBytes: 30 });
+    const first = await admission.reserve();
+    await expect(admission.reserve()).rejects.toMatchObject({ code: 'SCHEMA_CAPACITY_EXHAUSTED' });
+    first.release();
+    const next = await admission.reserve();
+    next.release();
+  });
+
+  it('refuses publication after a build exceeds the heap watermark', async () => {
+    let heap = 10;
+    const admission = new GraphileAdmission({ occupied: () => 0, evict: async () => false }, 2, () => heap, 100);
+    admission.configure({ buildReserveBytes: 10 });
+    const reservation = await admission.reserve();
+    heap = 101;
+    expect(() => reservation.assertPublishable()).toThrow('Schema capacity');
+    reservation.release();
+  });
+
+  it('does not recover from failed disposal when ownership counters reach zero', async () => {
+    const error = new Error('release failed');
+    const admission = new GraphileAdmission({ occupied: () => 0, evict: async () => false }, 2, () => 10, 100);
+    admission.configure({ buildReserveBytes: 10 });
+    const reservation = await admission.reserve();
+    admission.fail(error);
+    expect(() => reservation.assertPublishable()).toThrow(error);
+    await expect(admission.reserve()).rejects.toBe(error);
+    reservation.release();
+    expect(admission.stats).toMatchObject({ reserved: 0, failed: true });
+    admission.configure({ buildReserveBytes: 10 });
+    await expect(admission.reserve()).rejects.toBe(error);
+  });
+
+  it('combines process owners conservatively and rejects impossible or malformed budgets', () => {
+    const admission = new GraphileAdmission({ occupied: () => 0, evict: async () => false }, 10, () => 10, 100);
+    admission.configure({ max: 4, buildReserveBytes: 10 });
+    admission.configure({ max: 8, heapMaxBytes: 90, buildReserveBytes: 20 });
+    expect(admission.stats).toMatchObject({ max: 4, heapMaxBytes: 90, buildReserveBytes: 20 });
+    expect(() => admission.configure({ max: 0 })).toThrow();
+    expect(() => admission.configure({ buildReserveBytes: 90 })).toThrow();
+  });
+});