diff --git a/ts/packages/agents/browser/src/views/server/features/views/viewRoutes.mts b/ts/packages/agents/browser/src/views/server/features/views/viewRoutes.mts index 4d439e23c5..744aee0f66 100644 --- a/ts/packages/agents/browser/src/views/server/features/views/viewRoutes.mts +++ b/ts/packages/agents/browser/src/views/server/features/views/viewRoutes.mts @@ -12,7 +12,7 @@ import type { UnderstandingRequest, UnderstandingResult, } from "@typeagent/memory-service"; -import { memoryPerformanceExperiments } from "@typeagent/memory-service"; +import { memoryPerformanceExperiments } from "@typeagent/memory-service/performance-experiments"; import { understandingRequestSchema, understandingResultSchema, diff --git a/ts/packages/agents/browser/test/viewLifecycle.test.mjs b/ts/packages/agents/browser/test/viewLifecycle.test.mjs index bd880630e4..03f5ed6e62 100644 --- a/ts/packages/agents/browser/test/viewLifecycle.test.mjs +++ b/ts/packages/agents/browser/test/viewLifecycle.test.mjs @@ -2,6 +2,7 @@ // Licensed under the MIT License. import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; import { once } from "node:events"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -39,6 +40,52 @@ function createContext(folder) { }; } +test("view routes load experiment configuration without the memory runtime", () => { + const routes = new URL( + "../dist/views/server/features/views/viewRoutes.mjs", + import.meta.url, + ); + execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import assert from "node:assert/strict"; + import { registerHooks } from "node:module"; + registerHooks({ + resolve(specifier, context, nextResolve) { + assert.ok( + ![ + "@typeagent/memory-service", + "@typeagent/knowpro", + "@typeagent/aiclient", + ].includes(specifier), + "View startup must not load " + specifier + + " from " + context.parentURL, + ); + return nextResolve(specifier, context); + }, + }); + const routes = await import(${JSON.stringify(routes.href)}); + assert.equal(typeof routes.registerViewRoutes, "function"); + const { memoryPerformanceExperiments } = await import( + "@typeagent/memory-service/performance-experiments" + ); + assert.deepEqual( + [...memoryPerformanceExperiments("source-preview")], + ["source-preview"], + ); + assert.throws( + () => memoryPerformanceExperiments("topic-parallel"), + /Unknown memory performance experiment/, + ); + `, + ], + { cwd: new URL("..", import.meta.url), timeout: 30_000 }, + ); +}); + test( "view host readiness shares startup and cleans up after restart", { timeout: 20_000 }, diff --git a/ts/packages/memory/client/src/viewDetailProtocol.ts b/ts/packages/memory/client/src/viewDetailProtocol.ts index 0d18606f12..3e891a225c 100644 --- a/ts/packages/memory/client/src/viewDetailProtocol.ts +++ b/ts/packages/memory/client/src/viewDetailProtocol.ts @@ -6,10 +6,12 @@ import { z } from "zod"; import { viewDetailMaxChars, viewDetailPageChars, - type ViewDetailPage, - type ViewDetailPageRequest, - type ViewDetailReadRequest, - type ViewVersion, +} from "@typeagent/memory-service/view-detail"; +import type { + ViewDetailPage, + ViewDetailPageRequest, + ViewDetailReadRequest, + ViewVersion, } from "@typeagent/memory-service"; import { viewReadRequestSchema, viewVersionSchema } from "./viewProtocol.js"; diff --git a/ts/packages/memory/service/README.md b/ts/packages/memory/service/README.md index c3d2a1d253..363f601fa2 100644 --- a/ts/packages/memory/service/README.md +++ b/ts/packages/memory/service/README.md @@ -590,21 +590,26 @@ These switches do not alter models or relax required readiness, publication, editorial, correction or forgetting checks. They are not included in the existing experiment aliases. A combined treatment is not an isolated-factor measurement. -| Switch | Behavior | -| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `fast-models` | Separate fast query translation and initial reading; refinement and ingestion extraction retain their existing models. Requires an explicitly configured `TYPEAGENT_MEMORY_FAST_MODEL` or `fastQueryModelEndpoint`. Injected reader models retain precedence. Custom index factories are rejected rather than silently ignoring translation experiments. | -| `brief-reader` | A 700-token soft initial target with unchanged structured output and grounding requirements. | -| `source-preview` | An optional model-free, provisional lexical source phase before semantic retrieval. At most six 600-character fragments; not a validated answer or a usable follow-on/module handle. | -| `refinement-fallback` | Revalidate and retain an initial explanation only after an explicitly retryable model failure/deadline. Return `refinementFailed: true`, incomplete/partial status and warnings. Privacy, cancellation, scope and malformed-output errors still fail. | -| `translation-cache` | Completed successes only, keyed by corpus, fixed model, schema, exact question and full prompt context. Five-minute expiry; at most 128 entries / 1 MiB payload / 64 KiB per entry. Relative-date questions bypass caching; privacy changes and close purge it. Concurrent callers retain independent cancellation. | -| `evidence-leases` | Tolerate unrelated index-version changes for at most 120 seconds from original preparation, only after complete exact-evidence revalidation. Later phases cannot renew the lease. Forget, clear, suppression and changed references remain immediate. This is not whole-index MVCC. | -| `graph-parallel` | Existing graph projections plus ordered support queries at concurrency two. All siblings settle before a failed batch leaves scope. | -| `foreground-priority` | Defer new background maintenance for a shared maximum ten seconds, with a one-second cooldown under sustained requests. Does not cancel running work or defer explicit writes. | -| `lazy-collections` | Single-flight per-collection recovery on first access, with pending removal/clear recovery still eager. Unopened collections join completion-paced background warmup rather than all loading during discovery. | -| `indexing-parallel` | Default generative topic speculation at concurrency two (explicit topic settings retain precedence); two independent source synopses per batch; isolated prepared message embeddings overlap knowledge extraction. Source publication and topic commits remain serialized, stale inputs are revalidated, and started siblings settle before failure leaves scope. Embedding overlap requires `TYPEAGENT_MEMORY_PREPARED_INDEXING=1` and the built-in corpus index; legacy indexing and custom index factories are unchanged. | -| `topic-parallel` | Only the topic-speculation component of `indexing-parallel`; explicit topic settings still take precedence. | -| `synopsis-parallel` | Only the bounded two-source synopsis component of `indexing-parallel`. | -| `embedding-parallel` | Only the prepared-message embedding overlap component of `indexing-parallel`; prepared indexing and the built-in index remain prerequisites. | +Consumers that only parse these switches can import +`memoryPerformanceExperiments` from `@typeagent/memory-service/performance-experiments`. +This shares the same validation without loading the memory/indexing runtime during +browser view-server startup. The `@typeagent/memory-service/view-detail` entry point +similarly exposes the paging contract and bounds without loading the service. + +| Switch | Behavior | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `fast-models` | Separate fast query translation and initial reading; refinement and ingestion extraction retain their existing models. Requires an explicitly configured `TYPEAGENT_MEMORY_FAST_MODEL` or `fastQueryModelEndpoint`. Injected reader models retain precedence. Custom index factories are rejected rather than silently ignoring translation experiments. | +| `brief-reader` | A 700-token soft initial target with unchanged structured output and grounding requirements. | +| `source-preview` | An optional model-free, provisional lexical source phase before semantic retrieval. At most six 600-character fragments; not a validated answer or a usable follow-on/module handle. | +| `refinement-fallback` | Revalidate and retain an initial explanation only after an explicitly retryable model failure/deadline. Return `refinementFailed: true`, incomplete/partial status and warnings. Privacy, cancellation, scope and malformed-output errors still fail. | +| `translation-cache` | Completed successes only, keyed by corpus, fixed model, schema, exact question and full prompt context. Five-minute expiry; at most 128 entries / 1 MiB payload / 64 KiB per entry. Relative-date questions bypass caching; privacy changes and close purge it. Concurrent callers retain independent cancellation. | +| `evidence-leases` | Tolerate unrelated index-version changes for at most 120 seconds from original preparation, only after complete exact-evidence revalidation. Later phases cannot renew the lease. Forget, clear, suppression and changed references remain immediate. This is not whole-index MVCC. | +| `graph-parallel` | Existing graph projections plus ordered support queries at concurrency two. All siblings settle before a failed batch leaves scope. | +| `foreground-priority` | Defer new background maintenance for a shared maximum ten seconds, with a one-second cooldown under sustained requests. Does not cancel running work or defer explicit writes. | +| `lazy-collections` | Single-flight per-collection recovery on first access, with pending removal/clear recovery still eager. Unopened collections join completion-paced background warmup rather than all loading during discovery. | +| `indexing-parallel` | Two independent source synopses per batch; isolated prepared message embeddings overlap knowledge extraction. Generative topics always use frozen two-packet preparation independently of this switch. Source publication and topic commits remain serialized, stale inputs are revalidated, and started siblings settle before failure leaves scope. Embedding overlap requires `TYPEAGENT_MEMORY_PREPARED_INDEXING=1` and the built-in corpus index; legacy indexing and custom index factories are unchanged. | +| `synopsis-parallel` | Only the bounded two-source synopsis component of `indexing-parallel`. | +| `embedding-parallel` | Only the prepared-message embedding overlap component of `indexing-parallel`; prepared indexing and the built-in index remain prerequisites. | Fast model deployments also need verified `modelRequestConstraints` context, input and full output capacities for uncapped request admission. A soft reader @@ -827,6 +832,19 @@ retains the legacy raw-summary path for explicit diagnostic/custom-host use. The recipe/fingerprint distinguishes these policies, so old raw snapshots cannot silently qualify as current curated knowledge. +Generative curation always prepares up to two frozen raw-only packets concurrently, +then reconciles and commits each complete window serially. Prior deltas and compact +wire IDs default to true. Packet input-token targets and reserved prior fractions +remain optional; without them, existing node/character packing applies. +The former serial/speculative scheduler selection and `topic-parallel` experiment +have been removed. The `concurrency` and `frozenPreparation` programmatic settings, +`TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` and +`TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION` now fail explicitly. +Remove these controls instead of selecting another scheduling mode. +The `curated-lazy-v10:` recipe invalidates older-policy progress and receipts; +do not relabel saved artifacts. See the [topic curation guide](../../../tools/memory-maintenance-qualification/topic-curation-README.md) +for remaining settings, checkpoint reuse and replay migration. + The LLM decides semantic equivalence, compatible scope and whether a concept has a substantive, specific consumer purpose. Host rules validate supplied identities, exact membership accounting, cited sources and bounded payloads; @@ -876,7 +894,7 @@ catalog synchronization is still full-catalog work, not an incremental operation Set `FileMemoryServiceOptions.decisionTopics: true`, or `TYPEAGENT_MEMORY_DECISION_TOPICS=true` in the native/MCP/CLI/benchmark host, to select the experimental `curated-decision-v2` pipeline. The default remains -the generative `curated-lazy-v6` policy. Decision curation requires curated +the generative `curated-lazy-v10:` policy. Decision curation requires curated topics and explicitly rejects a conflicting custom `generateTopics` adapter; ordinary custom-adapter receiver/cancellation behavior is unchanged. diff --git a/ts/packages/memory/service/package.json b/ts/packages/memory/service/package.json index ef8e82f509..5c37aec7cc 100644 --- a/ts/packages/memory/service/package.json +++ b/ts/packages/memory/service/package.json @@ -15,10 +15,12 @@ ".": "./dist/index.js", "./rpc": "./dist/rpcFacade.js", "./benchmark": "./dist/benchmarkBridge.js", + "./performance-experiments": "./dist/memoryPerformanceExperimentConfig.js", "./agent-edition-validation": "./dist/agentEditionValidation.js", "./procedure-markdown": "./dist/procedureMarkdown.js", "./procedureMarkdown": "./dist/procedureMarkdown.js", "./view-text": "./dist/viewText.js", + "./view-detail": "./dist/viewDetail.js", "./view-types": "./dist/viewTypes.js", "./view-packages": "./dist/viewPackages.js" }, diff --git a/ts/packages/memory/service/src/fileMemoryService.ts b/ts/packages/memory/service/src/fileMemoryService.ts index 35576dc9af..d4bb8fd929 100644 --- a/ts/packages/memory/service/src/fileMemoryService.ts +++ b/ts/packages/memory/service/src/fileMemoryService.ts @@ -904,18 +904,9 @@ function configuredTopicCuration( options: FileMemoryServiceOptions, decisionTopics: boolean, curatedTopics: boolean, - parallel: boolean, ): TopicCurationSettings | undefined { - const parallelDefaults = - parallel && - !decisionTopics && - curatedTopics && - !options.viewSynthesisAdapter?.generateTopics; - if (!options.topicCuration && !parallelDefaults) return undefined; - const settings: TopicCurationSettings = { - ...(parallelDefaults ? { concurrency: 2 } : {}), - ...options.topicCuration, - }; + if (!options.topicCuration) return undefined; + const settings = { ...options.topicCuration }; validateTopicCurationSettings(settings); if ( decisionTopics || @@ -1481,18 +1472,11 @@ function configuredTopicWatermark( options: FileMemoryServiceOptions, curated: boolean, decision: boolean, - settings: TopicCurationSettings | undefined, ): boolean { if (options.topicWatermark !== true) return false; - if ( - !options.topicCurationReuse || - !curated || - decision || - ((settings?.concurrency ?? 1) !== 1 && - settings?.frozenPreparation !== true) - ) + if (!options.topicCurationReuse || !curated || decision) throw new Error( - "Topic watermark requires curation reuse and ordered generative curated topics; parallelism requires frozen preparation", + "Topic watermark requires curation reuse and generative curated topics", ); return true; } @@ -2035,13 +2019,11 @@ export class FileMemoryService options, this.decisionTopics, this.curatedTopics, - this.experiments.has("topic-parallel"), ); this.topicWatermark = configuredTopicWatermark( options, this.curatedTopics, this.decisionTopics, - this.topicCuration, ); this.topicPolicy = topicCurationPolicy( this.curatedTopics, @@ -11862,9 +11844,7 @@ export class FileMemoryService model: this.topicModelIdentity(), curationRecipe: this.decisionTopics ? topicDecisionRecipe - : this.curatedTopics - ? this.topicPolicy - : "curated-lazy-v6", + : this.topicPolicy, topicCurationReuse: this.topicCurationReuse, topicWatermark: this.topicWatermark, ...(this.topicWatermark && state.continuation diff --git a/ts/packages/memory/service/src/memoryPerformanceExperimentConfig.ts b/ts/packages/memory/service/src/memoryPerformanceExperimentConfig.ts new file mode 100644 index 0000000000..778bf02204 --- /dev/null +++ b/ts/packages/memory/service/src/memoryPerformanceExperimentConfig.ts @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const memoryPerformanceExperimentNames = [ + "fast-models", + "brief-reader", + "source-preview", + "refinement-fallback", + "translation-cache", + "evidence-leases", + "graph-parallel", + "foreground-priority", + "lazy-collections", + "indexing-parallel", + "synopsis-parallel", + "embedding-parallel", + "synopsis-checkpoint-priority", + "early-required-failure", + "wiki-full-context", + "wiki-stable-evidence", + "wiki-utility-first", + "wiki-coalesced-discovery", + "wiki-grouped-contributions", + "family-frozen-preparation", +] as const; + +export type MemoryPerformanceExperiment = + (typeof memoryPerformanceExperimentNames)[number]; + +export function memoryPerformanceExperiments( + value = process.env.TYPEAGENT_MEMORY_PERF_EXPERIMENTS ?? "", +): ReadonlySet { + const result = new Set(); + for (const name of value + .split(",") + .map((item) => item.trim()) + .filter(Boolean)) { + const names = + name === "p2" + ? memoryPerformanceExperimentNames.slice(0, 5) + : name === "p3" + ? memoryPerformanceExperimentNames.slice(5, 9) + : memoryPerformanceExperimentNames.filter( + (item) => item === name, + ); + if (!names.length) + throw new Error(`Unknown memory performance experiment: ${name}`); + for (const item of names) result.add(item); + } + if (result.has("indexing-parallel")) { + result.add("synopsis-parallel"); + result.add("embedding-parallel"); + } + return result; +} diff --git a/ts/packages/memory/service/src/memoryPerformanceExperiments.ts b/ts/packages/memory/service/src/memoryPerformanceExperiments.ts index 075f33abd5..251f27b309 100644 --- a/ts/packages/memory/service/src/memoryPerformanceExperiments.ts +++ b/ts/packages/memory/service/src/memoryPerformanceExperiments.ts @@ -3,32 +3,12 @@ import { timeSearchStage } from "@typeagent/knowpro"; import type { MaintenanceScheduler } from "./maintenanceScheduler.js"; -export const memoryPerformanceExperimentNames = [ - "fast-models", - "brief-reader", - "source-preview", - "refinement-fallback", - "translation-cache", - "evidence-leases", - "graph-parallel", - "foreground-priority", - "lazy-collections", - "indexing-parallel", - "topic-parallel", - "synopsis-parallel", - "embedding-parallel", - "synopsis-checkpoint-priority", - "early-required-failure", - "wiki-full-context", - "wiki-stable-evidence", - "wiki-utility-first", - "wiki-coalesced-discovery", - "wiki-grouped-contributions", - "family-frozen-preparation", -] as const; - -export type MemoryPerformanceExperiment = - (typeof memoryPerformanceExperimentNames)[number]; +import type { MemoryPerformanceExperiment } from "./memoryPerformanceExperimentConfig.js"; +export { + memoryPerformanceExperimentNames, + memoryPerformanceExperiments, + type MemoryPerformanceExperiment, +} from "./memoryPerformanceExperimentConfig.js"; export function deferSynopsisForCheckpoint( scheduler: Pick, @@ -66,34 +46,6 @@ export function resolveInitialReaderModelEndpoint( ); } -export function memoryPerformanceExperiments( - value = process.env.TYPEAGENT_MEMORY_PERF_EXPERIMENTS ?? "", -): ReadonlySet { - const result = new Set(); - for (const name of value - .split(",") - .map((item) => item.trim()) - .filter(Boolean)) { - const names = - name === "p2" - ? memoryPerformanceExperimentNames.slice(0, 5) - : name === "p3" - ? memoryPerformanceExperimentNames.slice(5, 9) - : memoryPerformanceExperimentNames.filter( - (item) => item === name, - ); - if (!names.length) - throw new Error(`Unknown memory performance experiment: ${name}`); - for (const item of names) result.add(item); - } - if (result.has("indexing-parallel")) { - result.add("topic-parallel"); - result.add("synopsis-parallel"); - result.add("embedding-parallel"); - } - return result; -} - export async function boundedSupportMap( items: readonly T[], concurrency: 1 | 2, diff --git a/ts/packages/memory/service/src/topicCapture.ts b/ts/packages/memory/service/src/topicCapture.ts index 6b24a9004f..deae4cbdd1 100644 --- a/ts/packages/memory/service/src/topicCapture.ts +++ b/ts/packages/memory/service/src/topicCapture.ts @@ -17,6 +17,7 @@ import { viewHash } from "./viewMerge.js"; import { initialTopicNodes, validateTopicSnapshot } from "./topicSnapshot.js"; import { topicClusterSignature } from "./topicDecision.js"; import { topicMaintenanceSignature } from "./topicClusterMaintenance.js"; +import { topicCurationPolicy } from "./topicCurationSettings.js"; export function supportsTopicIndex( index: CorpusIndex, @@ -113,7 +114,7 @@ function reusableTopicPackets( !previous.curationInputFingerprint || previous.corpusId !== options.corpusId || previous.recipe !== - `${options.model}:editorial-v1:${options.curationRecipe ?? "curated-lazy-v6"}` || + `${options.model}:editorial-v1:${options.curationRecipe ?? topicCurationPolicy(true)}` || previous.curationSuppressionFingerprint !== viewHash(options.suppressions) || !topicSourceClosurePreserved( @@ -171,7 +172,7 @@ function curationDependencyFingerprint( occurrences, suppressions: options.suppressions, model: options.model, - curationRecipe: options.curationRecipe ?? "curated-lazy-v6", + curationRecipe: options.curationRecipe ?? topicCurationPolicy(true), events: options.influenceEvents ?? [], }); } @@ -188,7 +189,7 @@ function reusableTopicCuration( !current || current.corpusId !== corpusId || current.recipe !== - `${model}:editorial-v1:${options.curationRecipe ?? "curated-lazy-v6"}` || + `${model}:editorial-v1:${options.curationRecipe ?? topicCurationPolicy(true)}` || viewHash(current.occurrences) !== viewHash(occurrences) || !current.nodes.every((node) => node.authorship === "generated") ) diff --git a/ts/packages/memory/service/src/topicCuration.ts b/ts/packages/memory/service/src/topicCuration.ts index 2bb8152270..a3ba524e44 100644 --- a/ts/packages/memory/service/src/topicCuration.ts +++ b/ts/packages/memory/service/src/topicCuration.ts @@ -24,7 +24,19 @@ import { TopicNominationIndex, topicSourceKey, } from "./topicLookup.js"; -import { runOrderedTopicBatches } from "./topicCurationScheduler.js"; +import { yieldMaintenanceBuild } from "./maintenanceConcurrency.js"; +import { + resolveTopicCurationSettings, + topicCurationPolicy, + validateTopicCurationSettings, + type TopicCurationSettings, +} from "./topicCurationSettings.js"; +export { + resolveTopicCurationSettings, + topicCurationPolicy, + validateTopicCurationSettings, + type TopicCurationSettings, +} from "./topicCurationSettings.js"; import { compactTopicCompletion, TopicCurationTransportError, @@ -59,14 +71,6 @@ export type TopicCurationCompletion = (( inspectRequest?: TopicCurationOptions["inspectRequest"]; }; type Complete = TopicCurationCompletion; -export interface TopicCurationSettings { - inputTokenTarget?: number; - concurrency?: 1 | 2; - priorContextFraction?: number; - priorDeltas?: boolean; - compactIds?: boolean; - frozenPreparation?: boolean; -} export interface TopicCurationCall { ordinal?: number; phase?: "preparation" | "reconciliation"; @@ -103,57 +107,6 @@ export type TopicCurationCandidate = Pick< prior?: TopicConcept; }; type Candidate = TopicCurationCandidate; -export function validateTopicCurationSettings( - options: TopicCurationSettings, -): void { - if ( - (options.inputTokenTarget !== undefined && - (!Number.isSafeInteger(options.inputTokenTarget) || - options.inputTokenTarget < 1)) || - (options.concurrency !== undefined && - ![1, 2].includes(options.concurrency)) || - [ - options.priorDeltas, - options.compactIds, - options.frozenPreparation, - ].some((value) => value !== undefined && typeof value !== "boolean") || - (options.frozenPreparation === true && options.concurrency !== 2) || - (options.priorContextFraction !== undefined && - (!Number.isFinite(options.priorContextFraction) || - options.priorContextFraction < 0 || - options.priorContextFraction > 0.5 || - options.inputTokenTarget === undefined)) - ) - throw new Error("Invalid topic curation settings"); -} - -export function topicCurationPolicy( - curated: boolean, - settings?: TopicCurationSettings, -): string { - if (!curated) return "raw-v1"; - if (!settings) return "curated-lazy-v6"; - validateTopicCurationSettings(settings); - if ( - settings.priorDeltas || - settings.compactIds || - settings.frozenPreparation - ) - return `curated-lazy-v9:${viewHash([ - settings.inputTokenTarget ?? null, - settings.concurrency ?? 1, - settings.priorContextFraction ?? 0, - settings.priorDeltas ?? false, - settings.compactIds ?? false, - settings.frozenPreparation ?? false, - ]).slice(0, 16)}`; - return `curated-lazy-v8:${viewHash([ - settings.inputTokenTarget ?? null, - settings.concurrency ?? 1, - settings.priorContextFraction ?? 0, - ]).slice(0, 16)}`; -} - export function orderedTopicNodes< T extends Pick, >(nodes: readonly T[], lazy = true) { @@ -573,7 +526,7 @@ function normalizedPacketResponse( }; } -export async function curateTopics( +function createTopicCuration( snapshot: TopicSnapshot, evidenceFor: (node: TopicNode) => ViewCitation[], complete: Complete, @@ -582,15 +535,9 @@ export async function curateTopics( checkpoint?: (curation: TopicCuration) => Promise, reusable?: TopicCuration, options: TopicCurationOptions = {}, -): Promise { +) { validateTopicCurationSettings(options); - if (options.frozenPreparation && !lazy) - throw new Error("Frozen topic preparation requires lazy curation"); - if ( - (options.onPacket || options.replayPackets) && - (!lazy || - ((options.concurrency ?? 1) !== 1 && !options.frozenPreparation)) - ) + if ((options.onPacket || options.replayPackets) && !lazy) throw new Error( "Topic packet replay requires lazy curation with ordered installation", ); @@ -935,8 +882,7 @@ ${options.priorDeltas ? "Each role new node must appear EXACTLY once; unchanged } if ( checkpoint && - (options.frozenPreparation || - completed === 1 || + (completed === 1 || completed % 8 === 0 || packet.end === nodes.length) ) { @@ -945,124 +891,178 @@ ${options.priorDeltas ? "Each role new node must appear EXACTLY once; unchanged } return false; }; - if (options.frozenPreparation) - await runFrozenTopicWindows( - nodes.length, - signal, - (start, ordinal) => - prepareTopicPacket( - nodes, - start, - ordinal, - new TopicNominationIndex(), - evidenceFor, - lazy, - evidenceIndex, - options, - ), - (packets) => { - recordSearchTiming("topic.curation.frozen.window", 0, { - start: packets[0].start, - end: packets[packets.length - 1].end, - }); - for (const [dependency, value] of Object.entries({ - recipe: snapshot.recipe, - prior: result, - preparation: packets.map((packet) => packet.fingerprint), - })) - recordSearchTiming( - `topic.curation.frozen.window.${dependency}`, - 0, - { fingerprint: viewHash(value) }, - ); - const batch: Candidate[] = [ - ...packets.flatMap((packet) => packet.batch), - ...result.concepts.map((concept) => ({ - ...concept, - prior: concept, - })), - ]; - const supplied = [ - ...new Map( - [ - ...packets.flatMap((packet) => packet.supplied), - ...result.concepts.flatMap( - (concept) => concept.citations, - ), - ].map((citation) => [viewHash(citation), citation]), - ).values(), - ]; - const payload = curationPayload( - batch, - supplied, - evidenceIndex, - options.priorDeltas, + return { + nodes, + result, + decide, + commit, + validate: (curation: TopicCuration) => + validateTopicCurationProgress( + { ...snapshot, curationProgress: curation }, + evidenceIndex, + ), + prepare: (start: number, ordinal: number, priors = nominations) => + prepareTopicPacket( + nodes, + start, + ordinal, + priors, + evidenceFor, + lazy, + evidenceIndex, + options, + ), + combine: (packets: Packet[]) => { + recordSearchTiming("topic.curation.frozen.window", 0, { + start: packets[0].start, + end: packets[packets.length - 1].end, + }); + for (const [dependency, value] of Object.entries({ + recipe: snapshot.recipe, + prior: result, + preparation: packets.map((packet) => packet.fingerprint), + })) + recordSearchTiming( + `topic.curation.frozen.window.${dependency}`, + 0, + { fingerprint: viewHash(value) }, ); - const schema = curationSchema(batch, supplied, lazy); - return { - ...packets[0], - end: packets[packets.length - 1].end, - batch, - reconsidered: [...result.concepts], - supplied, + const batch: Candidate[] = [ + ...packets.flatMap((packet) => packet.batch), + ...result.concepts.map((concept) => ({ + ...concept, + prior: concept, + })), + ]; + const supplied = [ + ...new Map( + [ + ...packets.flatMap((packet) => packet.supplied), + ...result.concepts.flatMap( + (concept) => concept.citations, + ), + ].map((citation) => [viewHash(citation), citation]), + ).values(), + ]; + const payload = curationPayload( + batch, + supplied, + evidenceIndex, + options.priorDeltas, + ); + const schema = curationSchema(batch, supplied, lazy); + return { + ...packets[0], + end: packets[packets.length - 1].end, + batch, + reconsidered: [...result.concepts], + supplied, + payload, + schema, + ordinal: Math.ceil(packets[0].ordinal / 2), + fingerprint: viewHash([ + "frozen-window-v1", + topicCurationPolicy(true, options), + snapshot.recipe, + rules, payload, schema, - ordinal: Math.ceil(packets[0].ordinal / 2), - fingerprint: viewHash([ - "frozen-window-v1", - topicCurationPolicy(true, options), - snapshot.recipe, - rules, - payload, - schema, - result, - batch.map((node) => [ - node.id, - node.memberIds ?? [node.id], - ]), - packets.map((packet) => packet.fingerprint), - ]), - }; - }, - decide, - (packets, preparationSignal) => - reconcileFrozenTopics( - snapshot, - packets.map((packet) => packet.batch), result, - evidenceFor, - complete, - preparationSignal, - options, - curateTopics, - (curation) => - validateTopicCurationProgress( - { ...snapshot, curationProgress: curation }, - evidenceIndex, - ), - ), - commit, - ); - else - await runOrderedTopicBatches( - nodes.length, - options.concurrency ?? 1, - signal, - (start, ordinal) => - prepareTopicPacket( - nodes, - start, - ordinal, - nominations, - evidenceFor, - lazy, - evidenceIndex, - options, - ), - decide, - commit, - ); - return result; + batch.map((node) => [node.id, node.memberIds ?? [node.id]]), + packets.map((packet) => packet.fingerprint), + ]), + }; + }, + }; +} + +// Single-pass evidence work is shared by frozen preparation, reconciliation and native resolution. +export async function curateTopicEvidence( + snapshot: TopicSnapshot, + evidenceFor: (node: TopicNode) => ViewCitation[], + complete: Complete, + signal: AbortSignal, + lazy = false, + checkpoint?: (curation: TopicCuration) => Promise, + reusable?: TopicCuration, + options: TopicCurationOptions = {}, +): Promise { + const work = createTopicCuration( + snapshot, + evidenceFor, + complete, + signal, + lazy, + checkpoint, + reusable, + options, + ); + let position = 0; + let ordinal = 1; + while (position < work.nodes.length) { + signal.throwIfAborted(); + const packet = work.prepare(position, ordinal); + const value = await work.decide(packet, signal); + signal.throwIfAborted(); + if (await work.commit(packet, value, ordinal)) + await yieldMaintenanceBuild(signal); + position = packet.end; + ordinal++; + } + return work.result; +} + +export async function curateTopics( + snapshot: TopicSnapshot, + evidenceFor: (node: TopicNode) => ViewCitation[], + complete: Complete, + signal: AbortSignal, + lazy = true, + checkpoint?: (curation: TopicCuration) => Promise, + reusable?: TopicCuration, + options: TopicCurationOptions = {}, +): Promise { + if (!lazy) + throw new Error("Frozen topic preparation requires lazy curation"); + const resolved = { ...options, ...resolveTopicCurationSettings(options) }; + const work = createTopicCuration( + snapshot, + evidenceFor, + complete, + signal, + true, + undefined, + reusable, + resolved, + ); + await runFrozenTopicWindows( + work.nodes.length, + signal, + (start, ordinal) => + work.prepare(start, ordinal, new TopicNominationIndex()), + work.combine, + work.decide, + (packets, preparationSignal) => + reconcileFrozenTopics( + snapshot, + packets.map((packet) => packet.batch), + work.result, + evidenceFor, + complete, + preparationSignal, + resolved, + curateTopicEvidence, + work.validate, + ), + async (packet, value, ordinal) => { + await work.commit(packet, value, ordinal); + if (checkpoint) { + await checkpoint(structuredClone(work.result)); + await yieldMaintenanceBuild(signal); + } + }, + ); + return work.result; } export function validateTopicCurationProgress( diff --git a/ts/packages/memory/service/src/topicCurationScheduler.ts b/ts/packages/memory/service/src/topicCurationScheduler.ts deleted file mode 100644 index 21b6189447..0000000000 --- a/ts/packages/memory/service/src/topicCurationScheduler.ts +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { recordSearchTiming, timeSearchStage } from "@typeagent/knowpro"; -import { yieldMaintenanceBuild } from "./maintenanceConcurrency.js"; - -export interface OrderedTopicPacket { - start: number; - end: number; - fingerprint: string; -} - -type Outcome = - | { success: true; value: T } - | { success: false; error: unknown }; - -export async function runOrderedTopicBatches

( - count: number, - concurrency: 1 | 2, - signal: AbortSignal, - prepare: (start: number, ordinal: number) => P, - execute: (packet: P, signal: AbortSignal) => Promise, - commit: (packet: P, value: T, ordinal: number) => Promise, -): Promise { - const pending = new Map< - number, - { - packet: P; - controller: AbortController; - promise: Promise>; - attempt: number; - state: { executed: boolean; completed: boolean }; - } - >(); - const started = new Set>>(); - let failure: { error: unknown } | undefined; - let attempts = 0; - const abortEntry = ( - entry: NonNullable>, - reason: unknown, - ) => { - if (entry.controller.signal.aborted) return; - entry.controller.abort(reason); - recordSearchTiming("topic.curation.speculation.abort", 0, { - attempt: entry.attempt, - fingerprint: entry.packet.fingerprint, - executed: entry.state.executed, - completed: entry.state.completed, - }); - }; - const abort = (reason: unknown) => - pending.forEach((entry) => abortEntry(entry, reason)); - const cancelled = () => abort(signal.reason); - const checkFailure = () => { - if (failure) throw failure.error; - }; - signal.addEventListener("abort", cancelled, { once: true }); - function launch(packet: P) { - signal.throwIfAborted(); - const controller = new AbortController(); - const attempt = ++attempts; - const attributes = { attempt, fingerprint: packet.fingerprint }; - recordSearchTiming( - "topic.curation.speculation.proposed", - 0, - attributes, - ); - const state = { executed: false, completed: false }; - const promise: Promise> = Promise.resolve() - .then(() => { - controller.signal.throwIfAborted(); - state.executed = true; - recordSearchTiming( - "topic.curation.speculation.started", - 0, - attributes, - ); - return timeSearchStage( - "topic.curation.speculation.execute", - () => execute(packet, controller.signal), - attributes, - ); - }) - .then( - (value) => { - state.completed = true; - recordSearchTiming( - "topic.curation.speculation.completed", - 0, - { - ...attributes, - aborted: controller.signal.aborted, - }, - ); - return { success: true, value }; - }, - (error: unknown) => { - recordSearchTiming( - "topic.curation.speculation.failed", - 0, - { - ...attributes, - executed: state.executed, - aborted: controller.signal.aborted, - }, - "error", - ); - if (!controller.signal.aborted) { - failure ??= { error }; - abort(error); - } - return { success: false, error }; - }, - ); - const launched = { packet, controller, promise, attempt, state }; - pending.set(packet.start, launched); - started.add(promise); - void promise.then(() => started.delete(promise)); - return launched; - } - try { - let position = 0; - let ordinal = 1; - while (position < count) { - signal.throwIfAborted(); - checkFailure(); - const packet = prepare(position, ordinal); - let entry = pending.get(position); - if (entry && entry.packet.fingerprint !== packet.fingerprint) { - for (const [start, queued] of pending) { - if (start < position) continue; - recordSearchTiming( - "topic.curation.speculation.discard", - 0, - { - ordinal, - attempt: queued.attempt, - fingerprint: queued.packet.fingerprint, - executed: queued.state.executed, - completed: queued.state.completed, - reason: "packet-or-expansion-changed", - }, - ); - abortEntry(queued, new Error("Topic proposal superseded")); - pending.delete(start); - } - // Drain invalidated calls before reusing their worker slots. - await Promise.all(started); - checkFailure(); - entry = undefined; - } - entry ??= launch(packet); - let next = packet.end; - for (let slot = 1; slot < concurrency && next < count; slot++) { - const queued = - pending.get(next) ?? launch(prepare(next, ordinal + slot)); - next = queued.packet.end; - } - const outcomePromise = entry.promise; - const outcome = await timeSearchStage( - "topic.curation.wait", - () => outcomePromise, - { ordinal }, - ); - signal.throwIfAborted(); - checkFailure(); - if (!outcome.success) throw outcome.error; - const checkpointed = await timeSearchStage( - "topic.curation.commit", - () => commit(packet, outcome.value, ordinal), - { ordinal }, - ); - recordSearchTiming("topic.curation.speculation.commit", 0, { - ordinal, - attempt: entry.attempt, - fingerprint: packet.fingerprint, - outstanding: pending.size, - checkpointed: checkpointed === true, - }); - if (checkpointed) - await yieldMaintenanceBuild(signal, async () => { - await Promise.all(started); - checkFailure(); - }); - pending.delete(position); - position = packet.end; - ordinal++; - } - } finally { - abort(new Error("Topic curation stopped")); - signal.removeEventListener("abort", cancelled); - await Promise.all(started); - } -} diff --git a/ts/packages/memory/service/src/topicCurationSettings.ts b/ts/packages/memory/service/src/topicCurationSettings.ts new file mode 100644 index 0000000000..46d4ac6e98 --- /dev/null +++ b/ts/packages/memory/service/src/topicCurationSettings.ts @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { viewHash } from "./viewMerge.js"; + +export interface TopicCurationSettings { + inputTokenTarget?: number; + priorContextFraction?: number; + priorDeltas?: boolean; + compactIds?: boolean; +} + +export function validateTopicCurationSettings( + options: TopicCurationSettings, +): void { + if ( + "concurrency" in options || + "frozenPreparation" in options || + (options.inputTokenTarget !== undefined && + (!Number.isSafeInteger(options.inputTokenTarget) || + options.inputTokenTarget < 1)) || + [options.priorDeltas, options.compactIds].some( + (value) => value !== undefined && typeof value !== "boolean", + ) || + (options.priorContextFraction !== undefined && + (!Number.isFinite(options.priorContextFraction) || + options.priorContextFraction < 0 || + options.priorContextFraction > 0.5 || + options.inputTokenTarget === undefined)) + ) + throw new Error("Invalid topic curation settings"); +} + +export function resolveTopicCurationSettings( + settings: TopicCurationSettings = {}, +): TopicCurationSettings { + validateTopicCurationSettings(settings); + return { + ...settings, + priorDeltas: settings.priorDeltas ?? true, + compactIds: settings.compactIds ?? true, + }; +} + +export function topicCurationPolicy( + curated: boolean, + settings?: TopicCurationSettings, +): string { + if (!curated) return "raw-v1"; + const resolved = resolveTopicCurationSettings(settings); + return `curated-lazy-v10:${viewHash([ + resolved.inputTokenTarget ?? null, + resolved.priorContextFraction ?? 0, + resolved.priorDeltas, + resolved.compactIds, + "frozen-two-packet-windows", + ]).slice(0, 16)}`; +} diff --git a/ts/packages/memory/service/src/topicDecisionCuration.ts b/ts/packages/memory/service/src/topicDecisionCuration.ts index ae346dc672..78a143ee10 100644 --- a/ts/packages/memory/service/src/topicDecisionCuration.ts +++ b/ts/packages/memory/service/src/topicDecisionCuration.ts @@ -23,7 +23,7 @@ import { createTopicTransaction, } from "./topicDecisionTransactions.js"; import { - curateTopics, + curateTopicEvidence, validateTopicCurationProgress, type TopicCurationCompletion, } from "./topicCuration.js"; @@ -768,7 +768,7 @@ async function resolveUncertainty( return output; }; resolutionComplete.inspectRequest = complete.inspectRequest; - const resolution = await curateTopics( + const resolution = await curateTopicEvidence( { ...withoutCuration(snapshot), nodes: snapshot.nodes.filter((node) => ids.has(node.id)), diff --git a/ts/packages/memory/service/src/topicFrozenPreparation.ts b/ts/packages/memory/service/src/topicFrozenPreparation.ts index 356604332a..628eecff8c 100644 --- a/ts/packages/memory/service/src/topicFrozenPreparation.ts +++ b/ts/packages/memory/service/src/topicFrozenPreparation.ts @@ -6,13 +6,18 @@ import type { TopicCurationCandidate, TopicCurationCompletion, TopicCurationOptions, - curateTopics, + curateTopicEvidence, } from "./topicCuration.js"; import type { TopicCuration, TopicNode, TopicSnapshot } from "./topicTypes.js"; import type { ViewCitation } from "./viewTypes.js"; -import type { OrderedTopicPacket } from "./topicCurationScheduler.js"; import { viewHash } from "./viewMerge.js"; +export interface OrderedTopicPacket { + start: number; + end: number; + fingerprint: string; +} + async function parallel( work: Array<(signal: AbortSignal) => Promise>, signal: AbortSignal, @@ -83,13 +88,11 @@ export async function reconcileFrozenTopics( complete: TopicCurationCompletion, signal: AbortSignal, options: TopicCurationOptions, - curate: typeof curateTopics, + curate: typeof curateTopicEvidence, validate: (curation: TopicCuration) => void, ): Promise { const inner: TopicCurationOptions = { ...options, - frozenPreparation: false, - concurrency: 1, }; delete inner.replayPackets; delete inner.onPacket; diff --git a/ts/packages/memory/service/src/topicGeneration.ts b/ts/packages/memory/service/src/topicGeneration.ts index 17ce8f434a..18891b20b3 100644 --- a/ts/packages/memory/service/src/topicGeneration.ts +++ b/ts/packages/memory/service/src/topicGeneration.ts @@ -110,11 +110,9 @@ function configuredGenerativeSettings( (key) => ![ "inputTokenTarget", - "concurrency", "priorContextFraction", "priorDeltas", "compactIds", - "frozenPreparation", ].includes(key), ) ) diff --git a/ts/packages/memory/service/src/viewHostOptions.ts b/ts/packages/memory/service/src/viewHostOptions.ts index 3de0aad905..67990143de 100644 --- a/ts/packages/memory/service/src/viewHostOptions.ts +++ b/ts/packages/memory/service/src/viewHostOptions.ts @@ -19,6 +19,14 @@ import { function curationSettings( environment: Readonly>, ): TopicCurationSettings | undefined { + for (const name of [ + "TYPEAGENT_MEMORY_TOPIC_CONCURRENCY", + "TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION", + ]) + if (environment[name] !== undefined) + throw new Error( + `${name} has been removed; generative topics always use frozen two-packet preparation`, + ); function numeric(name: string): number | undefined { const raw = environment[name]; if (raw === undefined) return undefined; @@ -27,18 +35,13 @@ function curationSettings( return Number(raw); } const inputTokenTarget = numeric("TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS"); - const concurrency = numeric("TYPEAGENT_MEMORY_TOPIC_CONCURRENCY"); const priorContextFraction = numeric( "TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION", ); - const flags: Pick< - TopicCurationSettings, - "priorDeltas" | "compactIds" | "frozenPreparation" - > = {}; + const flags: Pick = {}; for (const [field, variable] of [ ["priorDeltas", "TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS"], ["compactIds", "TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS"], - ["frozenPreparation", "TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION"], ] as const) if (environment[variable] !== undefined) flags[field] = parseViewDraftCapability( @@ -47,17 +50,13 @@ function curationSettings( ); if ( inputTokenTarget === undefined && - concurrency === undefined && priorContextFraction === undefined && Object.keys(flags).length === 0 ) return undefined; - if (concurrency !== undefined && concurrency !== 1 && concurrency !== 2) - throw new Error("Invalid TYPEAGENT_MEMORY_TOPIC_CONCURRENCY"); const settings: TopicCurationSettings = { ...flags, ...(inputTokenTarget !== undefined ? { inputTokenTarget } : {}), - ...(concurrency !== undefined ? { concurrency } : {}), ...(priorContextFraction !== undefined ? { priorContextFraction } : {}), }; validateTopicCurationSettings(settings); diff --git a/ts/packages/memory/service/test/maintenanceBuildPriority.spec.ts b/ts/packages/memory/service/test/maintenanceBuildPriority.spec.ts index c5bd79215d..e9e948c3f3 100644 --- a/ts/packages/memory/service/test/maintenanceBuildPriority.spec.ts +++ b/ts/packages/memory/service/test/maintenanceBuildPriority.spec.ts @@ -5,7 +5,7 @@ import { createMaintenanceBuildScheduler, yieldMaintenanceBuild, } from "../src/maintenanceConcurrency.js"; -import { runOrderedTopicBatches } from "../src/topicCurationScheduler.js"; +import { runFrozenTopicWindows } from "../src/topicFrozenPreparation.js"; import { ViewBuildRunner } from "../src/viewBuilds.js"; import type { ViewBuildJob, ViewBuildSnapshot } from "../src/viewTypes.js"; @@ -362,7 +362,7 @@ test("failed checkpoint or drain does not admit a short job until native cleanup expect(entered).toEqual(["cleanup", "short"]); }); -test("generative curation drains started lookahead after a checkpoint before lending capacity", async () => { +test("frozen curation drains its window before checkpointing and lending capacity", async () => { const schedule = createMaintenanceBuildScheduler(1, false, true); const both = gate(); const slow = gate(); @@ -372,28 +372,48 @@ test("generative curation drains started lookahead after a checkpoint before len const order: string[] = []; const native = schedule( () => - runOrderedTopicBatches( + runFrozenTopicWindows( 3, - 2, new AbortController().signal, (start) => ({ start, end: start + 1, fingerprint: String(start), }), - async ({ start }) => { - active++; - if (start === 1) both.resolve(); - await both.promise; - if (start === 1) await slow.promise; - active--; - return start; + (packets) => ({ + start: packets[0].start, + end: packets[packets.length - 1].end, + fingerprint: packets + .map((packet) => packet.fingerprint) + .join(","), + }), + (_packet, _signal, work) => work(), + async (packets) => { + await Promise.all( + packets.map(async ({ start }) => { + active++; + if (start === 1) both.resolve(); + await both.promise; + if (start === 1) await slow.promise; + active--; + }), + ); + return { + concepts: [], + deferred: [ + { + nodeId: String(packets[0].start), + reason: "Synthetic checkpoint", + }, + ], + }; }, async (_packet, value) => { durable++; - order.push(`commit-${value}`); - if (value === 0) committed.resolve(); - return true; + const start = Number(value.deferred[0].nodeId); + order.push(`commit-${start}`); + if (start === 0) committed.resolve(); + await yieldMaintenanceBuild(new AbortController().signal); }, ), true, @@ -404,12 +424,12 @@ test("generative curation drains started lookahead after a checkpoint before len expect(durable).toBe(1); order.push("short"); }, false); - await committed.promise; await turn(); - expect(order).toEqual(["commit-0"]); + expect(order).toEqual([]); slow.resolve(); + await committed.promise; await Promise.all([native, short]); - expect(order).toEqual(["commit-0", "short", "commit-1", "commit-2"]); + expect(order).toEqual(["commit-0", "short", "commit-2"]); }); test("conflicting explicit reservation and priority fail rather than reserving capacity silently", () => { diff --git a/ts/packages/memory/service/test/memoryPerformanceExperiments.spec.ts b/ts/packages/memory/service/test/memoryPerformanceExperiments.spec.ts index 2397e6f681..39f6e4026e 100644 --- a/ts/packages/memory/service/test/memoryPerformanceExperiments.spec.ts +++ b/ts/packages/memory/service/test/memoryPerformanceExperiments.spec.ts @@ -105,7 +105,6 @@ describe("bounded serving experiments", () => { test.each([ "fast-models", "fast-models,embedding-parallel", - "fast-models,topic-parallel", "fast-models,synopsis-parallel", "fast-models,indexing-parallel", ])( @@ -212,7 +211,6 @@ describe("bounded serving experiments", () => { expect(memoryPerformanceExperiments("p3").size).toBe(4); expect(memoryPerformanceExperiments("p2,p3").size).toBe(9); for (const treatment of [ - "topic-parallel", "synopsis-parallel", "embedding-parallel", ] as const) { diff --git a/ts/packages/memory/service/test/topicCuration.spec.ts b/ts/packages/memory/service/test/topicCuration.spec.ts index d8145f570a..5268de5769 100644 --- a/ts/packages/memory/service/test/topicCuration.spec.ts +++ b/ts/packages/memory/service/test/topicCuration.spec.ts @@ -7,7 +7,10 @@ import { queryTopicSnapshot, validateTopicSnapshot, } from "../src/topicSnapshot.js"; -import { curateTopics } from "../src/topicCuration.js"; +import { + curateTopicEvidence as curateTopics, + curateTopics as curateFrozenTopics, +} from "../src/topicCuration.js"; import type { TopicSnapshot, TopicCuration } from "../src/topicTypes.js"; import { openai } from "@typeagent/aiclient"; import { @@ -186,7 +189,7 @@ test("context packing exceeds 24 raw nodes and accounts for all tail nodes", asy validateTopicSnapshot(value); }); -test("context-sized concurrent packets preserve all nodes when completions finish out of order", async () => { +test("context-sized frozen preparation preserves nodes when completions finish out of order", async () => { const value = snapshot(105); const sizes: number[] = []; let active = 0; @@ -195,7 +198,7 @@ test("context-sized concurrent packets preserve all nodes when completions finis const first = new Promise((resolve) => { releaseFirst = resolve; }); - value.curation = await curateTopics( + value.curation = await curateFrozenTopics( value, (node) => node.citations, async (_rules, payload) => { @@ -218,7 +221,7 @@ test("context-sized concurrent packets preserve all nodes when completions finis true, undefined, undefined, - { ...contextOptions(), concurrency: 2 }, + contextOptions(), ); expect(maxActive).toBe(2); expect(Math.max(...sizes)).toBeGreaterThan(24); @@ -253,59 +256,61 @@ test("context packing rejects an oversized item or insufficient output capacity expect(complete).not.toHaveBeenCalled(); }); -test("speculative curation recomputes newly created targets and hidden membership before commit", async () => { - async function run(concurrency: 1 | 2) { - const value = snapshot(55); - let calls = 0; - const curation = await curateTopics( - value, - (node) => node.citations, - async (_rules, payload) => { - calls++; - const input = payload as { - nodes: { id: string; evidenceIds: string[] }[]; - }; - return { - concepts: [ - { - label: "Mercury", - scope: "Synthetic test", - purpose: "Mercury reference", - memberIds: input.nodes.map((node) => node.id), - evidenceIds: [ - ...new Set( - input.nodes.flatMap( - (node) => node.evidenceIds, - ), - ), - ], - salience: 6, - }, - ], - deferred: [], - }; - }, - new AbortController().signal, - true, - undefined, - undefined, - { concurrency }, - ); - validateTopicSnapshot({ ...value, curation }); - return { curation, calls }; - } - const serial = await run(1), - parallel = await run(2); - expect(parallel.curation).toEqual(serial.curation); - expect(parallel.calls).toBeGreaterThan(serial.calls); - expect(parallel.curation.concepts[0].memberIds).toHaveLength(55); +test("frozen curation prepares raw evidence once across changing prior concepts", async () => { + const value = snapshot(55); + let calls = 0; + let prepared = 0; + const curation = await curateFrozenTopics( + value, + (node) => node.citations, + async (_rules, payload, _schema, call) => { + calls++; + if (call?.phase === "preparation") prepared += call.rawNodes ?? 0; + const input = payload as { + nodes: { id: string; evidenceIds: string[] }[]; + }; + return { + concepts: [ + { + label: "Mercury", + scope: "Synthetic test", + purpose: "Mercury reference", + memberIds: input.nodes.map((node) => node.id), + evidenceIds: [ + ...new Set( + input.nodes.flatMap((node) => node.evidenceIds), + ), + ], + salience: 6, + }, + ], + deferred: [], + }; + }, + new AbortController().signal, + true, + undefined, + undefined, + ); + validateTopicSnapshot({ ...value, curation }); + expect(prepared).toBe(55); + expect(calls).toBe(5); + expect(curation.concepts[0].memberIds).toHaveLength(55); }); -test("experimental policies change input identity and preserve legacy defaults", () => { - expect(topicCurationPolicy(true)).toBe("curated-lazy-v6"); - expect(topicCurationPolicy(true, { concurrency: 2 })).not.toBe( - topicCurationPolicy(true), - ); +test("frozen policy canonicalizes defaults and invalidates legacy scheduler identities", () => { + expect(topicCurationPolicy(true)).toMatch(/^curated-lazy-v10:/); + expect( + topicCurationPolicy(true, { priorDeltas: true, compactIds: true }), + ).toBe(topicCurationPolicy(true)); + for (const settings of [ + Object.assign({ priorDeltas: true }, { concurrency: 1 }), + Object.assign({ priorDeltas: true }, { concurrency: 2 }), + Object.assign({ priorDeltas: true }, { frozenPreparation: false }), + ]) + expect(() => topicCurationPolicy(true, settings)).toThrow( + "Invalid topic curation settings", + ); expect(topicCurationPolicy(true, { inputTokenTarget: 32000 })).not.toBe( topicCurationPolicy(true, { inputTokenTarget: 64000 }), ); @@ -688,80 +693,71 @@ test("lazy curation packs interleaved labels by source without discarding origin validateTopicSnapshot(value); }); -test.each([1, 2] as const)( - "curation resumes validated checkpoint membership after a later failure at concurrency %i", - async (concurrency) => { - const value = snapshot(55); - let checkpoint: TopicCuration | undefined; - let calls = 0; - const decide = async (_rules: string, payload: unknown) => { - const batch = payload as { nodes: Array<{ id: string }> }; - return { - concepts: [], - deferred: batch.nodes.map(({ id }) => ({ - nodeId: id, - reason: "Incidental mention", - })), - }; +test("single-pass evidence resumes validated checkpoint membership after a later failure", async () => { + const value = snapshot(55); + let checkpoint: TopicCuration | undefined; + let calls = 0; + const decide = async (_rules: string, payload: unknown) => { + const batch = payload as { nodes: Array<{ id: string }> }; + return { + concepts: [], + deferred: batch.nodes.map(({ id }) => ({ + nodeId: id, + reason: "Incidental mention", + })), }; - await expect( - curateTopics( - value, - (node) => node.citations, - async (rules, payload) => { - if (++calls === 3) throw new Error("Provider interrupted"); - return decide(rules, payload); - }, - new AbortController().signal, - true, - async (progress) => { - validateTopicSnapshot({ - ...value, - curationProgress: progress, - }); - checkpoint = progress; - }, - undefined, - { concurrency }, - ), - ).rejects.toThrow("Provider interrupted"); - expect(checkpoint!.deferred).toHaveLength(24); - const saved = new Set( - checkpoint!.deferred.map((entry) => entry.nodeId), - ); - const resumed = await curateTopics( + }; + await expect( + curateTopics( value, (node) => node.citations, async (rules, payload) => { - const batch = payload as { nodes: Array<{ id: string }> }; - expect(batch.nodes.every(({ id }) => !saved.has(id))).toBe( - true, - ); + if (++calls === 3) throw new Error("Provider interrupted"); return decide(rules, payload); }, new AbortController().signal, true, + async (progress) => { + validateTopicSnapshot({ + ...value, + curationProgress: progress, + }); + checkpoint = progress; + }, undefined, - checkpoint, - { concurrency }, - ); - expect(resumed.deferred).toHaveLength(55); - validateTopicSnapshot({ ...value, curation: resumed }); - const invalid = structuredClone(checkpoint!); - invalid.deferred[0].nodeId = "invented"; - await expect( - curateTopics( - value, - (node) => node.citations, - decide, - new AbortController().signal, - true, - undefined, - invalid, - ), - ).rejects.toThrow("Invalid partial topic curation membership"); - }, -); + ), + ).rejects.toThrow("Provider interrupted"); + expect(checkpoint!.deferred).toHaveLength(24); + const saved = new Set(checkpoint!.deferred.map((entry) => entry.nodeId)); + const resumed = await curateTopics( + value, + (node) => node.citations, + async (rules, payload) => { + const batch = payload as { nodes: Array<{ id: string }> }; + expect(batch.nodes.every(({ id }) => !saved.has(id))).toBe(true); + return decide(rules, payload); + }, + new AbortController().signal, + true, + undefined, + checkpoint, + ); + expect(resumed.deferred).toHaveLength(55); + validateTopicSnapshot({ ...value, curation: resumed }); + const invalid = structuredClone(checkpoint!); + invalid.deferred[0].nodeId = "invented"; + await expect( + curateTopics( + value, + (node) => node.citations, + decide, + new AbortController().signal, + true, + undefined, + invalid, + ), + ).rejects.toThrow("Invalid partial topic curation membership"); +}); test("curation checkpoint persistence failure stops before another model request", async () => { const value = snapshot(55); diff --git a/ts/packages/memory/service/test/topicCurationScheduler.spec.ts b/ts/packages/memory/service/test/topicCurationScheduler.spec.ts deleted file mode 100644 index 78974fc3e2..0000000000 --- a/ts/packages/memory/service/test/topicCurationScheduler.spec.ts +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { runOrderedTopicBatches } from "../src/topicCurationScheduler.js"; -import { runWithSearchTiming } from "@typeagent/knowpro"; - -function deferred() { - let resolve!: () => void; - const promise = new Promise((done) => { - resolve = done; - }); - return { promise, resolve }; -} -const packet = (start: number) => ({ - start, - end: start + 1, - fingerprint: `${start}`, -}); - -test("disjoint calls overlap, finish out of order and commit in canonical order", async () => { - const first = deferred(), - both = deferred(); - const commits: number[] = [], - calls: number[] = []; - const run = runOrderedTopicBatches( - 2, - 2, - new AbortController().signal, - packet, - async ({ start }) => { - calls.push(start); - if (calls.length === 2) both.resolve(); - if (start === 0) await first.promise; - return start; - }, - async (_packet, value) => { - commits.push(value); - }, - ); - await both.promise; - expect(commits).toEqual([]); - first.resolve(); - await run; - expect(calls).toEqual([0, 1]); - expect(commits).toEqual([0, 1]); -}); - -test("changed packet/root dependencies discard and drain stale proposals before retry", async () => { - const first = deferred(), - stale = deferred(), - both = deferred(), - aborted = deferred(); - let revision = 0; - const calls: string[] = [], - commits: string[] = []; - const run = runOrderedTopicBatches( - 3, - 2, - new AbortController().signal, - (start) => ({ - start, - end: start === 1 && revision ? 3 : start + 1, - fingerprint: `${start}:${revision}`, - }), - async (packet, signal) => { - calls.push(packet.fingerprint); - if (calls.length === 2) both.resolve(); - if (packet.start === 0) await first.promise; - if (packet.fingerprint === "1:0") { - signal.addEventListener("abort", aborted.resolve, { - once: true, - }); - await stale.promise; - } - return packet.fingerprint; - }, - async (_packet, value) => { - commits.push(value); - revision++; - }, - ); - await both.promise; - first.resolve(); - await aborted.promise; - expect(calls).toEqual(["0:0", "1:0"]); - stale.resolve(); - await run; - expect(calls).toEqual(["0:0", "1:0", "1:1"]); - expect(commits).toEqual(["0:0", "1:1"]); -}); - -test("provider failure aborts and drains siblings without committing pending results", async () => { - const both = deferred(), - fail = deferred(); - let active = 0, - drained = false, - committed = false; - const run = runOrderedTopicBatches( - 2, - 2, - new AbortController().signal, - packet, - async ({ start }, signal) => { - if (++active === 2) both.resolve(); - if (!start) { - await fail.promise; - throw new Error("Provider failed"); - } - await new Promise((resolve) => - signal.addEventListener( - "abort", - () => { - drained = true; - resolve(); - }, - { once: true }, - ), - ); - signal.throwIfAborted(); - return start; - }, - async () => { - committed = true; - }, - ); - await both.promise; - fail.resolve(); - await expect(run).rejects.toThrow("Provider failed"); - expect(drained).toBe(true); - expect(committed).toBe(false); -}); - -test("cancelled signal-ignoring providers are drained and cannot commit", async () => { - const both = deferred(), - release = deferred(); - const controller = new AbortController(); - let calls = 0, - commits = 0; - const run = runOrderedTopicBatches( - 4, - 2, - controller.signal, - packet, - async () => { - if (++calls === 2) both.resolve(); - await release.promise; - return 0; - }, - async () => { - commits++; - }, - ); - await both.promise; - controller.abort(new Error("Cancelled")); - release.resolve(); - await expect(run).rejects.toThrow("Cancelled"); - expect(calls).toBe(2); - expect(commits).toBe(0); -}); - -test("checkpoint publication failure drains speculative calls without publishing them", async () => { - const both = deferred(); - let calls = 0, - drained = false; - const commits: number[] = []; - await expect( - runOrderedTopicBatches( - 3, - 2, - new AbortController().signal, - packet, - async ({ start }, signal) => { - if (++calls === 2) both.resolve(); - if (!start) await both.promise; - else { - await new Promise((resolve) => - signal.addEventListener( - "abort", - () => { - drained = true; - resolve(); - }, - { once: true }, - ), - ); - signal.throwIfAborted(); - } - return start; - }, - async (_packet, value) => { - commits.push(value); - throw new Error("Checkpoint persistence failed"); - }, - ), - ).rejects.toThrow("Checkpoint persistence failed"); - expect(drained).toBe(true); - expect(calls).toBe(2); - expect(commits).toEqual([0]); -}); - -describe("speculative work accounting", () => { - const originalFlag = process.env.TYPEAGENT_MEMORY_SEARCH_TRACE; - const originalWarn = console.warn; - let events: Array<{ - stage: string; - metadata: { attempt?: number; aborted?: boolean }; - }>; - beforeEach(() => { - process.env.TYPEAGENT_MEMORY_SEARCH_TRACE = "1"; - events = []; - console.warn = (line: unknown) => { - const text = String(line); - if (text.startsWith("[memory-search-timing] ")) - events.push( - JSON.parse(text.slice("[memory-search-timing] ".length)), - ); - }; - }); - afterEach(() => { - console.warn = originalWarn; - if (originalFlag === undefined) - delete process.env.TYPEAGENT_MEMORY_SEARCH_TRACE; - else process.env.TYPEAGENT_MEMORY_SEARCH_TRACE = originalFlag; - }); - - test("late post-abort completions remain measured and never become commits", async () => { - const both = deferred(), - stale = deferred(), - aborted = deferred(); - let revision = 0, - calls = 0; - const run = runWithSearchTiming("accounting", "test", () => - runOrderedTopicBatches( - 2, - 2, - new AbortController().signal, - (start) => ({ - start, - end: start + 1, - fingerprint: `${start}:${revision}`, - }), - async ({ start, fingerprint }, signal) => { - if (++calls === 2) both.resolve(); - if (start === 0) await both.promise; - if (fingerprint === "1:0") { - signal.addEventListener("abort", aborted.resolve, { - once: true, - }); - await stale.promise; - } - return fingerprint; - }, - async () => { - revision++; - }, - ), - ); - await aborted.promise; - stale.resolve(); - await run; - const at = (stage: string) => - events.filter( - (event) => - event.stage === `topic.curation.speculation.${stage}`, - ); - expect(at("proposed")).toHaveLength(3); - expect(at("started")).toHaveLength(3); - expect(at("completed")).toHaveLength(3); - expect(at("discard")).toHaveLength(1); - expect(at("abort")).toHaveLength(1); - expect( - at("completed").filter((event) => event.metadata.aborted), - ).toHaveLength(1); - expect(at("commit").map((event) => event.metadata.attempt)).toEqual([ - 1, 3, - ]); - }); - - test("failed publication has no success-shaped commit event", async () => { - await expect( - runWithSearchTiming("accounting", "test", () => - runOrderedTopicBatches( - 1, - 1, - new AbortController().signal, - packet, - async () => 1, - async () => { - throw new Error("Checkpoint publication failed"); - }, - ), - ), - ).rejects.toThrow("Checkpoint publication failed"); - expect( - events.filter( - (event) => - event.stage === "topic.curation.speculation.completed", - ), - ).toHaveLength(1); - expect( - events.filter( - (event) => event.stage === "topic.curation.speculation.commit", - ), - ).toHaveLength(0); - }); -}); diff --git a/ts/packages/memory/service/test/topicFrozenPreparation.spec.ts b/ts/packages/memory/service/test/topicFrozenPreparation.spec.ts index 836fef5a40..e94c1311d5 100644 --- a/ts/packages/memory/service/test/topicFrozenPreparation.spec.ts +++ b/ts/packages/memory/service/test/topicFrozenPreparation.spec.ts @@ -3,6 +3,7 @@ import { curateTopics, + curateTopicEvidence, type TopicCurationCompletion, type TopicCurationOptions, } from "../src/topicCuration.js"; @@ -84,8 +85,6 @@ function merged(payload: unknown) { } const settings: TopicCurationOptions = { - concurrency: 2, - frozenPreparation: true, priorDeltas: true, compactIds: true, }; @@ -129,7 +128,7 @@ test("frozen preparation overlaps, reconciles serially and installs complete win ); }, undefined, - { ...settings, onPacket: (packet) => packets.push(packet) }, + { onPacket: (packet) => packets.push(packet) }, ); expect(maximum).toBe(2); expect(checkpoints).toEqual([48, 50]); @@ -259,7 +258,7 @@ test("cold continuation retains an installed window and resumes only unprocessed test("delta responses retain omitted priors exactly but still repair omitted new nodes", async () => { const value = snapshot(3); const first = snapshot(1); - const prior = await curateTopics( + const prior = await curateTopicEvidence( first, (node) => node.citations, async (_rules, payload) => merged(payload), @@ -267,7 +266,7 @@ test("delta responses retain omitted priors exactly but still repair omitted new true, ); let calls = 0; - const result = await curateTopics( + const result = await curateTopicEvidence( value, (node) => node.citations, async (_rules, payload, _schema, call) => { @@ -387,7 +386,7 @@ test("delta window receipts retain audited prior summaries exactly on replay", a test("unknown compact response IDs use the existing bounded contract repair", async () => { let calls = 0; const value = snapshot(1); - const result = await curateTopics( + const result = await curateTopicEvidence( value, (node) => node.citations, async (_rules, payload, _schema, call) => { @@ -409,11 +408,12 @@ test("unknown compact response IDs use the existing bounded contract repair", as expect(result.concepts[0].memberIds).toEqual([value.nodes[0].id]); }); -test("parallel preparation rejects non-lazy use and ordinary speculative watermark replay", async () => { +test("frozen preparation rejects eager summaries and removed scheduler settings", async () => { const value = snapshot(1); const cases: Array<[boolean, TopicCurationOptions]> = [ [false, settings], - [true, { concurrency: 2, replayPackets: [] }], + [true, Object.assign({}, settings, { concurrency: 1 })], + [true, Object.assign({}, settings, { frozenPreparation: false })], ]; for (const [lazy, options] of cases) await expect( diff --git a/ts/packages/memory/service/test/topicGeneration.spec.ts b/ts/packages/memory/service/test/topicGeneration.spec.ts index 616994cd6d..0feb8ab594 100644 --- a/ts/packages/memory/service/test/topicGeneration.spec.ts +++ b/ts/packages/memory/service/test/topicGeneration.spec.ts @@ -348,11 +348,11 @@ test("named generative settings and Decision factories retain distinct recipes", }; }, }, - mode: { kind: "generative", settings: { concurrency: 2 } }, + mode: { kind: "generative" }, }); const result = await generate(input(1), new AbortController().signal); - expect(curationCalls).toBe(1); - expect(result.recipe).toMatch(/:curated-lazy-v8:[0-9a-f]{16}$/); + expect(curationCalls).toBe(2); + expect(result.recipe).toMatch(/:curated-lazy-v10:[0-9a-f]{16}$/); expect(result.curation?.deferred).toHaveLength(1); }); @@ -544,7 +544,7 @@ test.each([false, true])( input(7), new AbortController().signal, ); - expect(calls).toBe(1); + expect(calls).toBe(2); expect(snapshot.curation?.deferred).toHaveLength(7); expect( snapshot.nodes.every( @@ -681,8 +681,8 @@ test.each([ const snapshot = await outcome; expect(stages).toEqual( mode === "valid" - ? ["curation", "organization"] - : ["curation", "organization", "organization"], + ? ["curation", "curation", "organization"] + : ["curation", "curation", "organization", "organization"], ); expect(snapshot.parents).toHaveLength(1); expect(snapshot.parents[0].parentId).toMatch(/^concept:/); @@ -752,7 +752,7 @@ test("curation checkpoints seed only unchanged policy and exact captured evidenc model: { completionSettings: {}, complete: async (messages) => { - if (++calls === 2) + if (++calls === 5) throw new Error("Curation provider interrupted"); const payload = JSON.parse(promptText(messages, 1)); return { @@ -777,7 +777,7 @@ test("curation checkpoints seed only unchanged policy and exact captured evidenc expect(checkpoints).toHaveLength(1); const partial = checkpoints[0]; expect(partial.curation).toBeUndefined(); - expect(partial.curationProgress!.deferred).toHaveLength(24); + expect(partial.curationProgress!.deferred).toHaveLength(48); const resumed = await capture(frozen.topicInput!.occurrences, partial); expect(resumed.topicInput!.reusableCuration).toEqual( partial.curationProgress, diff --git a/ts/packages/memory/service/test/topicService.spec.ts b/ts/packages/memory/service/test/topicService.spec.ts index 6aee01703b..8116ad3338 100644 --- a/ts/packages/memory/service/test/topicService.spec.ts +++ b/ts/packages/memory/service/test/topicService.spec.ts @@ -104,7 +104,7 @@ describe("production topic service vertical slice", () => { ]; } if (curated) { - expect(input.model).toContain("topics-curated-lazy-v6"); + expect(input.model).toContain("topics-curated-lazy-v10:"); snapshot.curation = { concepts: [ { diff --git a/ts/packages/memory/service/test/topicWorkReuse.spec.ts b/ts/packages/memory/service/test/topicWorkReuse.spec.ts index c152860e39..a68c0d1743 100644 --- a/ts/packages/memory/service/test/topicWorkReuse.spec.ts +++ b/ts/packages/memory/service/test/topicWorkReuse.spec.ts @@ -17,6 +17,7 @@ import { import { indexedTopicOccurrence } from "../src/topicIncidence.js"; import { validateTopicSnapshot } from "../src/topicSnapshot.js"; import { topicClusterSignature } from "../src/topicDecision.js"; +import { topicCurationPolicy } from "../src/topicCuration.js"; import { prioritizeTopicMaintenance } from "../src/topicClusterMaintenance.js"; import { FileMemoryService } from "../src/fileMemoryService.js"; import { TypedViewStore } from "../src/viewStore.js"; @@ -193,7 +194,7 @@ test.each([false, true])( if (reuse) expect( state.continuation?.curationPackets?.length, - ).toBeGreaterThanOrEqual(2); + ).toBeGreaterThanOrEqual(1); else expect(state.continuation).toBeUndefined(); } await service.close(); @@ -215,8 +216,8 @@ test.each([false, true])( (await service.queryTopics({ corpusId, pageSize: 100 })) .totalNodes, ).toBe(54); - expect(calls).toBe(reuse ? 5 : 6); - expect(processedNodes).toBe(reuse ? 60 : 126); + expect(calls).toBe(reuse ? 10 : 18); + expect(processedNodes).toBe(reuse ? 120 : 396); const current = await new TypedViewStore(root).getTopicState( corpusId, ); @@ -270,7 +271,7 @@ async function priorSnapshot() { const witness = input.topicInput!.curationInputFingerprint; if (!witness) throw new Error("Expected captured dependency witness"); snapshot.curationInputFingerprint = witness; - snapshot.recipe += ":curated-lazy-v6"; + snapshot.recipe += `:${topicCurationPolicy(true)}`; snapshot.curationProgress = { concepts: [], deferred: [ @@ -479,9 +480,9 @@ function deferred() { } test.each([ - [48, "zz-added", 1], - [55, "zz-added", 1], - [48, "aa-added", 3], + [48, "zz-added", 2], + [55, "zz-added", 2], + [48, "aa-added", 6], ] as const)( "packet prefix replay preserves current output and respects ordering and tail boundaries: %s %s", async (count, sourceId, remainingCalls) => { @@ -528,7 +529,7 @@ test.each([ topicWatermark: true, }); const previous = await generate(input, new AbortController().signal); - expect(previous.curationPackets).toHaveLength(Math.ceil(count / 24)); + expect(previous.curationPackets).toHaveLength(Math.ceil(count / 48)); const added = indexedTopicOccurrence( { sourceId, @@ -551,7 +552,7 @@ test.each([ }); expect(next.topicInput?.reusableCuration).toBeUndefined(); expect(next.topicInput?.reusableCurationPackets).toHaveLength( - Math.ceil(count / 24), + Math.ceil(count / 48), ); const before = calls; const candidate = await generate(next, new AbortController().signal); @@ -601,7 +602,7 @@ test.each([ ); test("packet receipts compact prior concept membership and replay exact consolidation", async () => { - const seed = Array.from({ length: 48 }, (_, ordinal) => + const seed = Array.from({ length: 72 }, (_, ordinal) => indexedTopicOccurrence( { sourceId: "seed", @@ -652,7 +653,7 @@ test("packet receipts compact prior concept membership and replay exact consolid }), new AbortController().signal, ); - expect(previous.curation?.concepts[0].memberIds).toHaveLength(48); + expect(previous.curation?.concepts[0].memberIds).toHaveLength(72); expect( previous.curationPackets?.[1].response.concepts[0].memberIds, ).toHaveLength(25); @@ -678,19 +679,15 @@ test("packet receipts compact prior concept membership and replay exact consolid }); const before = calls; const candidate = await generate(next, new AbortController().signal); - expect(calls - before).toBe(1); + expect(calls - before).toBe(3); delete next.topicInput!.reusableCurationPackets; const control = await generate(next, new AbortController().signal); expect(candidate.curation).toEqual(control.curation); }); -test.each( - (["append", "correction", "forget", "staged-correction"] as const).flatMap( - (change) => [[change, false] as const, [change, true] as const], - ), -)( - "production private watermark is durable, unpublished, and fails closed for %s (frozen: %s)", - async (change, frozen) => { +test.each(["append", "correction", "forget", "staged-correction"] as const)( + "production frozen watermark is durable, unpublished, and fails closed for %s", + async (change) => { const directory = await mkdtemp( path.join(os.tmpdir(), "topic-watermark-"), ); @@ -723,16 +720,6 @@ test.each( }, mode: { kind: "generative", - ...(frozen - ? { - settings: { - concurrency: 2, - frozenPreparation: true, - priorDeltas: true, - compactIds: true, - }, - } - : {}), }, }); const adapter: ViewSynthesisAdapter = { @@ -821,14 +808,12 @@ test.each( corpusId, ); if (change !== "correction") { - expect(state.continuation?.curationPackets).toHaveLength( - frozen ? 1 : 2, - ); + expect(state.continuation?.curationPackets).toHaveLength(1); expect(state.snapshot?.curationProgress?.deferred).toHaveLength( - frozen ? 48 : 24, + 48, ); expect(state.family.dirty).toBe(true); - expect(calls).toBe(frozen ? 4 : 2); + expect(calls).toBe(4); if (change === "forget") { const preview = await service.previewForgetSource( corpusId, @@ -870,7 +855,7 @@ test.each( corpusId, ) ).continuation?.curationPackets?.length ?? 0, - ).toBe(change === "append" ? (frozen ? 1 : 2) : 0); + ).toBe(change === "append" ? 1 : 0); } else expect(state.continuation).toBeUndefined(); const before = calls; if (change === "append") { @@ -905,12 +890,7 @@ test.each( ? 2 : 1; expect(calls - before).toBe( - // Custom adapters cannot certify a different host curation policy. - frozen && change === "append" - ? 6 - : frozen - ? expectedCalls * 2 - : expectedCalls, + change === "append" ? 2 : expectedCalls * 2, ); const page = await service.queryTopics({ corpusId, pageSize: 100 }); expect(page.totalNodes).toBe( @@ -1027,10 +1007,10 @@ async function waitTopicJob(service: FileMemoryService, job: ViewBuildJob) { } test.each([ - [false, "unrelated", 3], + [false, "unrelated", 6], [true, "unrelated", 2], - [true, "correction", 3], - [true, "new-topic", 3], + [true, "correction", 6], + [true, "new-topic", 6], ] as const)( "production checkpoint churn preserves only unaffected work and reaches current durable output: reuse=%s change=%s", async (reuse, change, expectedRemainingCalls) => { @@ -1118,7 +1098,7 @@ test.each([ await ingest("seed", "The pilot remains blocked."); const obsolete = await service.rebuildTopics(corpusId); await checkpointed.promise; - expect(calls).toBe(1); + expect(calls).toBe(4); await ingest( change === "correction" ? "seed" : "additional", change === "unrelated" @@ -1165,7 +1145,7 @@ test.each([ ) ).state, ).toBe("complete"); - expect(calls - completedCalls).toBe(reuse ? 0 : 3); + expect(calls - completedCalls).toBe(reuse ? 0 : 6); expect( (await service.queryTopics({ corpusId, pageSize: 100 })).nodes, ).toEqual(page.nodes); diff --git a/ts/packages/memory/service/test/viewHostOptions.spec.ts b/ts/packages/memory/service/test/viewHostOptions.spec.ts index e9c3973946..765a80cfa7 100644 --- a/ts/packages/memory/service/test/viewHostOptions.spec.ts +++ b/ts/packages/memory/service/test/viewHostOptions.spec.ts @@ -59,19 +59,17 @@ test("short-job priority is explicitly configurable and conflicts with reservati ).toThrow("cannot both be enabled"); }); -test("topic experiments are opt-in and reject invalid host settings", () => { +test("topic packet controls reject invalid and removed scheduler settings", () => { expect(getConfiguredMemoryViewOptions({})).not.toHaveProperty( "topicCuration", ); expect( getConfiguredMemoryViewOptions({ TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "32000", - TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "2", TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION: "0.25", }).topicCuration, ).toEqual({ inputTokenTarget: 32000, - concurrency: 2, priorContextFraction: 0.25, }); @@ -79,6 +77,10 @@ test("topic experiments are opt-in and reject invalid host settings", () => { { TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "" }, { TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "NaN" }, { TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "3" }, + { TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "1" }, + { TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "2" }, + { TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION: "false" }, + { TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION: "true" }, { TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION: "0.2" }, { TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "-1" }, ]) @@ -108,16 +110,12 @@ test("unchanged topic curation reuse is opt-in for both generation engines", () ).toThrow("TYPEAGENT_MEMORY_TOPIC_CURATION_REUSE"); }); -test("frozen topic preparation is explicit and compatible with ordered watermarks", () => { +test("default frozen topic preparation is compatible with ordered watermarks", () => { const options = getConfiguredMemoryViewOptions({ - TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "2", - TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION: "true", TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS: "true", TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS: "true", }); expect(options.topicCuration).toEqual({ - concurrency: 2, - frozenPreparation: true, priorDeltas: true, compactIds: true, }); @@ -134,7 +132,7 @@ test("frozen topic preparation is explicit and compatible with ordered watermark getConfiguredMemoryViewOptions({ TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION: "true", }), - ).toThrow("Invalid topic curation settings"); + ).toThrow("has been removed"); expect(() => getConfiguredMemoryViewOptions({ TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS: "maybe", @@ -146,17 +144,17 @@ test("generative and native execution families retain distinct capacities and re expect( getConfiguredMemoryViewOptions({ TYPEAGENT_MEMORY_DECISION_CONCURRENCY: "24", - TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "2", + TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "32000", }), ).toMatchObject({ decisionTopicExecution: { concurrency: 24 }, - topicCuration: { concurrency: 2 }, + topicCuration: { inputTokenTarget: 32000 }, }); expect(() => getConfiguredMemoryViewOptions({ TYPEAGENT_MEMORY_DECISION_TOPICS: "true", - TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "2", + TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "32000", }), ).toThrow("cannot configure Decision"); for (const options of [{ decisionTopics: true }, { curatedTopics: false }]) @@ -164,12 +162,12 @@ test("generative and native execution families retain distinct capacities and re () => new FileMemoryService("unused", { ...options, - topicCuration: { concurrency: 2 }, + topicCuration: { inputTokenTarget: 32000 }, }), ).toThrow("Generative topic settings require"); }); -test("private topic watermark requires explicit single-worker generative reuse", () => { +test("private topic watermark requires explicit generative reuse", () => { expect(getConfiguredMemoryViewOptions({})).not.toHaveProperty( "topicWatermark", ); @@ -193,7 +191,6 @@ test("private topic watermark requires explicit single-worker generative reuse", { topicCurationReuse: false }, { decisionTopics: true }, { curatedTopics: false }, - { topicCuration: { concurrency: 2 as const } }, ]) expect( () => diff --git a/ts/tools/memory-maintenance-qualification/harness.test.mjs b/ts/tools/memory-maintenance-qualification/harness.test.mjs index d5ca850ac4..a9d1d54a55 100644 --- a/ts/tools/memory-maintenance-qualification/harness.test.mjs +++ b/ts/tools/memory-maintenance-qualification/harness.test.mjs @@ -60,6 +60,22 @@ async function runHarnessCli(args) { return { code, stdout, stderr }; } +test("compiled checkpoints include the shared topic curation settings module", async () => { + const hashes = await snapshotCompiledCheckpoint(productRoot); + assert.match( + hashes[ + path.join( + "packages", + "memory", + "service", + "dist", + "topicCurationSettings.js", + ) + ], + /^[a-f0-9]{64}$/, + ); +}); + test("offline worker CLI preserves newline output and captures final diagnostics before owned failure cleanup", async () => { const help = await runHarnessCli(["--help"]); assert.equal(help.code, 0, help.stderr); diff --git a/ts/tools/memory-maintenance-qualification/indexing-README.md b/ts/tools/memory-maintenance-qualification/indexing-README.md index d52a659959..705a682539 100644 --- a/ts/tools/memory-maintenance-qualification/indexing-README.md +++ b/ts/tools/memory-maintenance-qualification/indexing-README.md @@ -214,13 +214,22 @@ Time and question limits are **not financial limits**. ## Run the two live production arms -For a same-build comparison of the three parallelization changes, prepare a +For a same-build comparison of synopsis and embedding parallelization, prepare a **new** pilot with `--comparison indexing-parallel`. Both arms then enable -prepared indexing. A0 removes the bundled and all three individual indexing +prepared indexing. A0 removes the bundled and both individual indexing parallel experiments; A1 enables the bundle. Other inherited performance experiments remain identical. -An explicit `TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` is rejected rather than -silently overriding the treatment. Each recorded arm includes `indexingParallel`. +Removed `TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` and +`TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION` variables are rejected in every arm. +Generative topics always use frozen two-packet preparation and serial +reconciliation/commit, with input target and prior fraction unset by default +(preserving node/character packing and no reserved prior fraction), prior deltas +true and compact IDs true. The historical 24000-token / 0.25-fraction settings +remain explicit treatments, not defaults. These four settings remain configurable; +each arm records their effective values. Native Decision scheduling is unchanged. +Old approvals, recordings and receipts keep their original identities and must +not be relabeled; prepare fresh artifacts for the current implementation. +Each recorded arm also includes `indexingParallel`. Add `--first-case-only` to freeze one complete history for this comparison. This schedules the first deterministically selected case in both arms, without @@ -241,8 +250,11 @@ and completed ingestion timings remain distinct from successful readiness. ### Prepare independent treatments without running inference Use a new directory and `--comparison indexing-ablation --first-case-only` to -freeze four sequential conditions: `A0` (all parallel treatments off), `T1` -(topic-only), `S1` (synopsis-only), `E1` (embedding-only). Every condition enables +freeze three sequential conditions: `A0` (synopsis/embedding treatments off), +`S1` (synopsis-only), `E1` (embedding-only). Frozen topic preparation stays enabled +in every condition. The removed `topic-parallel` experiment fails explicitly; +older four-condition pilot schedules containing `T1` are rejected, not relabeled. +Every condition enables prepared indexing and uses the same complete first history with fresh stores. The runner refuses another arm order or an inherited topic-concurrency override. It removes inherited individual/bundled indexing treatments before selecting diff --git a/ts/tools/memory-maintenance-qualification/longmemeval-indexing-contract.mjs b/ts/tools/memory-maintenance-qualification/longmemeval-indexing-contract.mjs index 3516d54004..626750db46 100644 --- a/ts/tools/memory-maintenance-qualification/longmemeval-indexing-contract.mjs +++ b/ts/tools/memory-maintenance-qualification/longmemeval-indexing-contract.mjs @@ -19,7 +19,6 @@ export const categories = [ "multi-session", ]; export const parallelTreatmentNames = [ - "topic-parallel", "synopsis-parallel", "embedding-parallel", ]; @@ -27,6 +26,10 @@ export function indexingTreatments(environment = process.env) { const names = (environment.TYPEAGENT_MEMORY_PERF_EXPERIMENTS ?? "") .split(",") .map((name) => name.trim()); + assert.ok( + !names.includes("topic-parallel"), + "topic-parallel has been removed; generative topics always use frozen preparation and serial reconciliation", + ); return parallelTreatmentNames.filter( (name) => names.includes("indexing-parallel") || names.includes(name), ); @@ -41,7 +44,7 @@ export function productionSchedule(comparison = "prepared-indexing") { "Unknown production comparison", ); return comparison === "indexing-ablation" - ? ["A0", "T1", "S1", "E1"] + ? ["A0", "S1", "E1"] : ["A0", "A1"]; } export const digest = (value) => diff --git a/ts/tools/memory-maintenance-qualification/longmemeval-indexing.mjs b/ts/tools/memory-maintenance-qualification/longmemeval-indexing.mjs index fd0f338ee8..645cc3fd1f 100644 --- a/ts/tools/memory-maintenance-qualification/longmemeval-indexing.mjs +++ b/ts/tools/memory-maintenance-qualification/longmemeval-indexing.mjs @@ -21,6 +21,7 @@ import { benchmarkProviderConfiguration, benchmarkFailure, loadPrepared, + benchmarkTopicCuration, } from "./longmemeval-live.mjs"; import { digest, @@ -220,12 +221,9 @@ export function productionArmEnvironment( const { comparison, schedule } = productionPlan(pilot); assert.ok(schedule.includes(arm), "Choose a frozen production arm"); const env = { ...environment, TYPEAGENT_MEMORY_SEARCH_TRACE: "1" }; + benchmarkTopicCuration(env); + indexingTreatments(env); if (comparison !== "prepared-indexing") { - assert.equal( - env.TYPEAGENT_MEMORY_TOPIC_CONCURRENCY, - undefined, - "Parallel indexing comparison requires default topic concurrency", - ); env.TYPEAGENT_MEMORY_PREPARED_INDEXING = "1"; const experiments = (env.TYPEAGENT_MEMORY_PERF_EXPERIMENTS ?? "") .split(",") @@ -238,7 +236,6 @@ export function productionArmEnvironment( ); const treatment = { A1: "indexing-parallel", - T1: "topic-parallel", S1: "synopsis-parallel", E1: "embedding-parallel", }[arm]; @@ -750,10 +747,7 @@ export function pairedPerformance(arms, ids) { }; } export function treatmentPerformance(arms, ids, treatment) { - assert.ok( - ["T1", "S1", "E1"].includes(treatment), - "Unknown singleton treatment", - ); + assert.ok(["S1", "E1"].includes(treatment), "Unknown singleton treatment"); const paired = pairedPerformance( arms .filter((arm) => ["A0", treatment].includes(arm.arm)) diff --git a/ts/tools/memory-maintenance-qualification/longmemeval-indexing.test.mjs b/ts/tools/memory-maintenance-qualification/longmemeval-indexing.test.mjs index b31ab44016..2b841bee01 100644 --- a/ts/tools/memory-maintenance-qualification/longmemeval-indexing.test.mjs +++ b/ts/tools/memory-maintenance-qualification/longmemeval-indexing.test.mjs @@ -25,6 +25,7 @@ import { embeddingConfigurationHash, encode, freezePilot, + indexingTreatments, openPilot, readJson, replayTape, @@ -72,6 +73,23 @@ import { createTraceSummary } from "./longmemeval-indexing-traces.mjs"; const product = fileURLToPath(new URL("../../", import.meta.url)); +test("indexing treatment identity excludes removed topic scheduling", () => { + assert.deepEqual( + indexingTreatments({ + TYPEAGENT_MEMORY_PERF_EXPERIMENTS: "indexing-parallel", + }), + ["synopsis-parallel", "embedding-parallel"], + ); + assert.throws( + () => + indexingTreatments({ + TYPEAGENT_MEMORY_PERF_EXPERIMENTS: + "indexing-parallel,topic-parallel", + }), + /topic-parallel has been removed/, + ); +}); + test("pilot approval and worker share a sanitized identity for every pool member", () => { const primary = { provider: "offline", @@ -140,20 +158,19 @@ test("singleton ablations remove inherited treatments and preserve frozen order" const pilot = { manifest: { questionIds: ["one", "two", "three"], - schedule: ["A0", "T1", "S1", "E1"], + schedule: ["A0", "S1", "E1"], }, inputs: { comparison: "indexing-ablation", firstCaseOnly: true }, }; assert.deepEqual(productionPlan(pilot).schedule, pilot.manifest.schedule); for (const [arm, expected] of [ ["A0", ""], - ["T1", "topic-parallel"], ["S1", "synopsis-parallel"], ["E1", "embedding-parallel"], ]) { const env = productionArmEnvironment(pilot, arm, { TYPEAGENT_MEMORY_PERF_EXPERIMENTS: - "indexing-parallel,topic-parallel,synopsis-parallel,embedding-parallel,translation-cache", + "indexing-parallel,synopsis-parallel,embedding-parallel,translation-cache", }); assert.equal(env.TYPEAGENT_MEMORY_PREPARED_INDEXING, "1"); assert.equal( @@ -168,12 +185,12 @@ test("singleton ablations remove inherited treatments and preserve frozen order" ...pilot, manifest: { ...pilot.manifest, - schedule: ["A0", "S1", "T1", "E1"], + schedule: ["A0", "T1", "S1", "E1"], }, }), /schedule/, ); - const arms = ["A0", "T1", "S1", "E1"].map((arm) => ({ + const arms = ["A0", "S1", "E1"].map((arm) => ({ arm, status: arm === "E1" ? "failed" : "completed", cases: [ @@ -185,7 +202,7 @@ test("singleton ablations remove inherited treatments and preserve frozen order" ], })); assert.equal( - treatmentPerformance(arms, ["one"], "T1").medians.ingestionMs + treatmentPerformance(arms, ["one"], "S1").medians.ingestionMs .medianRatioControlOverTreatment, 2, ); @@ -226,7 +243,14 @@ test("parallel comparison freezes one full history and holds preparation constan productionArmEnvironment(pilot, "A1", { TYPEAGENT_MEMORY_TOPIC_CONCURRENCY: "1", }), - /default topic concurrency/, + /TYPEAGENT_MEMORY_TOPIC_CONCURRENCY/, + ); + assert.throws( + () => + productionArmEnvironment(pilot, "A1", { + TYPEAGENT_MEMORY_PERF_EXPERIMENTS: "topic-parallel", + }), + /topic-parallel has been removed/, ); assert.throws( () => productionPlan({ ...pilot, inputs: { comparison: "typo" } }), @@ -376,7 +400,7 @@ test("selection freezes three categories without using question/reference labels await assert.rejects(openPilot(directory), /Assertion/); })); -test("freezing an ablation persists four independent conditions and one complete production history", async () => +test("freezing an ablation persists three independent conditions and one complete production history", async () => temporary(async (root) => { const inputs = { comparison: "indexing-ablation", firstCaseOnly: true }; const directory = path.join(root, "ablation"); @@ -388,7 +412,7 @@ test("freezing an ablation persists four independent conditions and one complete inputs, ); const pilot = await openPilot(directory); - assert.deepEqual(pilot.manifest.schedule, ["A0", "T1", "S1", "E1"]); + assert.deepEqual(pilot.manifest.schedule, ["A0", "S1", "E1"]); assert.equal(productionPlan(pilot).questionIds.length, 1); assert.equal(pilot.construction.length, 3); assert.equal(pilot.construction[0].sessions.length, 2); diff --git a/ts/tools/memory-maintenance-qualification/longmemeval-live.mjs b/ts/tools/memory-maintenance-qualification/longmemeval-live.mjs index b52b3e0625..8df65f76c4 100644 --- a/ts/tools/memory-maintenance-qualification/longmemeval-live.mjs +++ b/ts/tools/memory-maintenance-qualification/longmemeval-live.mjs @@ -8,6 +8,8 @@ import { pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; import { randomUUID } from "node:crypto"; import { appendFileSync } from "node:fs"; +import { getConfiguredMemoryViewOptions } from "../../packages/memory/service/dist/viewHostOptions.js"; +import { resolveTopicCurationSettings } from "../../packages/memory/service/dist/topicCurationSettings.js"; import { assertCheckpoint, checkpointExpectationHash, @@ -78,13 +80,7 @@ export function recordBenchmarkArm(state, capabilities, resume, options = {}) { ), ingestion: "One Markdown source per original session position; dates, roles and turn text preserved", - topicCuration: { - inputTokenTarget: - process.env.TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS ?? null, - concurrency: process.env.TYPEAGENT_MEMORY_TOPIC_CONCURRENCY ?? null, - priorContextFraction: - process.env.TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION ?? null, - }, + topicCuration: benchmarkTopicCuration(), wikiAssessmentConcurrency: Number( process.env.TYPEAGENT_MEMORY_WIKI_ASSESSMENT_CONCURRENCY ?? 1, ), @@ -133,6 +129,20 @@ export function recordBenchmarkArm(state, capabilities, resume, options = {}) { state.arm = arm; } +export function benchmarkTopicCuration(environment = process.env) { + const configured = getConfiguredMemoryViewOptions(environment); + return configured.decisionTopics + ? { + engine: "decision", + execution: configured.decisionTopicExecution ?? null, + } + : { + engine: "generative", + scheduling: "frozen-two-packet-serial-reconciliation-v1", + ...resolveTopicCurationSettings(configured.topicCuration), + }; +} + async function benchmarkCheckpoint(productRoot) { const hashes = await snapshotCompiledCheckpoint(productRoot); for (const file of [ diff --git a/ts/tools/memory-maintenance-qualification/longmemeval-runner.test.mjs b/ts/tools/memory-maintenance-qualification/longmemeval-runner.test.mjs index 42da3a733d..b27d6cce21 100644 --- a/ts/tools/memory-maintenance-qualification/longmemeval-runner.test.mjs +++ b/ts/tools/memory-maintenance-qualification/longmemeval-runner.test.mjs @@ -30,6 +30,7 @@ import { createBenchmarkMemoryService, createRetainedKnowledgeReader, recordBenchmarkArm, + benchmarkTopicCuration, recordKnowledgeAllowance, continuedBenchmarkState, createBenchmarkStateWriter, @@ -303,6 +304,41 @@ test("effective product capabilities are recorded and configuration changes cann assert.equal(state.arm.informationalViews, true); }); +test("benchmark topic identity records frozen defaults and rejects removed selectors", () => { + assert.deepEqual(benchmarkTopicCuration({}), { + engine: "generative", + scheduling: "frozen-two-packet-serial-reconciliation-v1", + priorDeltas: true, + compactIds: true, + }); + const custom = benchmarkTopicCuration({ + TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS: "12000", + TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION: "0.5", + TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS: "false", + TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS: "false", + }); + assert.equal(custom.inputTokenTarget, 12000); + assert.equal(custom.priorContextFraction, 0.5); + assert.equal(custom.priorDeltas, false); + assert.equal(custom.compactIds, false); + for (const [key, value] of [ + ["TYPEAGENT_MEMORY_TOPIC_CONCURRENCY", "1"], + ["TYPEAGENT_MEMORY_TOPIC_CONCURRENCY", "2"], + ["TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION", "true"], + ["TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION", "false"], + ]) + assert.throws( + () => benchmarkTopicCuration({ [key]: value }), + new RegExp(key), + ); + const decision = benchmarkTopicCuration({ + TYPEAGENT_MEMORY_DECISION_TOPICS: "true", + TYPEAGENT_MEMORY_DECISION_CONCURRENCY: "4", + }); + assert.equal(decision.engine, "decision"); + assert.equal(decision.execution.concurrency, 4); +}); + for (const [field, key, lanes] of [ [ "wikiAssessmentConcurrency", diff --git a/ts/tools/memory-maintenance-qualification/quality-README.md b/ts/tools/memory-maintenance-qualification/quality-README.md index 7f228871c9..8baa63f8e7 100644 --- a/ts/tools/memory-maintenance-qualification/quality-README.md +++ b/ts/tools/memory-maintenance-qualification/quality-README.md @@ -652,12 +652,19 @@ changed member models, endpoints or request constraints invalidate approval. Set `TYPEAGENT_MEMORY_SEARCH_TRACE=1` before an explicitly authorized run to retain the existing native stage spans in worker stderr. The arm records this setting -and the existing opt-in `TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS`, -`TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` and `TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION` -settings; changing them is a different arm and cannot silently resume an older -run. Production topic defaults are unchanged. Qualify packet admission and -coverage before increasing concurrency; context capacity does not establish -provider tokens-per-minute capacity. +and effective topic settings: input target and prior fraction remain unset +(existing node/character packing, no reserved prior fraction), while prior +deltas and compact IDs default to true. Configure explicit token packing with +`TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS`, `TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION`, +`TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS` and `TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS`. +The historical 24000-token / 0.25-fraction candidate settings are not defaults. +Generative topics always use frozen two-packet preparation and serial +reconciliation/commit. Removed `TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` and +`TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION` settings fail explicitly. +Changing settings creates a different arm; older arm identities cannot silently +resume under the new defaults. Retain old reports as historical evidence and +create fresh run artifacts. Native Decision scheduling remains unchanged. +Context capacity does not establish provider tokens-per-minute capacity. During required-knowledge waits, an independent observer reads the authoritative checkpoint every 30 seconds. Progress records separate the wait outcome from diff --git a/ts/tools/memory-maintenance-qualification/topic-curation-README.md b/ts/tools/memory-maintenance-qualification/topic-curation-README.md index 5e3b0013d6..9cb80523b4 100644 --- a/ts/tools/memory-maintenance-qualification/topic-curation-README.md +++ b/ts/tools/memory-maintenance-qualification/topic-curation-README.md @@ -13,7 +13,8 @@ installation. Native receipt validation remains unchanged. The LLM driver remains an LLM engine adapter. The separate [`topic-decision-replay.mjs`](./topic-decision-replay.mjs) uses the existing -Decision generator and its native scheduling, not the LLM speculative scheduler. +Decision generator and its unchanged native scheduling, not generative frozen +preparation. Both use small shared loading, hashing, implementation-identity, atomic-progress and disposable-storage utilities. No plugin/scheduler framework is introduced. @@ -21,41 +22,60 @@ Run commands from `ts` after `pnpm run build`. Use private, **new** output directories outside the original corpus store. Captured requests/responses contain source evidence; do not upload them to public systems. -## Production defaults and experimental options +## Production defaults and migration Evidence selection, ownership validation and lexical nomination use snapshot-local indexes while retaining citation ordering and the existing top-24 nomination policy. Similarity nominates comparisons; the model still judges equivalence. -Default curation retains legacy packing (up to 24 raw nodes, 32,000 payload -characters) and one worker. Hosts using `getConfiguredMemoryViewOptions` can opt -into the following settings; programmatic callers can use `topicCuration` on -`FileMemoryServiceOptions`. - -| Environment variable | Values | Effect | -| --------------------------------------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------- | -| `TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS` | Positive integer | Packs against the complete serialized request, including instructions and schema, using native model inspection | -| `TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION` | 0 through 0.5; requires input target | Reserves input and estimated output room for prior-concept comparisons | -| `TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` | 1 or 2 | Bounds speculative proposals; commits remain in canonical order | - -Invalid settings fail explicitly. Experimental settings change policy/input -identity. Unset the variables to retain defaults. - -The generative replay CLI accepts `--input-tokens`, `--prior-fraction` and -`--concurrency` for those same production settings. It also passes these opt-in -production treatments through unchanged: - -| CLI flag | `TopicCurationSettings` field | -| ---------------------- | ----------------------------- | -| `--prior-deltas` | `priorDeltas: true` | -| `--compact-ids` | `compactIds: true` | -| `--frozen-preparation` | `frozenPreparation: true` | - -Omitted treatments remain unset, preserving production defaults. Negative -forms (`--no-prior-deltas`, `--no-compact-ids`, `--no-frozen-preparation`) record -explicit false settings. Production validates combinations; frozen preparation -requires concurrency 2. Effective settings are saved in both result and replay -identity, so recordings/checkpoints from different policies cannot be mixed. +Generative curation always prepares up to two raw-only packets concurrently, +then reconciles and commits each window serially. There is no serial or +speculative scheduler selection. Production policy is `curated-lazy-v10:`; +`resolveTopicCurationSettings(settings?)` resolves the configurable defaults +below. Hosts use `getConfiguredMemoryViewOptions`; programmatic callers use +`topicCuration` on `FileMemoryServiceOptions`. +Packet budgets preserve existing behavior: without an explicit input-token +target, packets retain the up-to-24-node / 32,000-payload-character bounds, +with no reserved prior fraction. Enabling frozen scheduling does not require +request inspection on an existing injected model. Explicit context-aware token +budgets still require the model's configured request inspection. + +| Environment variable | Values (default) | Effect | +| --------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `TYPEAGENT_MEMORY_TOPIC_INPUT_TOKENS` | Positive integer (unset) | Opts into packing the complete serialized request, including instructions and schema, using native model inspection | +| `TYPEAGENT_MEMORY_TOPIC_PRIOR_FRACTION` | 0 through 0.5 (unset / 0) | Reserves input and estimated output room for prior-concept comparisons with explicit token packing | +| `TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS` | Explicit boolean (true) | Allows unchanged prior concepts to remain read-only context | +| `TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS` | Explicit boolean (true) | Uses compact wire identifiers while preserving host IDs | + +Invalid settings fail explicitly. Settings change policy/input identity. +Unset the variables to use the frozen defaults. +The 24000-token / 0.25-fraction settings used in historical live candidate +trials remain explicit configurable treatments, not production defaults. + +The generative replay CLI accepts `--input-tokens`, `--prior-fraction`, +`--prior-deltas` and `--compact-ids` for those same production settings: + +| CLI flag | `TopicCurationSettings` field | +| ---------------- | ----------------------------- | +| `--prior-deltas` | `priorDeltas: true` | +| `--compact-ids` | `compactIds: true` | + +Omitted options resolve to production defaults. Negative forms +`--no-prior-deltas` and `--no-compact-ids` record explicit false settings. +`--concurrency`, `--frozen-preparation` and `--no-frozen-preparation` have been +removed and fail explicitly. Remove them rather than substituting another mode. +The host variables `TYPEAGENT_MEMORY_TOPIC_CONCURRENCY` and +`TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION` likewise fail even when set to their +former candidate values. The corresponding programmatic fields `concurrency` +and `frozenPreparation` are no longer accepted. + +Effective settings are saved in result and replay identity. Recorded replay +requires the original `result.json` identity as well as exact request bindings. +Old tapes, progress files, captured inputs and native receipts retain their +original identities: do not edit their policy labels, fingerprints or settings +to make them appear compatible. Generate fresh artifacts for the new policy. +The low-level `curateTopicEvidence` parser supports native resolution and frozen +preparation/reconciliation only; it is not an alternate host build mode. Frozen preparation prepares two raw-only packets concurrently, then serially reconciles their proposed groups and every deferred raw candidate against the @@ -64,9 +84,8 @@ atomically, with one final fingerprinted receipt per window. It is not two concurrent canonical commits. Prior deltas make unchanged prior concepts read-only context that can be omitted from responses; compact IDs change only wire node identifiers and schema enums, preserving factual fields and host IDs. -The host equivalents are `TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS`, -`TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS`, and -`TYPEAGENT_MEMORY_TOPIC_FROZEN_PREPARATION` (explicit booleans, default unset). +The configurable host equivalents are `TYPEAGENT_MEMORY_TOPIC_PRIOR_DELTAS` +and `TYPEAGENT_MEMORY_TOPIC_COMPACT_IDS` (explicit booleans, default true). Unchanged prior objects, including audited summaries, also survive receipt replay without being reconstructed as new pending concepts. @@ -84,10 +103,8 @@ curated by this first treatment. `TYPEAGENT_MEMORY_TOPIC_WATERMARK=true` (programmatic `topicWatermark: true`) adds private pinned-generation checkpoints and exact packet-prefix replay. -It requires curation reuse and ordered generative curated topics: either one -worker or explicit frozen preparation with two workers. Native Decision and -ordinary speculative two-worker execution still reject this option. Defaults -remain unchanged. +It requires curation reuse and generative curated topics. Native Decision still +rejects this option. Reuse and watermark remain opt-in. Source additions may leave one old build running while the existing maintenance scheduler coalesces one latest topic demand. Old curation checkpoints are durable @@ -98,8 +115,7 @@ prefix. Instructions, payload, schema and hidden original membership must match; the first changed ordering, packet boundary, qualification or nomination ends replay. Cached responses pass the production parser without provider repairs. Receipts store packet-local candidate/evidence IDs rather than repeated expanded -concept membership. Legacy first/every-eighth/final checkpoint frequency stays unchanged. -Frozen preparation checkpoints every completed reconciliation window; partially +concept membership. Frozen curation checkpoints every completed reconciliation window; partially prepared windows never become resumable installed decisions. Source removal/replacement, suppression, events and model identity changes fail @@ -177,11 +193,9 @@ admission guard. Packing also checks an experimental output estimate of This is not a proven worst-case schema bound or a new provider output cap. A larger input target cannot cure output-bound prior starvation. -Speculative responses are reusable only if rebuilt instructions, payload, -schema, packet boundary and hidden original membership still match. Newly -created targets can invalidate later proposals even across disjoint sources. -Stale calls are aborted/drained and rerun. Failure and cancellation cannot -publish pending speculative work. +Frozen receipts are reusable only if rebuilt instructions, payload, schema, +packet boundary and hidden original membership still match. Failure and +cancellation cannot publish a partially prepared or unreconciled window. Context-budgeted packets explicitly bind permitted evidence to all original member sources, not only a prior concept's retained citations. Strict validation @@ -199,7 +213,8 @@ and bounded repairs remain in force. Detailed summaries stay lazy. | `live` | Bounded configured-model curation | Requires `--live` | Choose `--history --commit ` (and `--job` if needed) -or `--input `. Capture uses one worker. +or `--input `. Capture saves the first frozen +preparation request without initializing a provider; it does not install a window. ```powershell if (Test-Path $PROFILE) { . $PROFILE } @@ -211,12 +226,12 @@ node $tool --mode cpu --history $history --commit $commit ` --samples 5 --warmups 1 --output $newCpuDirectory node $tool --mode recorded --history $history --commit $commit ` - --nodes 96 --input-tokens 32000 --prior-fraction 0.25 --concurrency 1 ` + --nodes 96 --input-tokens 24000 --prior-fraction 0.25 ` --max-calls 8 --responses $matchingResponses --output $newReplayDirectory # Only after authorizing provider calls: node $tool --mode live --live --history $history --commit $commit ` - --nodes 96 --input-tokens 32000 --prior-fraction 0.25 --concurrency 1 ` + --nodes 96 --input-tokens 24000 --prior-fraction 0.25 ` --max-calls 8 --timeout-ms 240000 --output $newLiveDirectory ``` @@ -229,7 +244,9 @@ across machines when provider capacity is shared. ## Progress, organization and persistence - `--nodes` selects a canonical subset but retains full source evidence context. -- `--resume` uses historical validated progress and cannot combine with `--nodes`. +- `--resume` uses historical validated progress only when its saved recipe + matches the current frozen policy and effective settings; it cannot combine + with `--nodes`. Legacy-policy progress is rejected rather than relabeled. - `--progress ` resumes atomic private checkpoints only under matching input, node selection, implementation, endpoint and settings; cannot combine with `--resume`. Progress includes a curation/state digest. @@ -332,14 +349,15 @@ $common = @('--mode', 'live', '--live', '--history', $history, '--commit', $commit, '--job', $job, '--deadline', $deadline, '--endpoint', $endpoint, '--expected-model-binding', $modelBinding.hash, '--timeout-ms', '10800000', '--max-calls', '10000', '--organize') -node $tool @common --output $newBaselineDirectory -node $tool @common --output $newCandidateDirectory ` - --concurrency 2 --input-tokens 24000 --prior-fraction 0.25 ` - --prior-deltas --compact-ids --frozen-preparation +node $tool @common --output $newDefaultDirectory +node $tool @common --output $newSettingsDirectory ` + --input-tokens 12000 --prior-fraction 0.25 ` + --no-prior-deltas --no-compact-ids ``` -The candidate changes multiple policies; it is a paired treatment, not an -attribution experiment for individual flags. An incomplete arm is reported as +Both arms use frozen preparation. The second changes multiple configurable +settings; it is not a legacy-serial baseline or an attribution experiment for +individual flags. An incomplete arm is reported as incomplete even if structural coverage is high. No live trial is part of the offline regression tests: @@ -363,7 +381,7 @@ Thus metadata, local adjudication, maintenance and organization use the same generator as production, and every native or generative completion consumes the same explicit call budget/deadline. Native execution settings are passed unchanged; concurrency 24 is supported and never -replaced with the LLM one/two-worker scheduler. +replaced with the generative frozen preparation schedule. Supply `--identity-file` with a frozen JSON manifest: @@ -476,10 +494,11 @@ semantic quality or fresh end-to-end readiness. Corrected-contract 96-node ABBA trials averaged 133.229 seconds with one worker and 134.979 with two; stale proposals increased work. Two samples per condition -are not enough for a general performance conclusion. No experimental defaults -were promoted. Human admission/merge review, fresh full-input runs and native +are not enough for a general performance conclusion. At that historical +checkpoint no experimental defaults were promoted; it does not describe the +current frozen defaults. Human admission/merge review, fresh full-input runs and native readiness/answer qualification remain required. Run offline regression coverage with `pnpm run test:memory-quality`. Relevant service tests cover budget boundaries, indexed parity, hidden-member evidence, -ordered commits, stale proposals, cancellation and checkpoint failures. +frozen preparation, serial reconciliation/commit, cancellation and checkpoint failures. diff --git a/ts/tools/memory-maintenance-qualification/topic-curation-options.test.mjs b/ts/tools/memory-maintenance-qualification/topic-curation-options.test.mjs index fdc2108564..bc0fc778d8 100644 --- a/ts/tools/memory-maintenance-qualification/topic-curation-options.test.mjs +++ b/ts/tools/memory-maintenance-qualification/topic-curation-options.test.mjs @@ -78,7 +78,7 @@ test("JavaScript callers reject obsolete positional arguments before model or en assert.equal(calls, 0); for (const mode of [ { kind: "raw" }, - { kind: "generative", settings: { concurrency: 2 } }, + { kind: "generative", settings: { compactIds: false } }, { kind: "decision", createOptions: factory }, ]) assert.equal( @@ -107,6 +107,15 @@ test("named mode validation rejects mixed or malformed settings without silently { mode: { kind: "generative", settings: null } }, { mode: { kind: "generative", settings: { unknown: true } } }, { mode: { kind: "generative", settings: { concurrency: 24 } } }, + { mode: { kind: "generative", settings: { concurrency: 1 } } }, + { mode: { kind: "generative", settings: { concurrency: 2 } } }, + { mode: { kind: "generative", settings: { frozenPreparation: true } } }, + { + mode: { + kind: "generative", + settings: { frozenPreparation: false }, + }, + }, { mode: { kind: "decision" } }, { mode: { kind: "decision", createOptions: true } }, { mode: { kind: "decision", createOptions: () => {}, settings: {} } }, diff --git a/ts/tools/memory-maintenance-qualification/topic-curation-replay.mjs b/ts/tools/memory-maintenance-qualification/topic-curation-replay.mjs index 6baaaefd16..cd38610dbb 100644 --- a/ts/tools/memory-maintenance-qualification/topic-curation-replay.mjs +++ b/ts/tools/memory-maintenance-qualification/topic-curation-replay.mjs @@ -27,8 +27,11 @@ import { orderedTopicNodes, validateTopicCuration, validateTopicCurationProgress, - validateTopicCurationSettings, } from "../../packages/memory/service/dist/topicCuration.js"; +import { + resolveTopicCurationSettings, + topicCurationPolicy, +} from "../../packages/memory/service/dist/topicCurationSettings.js"; import { TopicEvidenceIndex, TopicNominationIndex, @@ -41,6 +44,7 @@ import { loadCapturedTopicInput, positiveInteger as integer, compiledFileHashes, + assertReplayIdentity, readReplayProgress, writeReplayProgress, } from "./replay-artifacts.mjs"; @@ -137,6 +141,12 @@ export async function inspectConfiguredTopicReplayModelBinding(endpoint) { } export function topicReplaySettings(values) { + for (const option of ["concurrency", "frozen-preparation"]) + assert.equal( + values[option], + undefined, + `--${option} has been removed; generative topics always use frozen preparation and serial reconciliation`, + ); const settings = { ...(values["input-tokens"] !== undefined ? { @@ -147,7 +157,6 @@ export function topicReplaySettings(values) { ), } : {}), - concurrency: integer(values.concurrency, 1, "concurrency"), ...(values["prior-fraction"] !== undefined ? { priorContextFraction: Number(values["prior-fraction"]) } : {}), @@ -155,15 +164,13 @@ export function topicReplaySettings(values) { for (const [option, field] of [ ["prior-deltas", "priorDeltas"], ["compact-ids", "compactIds"], - ["frozen-preparation", "frozenPreparation"], ]) { if (values[option] !== undefined) { assert.ok(typeof values[option] === "boolean", `Invalid ${option}`); settings[field] = values[option]; } } - validateTopicCurationSettings(settings); - return settings; + return resolveTopicCurationSettings(settings); } export function topicReplayDeadline(values, now = Date.now) { @@ -278,7 +285,7 @@ export async function loadTopicReplay(values) { return { input, snapshot, state, inputHash, retainedCurationHash }; } -function offlineInspection(rules, payload, schema) { +export function offlineInspection(rules, payload, schema) { const serializedRequest = JSON.stringify( wire.buildRequestBody({ messages: [ @@ -579,12 +586,12 @@ async function callReplay( !recorded || values.responses, "Recorded mode requires --responses", ); - assert.ok( - values.mode !== "capture" || - settings.concurrency === 1 || - settings.frozenPreparation, - "Capture mode requires concurrency 1 unless frozen preparation is enabled", - ); + if (recorded) { + const recording = JSON.parse( + await readFile(path.join(values.responses, "result.json"), "utf8"), + ); + assertReplayIdentity(recording.identity, identity); + } const maxCalls = integer(values["max-calls"], 1, "max-calls"); const { controller } = budget; let model; @@ -614,6 +621,10 @@ async function callReplay( const complete = async (rules, payload, schema, context) => { budget.check(); context.signal.throwIfAborted(); + if (!live && !recorded && calls.length > 0) + throw new CaptureComplete( + "First frozen preparation request captured without provider initialization", + ); if (calls.length >= maxCalls) throw new CallLimit("Replay call limit reached"); const request = inspect(rules, payload, schema); @@ -914,8 +925,8 @@ export async function implementationIdentity() { identity.compiled = await compiledFileHashes(productRoot, [ ...[ "topicCuration", + "topicCurationSettings", "topicLookup", - "topicCurationScheduler", "topicGeneration", "topicCurationTransport", "topicFrozenPreparation", @@ -994,6 +1005,12 @@ export async function runTopicReplay( ); const { input, snapshot, state, inputHash, retainedCurationHash } = await loadTopicReplay(values); + if (values.resume) + assert.equal( + state.topics.recipe, + `${input.model}:editorial-v1:${topicCurationPolicy(true, settings)}`, + "Historical curation policy does not match frozen replay settings; capture fresh input instead of relabeling retained progress", + ); const implementation = await implementationIdentity(); const identity = { ...implementation, diff --git a/ts/tools/memory-maintenance-qualification/topic-curation-replay.test.mjs b/ts/tools/memory-maintenance-qualification/topic-curation-replay.test.mjs index a3fcc5b56f..48a29f708a 100644 --- a/ts/tools/memory-maintenance-qualification/topic-curation-replay.test.mjs +++ b/ts/tools/memory-maintenance-qualification/topic-curation-replay.test.mjs @@ -16,6 +16,7 @@ import { topicReplayDeadline, finalTopicCoverage, inspectTopicReplayModelBinding, + offlineInspection, } from "./topic-curation-replay.mjs"; import { ViewHistory } from "../../packages/memory/service/dist/viewHistory.js"; import { writeSegmentedJson } from "../../packages/knowPro/dist/segmentedJson.js"; @@ -25,16 +26,33 @@ function syntheticModel(complete) { return { completionSettings: { response_format: { type: "json_object" } }, inspectRequest(messages, schema) { - return { - serializedRequest: JSON.stringify({ messages, schema }), - estimatedInputTokens: 100, - constraints: { outputTokens: 1000 }, - }; + return offlineInspection( + messages[0].content, + JSON.parse(messages[1].content), + schema, + ); }, complete, }; } +async function recordSynthetic(values, complete = deferResponse) { + return runTopicReplay( + { ...values, mode: "live", live: true, "max-calls": "1000" }, + { initializeModel: () => syntheticModel(complete) }, + ); +} + +function firstWindowCalls(result) { + let reconciled = false; + const next = result.observations.calls.findIndex(({ phase }) => { + if (phase === "reconciliation") reconciled = true; + return reconciled && phase === "preparation"; + }); + assert.ok(next > 0, "Fixture must span multiple frozen windows"); + return String(next); +} + function deferResponse(messages) { const payload = JSON.parse(messages[1].content); return { @@ -173,7 +191,6 @@ test("binding is rechecked immediately before normal, repair and organization pr const input = await fixture(root); for (const boundary of ["normal", "contract-repair", "organization"]) { let providerCalls = 0; - let inspections = 0; const model = syntheticModel((messages) => { providerCalls++; if (boundary === "organization") { @@ -181,22 +198,23 @@ test("binding is rechecked immediately before normal, repair and organization pr return { success: true, data: JSON.stringify({ - concepts: [nodes.slice(0, 4), nodes.slice(4)].map( - (group, index) => ({ - label: `Synthetic concept ${index}`, - scope: "Fixture", - purpose: "A scoped fixture reference", - salience: 6, - memberIds: group.map(({ id }) => id), - evidenceIds: [ - ...new Set( - group.flatMap( - (node) => node.evidenceIds, - ), + concepts: [ + nodes.slice(0, Math.ceil(nodes.length / 2)), + nodes.slice(Math.ceil(nodes.length / 2)), + ].map((group, index) => ({ + label: `Synthetic concept ${index}`, + scope: "Fixture", + purpose: "A scoped fixture reference", + salience: 6, + memberIds: group.map(({ id }) => id), + evidenceIds: [ + ...new Set( + group.flatMap( + (node) => node.evidenceIds, ), - ], - }), - ), + ), + ], + })), deferred: [], }), }; @@ -212,12 +230,14 @@ test("binding is rechecked immediately before normal, repair and organization pr const inspect = model.inspectRequest; model.inspectRequest = (...args) => { const result = inspect(...args); - inspections++; if ( (boundary === "organization" && Array.isArray(JSON.parse(args[0][1].content))) || - (boundary !== "organization" && - inspections === (boundary === "normal" ? 1 : 2)) + boundary === "normal" || + (boundary === "contract-repair" && + args[0][0].content.includes( + "Repair the rejected curation contract", + )) ) model.completionSettings.temperature = 0.8; return result; @@ -239,7 +259,10 @@ test("binding is rechecked immediately before normal, repair and organization pr ), /Replay model binding does not match/, ); - assert.equal(providerCalls, boundary === "normal" ? 0 : 1); + assert.equal( + providerCalls, + boundary === "normal" ? 0 : boundary === "organization" ? 2 : 1, + ); const calls = JSON.parse( await readFile(path.join(output, "calls.json"), "utf8"), ); @@ -267,11 +290,12 @@ test("matching model binding is saved in result metrics and each provider receip live: true, endpoint: "gpt-6-luna", "expected-model-binding": expected.hash.toUpperCase(), + "max-calls": "10", }, { initializeModel: () => model }, ); assert.equal(result.observations.completed, true); - assert.equal(result.observations.providerCalls, 1); + assert.equal(result.observations.providerCalls, 2); assert.equal(result.observations.modelBindingHash, expected.hash); assert.deepEqual(result.observations.modelBinding, expected.binding); assert.equal( @@ -397,23 +421,22 @@ test("payload history reads the selected build at a pinned commit and never adva }); test("effective topic replay settings preserve defaults and every explicit treatment", () => { - assert.deepEqual(topicReplaySettings({}), { concurrency: 1 }); + assert.deepEqual(topicReplaySettings({}), { + priorDeltas: true, + compactIds: true, + }); assert.deepEqual( topicReplaySettings({ - concurrency: "2", "input-tokens": "12000", "prior-fraction": "0.25", "prior-deltas": true, "compact-ids": false, - "frozen-preparation": true, }), { - concurrency: 2, inputTokenTarget: 12000, priorContextFraction: 0.25, priorDeltas: true, compactIds: false, - frozenPreparation: true, }, ); assert.throws( @@ -421,9 +444,16 @@ test("effective topic replay settings preserve defaults and every explicit treat /Invalid compact-ids/, ); assert.throws( - () => topicReplaySettings({ "prior-fraction": "0.25" }), + () => topicReplaySettings({ "prior-fraction": "0.5" }), /Invalid topic curation settings/, ); + for (const values of [ + { concurrency: "1" }, + { concurrency: "2" }, + { "frozen-preparation": true }, + { "frozen-preparation": false }, + ]) + assert.throws(() => topicReplaySettings(values), /has been removed/); }); test("compact wire IDs retain raw counts and candidate capture saves effective production settings", async () => { @@ -435,8 +465,6 @@ test("compact wire IDs retain raw counts and candidate capture saves effective p const settings = { "compact-ids": true, "prior-deltas": true, - "frozen-preparation": true, - concurrency: "2", "input-tokens": "24000", "prior-fraction": "0.25", }; @@ -577,11 +605,10 @@ test("deadline is checked immediately before every provider call, including repa data.deferred[1] = data.deferred[0]; return { success: true, data: JSON.stringify(data) }; }); - let inspections = 0; const inspect = model.inspectRequest; model.inspectRequest = (...args) => { const result = inspect(...args); - if (++inspections === 2) now += 1001; + if (args[0][0].content.includes("Repair")) now += 1001; return result; }; const output = path.join(root, "cancelled"); @@ -613,7 +640,7 @@ test("provider gets an abort signal and cancellation persists metrics and valida path.join(testDirectory, "topic-provider-abort-"), ); try { - const input = await fixture(root, 55); + const input = await fixture(root, 200); let receivedSignal; let calls = 0; const output = path.join(root, "cancelled"); @@ -624,6 +651,7 @@ test("provider gets an abort signal and cancellation persists metrics and valida mode: "live", live: true, "max-calls": "10", + "input-tokens": "6000", "timeout-ms": "3000", }, { @@ -639,17 +667,21 @@ test("provider gets an abort signal and cancellation persists metrics and valida ); assert.equal(receivedSignal.aborted, true); assert.equal(result.observations.disposition, "deadline-incomplete"); - assert.equal(result.observations.providerCalls, 2); - assert.ok(result.observations.coverage.deferredNodes > 0); + assert.ok(result.observations.providerCalls >= 2); + assert.equal(result.observations.coverage.deferredNodes, 0); assert.ok(result.observations.coverage.unaccountedNodes > 0); - const progress = JSON.parse( - await readFile(path.join(output, "progress.json"), "utf8"), - ); const partial = JSON.parse( await readFile(path.join(output, "partial-curation.json"), "utf8"), ); - assert.deepEqual(partial, progress.curation); - assert.equal(result.observations.calls[1].status, "aborted"); + assert.deepEqual(partial, { concepts: [], deferred: [] }); + assert.ok( + result.observations.calls.some( + ({ status }) => status === "aborted", + ), + ); + await assert.rejects(readFile(path.join(output, "progress.json")), { + code: "ENOENT", + }); } finally { await rm(root, { recursive: true, force: true }); } @@ -669,20 +701,21 @@ test("organization shares the original deadline and cannot invoke a provider aft return { success: true, data: JSON.stringify({ - concepts: [nodes.slice(0, 4), nodes.slice(4)].map( - (group, index) => ({ - label: `Synthetic concept ${index}`, - scope: "Fixture", - purpose: "A scoped fixture reference", - salience: 6, - memberIds: group.map(({ id }) => id), - evidenceIds: [ - ...new Set( - group.flatMap((node) => node.evidenceIds), - ), - ], - }), - ), + concepts: [ + nodes.slice(0, Math.ceil(nodes.length / 2)), + nodes.slice(Math.ceil(nodes.length / 2)), + ].map((group, index) => ({ + label: `Synthetic concept ${index}`, + scope: "Fixture", + purpose: "A scoped fixture reference", + salience: 6, + memberIds: group.map(({ id }) => id), + evidenceIds: [ + ...new Set( + group.flatMap((node) => node.evidenceIds), + ), + ], + })), deferred: [], }), }; @@ -707,7 +740,7 @@ test("organization shares the original deadline and cannot invoke a provider aft }, { now: () => now, initializeModel: () => model }, ); - assert.equal(providerCalls, 1); + assert.equal(providerCalls, 2); assert.equal(result.observations.disposition, "deadline-incomplete"); assert.equal(result.observations.coverage.admittedNodes, 8); assert.equal(result.observations.coverage.unaccountedNodes, 0); @@ -794,24 +827,14 @@ test("capture, indexed parity and recorded parser replay require no provider", a assert.equal(captured.observations.providerCalls, 0); assert.equal(captured.observations.disposition, "capture-only"); assert.equal(captured.observations.completed, false); - const binding = JSON.parse( - await readFile(path.join(output, "binding-1.json"), "utf8"), - ); - await writeFile( - path.join(output, "response-1.json"), - JSON.stringify({ - concepts: [], - deferred: binding.payload.nodes.map(({ id }) => ({ - nodeId: id, - reason: "Synthetic replay fixture", - })), - }), - ); + const responses = path.join(root, "responses"); + await recordSynthetic({ input, output: responses }); const recorded = await runTopicReplay({ input, output: path.join(root, "recorded"), mode: "recorded", - responses: output, + responses, + "max-calls": "10", }); assert.equal(recorded.observations.completed, true); assert.equal(recorded.observations.providerCalls, 0); @@ -820,7 +843,7 @@ test("capture, indexed parity and recorded parser replay require no provider", a input, output: path.join(root, "resumed"), mode: "recorded", - responses: output, + responses, progress: path.join(root, "recorded", "progress.json"), organize: true, }); @@ -865,7 +888,7 @@ test("replay rejects invalid budgets and mismatched response bindings", async () output: path.join(root, "invalid"), concurrency: "3", }), - /Invalid topic curation/, + /has been removed/, ); const output = path.join(root, "capture"); await runTopicReplay({ input, output }); @@ -884,6 +907,79 @@ test("replay rejects invalid budgets and mismatched response bindings", async () } }); +test("legacy recording identities are rejected without rewriting artifacts", async () => { + const root = await mkdtemp(path.join(testDirectory, "topic-legacy-tape-")); + try { + const input = await fixture(root); + const output = path.join(root, "tape"); + await recordSynthetic({ input, output }); + const file = path.join(output, "result.json"); + const recording = JSON.parse(await readFile(file, "utf8")); + recording.identity.settings = { concurrency: 1 }; + const legacy = JSON.stringify(recording); + await writeFile(file, legacy); + await assert.rejects( + runTopicReplay({ + input, + mode: "recorded", + responses: output, + output: path.join(root, "rejected"), + }), + /Replay identity does not match/, + ); + assert.equal(await readFile(file, "utf8"), legacy); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("historical progress from a legacy curation policy is not relabeled for resume", async () => { + const root = await mkdtemp( + path.join(testDirectory, "topic-legacy-progress-"), + ); + try { + const file = await fixture(root); + const input = JSON.parse(await readFile(file, "utf8")); + const { snapshot } = await loadTopicReplay({ input: file }); + const directory = path.join(root, "store", "views"); + const history = new ViewHistory(directory, () => ({})); + const head = await history.commit( + null, + { + builds: { selected: { results: [{ snapshot: input }] } }, + topics: { + inputFingerprint: input.fingerprint, + recipe: `${input.model}:editorial-v1:curated-lazy-v6`, + curationProgress: { + concepts: [], + deferred: snapshot.nodes.map(({ id }) => ({ + nodeId: id, + reason: "Historical fixture", + })), + }, + }, + }, + {}, + "offline", + "Legacy progress", + ); + await assert.rejects( + runTopicReplay({ + history: path.join(directory, "view-history.git"), + commit: head, + job: "selected", + resume: true, + output: path.join(root, "rejected"), + }), + /Historical curation policy does not match/, + ); + assert.equal(await history.readHead(), head); + assert.equal(await readFile(file, "utf8"), JSON.stringify(input)); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("implementation identity excludes generated tool data artifacts", async () => { const root = await mkdtemp( path.join( @@ -898,6 +994,12 @@ test("implementation identity excludes generated tool data artifacts", async () output: path.join(root, "capture"), }); assert.equal(result.identityVerified, true); + assert.match( + result.identity.compiled[ + "packages/memory/service/dist/topicCurationSettings.js" + ], + /^[a-f0-9]{64}$/, + ); assert.ok( Object.keys(result.identity.untracked).every( (file) => !file.includes(path.basename(root)), @@ -913,43 +1015,40 @@ test("bounded replay resumes only matching validated durable progress", async () path.join(testDirectory, "topic-replay-resume-"), ); try { - const input = await fixture(root, 55); + const input = await fixture(root, 200); + const settings = { "input-tokens": "6000" }; const captured = path.join(root, "captured"); - await runTopicReplay({ input, output: captured }); - const binding = JSON.parse( - await readFile(path.join(captured, "binding-1.json"), "utf8"), - ); - await writeFile( - path.join(captured, "response-1.json"), - JSON.stringify({ - concepts: [], - deferred: binding.payload.nodes.map(({ id }) => ({ - nodeId: id, - reason: "Synthetic replay fixture", - })), - }), - ); + const tape = await recordSynthetic({ + input, + output: captured, + ...settings, + }); const partial = await runTopicReplay({ input, + ...settings, output: path.join(root, "partial"), mode: "recorded", responses: captured, + "max-calls": firstWindowCalls(tape), }); assert.equal(partial.observations.completed, false); assert.equal(partial.observations.disposition, "call-limit-incomplete"); const progress = path.join(root, "partial", "progress.json"); const remaining = path.join(root, "remaining"); - await runTopicReplay({ input, progress, output: remaining }); - const next = JSON.parse( - await readFile(path.join(remaining, "binding-1.json"), "utf8"), - ); - const saved = new Set(binding.payload.nodes.map(({ id }) => id)); - assert.ok(next.payload.nodes.every(({ id }) => !saved.has(id))); + const next = await recordSynthetic({ + input, + progress, + output: remaining, + ...settings, + }); + assert.equal(next.observations.completed, true); + assert.equal(next.observations.accountedNodes, 200); await assert.rejects( runTopicReplay({ input, progress, - concurrency: "2", + ...settings, + "prior-deltas": false, output: path.join(root, "wrong-policy"), }), /does not match/, @@ -958,7 +1057,8 @@ test("bounded replay resumes only matching validated durable progress", async () runTopicReplay({ input, progress, - nodes: "54", + ...settings, + nodes: "199", output: path.join(root, "wrong-selection"), }), /does not match/, @@ -973,48 +1073,31 @@ test("subset replay resumes partial and completed progress with complete source path.join(testDirectory, "topic-replay-subset-resume-"), ); try { - const input = await fixture(root, 55); - const selected = { input, nodes: "30" }; + const input = await fixture(root, 250); + const selected = { input, nodes: "200", "input-tokens": "6000" }; const captured = path.join(root, "captured"); - const capture = await runTopicReplay({ + const capture = await recordSynthetic({ ...selected, output: captured, }); - assert.equal(capture.nodes, 30); - assert.equal(capture.occurrences, 55); - const deferCaptured = async (directory) => { - const binding = JSON.parse( - await readFile(path.join(directory, "binding-1.json"), "utf8"), - ); - await writeFile( - path.join(directory, "response-1.json"), - JSON.stringify({ - concepts: [], - deferred: binding.payload.nodes.map(({ id }) => ({ - nodeId: id, - reason: "Synthetic subset replay fixture", - })), - }), - ); - return binding.payload.nodes.length; - }; - const processed = await deferCaptured(captured); + assert.equal(capture.nodes, 200); + assert.equal(capture.occurrences, 250); const partial = path.join(root, "partial"); const first = await runTopicReplay({ ...selected, mode: "recorded", responses: captured, output: partial, + "max-calls": firstWindowCalls(capture), }); assert.equal(first.observations.completed, false); const progress = path.join(partial, "progress.json"); const remaining = path.join(root, "remaining"); - await runTopicReplay({ + await recordSynthetic({ ...selected, progress, output: remaining, }); - assert.equal(processed + (await deferCaptured(remaining)), 30); const completed = path.join(root, "completed"); const result = await runTopicReplay({ ...selected, @@ -1022,24 +1105,25 @@ test("subset replay resumes partial and completed progress with complete source mode: "recorded", responses: remaining, output: completed, + "max-calls": "1000", }); assert.equal(result.observations.completed, true); - assert.equal(result.observations.accountedNodes, 30); - assert.equal(result.occurrences, 55); + assert.equal(result.observations.accountedNodes, 200); + assert.equal(result.occurrences, 250); const resumed = await runTopicReplay({ ...selected, progress: path.join(completed, "progress.json"), mode: "recorded", - responses: path.join(root, "unused-responses"), + responses: remaining, output: path.join(root, "resumed"), }); assert.equal(resumed.observations.completed, true); - assert.equal(resumed.observations.accountedNodes, 30); + assert.equal(resumed.observations.accountedNodes, 200); assert.equal(resumed.observations.calls.length, 0); await assert.rejects( runTopicReplay({ ...selected, - nodes: "31", + nodes: "201", progress, output: path.join(root, "wrong-selection"), }), @@ -1128,36 +1212,38 @@ test("completed curation resumes through the real organization generator without try { const input = await fixture(root); const captured = path.join(root, "captured"); - await runTopicReplay({ input, output: captured }); - const binding = JSON.parse( - await readFile(path.join(captured, "binding-1.json"), "utf8"), - ); - const groups = [ - binding.payload.nodes.slice(0, 4), - binding.payload.nodes.slice(4), - ]; - await writeFile( - path.join(captured, "response-1.json"), - JSON.stringify({ - concepts: groups.map((nodes, index) => ({ - label: `Synthetic concept ${index}`, - scope: "Fixture", - purpose: "A scoped fixture reference", - salience: 6, - memberIds: nodes.map(({ id }) => id), - evidenceIds: [ - ...new Set(nodes.flatMap((node) => node.evidenceIds)), - ], - })), - deferred: [], - }), - ); + await recordSynthetic({ input, output: captured }, (messages) => { + const nodes = JSON.parse(messages[1].content).nodes; + const groups = [ + nodes.slice(0, nodes.length / 2), + nodes.slice(nodes.length / 2), + ]; + return { + success: true, + data: JSON.stringify({ + concepts: groups.map((nodes, index) => ({ + label: `Synthetic concept ${index}`, + scope: "Fixture", + purpose: "A scoped fixture reference", + salience: 6, + memberIds: nodes.map(({ id }) => id), + evidenceIds: [ + ...new Set( + nodes.flatMap((node) => node.evidenceIds), + ), + ], + })), + deferred: [], + }), + }; + }); const curated = path.join(root, "curated"); await runTopicReplay({ input, mode: "recorded", responses: captured, output: curated, + "max-calls": "10", }); await test("frozen replay organizes captured watermark input without claiming native packet receipts", async () => { @@ -1218,12 +1304,10 @@ test("completed curation resumes through the real organization generator without mode: "live", live: true, organize: true, - concurrency: "2", "input-tokens": "24000", "prior-fraction": "0.25", "prior-deltas": true, "compact-ids": true, - "frozen-preparation": true, "max-calls": "10", }, { initializeModel: () => model }, @@ -1249,12 +1333,10 @@ test("completed curation resumes through the real organization generator without mode: { kind: "generative", settings: { - concurrency: 2, inputTokenTarget: 24000, priorContextFraction: 0.25, priorDeltas: true, compactIds: true, - frozenPreparation: true, }, }, })(nativeInput, new AbortController().signal), @@ -1268,16 +1350,26 @@ test("completed curation resumes through the real organization generator without }); const progress = path.join(curated, "progress.json"); const organizationCapture = path.join(root, "organization-capture"); - await assert.rejects( - runTopicReplay({ + await recordSynthetic( + { input, progress, - mode: "recorded", organize: true, - responses: path.join(root, "missing-responses"), output: organizationCapture, - }), - /ENOENT/, + }, + (messages) => { + const topics = JSON.parse(messages[1].content); + assert.equal(topics.length, 2); + return { + success: true, + data: JSON.stringify({ + parents: [ + { parentId: topics[0].id, childId: topics[1].id }, + ], + related: [], + }), + }; + }, ); const organizationBinding = JSON.parse( await readFile( @@ -1287,13 +1379,6 @@ test("completed curation resumes through the real organization generator without ); const topics = organizationBinding.payload; assert.equal(topics.length, 2); - await writeFile( - path.join(organizationCapture, "response-1.json"), - JSON.stringify({ - parents: [{ parentId: topics[0].id, childId: topics[1].id }], - related: [], - }), - ); const result = await runTopicReplay({ input, progress, diff --git a/ts/tools/memory-maintenance-qualification/topic-decision-replay.test.mjs b/ts/tools/memory-maintenance-qualification/topic-decision-replay.test.mjs index 5ae3b1cde0..2ec41df8ce 100644 --- a/ts/tools/memory-maintenance-qualification/topic-decision-replay.test.mjs +++ b/ts/tools/memory-maintenance-qualification/topic-decision-replay.test.mjs @@ -728,6 +728,12 @@ test("default Decision capture hashes shipped implementations and performs no ne }); assert.equal(result.disposition, "capture-only"); assert.equal(result.providerCalls, 0); + assert.match( + result.identity.implementation[ + "packages/memory/service/dist/topicCurationSettings.js" + ], + /^[a-f0-9]{64}$/, + ); assert.ok( Object.hasOwn( result.identity.implementation, diff --git a/ts/tools/memory-maintenance-qualification/topic-work-benchmark.mjs b/ts/tools/memory-maintenance-qualification/topic-work-benchmark.mjs index bc4a914688..37c0ef00ab 100644 --- a/ts/tools/memory-maintenance-qualification/topic-work-benchmark.mjs +++ b/ts/tools/memory-maintenance-qualification/topic-work-benchmark.mjs @@ -23,6 +23,7 @@ import { } from "./replay-artifacts.mjs"; const root = fileURLToPath(new URL("../../", import.meta.url)); + const incidence = Array.from({ length: 55 }, (_, ordinal) => indexedTopicOccurrence( { @@ -101,7 +102,8 @@ async function continuousArm(reuse) { const payload = JSON.parse(String(messages[1].content)); attempts++; processedNodes += payload.nodes.length; - for (const { id } of payload.nodes) visited.add(id); + for (const { label, scope } of payload.nodes) + visited.add(JSON.stringify([label, scope])); return { success: true, data: JSON.stringify({ @@ -256,7 +258,7 @@ async function arm(change, reuse) { indexGeneration: "generation-1", }); const current = await generate(captured, new AbortController().signal); - assert.equal(calls, 3); + assert.equal(calls, 6); calls = completed = 0; const start = performance.now(); const input = await captureTopicBuild({ @@ -281,7 +283,7 @@ async function arm(change, reuse) { const captureAndCurationMs = performance.now() - start; validateTopicSnapshot(generated); assert.notEqual(input.fingerprint, current.inputFingerprint); - assert.equal(calls, change === "append" ? (reuse ? 1 : 3) : reuse ? 0 : 3); + assert.equal(calls, change === "append" ? (reuse ? 2 : 6) : reuse ? 0 : 6); assert.equal(completed, calls); if (change !== "append") { assert.deepEqual(generated.nodes, current.nodes); @@ -299,7 +301,7 @@ async function arm(change, reuse) { packet.fingerprint === generated.curationPackets?.[index]?.fingerprint, ) ?? []; - if (change === "append") assert.equal(replayed.length, reuse ? 2 : 0); + if (change === "append") assert.equal(replayed.length, reuse ? 1 : 0); return { reuse, captureMs, diff --git a/ts/tools/memory-maintenance-qualification/topic-work-benchmark.test.mjs b/ts/tools/memory-maintenance-qualification/topic-work-benchmark.test.mjs index 2a919364b2..7892ebeb9b 100644 --- a/ts/tools/memory-maintenance-qualification/topic-work-benchmark.test.mjs +++ b/ts/tools/memory-maintenance-qualification/topic-work-benchmark.test.mjs @@ -27,9 +27,9 @@ test("append-only packet treatment measures against whole-input reuse and preser assert.equal(result.workloads.length, 1); assert.equal(result.workloads[0].change, "append"); for (const { baseline, candidate } of result.workloads[0].pairs) { - assert.equal(baseline.providerAttempts, 3); - assert.equal(candidate.providerAttempts, 1); - assert.equal(candidate.replayedPackets, 2); + assert.equal(baseline.providerAttempts, 6); + assert.equal(candidate.providerAttempts, 2); + assert.equal(candidate.replayedPackets, 1); assert.equal(candidate.replayedRawNodes, 48); assert.equal(candidate.accounting.deferred, 56); assert.equal(candidate.outputDigest, baseline.outputDigest); @@ -38,6 +38,12 @@ test("append-only packet treatment measures against whole-input reuse and preser test("offline paired work benchmark has exact disposition parity and never hides zero-adoption outcomes", async () => { const result = await runTopicWorkBenchmark(2); + assert.match( + result.identity.files[ + "packages/memory/service/dist/topicCurationSettings.js" + ], + /^[a-f0-9]{64}$/, + ); assert.equal(result.workloads.length, 2); for (const workload of result.workloads) { assert.deepEqual( @@ -48,7 +54,7 @@ test("offline paired work benchmark has exact disposition parity and never hides ], ); for (const { baseline, candidate } of workload.pairs) { - assert.equal(baseline.providerAttempts, 3); + assert.equal(baseline.providerAttempts, 6); assert.equal(candidate.providerAttempts, 0); assert.equal(candidate.reusedDecisions, 55); assert.equal(baseline.outputDigest, candidate.outputDigest);