diff --git a/ts/packages/memory/service/README.md b/ts/packages/memory/service/README.md index 0e8b2008d4..a61fc8231c 100644 --- a/ts/packages/memory/service/README.md +++ b/ts/packages/memory/service/README.md @@ -51,6 +51,33 @@ job files at once. Filtering, ordering and pagination remain unchanged; unreadable or corrupt job files still fail explicitly rather than producing an apparently empty or complete result. +## Recoverable Wiki construction planning + +Wiki and project construction planning validates the descriptors it uses or +changes, target revisions, original source revisions and construction access. +Retained admission and candidate-visit decisions also track every nominated +coverage/canonical page and its separation relationships, including decisions +that select no build. Inputs are checked again immediately before admission; +the existing build and publication guards remain in force. + +Unrelated View additions or revisions do not discard prepared work. Concurrent +unrelated descriptor changes are merged rather than overwritten and leave the +family dirty for another bounded pass. Changed consulted dependencies defer the +plan instead of recording a permanent technical rejection. Existing pages, +proposals, unprocessed cursors and newly discovered candidate identities are +retained; no stale proposal authorizes an overwrite or publication. +A concurrent family pause remains authoritative and does not create recovery +writes after the pause. + +Deferred planning is durable across restart and visible as pending/deferred +maintenance. The existing 30-second reconciliation wave resumes it, without an +immediate retry loop. Three consecutive dependency conflicts in one source-index +generation stop automatic retries with an explicit-maintenance-required error. +Explicit maintenance or a changed source-index generation resets the streak; +successful planning clears it without clearing unrelated target failures. +Configured automatic pass and candidate-read budgets still apply. This recovery +does not change model selection, admission choices or editorial policies. + ## Installed memory-view packs A memory-view pack is an npm package exporting a static JSON manifest at diff --git a/ts/packages/memory/service/src/fileMemoryService.ts b/ts/packages/memory/service/src/fileMemoryService.ts index 3542326b06..cdaee7fc08 100644 --- a/ts/packages/memory/service/src/fileMemoryService.ts +++ b/ts/packages/memory/service/src/fileMemoryService.ts @@ -634,6 +634,17 @@ function knowledgeBuildState( class SchemaMaintenanceBlockedError extends Error {} +interface WikiPlanningDependencies { + viewIds: Set; + inputs: ViewBuildSnapshot[]; + pages: Array<{ + input: ViewBuildSnapshot; + fingerprint: string; + candidateVisit?: boolean; + }>; + separations: Awaited>; +} + interface StoredRevision extends SourceRevision { content: string; synopsis?: SourceSynopsis & { attempts: number }; @@ -9252,26 +9263,34 @@ export class FileMemoryService private async wikiAdmissionPages( input: ViewBuildSnapshot, + dependencies?: WikiPlanningDependencies, ): Promise { const family = await this.viewStore.getWikiFamily(input.corpusId); const separations = await this.viewStore.listViewSeparations( input.corpusId, ); const participants = separationParticipants(separations); - if (participants.has(input.definition.viewId)) return []; - return nominateWikiPages( - input, - (await this.wikiAdmissionViews(input.corpusId)).filter( - (view) => !participants.has(view.viewId), - ), - new Map( - family.descriptors.map((descriptor) => [ - descriptor.viewId, - descriptor.identity, - ]), - ), - this.experiments.has("wiki-full-context"), - ); + const pages = participants.has(input.definition.viewId) + ? [] + : nominateWikiPages( + input, + (await this.wikiAdmissionViews(input.corpusId)).filter( + (view) => !participants.has(view.viewId), + ), + new Map( + family.descriptors.map((descriptor) => [ + descriptor.viewId, + descriptor.identity, + ]), + ), + this.experiments.has("wiki-full-context"), + ); + if (dependencies) { + dependencies.viewIds.add(input.definition.viewId); + for (const page of pages) dependencies.viewIds.add(page.viewId); + dependencies.pages.push({ input, fingerprint: viewHash(pages) }); + } + return pages; } private async wikiAdmissionViews(corpusId: string): Promise { @@ -10943,12 +10962,14 @@ export class FileMemoryService private scheduleWikiMaintenance( corpusId: string, kind: "wiki" | "projects" = "wiki", + retainDeferred = false, ): void { if ( !this.informationalWikiEnabled || this.closed || - this.ingestionTails.has(corpusId) || - this.knowledgeMaintenance.has(corpusId) + (!retainDeferred && + (this.ingestionTails.has(corpusId) || + this.knowledgeMaintenance.has(corpusId))) ) return; this.maintenance.request( @@ -10971,6 +10992,8 @@ export class FileMemoryService kind, ); if (!family.enabled) return; + if ((family.planningConflict?.attempts ?? 0) >= 3) + return; const snapshot = await this.viewStore.listViews(corpusId); if (this.maintenance.isDeferred(corpusId)) return; @@ -11134,6 +11157,12 @@ export class FileMemoryService ): Promise { const family = await this.viewStore.getWikiFamily(corpusId, kind); const before = JSON.stringify(family); + this.reconcileWikiPlanningConflict( + corpusId, + kind, + family, + indexGeneration, + ); for (const view of views.filter( (view) => view.state === "draft" && @@ -11377,6 +11406,10 @@ export class FileMemoryService expectedHead: snapshot.head, targets: [], }; + if (!automatic) { + this.clearWikiPlanningConflict(corpusId, kind, family); + delete family.planningConflict; + } await this.applyInformationalFamilyOperation(request, family); if (!family.enabled) return this.viewStore.recordViewMaintenance(plan); if (await this.hasActiveInformationalBuild(corpusId, kind)) @@ -11425,6 +11458,10 @@ export class FileMemoryService )) ) return this.viewStore.recordViewMaintenance(plan); + const capturedFamily = await this.viewStore.getWikiFamily( + corpusId, + kind, + ); await this.ensureDocumentIndex(corpusId, runtime); const eligibleIds = new Set(eligible.map((source) => source.sourceId)); const page = await this.readInformationalSubjects( @@ -11462,45 +11499,105 @@ export class FileMemoryService snapshot.views, kind, ); - const familyVersion = viewHash( - await this.viewStore.getWikiFamily(corpusId, kind), - ); - const { targets, snapshots } = await timeSearchStage( - "wiki.plan.targets", - () => - this.prepareWikiTargets( - corpusId, - family, - snapshot.views, - eligible, - request.operation === "rebuild", - plan, - kind, - automatic, - ), - ); - await this.assertWikiPlanCurrent( - corpusId, - kind, - familyVersion, - snapshot.views, + return this.prepareAndAdmitWikiFamily( + runtime, + request, + family, + capturedFamily, + plan, + { + views: snapshot.views, + eligible, + automatic, + nextContinuationToken: page.nextContinuationToken, + }, ); + } + + private async prepareAndAdmitWikiFamily( + runtime: CorpusRuntime, + request: Extract< + ViewMaintenanceRequest, + { family: "wiki" | "projects" } + >, + family: WikiFamilyState, + capturedFamily: WikiFamilyState, + plan: ViewMaintenancePlan, + context: { + views: ViewVersion[]; + eligible: StoredSource[]; + automatic: boolean; + nextContinuationToken: string | undefined; + }, + ): Promise { + const { views, eligible, automatic, nextContinuationToken } = context; + const corpusId = request.corpusId; + const kind = request.family; + const dependencies: WikiPlanningDependencies = { + viewIds: new Set(), + inputs: [], + pages: [], + separations: await this.viewStore.listViewSeparations(corpusId), + }; + let prepared: Awaited< + ReturnType + >; + let checked: Awaited< + ReturnType + >; + try { + prepared = await this.prepareWikiTargets( + corpusId, + family, + views, + eligible, + request.operation === "rebuild", + plan, + kind, + automatic, + dependencies, + ); + checked = await this.assertWikiPlanCurrent( + corpusId, + kind, + capturedFamily, + family, + views, + dependencies, + ); + } catch (error) { + if (!(error instanceof ViewBuildStaleError)) throw error; + return this.deferWikiPlanning( + corpusId, + kind, + capturedFamily, + family, + plan, + ); + } + const { targets, snapshots } = prepared; + const currentFamily = checked.family; // Save discovered identities while leaving unadmitted work retryable. - await this.viewStore.saveWikiFamily(corpusId, family, kind); + await this.viewStore.saveWikiFamily(corpusId, currentFamily, kind); if (this.deferBatchMaintenance(corpusId, automatic)) return this.viewStore.recordViewMaintenance(plan); - const nextFamily = structuredClone(family); + let nextFamily = structuredClone(currentFamily); + delete nextFamily.planningConflict; this.stageWikiFamilyAdmission(nextFamily, targets, automatic); - nextFamily.cursor = Number(page.nextContinuationToken ?? 0); + nextFamily.cursor = checked.contextChanged + ? currentFamily.cursor + : Number(nextContinuationToken ?? 0); if (runtime.manifest.indexGeneration) nextFamily.indexVersion = runtime.manifest.indexGeneration; else delete nextFamily.indexVersion; - nextFamily.dirty = this.wikiPassNeedsContinuation( - plan, - targets.length, - automatic, - !!page.nextContinuationToken, - ); + nextFamily.dirty = + checked.contextChanged || + this.wikiPassNeedsContinuation( + plan, + targets.length, + automatic, + !!nextContinuationToken, + ); if (!nextFamily.dirty) { const decisions = plan.targets.filter( (target) => @@ -11510,15 +11607,45 @@ export class FileMemoryService if (decisions.length) nextFamily.error = `${decisions[0].reason}; ${decisions.length} targets require decisions; independent maintenance continues`; } - return this.admitWikiFamilyTargets( - corpusId, - kind, - nextFamily, - plan, - targets, - snapshots, - automatic, - ); + try { + checked = await this.assertWikiPlanCurrent( + corpusId, + kind, + currentFamily, + nextFamily, + views, + dependencies, + ); + nextFamily = checked.family; + nextFamily.dirty ||= checked.contextChanged; + const result = await this.admitWikiFamilyTargets( + corpusId, + kind, + nextFamily, + plan, + targets, + snapshots, + automatic, + ); + this.clearWikiPlanningConflict(corpusId, kind, currentFamily); + return result; + } catch (error) { + if ( + !(error instanceof ViewBuildStaleError) && + !( + error instanceof Error && + error.message.startsWith("View history head conflict") + ) + ) + throw error; + return this.deferWikiPlanning( + corpusId, + kind, + currentFamily, + nextFamily, + plan, + ); + } } private async admitWikiFamilyTargets( @@ -11677,6 +11804,7 @@ export class FileMemoryService private async retainedScopedWikiAssessment( input: ViewBuildSnapshot, descriptor: WikiFamilyState["descriptors"][number], + dependencies?: WikiPlanningDependencies, ): Promise { const utilityKey = descriptor.admission?.utilityEvidenceKey; const key = utilityKey ?? descriptor.admission?.assessmentKey; @@ -11690,7 +11818,7 @@ export class FileMemoryService ) : wikiAssessmentKey( input, - await this.wikiAdmissionPages(input), + await this.wikiAdmissionPages(input, dependencies), this.wikiAdmissionContext([]), )) ) @@ -11709,6 +11837,7 @@ export class FileMemoryService private async refreshWikiCandidateVisits( corpusId: string, family: WikiFamilyState, + dependencies?: WikiPlanningDependencies, ): Promise { const cursor = family.candidateReadCursor; if (!cursor) return false; @@ -11731,6 +11860,7 @@ export class FileMemoryService ); const invalid = new Set(); for (const descriptor of deferred) { + dependencies?.viewIds.add(descriptor.viewId); const receipt = descriptor.admission!; const job = jobs.find((job) => receipt.assessmentJobId @@ -11744,6 +11874,36 @@ export class FileMemoryService const result = job?.results.find( (result) => result.viewId === descriptor.viewId, ); + if (result && dependencies) { + try { + await this.recheckWikiPlanInputs(corpusId, [ + result.snapshot, + ]); + } catch (error) { + if (!(error instanceof ViewBuildStaleError)) throw error; + delete descriptor.admission; + invalid.add(descriptor.viewId); + continue; + } + dependencies.inputs.push(result.snapshot); + } + const pages = + result && !receipt.utilityEvidenceKey + ? nominateWikiPages( + result.snapshot, + views, + identities, + this.experiments.has("wiki-full-context"), + ) + : []; + if (result && !receipt.utilityEvidenceKey && dependencies) { + for (const page of pages) dependencies.viewIds.add(page.viewId); + dependencies.pages.push({ + input: result.snapshot, + fingerprint: viewHash(pages), + candidateVisit: true, + }); + } if (!result) { delete descriptor.admission; invalid.add(descriptor.viewId); @@ -11760,12 +11920,7 @@ export class FileMemoryService : receipt.assessmentKey !== wikiAssessmentKey( result.snapshot, - nominateWikiPages( - result.snapshot, - views, - identities, - this.experiments.has("wiki-full-context"), - ), + pages, this.wikiAdmissionContext([]), ) ) { @@ -11829,18 +11984,20 @@ export class FileMemoryService plan: ViewMaintenancePlan, kind: "wiki" | "projects", automatic: boolean, + dependencies?: WikiPlanningDependencies, ): Promise<{ targets: ViewBuildTarget[]; snapshots: ViewBuildSnapshot[] }> { const targets: ViewBuildTarget[] = []; const snapshots: ViewBuildSnapshot[] = []; const generation = (await this.getCorpusRuntime(corpusId)).manifest .indexGeneration; const policy = this.wikiAdmissionContext(views); - await this.refreshWikiCandidateVisits(corpusId, family); + await this.refreshWikiCandidateVisits(corpusId, family, dependencies); const { targetLimit, candidateLimit, descriptors, recordRead } = this.wikiTargetCandidates(family, views, automatic, generation); let inspected = 0; for (const descriptor of descriptors) { if (targets.length === targetLimit) break; + dependencies?.viewIds.add(descriptor.viewId); const current = views.find( (view) => view.viewId === descriptor.viewId, ); @@ -11890,6 +12047,7 @@ export class FileMemoryService rebuild, plan, kind, + dependencies, ); if (!input) { if (automatic) @@ -11905,7 +12063,11 @@ export class FileMemoryService !rebuild && !current && kind === "wiki" && - (await this.retainedScopedWikiAssessment(input, descriptor)) + (await this.retainedScopedWikiAssessment( + input, + descriptor, + dependencies, + )) ) { recordRead(descriptor.viewId); continue; @@ -11999,6 +12161,7 @@ export class FileMemoryService rebuild: boolean, plan: ViewMaintenancePlan, kind: "wiki" | "projects", + dependencies?: WikiPlanningDependencies, ): Promise { const input = await this.snapshotInformationalTarget( corpusId, @@ -12007,6 +12170,7 @@ export class FileMemoryService undefined, true, ); + dependencies?.inputs.push(input); const query = input.definition.wiki?.query ?? input.definition.projectSummary?.query; @@ -12027,6 +12191,7 @@ export class FileMemoryService kind === "wiki" && !current ? await this.viewStore.pendingWikiContributionConflict(input) : undefined; + if (conflict) dependencies?.viewIds.add(conflict.human.viewId); if ( conflict && (await this.informationalVersionEligible(conflict.human)) @@ -12053,6 +12218,8 @@ export class FileMemoryService corpusId, target.definition.viewId, input, + true, + (viewId) => dependencies?.viewIds.add(viewId), )) ) return undefined; @@ -13246,7 +13413,11 @@ export class FileMemoryService validateMaintenanceRequest(request, true); const runtime = await this.requireViews(request.corpusId); if ("family" in request) { - this.maintenance.cancel(request.corpusId, request.family); + this.maintenance.cancel( + request.corpusId, + request.family, + request.family !== "topics", + ); const result = await this.enqueueWrite(request.corpusId, () => request.family === "topics" ? this.maintainTopics(runtime, request) @@ -19743,21 +19914,273 @@ export class FileMemoryService ); } + private async recheckWikiPlanInputs( + corpusId: string, + inputs: ViewBuildSnapshot[], + ): Promise { + const runtime = await this.getCorpusRuntime(corpusId); + for (const input of inputs) { + for (const selected of input.inputs) { + const source = runtime.manifest.sources.find( + (entry) => entry.sourceId === selected.sourceId, + ); + const revision = source?.revisions.find( + (entry) => entry.revisionId === selected.revisionId, + ); + if ( + !source || + !revision || + source.activeRevisionId !== selected.revisionId || + revision.state !== "ready" || + !(await this.sourceAllowed(source, revision, "build")) + ) + throw new ViewBuildStaleError( + "Wiki planning source revision or construction authorization changed", + ); + } + await this.recheckViewInput(input); + } + } + private async assertWikiPlanCurrent( corpusId: string, kind: "wiki" | "projects", - familyVersion: string, + capturedFamily: WikiFamilyState, + plannedFamily: WikiFamilyState, views: ViewVersion[], - ): Promise { + dependencies: WikiPlanningDependencies, + ): Promise<{ family: WikiFamilyState; contextChanged: boolean }> { + await this.recheckWikiPlanInputs(corpusId, dependencies.inputs); + for (const check of dependencies.pages) { + const pages = check.candidateVisit + ? nominateWikiPages( + check.input, + await this.wikiAdmissionViews(corpusId), + new Map( + plannedFamily.descriptors.map((descriptor) => [ + descriptor.viewId, + descriptor.identity, + ]), + ), + this.experiments.has("wiki-full-context"), + ) + : await this.wikiAdmissionPages(check.input); + if (viewHash(pages) !== check.fingerprint) + throw new ViewBuildStaleError( + "Wiki planning coverage or canonical context changed", + ); + } const family = await this.viewStore.getWikiFamily(corpusId, kind); const current = await this.viewStore.listViews(corpusId); + const changedDescriptors = new Set( + [...capturedFamily.descriptors, ...plannedFamily.descriptors] + .filter( + (descriptor) => + viewHash( + capturedFamily.descriptors.find( + (entry) => entry.viewId === descriptor.viewId, + ), + ) !== + viewHash( + plannedFamily.descriptors.find( + (entry) => entry.viewId === descriptor.viewId, + ), + ), + ) + .map((descriptor) => descriptor.viewId), + ); + const descriptorIds = new Set([ + ...changedDescriptors, + ...dependencies.viewIds, + ]); + const scopedFamily = (value: WikiFamilyState) => ({ + enabled: value.enabled, + indexVersion: value.indexVersion, + automaticPasses: value.automaticPasses, + planningConflict: value.planningConflict, + descriptors: value.descriptors + .filter((descriptor) => descriptorIds.has(descriptor.viewId)) + .sort((a, b) => a.viewId.localeCompare(b.viewId)), + }); + const relevantSeparations = ( + receipts: WikiPlanningDependencies["separations"], + ) => + receipts.filter((receipt) => + [...separationParticipants([receipt])].some((id) => + dependencies.viewIds.has(id), + ), + ); + const separations = await this.viewStore.listViewSeparations(corpusId); if ( - viewHash(family) !== familyVersion || - viewHash(current.views.map((view) => view.revisionId)) !== - viewHash(views.map((view) => view.revisionId)) + viewHash(scopedFamily(family)) !== + viewHash(scopedFamily(capturedFamily)) || + viewHash(relevantSeparations(separations)) !== + viewHash(relevantSeparations(dependencies.separations)) ) throw new ViewBuildStaleError( - "Informational maintenance plan changed during derivative preparation", + "Wiki planning descriptors or relationships changed", ); + for (const viewId of dependencies.viewIds) { + const captured = views.find((view) => view.viewId === viewId); + const latest = current.views.find((view) => view.viewId === viewId); + if ( + captured?.revisionId !== latest?.revisionId || + captured?.state !== latest?.state + ) + throw new ViewBuildStaleError( + "Wiki planning target or consulted page changed", + ); + } + // Preserve concurrent unrelated descriptor work instead of replacing its durable state. + const merged = structuredClone(plannedFamily); + for (const key of [ + "dirty", + "cursor", + "reviewCursor", + "candidateReadCursor", + "error", + ] as const) { + if (viewHash(family[key]) === viewHash(capturedFamily[key])) + continue; + if (family[key] === undefined) Reflect.deleteProperty(merged, key); + else Object.assign(merged, { [key]: family[key] }); + } + merged.descriptors = [ + ...plannedFamily.descriptors.flatMap((descriptor) => { + const latest = family.descriptors.find( + (entry) => entry.viewId === descriptor.viewId, + ); + return changedDescriptors.has(descriptor.viewId) + ? [descriptor] + : latest + ? [latest] + : []; + }), + ...family.descriptors.filter( + (descriptor) => + !changedDescriptors.has(descriptor.viewId) && + !plannedFamily.descriptors.some( + (entry) => entry.viewId === descriptor.viewId, + ), + ), + ]; + return { + family: merged, + contextChanged: + viewHash(family) !== viewHash(capturedFamily) || + viewHash(current.views.map((view) => view.revisionId)) !== + viewHash(views.map((view) => view.revisionId)), + }; + } + + private clearWikiPlanningConflict( + corpusId: string, + kind: "wiki" | "projects", + family: WikiFamilyState, + ): void { + if (family.planningConflict) + this.maintenance.resumeDeferred( + corpusId, + kind, + family.planningConflict.reason, + ); + this.maintenance.reportFailure(corpusId, kind, "planning-conflict"); + } + + private reconcileWikiPlanningConflict( + corpusId: string, + kind: "wiki" | "projects", + family: WikiFamilyState, + generation: string | undefined, + ): void { + const conflict = family.planningConflict; + if (!conflict) return; + if (conflict.generation !== generation) { + this.clearWikiPlanningConflict(corpusId, kind, family); + if (family.error === conflict.reason) delete family.error; + delete family.planningConflict; + return; + } + if (conflict.attempts >= 3) { + this.scheduleWikiMaintenance(corpusId, kind, true); + this.maintenance.park(corpusId, kind, conflict.reason); + this.maintenance.reportFailure( + corpusId, + kind, + "planning-conflict", + conflict.reason, + ); + } else { + this.maintenance.resumeDeferred(corpusId, kind, conflict.reason); + } + } + + private async deferWikiPlanning( + corpusId: string, + kind: "wiki" | "projects", + capturedFamily: WikiFamilyState, + plannedFamily: WikiFamilyState, + plan: ViewMaintenancePlan, + ): Promise { + const family = await this.viewStore.getWikiFamily(corpusId, kind); + if (!family.enabled) + throw new ViewBuildStaleError( + "Wiki maintenance was paused during planning", + ); + const generation = this.corpora.get(corpusId)?.manifest.indexGeneration; + const prior = family.planningConflict; + const attempts = + (prior?.generation === generation ? (prior?.attempts ?? 0) : 0) + 1; + const exhausted = attempts >= 3; + const reason = exhausted + ? "Repeated planning dependency conflicts; explicit maintenance is required" + : "Planning dependencies changed; retained work awaits the next maintenance wave"; + family.planningConflict = { + reason, + ...(generation ? { generation } : {}), + attempts, + }; + family.dirty = true; + // Retain newly discovered identities, but never restore a concurrently removed candidate. + for (const descriptor of plannedFamily.descriptors) + if ( + !capturedFamily.descriptors.some( + (entry) => entry.viewId === descriptor.viewId, + ) && + !family.descriptors.some( + (entry) => entry.viewId === descriptor.viewId, + ) + ) { + const retained = structuredClone(descriptor); + delete retained.admission; + family.descriptors.push(retained); + } + if (exhausted) family.error = reason; + await this.viewStore.saveWikiFamily(corpusId, family, kind); + for (const target of plan.targets) { + target.state = "blocked"; + target.reason = reason; + } + if (!plan.targets.length) + plan.targets.push( + ...plannedFamily.descriptors.map((descriptor) => ({ + viewId: descriptor.viewId, + expectedVersion: 0, + state: "blocked" as const, + reason, + })), + ); + if (!this.closed) { + this.scheduleWikiMaintenance(corpusId, kind, true); + this.maintenance.park(corpusId, kind, reason); + if (exhausted) + this.maintenance.reportFailure( + corpusId, + kind, + "planning-conflict", + reason, + ); + } + return this.viewStore.recordViewMaintenance(plan); } } diff --git a/ts/packages/memory/service/src/maintenanceScheduler.ts b/ts/packages/memory/service/src/maintenanceScheduler.ts index d14b30be5a..1e58289c40 100644 --- a/ts/packages/memory/service/src/maintenanceScheduler.ts +++ b/ts/packages/memory/service/src/maintenanceScheduler.ts @@ -221,14 +221,19 @@ export class MaintenanceScheduler { this.timer.unref(); } - public cancel(corpusId: string, family: string): void { + public cancel( + corpusId: string, + family: string, + preserveTargetFailures = false, + ): void { const key = `${corpusId}:${family}`; this.pending.delete(key); this.failures.delete(key); this.parked.delete(key); - for (const target of this.reportedFailures.keys()) - if (target.startsWith(`${key}:`)) - this.reportedFailures.delete(target); + if (!preserveTargetFailures) + for (const target of this.reportedFailures.keys()) + if (target.startsWith(`${key}:`)) + this.reportedFailures.delete(target); } public reportFailure( @@ -246,6 +251,15 @@ export class MaintenanceScheduler { this.parked.set(`${corpusId}:${family}`, reason); } + public resumeDeferred( + corpusId: string, + family: string, + reason: string, + ): void { + const key = `${corpusId}:${family}`; + if (this.parked.get(key) === reason) this.parked.delete(key); + } + public async waitForBuild(work: () => Promise): Promise { const key = this.workKey.getStore(); if (!this.prioritizeShortBuilds || !key || this.waitingBuilds.has(key)) diff --git a/ts/packages/memory/service/src/viewStore.ts b/ts/packages/memory/service/src/viewStore.ts index 82ef3fbcbc..bef8123319 100644 --- a/ts/packages/memory/service/src/viewStore.ts +++ b/ts/packages/memory/service/src/viewStore.ts @@ -133,6 +133,11 @@ export interface WikiFamilyState { reviewCursor?: string; indexVersion?: string; error?: string; + planningConflict?: { + reason: string; + generation?: string; + attempts: number; + }; automaticPasses?: { generation: string | undefined; policy: string; @@ -202,11 +207,13 @@ function timestamp(): string { function unchangedWikiAmbiguity( result: ViewBuildTargetResult, state: ViewStoreState, + consulted?: (viewId: string) => void, ): boolean { const dependencies = result.wikiAmbiguityDependencies; return ( !!dependencies?.length && dependencies.every((dependency) => { + consulted?.(dependency.viewId); const page = viewVersions(state, dependency.viewId).at(-1); return ( page?.state !== "archived" && @@ -1856,6 +1863,7 @@ export class TypedViewStore { viewId: string, input: ViewBuildSnapshot, includeParkedFailures = true, + consulted?: (viewId: string) => void, ): Promise { const { state } = await this.readSnapshot(corpusId); const pendingConflict = Object.values(state.conflicts ?? {}).some( @@ -1879,7 +1887,7 @@ export class TypedViewStore { result.state === "blocked" && result.wikiAssessment?.disposition === "ambiguous" ) - return unchangedWikiAmbiguity(result, state); + return unchangedWikiAmbiguity(result, state, consulted); return ["blocked", "conflicted", "cancelled"].includes( result.state, ); diff --git a/ts/packages/memory/service/test/informationalWiki.spec.ts b/ts/packages/memory/service/test/informationalWiki.spec.ts index 9e7be63b5f..6ef2b5b9f2 100644 --- a/ts/packages/memory/service/test/informationalWiki.spec.ts +++ b/ts/packages/memory/service/test/informationalWiki.spec.ts @@ -2589,8 +2589,13 @@ describe("informational wiki lifecycle (mock extraction and synthesis)", () => { release(); const outcome = await maintaining; if (suppress) { - expect(outcome.error).toBeInstanceOf(ViewBuildStaleError); - expect(outcome.result).toBeUndefined(); + expect(outcome.error).toBeUndefined(); + expect(outcome.result?.job).toBeUndefined(); + const family = await new TypedViewStore(root).getWikiFamily( + corpusId, + ); + expect(family.dirty).toBe(true); + expect(family.planningConflict?.attempts).toBe(1); } else { expect(outcome.error).toBeUndefined(); expect(outcome.result?.job).toBeDefined(); diff --git a/ts/packages/memory/service/test/viewSeparationService.spec.ts b/ts/packages/memory/service/test/viewSeparationService.spec.ts index 9e7ea4d757..fb2f7f5de0 100644 --- a/ts/packages/memory/service/test/viewSeparationService.spec.ts +++ b/ts/packages/memory/service/test/viewSeparationService.spec.ts @@ -238,12 +238,30 @@ test("confirmed separation survives partial generation and restart, checks real "stale", ); authorized = true; - await store.forgetSource(corpusId, source.sourceId); + const forget = await service.previewForgetSource( + corpusId, + source.sourceId, + ); + // Serialize the privacy purge with the live service's background maintenance. + const forgotten = await service.forgetSource({ + corpusId, + sourceId: source.sourceId, + confirmationToken: forget.confirmationToken, + }); + expect(forgotten.deletedRevisionCount).toBe(1); + await service.close(); + service = open(); + await service.initialize(); + expect(await service.listSources(corpusId)).toEqual([]); expect(await store.listViewSeparations(corpusId)).toEqual([]); - for (const child of proposal.destinations) + for (const child of proposal.destinations) { expect( await store.getView({ corpusId, viewId: child.viewId }), ).toBeUndefined(); + expect( + await service.getView({ corpusId, viewId: child.viewId }), + ).toBeUndefined(); + } } finally { await service.close(); await rm(root, { recursive: true, force: true }); diff --git a/ts/packages/memory/service/test/viewStore.spec.ts b/ts/packages/memory/service/test/viewStore.spec.ts index 59b4a4e1e7..143dd5cfe9 100644 --- a/ts/packages/memory/service/test/viewStore.spec.ts +++ b/ts/packages/memory/service/test/viewStore.spec.ts @@ -105,13 +105,17 @@ describe("derived-only View persistence", () => { }), ); }); + const consulted: string[] = []; expect( await store.hasPendingInformationalInput( corpusId, "guide", snapshot, + true, + (viewId) => consulted.push(viewId), ), ).toBe(true); + expect(consulted).toEqual(["first", "second"]); expect(await store.hasInvalidatedWikiAmbiguity(corpusId)).toBe(false); const unchanged = await store.listViews(corpusId); await store.saveViewDraft({ diff --git a/ts/packages/memory/service/test/wikiPlanningRecovery.spec.ts b/ts/packages/memory/service/test/wikiPlanningRecovery.spec.ts new file mode 100644 index 0000000000..7750f3ceb9 --- /dev/null +++ b/ts/packages/memory/service/test/wikiPlanningRecovery.spec.ts @@ -0,0 +1,647 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { randomUUID } from "node:crypto"; +import { mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { FileMemoryService } from "../src/fileMemoryService.js"; +import { hydrateInformational } from "../src/informationalWiki.js"; +import { + indexedLocator, + parseIndexedLocator, + selectIndexedFragments, +} from "../src/indexedFragments.js"; +import { MaintenanceScheduler } from "../src/maintenanceScheduler.js"; +import { waitForMemoryJob } from "../src/rpcFacade.js"; +import { ViewBuildStaleError } from "../src/viewBuilds.js"; +import type { TypedViewStore, WikiFamilyState } from "../src/viewStore.js"; +import type { ViewSeparationReceipt } from "../src/viewSeparation.js"; +import { + authoredRelationships, + derivedDefinitionInput, +} from "../src/viewRelationships.js"; +import type { IndexedFragmentRequest } from "../src/types.js"; +import type { + ViewBuildJob, + ViewVersion, + ViewMaintenanceResult, +} from "../src/viewTypes.js"; +import { FakeSemanticCorpusIndex } from "./fakeSemanticCorpusIndex.js"; + +class PlanningIndex extends FakeSemanticCorpusIndex { + public async listSubjects() { + return { + total: 1, + items: [ + { + term: "acquisition", + postingCount: this.documents.length, + sourceIds: this.documents.map( + (document) => document.source.sourceId, + ), + }, + ], + }; + } + + public async readFragments(request: IndexedFragmentRequest) { + return selectIndexedFragments( + this.documents.map((document) => ({ + sourceId: document.source.sourceId, + revisionId: document.revision.revisionId, + blockOrdinal: 0, + text: document.content, + score: 1, + })), + request, + ); + } + + public async resolveIndexedLocator( + sourceId: string, + revisionId: string, + locator: string, + ) { + const range = parseIndexedLocator(locator); + const document = this.documents.find( + (entry) => + entry.source.sourceId === sourceId && + entry.revision.revisionId === revisionId, + ); + if ( + !range || + !document || + indexedLocator( + { + sourceId, + revisionId, + blockOrdinal: 0, + text: document.content, + score: 1, + }, + range.start, + range.end, + ) !== locator + ) + throw new Error("Invalid indexed anchor"); + return document.content.slice(range.start, range.end); + } +} + +describe("dependency-scoped Wiki planning recovery", () => { + let root: string; + let service: FileMemoryService; + let corpusId: string; + let draft: ViewVersion; + let allowSources: boolean; + const open = () => + new FileMemoryService(root, { + maintenanceMode: "manual", + discoveryPolicy: "off", + indexFactory: (_corpusId, directory) => + new PlanningIndex(directory), + sourceAuthorization: async () => allowSources, + viewSynthesisAdapter: { + identity: "offline-planning-test", + checkEditorial: async () => ({ safe: true, failures: [] }), + generate: async (input) => + hydrateInformational(input, { + title: input.definition.wiki!.query, + summary: "Acquisition requires independent approval.", + status: "partial", + prose: input.inputs + .map((source) => source.content) + .join("\n\n"), + limits: "Owner approval is unknown.", + citationIds: input.inputs + .flatMap((source) => source.passages ?? []) + .map((_, index) => `f${index}`), + }), + validate: async () => { + throw new Error("Legacy validation must not run"); + }, + }, + }); + + async function waitBuild(job: ViewBuildJob) { + for ( + let attempt = 0; + job.state === "running" && attempt < 300; + attempt++ + ) { + await new Promise((resolve) => setTimeout(resolve, 10)); + job = (await service.getViewBuild({ corpusId, jobId: job.jobId }))!; + } + expect(job.state).toBe("complete"); + } + + async function editDirect(view: ViewVersion, viewId = view.viewId) { + const store: TypedViewStore = service["viewStore"]; + const snapshot = await store.listViews(corpusId); + const latest = snapshot.views.find((entry) => entry.viewId === viewId); + const content = structuredClone(view.content); + content.summary = `Retained human proposal ${randomUUID()}; approval remains unknown.`; + return ( + await store.saveViewDraft({ + corpusId, + viewId, + expectedHead: snapshot.head, + expectedVersion: latest?.version ?? 0, + definition: { ...derivedDefinitionInput(view), viewId }, + content, + relationships: authoredRelationships(view), + }) + ).version; + } + + async function maintain(automatic = false): Promise { + const runtime = await service["getCorpusRuntime"](corpusId); + return service["enqueueWrite"](corpusId, async () => + service["maintainWikiFamily"]( + runtime, + { + corpusId, + family: "wiki", + operation: "rebuild", + expectedHead: ( + await service["viewStore"].listViews(corpusId) + ).head, + }, + automatic, + ), + ); + } + + beforeEach(async () => { + allowSources = true; + root = path.resolve(".wiki-planning-unit-owned", randomUUID()); + await mkdir(root, { recursive: true }); + service = open(); + corpusId = (await service.createCorpus("Planning recovery")).corpusId; + const ingestion = await service.ingestDocument({ + corpusId, + source: { + sourceId: "original", + sourceType: "text", + title: "Original", + text: "Acquisition requires independent approval; owner approval is unknown.", + }, + }); + expect( + ( + await waitForMemoryJob(service, ingestion.jobId, { + pollIntervalMs: 10, + }) + ).state, + ).toBe("complete"); + const source = (await service.getSource(corpusId, "original"))!; + await waitBuild( + await service.buildViews({ + corpusId, + expectedHead: (await service.listViews(corpusId)).head, + publication: false, + targets: [ + { + expectedVersion: 0, + definition: { + viewId: "selected", + kind: "wiki", + selector: { + kind: "sources", + sources: [ + { + sourceId: "original", + revisionId: source.activeRevisionId!, + }, + ], + }, + wiki: { + construction: "informational-wiki-v1", + query: "acquisition", + aliases: [], + excludedSourceIds: [], + }, + }, + }, + ], + }), + ); + draft = (await service.listViews(corpusId)).views[0]; + const family = await service["viewStore"].getWikiFamily(corpusId); + family.descriptors = [ + { + viewId: draft.viewId, + query: "acquisition", + sourceIds: ["original"], + }, + ]; + family.dirty = true; + family.cursor = 0; + await service["viewStore"].saveWikiFamily(corpusId, family); + }); + + afterEach(async () => { + allowSources = true; + await service.close(); + await rm(root, { recursive: true, force: true }); + }); + + test("unrelated page additions, revisions and descriptor writes preserve useful work", async () => { + let unrelated = await editDirect(draft, "unrelated"); + const prepare = service["prepareWikiTargets"].bind(service); + service["prepareWikiTargets"] = async ( + ...args: Parameters + ) => { + const planningFamily = args[1] as WikiFamilyState; + planningFamily.descriptors = planningFamily.descriptors.filter( + (entry) => entry.viewId === "selected", + ); + const prepared = await prepare(...args); + expect(args[5]).toMatchObject({ targets: [{ state: "rebuild" }] }); + unrelated = await editDirect(unrelated); + await editDirect(draft, "new-unrelated"); + const family: WikiFamilyState = + await service["viewStore"].getWikiFamily(corpusId); + family.descriptors.push({ + viewId: "unrelated", + query: "separate topic", + sourceIds: ["original"], + }); + family.cursor = 7; + family.reviewCursor = "unrelated-review"; + await service["viewStore"].saveWikiFamily(corpusId, family); + return prepared; + }; + const result = await maintain(); + expect(result.job).toBeDefined(); + await waitBuild(result.job!); + const family: WikiFamilyState = + await service["viewStore"].getWikiFamily(corpusId); + expect(family.planningConflict).toBeUndefined(); + expect(family.dirty).toBe(true); + expect(family.cursor).toBe(7); + expect(family.reviewCursor).toBe("unrelated-review"); + expect( + family.descriptors.some((entry) => entry.viewId === "unrelated"), + ).toBe(true); + expect( + ( + await service["viewStore"].getView({ + corpusId, + viewId: "unrelated", + }) + )?.revisionId, + ).toBe(unrelated.revisionId); + }); + + test.each([ + "target", + "descriptor", + "source", + "authorization", + "removal", + ] as const)( + "%s dependency changes defer without overwriting pages or advancing discovery", + async (dependency) => { + const prepare = service["prepareWikiTargets"].bind(service); + service["prepareWikiTargets"] = async ( + ...args: Parameters + ) => { + const prepared = await prepare(...args); + if (dependency === "target") draft = await editDirect(draft); + if (dependency === "descriptor" || dependency === "removal") { + const family = + await service["viewStore"].getWikiFamily(corpusId); + if (dependency === "descriptor") + family.descriptors[0].query = "changed acquisition"; + else family.descriptors = []; + await service["viewStore"].saveWikiFamily(corpusId, family); + } + if (dependency === "source") { + const runtime = await service["getCorpusRuntime"](corpusId); + const manifest = structuredClone(runtime.manifest); + manifest.sources[0].activeRevisionId = "replaced"; + runtime.manifest = manifest; + } + if (dependency === "authorization") allowSources = false; + return prepared; + }; + const result = await maintain(); + allowSources = true; + expect(result.job).toBeUndefined(); + expect( + result.plan.targets.some((target) => + target.reason.includes("next maintenance wave"), + ), + ).toBe(true); + const family: WikiFamilyState = + await service["viewStore"].getWikiFamily(corpusId); + expect(family.dirty).toBe(true); + expect(family.cursor).toBe(0); + expect(family.candidateReadCursor).toBeUndefined(); + expect(family.planningConflict?.attempts).toBe(1); + expect(family.error).toBeUndefined(); + expect(service.getMaintenanceStatus().settled).toBe(false); + expect( + ( + await service["viewStore"].getView({ + corpusId, + viewId: draft.viewId, + }) + )?.revisionId, + ).toBe(draft.revisionId); + if (dependency === "removal") + expect( + family.descriptors.some( + (entry) => entry.viewId === "selected", + ), + ).toBe(false); + }, + ); + + test("new candidates survive early stale preparation, while technical failures still fail", async () => { + service["prepareWikiTargets"] = async ( + ...args: Parameters + ) => { + (args[1] as WikiFamilyState).descriptors.push({ + viewId: "fresh-candidate", + query: "new evidence topic", + sourceIds: ["original"], + }); + throw new ViewBuildStaleError("Source replaced"); + }; + expect((await maintain()).job).toBeUndefined(); + const family: WikiFamilyState = + await service["viewStore"].getWikiFamily(corpusId); + expect( + family.descriptors.some( + (entry) => entry.viewId === "fresh-candidate", + ), + ).toBe(true); + expect(family.planningConflict?.attempts).toBe(1); + service["prepareWikiTargets"] = async () => { + throw new Error("Injected technical failure"); + }; + await expect(maintain()).rejects.toThrow("Injected technical failure"); + }); + + test("the final admission check catches edits after saving discovered identities", async () => { + const store: TypedViewStore = service["viewStore"]; + const save = store.saveWikiFamily.bind(store); + let writes = 0; + store.saveWikiFamily = async ( + ...args: Parameters + ) => { + await save(...args); + if (++writes === 2) draft = await editDirect(draft); + }; + expect((await maintain()).job).toBeUndefined(); + expect( + (await store.getView({ corpusId, viewId: draft.viewId })) + ?.revisionId, + ).toBe(draft.revisionId); + expect((await store.getWikiFamily(corpusId)).dirty).toBe(true); + }); + + test.each(["mutation", "archive", "relationship"] as const)( + "consulted coverage witness %s defers even when no build is selected", + async (change) => { + let witness = await editDirect(draft, "coverage-witness"); + service["wikiAdmissionViews"] = async () => [witness]; + let separations: ViewSeparationReceipt[] = []; + service["viewStore"].listViewSeparations = async () => separations; + const prepare = service["prepareWikiTargets"].bind(service); + service["prepareWikiTargets"] = async ( + ...args: Parameters + ) => { + const prepared = await prepare(...args); + await service["retainedScopedWikiAssessment"]( + prepared.snapshots[0], + { + viewId: "selected", + query: "acquisition", + sourceIds: ["original"], + admission: { + generation: undefined, + policy: "fixture", + reason: "retained", + assessmentKey: "fixture", + }, + }, + args[8], + ); + if (change === "mutation") witness = await editDirect(witness); + if (change === "archive") { + const store: TypedViewStore = service["viewStore"]; + witness = ( + await store.archiveView({ + corpusId, + viewId: witness.viewId, + expectedHead: (await store.listViews(corpusId)) + .head!, + expectedVersion: witness.version, + }) + ).version; + } + if (change === "relationship") + separations = [ + { + receiptId: "separation", + corpusId, + viewId: witness.viewId, + acceptedRevisionId: witness.revisionId, + acceptedVersion: witness.version, + contentHash: "fixture", + definitionHash: "fixture", + assessmentHash: "fixture", + confirmationToken: "fixture", + reason: "Review page scope", + overview: { + title: witness.content.title, + retainedSectionIds: [], + retainedHumanUnits: [], + }, + destinations: [], + unresolved: [], + state: "proposed", + reasonForState: "Awaiting confirmation", + createdAt: new Date().toISOString(), + input: draft.generation!.input!, + privacySourceIds: ["original"], + }, + ]; + return { targets: [], snapshots: [] }; + }; + expect((await maintain()).job).toBeUndefined(); + expect( + (await service["viewStore"].getWikiFamily(corpusId)) + .planningConflict?.attempts, + ).toBe(1); + }, + ); + + test("one reconciliation wave retries once, and successful planning clears only its own failure", async () => { + const prepare = service["prepareWikiTargets"].bind(service); + let calls = 0; + service["prepareWikiTargets"] = async ( + ...args: Parameters + ) => { + if (args[6] === "wiki" && ++calls === 1) + throw new ViewBuildStaleError("Context changed"); + return prepare(...args); + }; + await maintain(); + expect(calls).toBe(1); + expect(service.getMaintenanceStatus().deferred).toHaveLength(1); + const runtime = await service["getCorpusRuntime"](corpusId); + await service["reconcileInformationalFamily"]( + corpusId, + "wiki", + runtime.manifest.indexGeneration, + (await service["viewStore"].listViews(corpusId)).views, + ); + expect(service.getMaintenanceStatus().deferred).toEqual([]); + await service.drainMaintenance(10_000); + expect(calls).toBe(2); + expect( + (await service["viewStore"].getWikiFamily(corpusId)) + .planningConflict, + ).toBeUndefined(); + }); + + test("restart retains conflicts, bounds repeated automatic attempts and permits explicit recovery", async () => { + const captured = await service["viewStore"].getWikiFamily(corpusId); + captured.indexVersion = ( + await service["getCorpusRuntime"](corpusId) + ).manifest.indexGeneration; + captured.cursor = 7; + captured.candidateReadCursor = { + generation: "retained", + policy: "retained", + visitedViewIds: ["done"], + }; + await service["viewStore"].saveWikiFamily(corpusId, captured); + const defer = () => + service["deferWikiPlanning"](corpusId, "wiki", captured, captured, { + corpusId, + expectedHead: null, + targets: [], + }); + await defer(); + await service.close(); + service = open(); + await service.initialize(); + expect( + (await service["viewStore"].getWikiFamily(corpusId)) + .planningConflict?.attempts, + ).toBe(1); + await defer(); + await defer(); + const exhausted = await service["viewStore"].getWikiFamily(corpusId); + expect(exhausted.planningConflict?.attempts).toBe(3); + expect(exhausted.error).toContain("explicit maintenance"); + expect(exhausted.cursor).toBe(7); + expect(exhausted.candidateReadCursor).toEqual( + captured.candidateReadCursor, + ); + await service.close(); + service = open(); + await service.initialize(); + expect( + (await service["viewStore"].getWikiFamily(corpusId)) + .planningConflict?.attempts, + ).toBe(3); + expect(service.getMaintenanceStatus().settled).toBe(false); + const prepare = service["prepareWikiTargets"].bind(service); + let automaticCalls = 0; + service["prepareWikiTargets"] = async ( + ...args: Parameters + ) => { + if (args[6] === "wiki") automaticCalls++; + return prepare(...args); + }; + await service.drainMaintenance(1000); + expect(automaticCalls).toBe(0); + service["maintenance"].reportFailure( + corpusId, + "wiki", + "other-target", + "Unrelated failure", + ); + const result = await service.maintainViews({ + corpusId, + family: "wiki", + operation: "rebuild", + expectedHead: (await service["viewStore"].listViews(corpusId)).head, + }); + if (result.job) await waitBuild(result.job); + expect( + (await service["viewStore"].getWikiFamily(corpusId)) + .planningConflict, + ).toBeUndefined(); + expect(service.getMaintenanceStatus().failures).toEqual([ + { + key: `${corpusId}:wiki:other-target`, + error: "Unrelated failure", + }, + ]); + }); + + test("a changed source-index generation resets an exhausted conflict without erasing other failures", async () => { + const store: TypedViewStore = service["viewStore"]; + const family = await store.getWikiFamily(corpusId); + family.planningConflict = { + attempts: 3, + generation: "old-generation", + reason: "Planning conflict", + }; + family.error = "Planning conflict"; + await store.saveWikiFamily(corpusId, family); + service["maintenance"].reportFailure( + corpusId, + "wiki", + "planning-conflict", + "Planning conflict", + ); + service["maintenance"].reportFailure( + corpusId, + "wiki", + "other", + "Keep this failure", + ); + const runtime = await service["getCorpusRuntime"](corpusId); + await service["reconcileInformationalFamily"]( + corpusId, + "wiki", + runtime.manifest.indexGeneration, + (await store.listViews(corpusId)).views, + ); + expect( + (await store.getWikiFamily(corpusId)).planningConflict, + ).toBeUndefined(); + expect(service.getMaintenanceStatus().failures).toEqual([ + { + key: `${corpusId}:wiki:other`, + error: "Keep this failure", + }, + ]); + }); +}); + +test("deferred resumption cannot clear a different park or target failure", () => { + const scheduler = new MaintenanceScheduler(() => false, "manual"); + scheduler.request("corpus", "wiki", "generation", async () => {}); + scheduler.reportFailure("corpus", "wiki", "target", "Keep this failure"); + scheduler.park("corpus", "wiki", "Pass budget exhausted"); + scheduler.resumeDeferred("corpus", "wiki", "Planning conflict"); + expect(scheduler.status().deferred).toEqual([ + { key: "corpus:wiki", reason: "Pass budget exhausted" }, + ]); + scheduler.park("corpus", "wiki", "Planning conflict"); + scheduler.resumeDeferred("corpus", "wiki", "Planning conflict"); + scheduler.resumeDeferred("corpus", "wiki", "Planning conflict"); + expect(scheduler.status().deferred).toEqual([]); + expect(scheduler.status().failures).toEqual([ + { key: "corpus:wiki:target", error: "Keep this failure" }, + ]); +});