From 68df7c40105f1321db84010a7c92c3ba2511705b Mon Sep 17 00:00:00 2001 From: danusha2345 Date: Thu, 15 Oct 2026 20:07:05 +0300 Subject: [PATCH] fix(index): a full index ends with complete planner statistics (#2571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maintenance after a bulk write ran `PRAGMA optimize` under `analysis_limit=1000`: statistics from the first rows of each index. On a repository with one generated file of thousands of symbols (an Android `R.java`, a bundled `.d.ts`) that sample misses the big file, and the planner takes every file for a few symbols. The dominant-file query behind every `codegraph_explore` is then planned as a scan of each file's symbols per symbol — ~100 s per explore on a 280k-node index, where exact statistics give 1.5 s; every CLI `explore` and every prompt hook paid it again, and the first MCP explore held the queue for the rest. A full index now ends with a full `ANALYZE` (under a second on a 1 GB index, once, after a run that took minutes). A sync keeps the incremental `optimize`, which re-analyzes only a table that grew by a lot. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 1 + __tests__/index-full-analyze-stats.test.ts | 52 ++++++++++++++++++++++ src/db/index.ts | 21 +++++++-- src/index.ts | 2 +- 4 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 __tests__/index-full-analyze-stats.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d1b80a152f..c1a7793036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -176,6 +176,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In TypeScript and JavaScript, a call on a field or variable whose class the file imports now goes through that class and the classes and interfaces it extends, each found the way its own file names it, before any other class of the same name. Before, when the class didn't declare the method itself, the call went to whichever same-named class had it: in vscode, `this._storage.dispose()` reached a test fixture's `Disposable`, and calls on a library's services in bitwarden reached the browser app's copies. A call like `this.logService.trace()`, whose `ILogService` gets `trace` from the `ILogger` it extends, linked nowhere and now links to `ILogger`'s method. A field's call whose class and everything it extends are known but lack the method, as when a package's base class provides it, now links to nothing rather than to an unrelated class. Re-index TypeScript and JavaScript projects after upgrading. - In TypeScript, a call on a value of an interface that extends one also declared as a constant, like vscode's `ITerminalLogService extends ILogService` where `ILogService` is its dependency-injection decorator too, now links to the method the extended interface declares, such as `ILogger`'s `trace`. Before, the constant was taken for the extended type, so the call linked nowhere. Re-index TypeScript projects after upgrading. - In JavaScript and TypeScript, an import written from a jsconfig or tsconfig `baseUrl`, like `import MainLayout from 'layout/MainLayout'` under `"baseUrl": "src"`, or through a catch-all `"paths": { "*": ["./src/*"] }`, now links to the file it names, read with the config of the app the importing file belongs to, and so does an import of a single name like `'config'` or `'themes'`. Create React App and Vite admin templates such as Berry, Mantis and MatX write nearly every import this way. Before, each one was taken for an npm package, so it linked nothing, its names were matched by name alone, which could reach a same-named component in another app of the same repository, and a React Router route whose page loads through `lazy(() => import('views/dashboard/Default'))` pointed at that loader instead of the page and its layout. A package the app's `package.json` declares, or a Node built-in, never links to a project module of the same name, only to type declarations the project keeps for it, and only a file the index holds counts, never a folder or something in `node_modules`. `codegraph sync` and the file watcher link such an import too when the file it names is added later or gains what it imports. Re-index JavaScript and TypeScript projects after upgrading. +- A full `codegraph index` now ends with complete planner statistics, so on a repository with one huge generated file (an Android `R.java`, a bundled `.d.ts`) `codegraph_explore` no longer takes a minute or more per call. (#2571) ## [1.6.2] - 2026-10-03 diff --git a/__tests__/index-full-analyze-stats.test.ts b/__tests__/index-full-analyze-stats.test.ts new file mode 100644 index 0000000000..31c06ad075 --- /dev/null +++ b/__tests__/index-full-analyze-stats.test.ts @@ -0,0 +1,52 @@ +/** + * A full index leaves the planner exact statistics (#2571). + * + * The maintenance after a bulk write ran `PRAGMA optimize` under + * `analysis_limit=1000`: statistics from the first rows of each index. On a + * repository with one generated file of thousands of symbols (an Android + * `R.java`, a bundled `.d.ts`) that sample misses the big file, the planner + * takes every file for a few symbols, and the dominant-file query behind + * every `codegraph_explore` is planned as a scan of each file's symbols per + * symbol — ~100 s per explore on a 280k-node index, 1.5 s with exact + * statistics. A full index now ends with a full `ANALYZE`; a sync keeps the + * incremental `optimize`, which re-analyzes only a table that grew by a lot. + */ +import { describe, it, expect } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { DatabaseConnection } from '../src/db'; + +describe('#2571 — a full index leaves the planner exact statistics', () => { + it('runMaintenance after a full index analyzes every row, not a sample', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-analyze-')); + const db = DatabaseConnection.initialize(path.join(dir, 'codegraph.db')); + try { + const raw = db.getDb(); + const insert = raw.prepare( + `INSERT INTO nodes (id, kind, name, qualified_name, file_path, language, start_line, end_line, start_column, end_column, updated_at) + VALUES (?, 'constant', ?, ?, ?, 'java', 1, 1, 0, 0, 0)`, + ); + // One generated file of 3,000 symbols beside 3,000 files of one: two symbols per file on average. + // The big file sorts after the small ones, where a sample of the index never reaches it. + raw.exec('BEGIN'); + for (let i = 0; i < 3000; i++) { + insert.run(`big-${i}`, `R${i}`, `R.R${i}`, 'zz/R.java'); + insert.run(`one-${i}`, `f${i}`, `f${i}`, `src/${String(i).padStart(4, '0')}.java`); + } + raw.exec('COMMIT'); + const stat = (): string | undefined => + (raw.prepare(`SELECT stat FROM sqlite_stat1 WHERE idx = 'idx_nodes_file_path'`).get() as { stat: string } | undefined)?.stat; + + // The statistics a sync's maintenance leaves: a sample of the first rows, which misses the big file. + await db.runMaintenance(); + expect(stat()).toBe('6000 1'); + + await db.runMaintenance({ fullAnalyze: true }); + expect(stat()).toBe('6000 2'); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/db/index.ts b/src/db/index.ts index cfe7c10a72..adc0810f2a 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -748,6 +748,16 @@ export class DatabaseConnection { * tables whose row counts changed materially since the last * ANALYZE. Without it, the query planner has no statistics on the * freshly-bulk-loaded tables and can pick suboptimal indexes. + * After a FULL index (`fullAnalyze`) it is a full `ANALYZE` instead: + * the statistics `optimize` gathers under `analysis_limit` come from + * the first rows of each index, and on a repository with one + * generated file of thousands of symbols (an Android `R.java`, a + * bundled `.d.ts`) that sample misses the big file. The planner + * then takes every file for a few symbols, and the dominant-file query + * behind every explore is planned as a scan of each file's symbols + * per symbol — ~100 s per explore on a 280k-node index, 1.5 s with + * exact statistics (#2571). A full ANALYZE of a 1 GB index is under + * a second, once, at the end of a run that took minutes. * * - `PRAGMA wal_checkpoint(PASSIVE)` — fold pending WAL pages back * into the main database file so the WAL file doesn't grow @@ -769,17 +779,20 @@ export class DatabaseConnection { * and SKIPS the checkpoint (the final close() checkpoints after the * CLI has already disarmed its watchdog). */ - async runMaintenance(): Promise { + async runMaintenance(options: { fullAnalyze?: boolean } = {}): Promise { + const analyze = options.fullAnalyze + ? ['PRAGMA analysis_limit=0', 'ANALYZE'] + : ['PRAGMA analysis_limit=1000', 'PRAGMA optimize']; // In-memory / test databases: nothing worth a worker round-trip. if (!this.dbPath || this.dbPath === ':memory:') { - try { this.db.exec('PRAGMA optimize'); } catch { /* ignore */ } + for (const p of analyze) { try { this.db.exec(p); } catch { /* ignore */ } } try { this.db.exec('PRAGMA wal_checkpoint(PASSIVE)'); } catch { /* ignore */ } return; } await this.runPragmasOffThread( - ['PRAGMA analysis_limit=1000', 'PRAGMA optimize', 'PRAGMA wal_checkpoint(PASSIVE)'], + [...analyze, 'PRAGMA wal_checkpoint(PASSIVE)'], // Worker threads unavailable — bounded in-line fallback, no checkpoint. - ['PRAGMA analysis_limit=1000', 'PRAGMA optimize'] + analyze ); } diff --git a/src/index.ts b/src/index.ts index 7a5a2d4252..fa39790427 100644 --- a/src/index.ts +++ b/src/index.ts @@ -717,7 +717,7 @@ export class CodeGraph { // maintenance checkpoint don't contend for the checkpointer lock // (the loser would silently no-op and leave the WAL unfolded). if (walValve) { walValve.stop(); await walValve.drain(); } - await this.db.runMaintenance(); + await this.db.runMaintenance({ fullAnalyze: true }); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] maintenance: ${Date.now() - tMaint}ms`); }