diff --git a/CHANGELOG.md b/CHANGELOG.md index db92767b1..3fc0c6815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -215,6 +215,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In Rust, a call like `String::from("…")`, `Vec::from([…])` or `NonNull::from(x)` no longer links to your project's own `impl From for String` (or for `Vec`, `NonNull`, …) unless what it passes is a `YourType`. A method call on a standard-library value, like `files.iter()` or `buf.extend(…)`, links to your trait's implementation for that type only where the trait is imported. Before, every `String::from("…")` call in dioxus's CLI linked to one such conversion, and tokio's `vec.iter()` calls to an internal trait's `Vec` impl. Re-index Rust projects after upgrading. - In Rust, a type alias, struct, enum or function declared without `pub` now links only from its own module and the modules inside it, and a name a `use` brings in from one of your workspace's crates, like `use tokio_test::task::spawn` or `use crate::error::Error`, links only to an item of that crate or of a crate it depends on. Before, a file's `use crate::Result` could link its `Result<…>` types to another module's private `type Result`, as in dioxus's CLI, and a call to an imported function could link to another crate's function of the same name, such as tokio's test `spawn(…)` calls to one in its examples, or to nothing because a private function of that name was in the way. Re-index Rust projects after upgrading. - In C++, every name a declaration lists now has the declaration's type, like `strings_found` in google/breakpad's `ByteBuffer symbols_found, strings_found;` or `data_` in a class's `Table *index_, *data_;`, so calls on it link to that class's methods. Before, only the first name had its type read, and a call on another name linked to nothing or to the class of an earlier variable with the same name. +- A full `codegraph index` now ends with complete planner statistics, so on a repository with one huge generated file (an Android `R.java`, a bundled `.d.ts`) `codegraph_explore` no longer takes a minute or more per call. (#2571) ## [1.6.2] - 2026-10-03 diff --git a/__tests__/index-full-analyze-stats.test.ts b/__tests__/index-full-analyze-stats.test.ts new file mode 100644 index 000000000..31c06ad07 --- /dev/null +++ b/__tests__/index-full-analyze-stats.test.ts @@ -0,0 +1,52 @@ +/** + * A full index leaves the planner exact statistics (#2571). + * + * The maintenance after a bulk write ran `PRAGMA optimize` under + * `analysis_limit=1000`: statistics from the first rows of each index. On a + * repository with one generated file of thousands of symbols (an Android + * `R.java`, a bundled `.d.ts`) that sample misses the big file, the planner + * takes every file for a few symbols, and the dominant-file query behind + * every `codegraph_explore` is planned as a scan of each file's symbols per + * symbol — ~100 s per explore on a 280k-node index, 1.5 s with exact + * statistics. A full index now ends with a full `ANALYZE`; a sync keeps the + * incremental `optimize`, which re-analyzes only a table that grew by a lot. + */ +import { describe, it, expect } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { DatabaseConnection } from '../src/db'; + +describe('#2571 — a full index leaves the planner exact statistics', () => { + it('runMaintenance after a full index analyzes every row, not a sample', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-analyze-')); + const db = DatabaseConnection.initialize(path.join(dir, 'codegraph.db')); + try { + const raw = db.getDb(); + const insert = raw.prepare( + `INSERT INTO nodes (id, kind, name, qualified_name, file_path, language, start_line, end_line, start_column, end_column, updated_at) + VALUES (?, 'constant', ?, ?, ?, 'java', 1, 1, 0, 0, 0)`, + ); + // One generated file of 3,000 symbols beside 3,000 files of one: two symbols per file on average. + // The big file sorts after the small ones, where a sample of the index never reaches it. + raw.exec('BEGIN'); + for (let i = 0; i < 3000; i++) { + insert.run(`big-${i}`, `R${i}`, `R.R${i}`, 'zz/R.java'); + insert.run(`one-${i}`, `f${i}`, `f${i}`, `src/${String(i).padStart(4, '0')}.java`); + } + raw.exec('COMMIT'); + const stat = (): string | undefined => + (raw.prepare(`SELECT stat FROM sqlite_stat1 WHERE idx = 'idx_nodes_file_path'`).get() as { stat: string } | undefined)?.stat; + + // The statistics a sync's maintenance leaves: a sample of the first rows, which misses the big file. + await db.runMaintenance(); + expect(stat()).toBe('6000 1'); + + await db.runMaintenance({ fullAnalyze: true }); + expect(stat()).toBe('6000 2'); + } finally { + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/db/index.ts b/src/db/index.ts index cfe7c10a7..adc0810f2 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -748,6 +748,16 @@ export class DatabaseConnection { * tables whose row counts changed materially since the last * ANALYZE. Without it, the query planner has no statistics on the * freshly-bulk-loaded tables and can pick suboptimal indexes. + * After a FULL index (`fullAnalyze`) it is a full `ANALYZE` instead: + * the statistics `optimize` gathers under `analysis_limit` come from + * the first rows of each index, and on a repository with one + * generated file of thousands of symbols (an Android `R.java`, a + * bundled `.d.ts`) that sample misses the big file. The planner + * then takes every file for a few symbols, and the dominant-file query + * behind every explore is planned as a scan of each file's symbols + * per symbol — ~100 s per explore on a 280k-node index, 1.5 s with + * exact statistics (#2571). A full ANALYZE of a 1 GB index is under + * a second, once, at the end of a run that took minutes. * * - `PRAGMA wal_checkpoint(PASSIVE)` — fold pending WAL pages back * into the main database file so the WAL file doesn't grow @@ -769,17 +779,20 @@ export class DatabaseConnection { * and SKIPS the checkpoint (the final close() checkpoints after the * CLI has already disarmed its watchdog). */ - async runMaintenance(): Promise { + async runMaintenance(options: { fullAnalyze?: boolean } = {}): Promise { + const analyze = options.fullAnalyze + ? ['PRAGMA analysis_limit=0', 'ANALYZE'] + : ['PRAGMA analysis_limit=1000', 'PRAGMA optimize']; // In-memory / test databases: nothing worth a worker round-trip. if (!this.dbPath || this.dbPath === ':memory:') { - try { this.db.exec('PRAGMA optimize'); } catch { /* ignore */ } + for (const p of analyze) { try { this.db.exec(p); } catch { /* ignore */ } } try { this.db.exec('PRAGMA wal_checkpoint(PASSIVE)'); } catch { /* ignore */ } return; } await this.runPragmasOffThread( - ['PRAGMA analysis_limit=1000', 'PRAGMA optimize', 'PRAGMA wal_checkpoint(PASSIVE)'], + [...analyze, 'PRAGMA wal_checkpoint(PASSIVE)'], // Worker threads unavailable — bounded in-line fallback, no checkpoint. - ['PRAGMA analysis_limit=1000', 'PRAGMA optimize'] + analyze ); } diff --git a/src/index.ts b/src/index.ts index 5a4ff39b7..b4e83a9a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -717,7 +717,7 @@ export class CodeGraph { // maintenance checkpoint don't contend for the checkpointer lock // (the loser would silently no-op and leave the WAL unfolded). if (walValve) { walValve.stop(); await walValve.drain(); } - await this.db.runMaintenance(); + await this.db.runMaintenance({ fullAnalyze: true }); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] maintenance: ${Date.now() - tMaint}ms`); }