merge: reconcile upstream through 7ba8eee4

CHANGELOG.md

@@ -571,6 +571,9 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - `codegraph sync` and the file watcher now link a component to the JSX that renders it, like `<Team />`, when the component is added in a file with no JSX of its own, as in `export default function Team() { return null; }`. Before, the link waited for an unrelated change to a file with JSX, or for a re-index, so the new component looked unused.
 
 - In JavaScript and TypeScript, a default import now links to what the module's `export default` statement exports, wherever the statement sits in the file. Before, a module that declares its default, like `export default function Vans() {…}` or `export default class Store {…}`, had its default import linked to the first function it exports instead: in a React Router data-router app, where a page exports its `loader` or `action` above the page component, `import Vans, { loader as vansLoader } from './pages/Vans'` made the route `/vans` and every use of `Vans` lead to the loader. A component file that also exports a styled or memoized component no longer hands its default import to that component, and an anonymous default, like `export default function () {…}` or `export default () => …`, is no longer linked to a function exported beside it or nested inside it. A module loaded with `require('./x').default` is read the same way. Re-index JavaScript and TypeScript projects after upgrading.
+
+- `codegraph sync` and the file watcher now update the files that import through a module that re-exports what it imports, like a package's `__init__.py` or an `index.ts` barrel, when that module is deleted or added, and in Python also the files that reach it through another package re-exporting it. Before, deleting `shop/__init__.py` left every `from shop import Order` linked to the class it re-exported, and deleting `components/index.ts` left every `import { Button } from './components'` linked, until each importing file changed or the project was re-indexed. When such a module came back, the imports stayed linked the way they were while it was missing, and in a large Python project an import of a module whose name is as common as `models` or `test`, like `from django.test import TestCase`, stayed unlinked. Python, JavaScript, TypeScript, Vue, Svelte and Astro.
+
 #### MCP / indexing
 
 - Full indexes of large projects no longer sometimes run two to three times slower than usual. A timing clash between two background database maintenance steps could switch reference resolution from several threads to one partway through.

README.md

@@ -52,7 +52,7 @@ Follow [@getcodegraph](https://x.com/getcodegraph) on X for updates.
 
 ## About this fork
 
-This is **bompus/codegraph**, a fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph). Its default branch, `fork/consolidated`, contains upstream `main` through [`a3fdb920`](https://github.com/colbymchenry/codegraph/commit/a3fdb920) (after v1.6.2) plus the fork's own work, and it takes upstream changes as they land. Changes that suit upstream are also offered there as pull requests.
+This is **bompus/codegraph**, a fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph). Its default branch, `fork/consolidated`, contains upstream `main` through [`7ba8eee4`](https://github.com/colbymchenry/codegraph/commit/7ba8eee4) (after v1.6.2) plus the fork's own work, and it takes upstream changes as they land. Changes that suit upstream are also offered there as pull requests.
 
 The fork publishes no releases. The install scripts, npm package, badges and `codegraph upgrade` further down this page install **upstream's** releases. To run the fork, build it from source (below).
 

__tests__/sync-module-importers.test.ts

@@ -0,0 +1,154 @@
+/**
+ * A sync re-resolves what an import statement binds when the module it names
+ * appears or goes away.
+ *
+ * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`,
+ * which re-exports `Order`; `import { Button } from './components'` reaches
+ * `components/Button.ts` through `components/index.ts`. Deleting the
+ * re-exporting file left every importer linked through it: those edges point
+ * into files that stay, so nothing removed them, while a fresh index of the
+ * tree without the file links them by name or not at all. And a module that
+ * appeared after its importers were bound by name in its absence left those
+ * edges as the name lookup made them, while a fresh index binds them through
+ * the module.
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import CodeGraph from '../src/index';
+
+type Files = Record<string, string>;
+
+/**
+ * Python: an absolute import, imports from inside and beside the package, a
+ * renamed re-export, a plain module, a subpackage imported from the package
+ * around it (`from app import orm`, then `orm.Model`), and an import inside a
+ * function, which has no import node. Two packages re-export the shop package
+ * in turn, one with a star import and one by name, and are imported in its
+ * stead. `vendored.py` makes `shop` as common a name among failed imports as
+ * `models` or `test` is in a large project: more than sync's failed-import
+ * retry looks up by name.
+ */
+const PYTHON: Files = {
+  'vendored.py': Array.from({ length: 501 }, (_, i) => `import vendor${i}.shop\n`).join(''),
+  'app/store/__init__.py': 'from app.shop import *\n',
+  'report.py': 'from app.store import Order\n\n\ndef report():\n    return Order()\n',
+  'app/front/__init__.py': 'from ..shop import Cart as Basket\n',
+  'kiosk.py': 'from app.front import Basket\n\n\ndef sell():\n    return Basket()\n',
+  'app/__init__.py': '',
+  'app/shop/models.py': 'class _Order:\n    pass\n\n\nclass Cart:\n    pass\n',
+  'app/shop/__init__.py': 'from .models import _Order as Order\nfrom .models import Cart\n',
+  'app/shop/api/views.py': 'from .. import Order, Cart\n\n\ndef view():\n    return Order(), Cart()\n',
+  'app/checkout.py': 'from .shop import Cart\n\n\ndef run():\n    return Cart()\n',
+  'main.py': 'from app.shop import Order\n\n\ndef go():\n    return Order()\n',
+  'lib/__init__.py': '',
+  'lib/impl.py': 'def helper():\n    return 1\n',
+  'lib/api.py': 'from .impl import helper as public_helper\n',
+  'tool.py': 'from lib.api import public_helper\n\n\ndef use():\n    return public_helper()\n',
+  'app/orm/base.py': 'class _Model:\n    pass\n',
+  'app/orm/__init__.py': 'from .base import _Model as Model\n',
+  'app/blog.py': 'from app import orm\n\n\nclass Post(orm.Model):\n    pass\n',
+  'test_orders.py': 'def test_order():\n    from app.shop import Order\n    return Order()\n',
+};
+const PYTHON_BARRELS = ['app/shop/__init__.py', 'lib/api.py', 'app/orm/__init__.py'];
+
+/** ES modules: an `index.ts` re-exporting under the same name and under another. */
+const TS: Files = {
+  'package.json': '{"name":"app"}',
+  'src/components/Button.ts': 'export function Button() { return 1; }\n',
+  'src/components/Icon.ts': 'export function BaseIcon() { return 2; }\n',
+  'src/components/index.ts': "export { Button } from './Button';\nexport { BaseIcon as Icon } from './Icon';\n",
+  'src/app.ts': "import { Button, Icon } from './components';\nexport function render() { return [Button(), Icon()]; }\n",
+};
+const TS_BARRELS = ['src/components/index.ts'];
+
+let roots: string[] = [];
+let graphs: CodeGraph[] = [];
+
+afterEach(() => {
+  for (const graph of graphs) graph.close();
+  graphs = [];
+  for (const root of roots) fs.rmSync(root, { recursive: true, force: true });
+  roots = [];
+});
+
+function tempRoot(): string {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-importers-'));
+  roots.push(root);
+  return root;
+}
+
+function write(root: string, files: Files): void {
+  for (const [file, text] of Object.entries(files)) {
+    fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+    fs.writeFileSync(path.join(root, file), text);
+  }
+}
+
+async function index(root: string): Promise<CodeGraph> {
+  const graph = await CodeGraph.init(root, { index: true });
+  graphs.push(graph);
+  return graph;
+}
+
+/** Every edge, by its ends' natural keys, with where it is written and how it resolved. */
+function edgesOf(graph: CodeGraph): string[] {
+  const keys = new Map(
+    graph.getFiles()
+      .flatMap((file) => graph.getNodesInFile(file.path))
+      .map((n) => [n.id, `${n.kind} ${n.filePath}:${n.qualifiedName}:${n.startLine}`] as const)
+  );
+  return graph.getOutgoingEdgesFrom([...keys.keys()])
+    .map((e) => `${e.kind} ${keys.get(e.source)} -> ${keys.get(e.target) ?? e.target} @${e.line}:${e.column} ${JSON.stringify(e.metadata ?? {})}`)
+    .sort();
+}
+
+/** The tree, indexed from scratch in a folder of its own: what the sync has to match. */
+async function fullIndexOf(files: Files): Promise<string[]> {
+  const root = tempRoot();
+  write(root, files);
+  return edgesOf(await index(root));
+}
+
+const without = (files: Files, gone: string[]): Files =>
+  Object.fromEntries(Object.entries(files).filter(([file]) => !gone.includes(file)));
+const only = (files: Files, kept: string[]): Files =>
+  Object.fromEntries(Object.entries(files).filter(([file]) => kept.includes(file)));
+
+describe.each([
+  ['Python package and module', PYTHON, PYTHON_BARRELS],
+  ['ES module barrel', TS, TS_BARRELS],
+])('sync re-resolves the imports through a re-exporting file: %s', (_label, files, barrels) => {
+  it('when it is deleted', async () => {
+    const root = tempRoot();
+    write(root, files);
+    const graph = await index(root);
+
+    for (const file of barrels) fs.rmSync(path.join(root, file));
+    expect((await graph.sync()).filesRemoved).toBe(barrels.length);
+    expect(edgesOf(graph)).toEqual(await fullIndexOf(without(files, barrels)));
+  }, 120_000);
+
+  it('when it comes back after it was deleted', async () => {
+    const root = tempRoot();
+    write(root, files);
+    const graph = await index(root);
+
+    for (const file of barrels) fs.rmSync(path.join(root, file));
+    await graph.sync();
+    write(root, only(files, barrels));
+    expect((await graph.sync()).filesAdded).toBe(barrels.length);
+    expect(edgesOf(graph)).toEqual(await fullIndexOf(files));
+  }, 120_000);
+
+  it('when it appears after its importers were bound without it', async () => {
+    const root = tempRoot();
+    write(root, without(files, barrels));
+    const graph = await index(root);
+
+    write(root, only(files, barrels));
+    expect((await graph.sync()).filesAdded).toBe(barrels.length);
+    expect(edgesOf(graph)).toEqual(await fullIndexOf(files));
+  }, 120_000);
+});

src/codegraph.ts

@@ -66,6 +66,7 @@ import { CodeGraphPackageVersion } from './mcp/version';
 import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments';
 import { seedLiteralsInQuery, seedRouteNamesInQuery } from './extraction/literal-capture';
 import { createYielder } from './resolution/cooperative-yield';
+import { createImportReader, reopenReferencesThroughAddedFiles } from './resolution/module-importers';
 import { minRefsForPool } from './resolution/resolver-pool';
 
 /** The extraction, resolution and watcher modules, loaded on first use. */
@@ -1023,6 +1024,9 @@ export class CodeGraph {
         // per-name ceiling counts only edges up to it (see
         // resurrectStaleResolutionEdges).
         let edgesBeforeRetry: number | undefined;
+        // And the last edge before this sync resolved anything: an importer's
+        // edge up to it was bound before a module this sync adds was there.
+        const edgesBeforeResolution = this.queries.getMaxEdgeId();
         if (filesChanged) {
           if (result.changedFilePaths) {
             // Scope resolution to changed files (git fast path — bounded set)
@@ -1088,6 +1092,21 @@ export class CodeGraph {
               });
             }
             if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-failed-ref-retry: ${Date.now() - tRetry}ms (${retryable.length} refs)`);
+
+            // A module that appeared, such as a package `__init__.py` or an
+            // `index.ts` re-exporting what it imports: what the files importing
+            // it bound by name in its absence, or failed, goes back to pending
+            // for the sweep below, which binds it through the module as a fresh
+            // index does. (The removal half runs before removed files are deleted.)
+            if (result.addedFilePaths && result.addedFilePaths.length > 0) {
+              const tAdded = Date.now();
+              const added = result.addedFilePaths.flatMap((filePath) => {
+                const file = this.queries.getFileByPath(filePath);
+                return file ? [{ path: filePath, language: file.language }] : [];
+              });
+              const reopened = reopenReferencesThroughAddedFiles(this.queries, createImportReader(this.projectRoot, this.queries), added, edgesBeforeResolution);
+              if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-added-module-importers: ${Date.now() - tAdded}ms (${reopened} refs re-opened)`);
+            }
           } else {
             // No git info — use batched resolution to avoid OOM
             const unresolvedCount = this.queries.getUnresolvedReferencesCount();

src/db/queries.ts

@@ -76,6 +76,16 @@ const IS_INTERFACE_MEMBER = (alias: string): string => `EXISTS (
    WHERE ce.target = ${alias}.id AND ce.kind = 'contains' AND owner.kind = 'interface'
 )`;
 
+/**
+ * A SQL predicate: is the reference name in `column` written through one of
+ * the names in the JSON array bound to the next two parameters, as the name
+ * itself or as a member read through it (`Order.objects` through `Order`)?
+ */
+const THROUGH_NAMES = (column: string): string => `(
+  ${column} IN (SELECT value FROM json_each(?))
+  OR (instr(${column}, '.') > 1 AND substr(${column}, 1, instr(${column}, '.') - 1) IN (SELECT value FROM json_each(?)))
+)`;
+
 /**
  * How much of the exact-name bonus a `deprioritize`d path keeps (#982). Damped
  * rather than zeroed: a query that genuinely targets that tree must still rank
@@ -3261,6 +3271,134 @@ export class QueryBuilder {
     }));
   }
 
+  /**
+   * Resolution edges out of the nodes of `filePath` whose reference was
+   * written through one of `names` (see {@link THROUGH_NAMES}), with the source
+   * file and language a resurrection needs and the file of each target.
+   * Synthesized (`provenance='heuristic'`) and `contains` edges are left out.
+   */
+  getResolutionEdgesThrough(
+    filePath: string,
+    names: readonly string[]
+  ): Array<Edge & { edgeId: number; sourceFilePath: string; sourceLanguage: Language; targetFilePath: string }> {
+    const rows = this.db
+      .prepare(
+        `SELECT * FROM (
+           SELECT e.*, src.file_path AS source_file_path, src.language AS source_language, tgt.file_path AS target_file_path,
+                  CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name
+             FROM nodes src
+             JOIN edges e ON e.source = src.id
+             JOIN nodes tgt ON tgt.id = e.target
+            WHERE src.file_path = ?
+              AND e.kind != 'contains'
+              AND (e.provenance IS NULL OR e.provenance != 'heuristic')
+         ) WHERE ${THROUGH_NAMES('ref_name')}`
+      )
+      .all(filePath, JSON.stringify(names), JSON.stringify(names)) as Array<
+        EdgeRow & { source_file_path: string; source_language: Language; target_file_path: string }
+      >;
+    return rows.map((row) => ({
+      ...rowToEdge(row),
+      edgeId: row.id,
+      sourceFilePath: row.source_file_path,
+      sourceLanguage: row.source_language,
+      targetFilePath: row.target_file_path,
+    }));
+  }
+
+  /**
+   * The row ids of the failed references of `filePath` that import one of
+   * `modules` or are written through one of `names` (see {@link THROUGH_NAMES}).
+   */
+  getFailedReferencesThrough(filePath: string, modules: readonly string[], names: readonly string[]): number[] {
+    return (this.db
+      .prepare(
+        `SELECT id FROM unresolved_refs
+          WHERE file_path = ? AND status = 'failed'
+            AND ((reference_kind = 'imports' AND reference_name IN (SELECT value FROM json_each(?)))
+                 OR ${THROUGH_NAMES('reference_name')})`
+      )
+      .all(filePath, JSON.stringify(modules), JSON.stringify(names), JSON.stringify(names)) as Array<{ id: number }>)
+      .map((row) => row.id);
+  }
+
+  /** The targets, in other files, of the non-synthesized `imports` edges out of the nodes of `filePath`. */
+  getImportTargetsFromFile(filePath: string): string[] {
+    return (this.db
+      .prepare(
+        `SELECT DISTINCT e.target AS target
+           FROM nodes src
+           JOIN edges e ON e.source = src.id
+           JOIN nodes tgt ON tgt.id = e.target
+          WHERE src.file_path = ?
+            AND e.kind = 'imports'
+            AND (e.provenance IS NULL OR e.provenance != 'heuristic')
+            AND tgt.file_path != ?`
+      )
+      .all(filePath, filePath) as Array<{ target: string }>)
+      .map((row) => row.target);
+  }
+
+  /**
+   * The `imports` edges into the node `targetId` from files other than
+   * `exceptFile`: each importing file, its language, and the module specifier
+   * it wrote (the edge's `refName`; null when it carries none).
+   */
+  getImportsInto(
+    targetId: string,
+    exceptFile: string
+  ): Array<{ filePath: string; language: Language; refName: string | null }> {
+    return (this.db
+      .prepare(
+        `SELECT src.file_path AS file_path, src.language AS language,
+                CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name
+           FROM edges e
+           JOIN nodes src ON src.id = e.source
+          WHERE e.target = ? AND e.kind = 'imports' AND src.file_path != ?`
+      )
+      .all(targetId, exceptFile) as Array<{ file_path: string; language: Language; ref_name: unknown }>)
+      .map((row) => ({
+        filePath: row.file_path,
+        language: row.language,
+        refName: typeof row.ref_name === 'string' ? row.ref_name : null,
+      }));
+  }
+
+  /** The files with a resolution edge into one of `targetIds`. */
+  getFilesWithEdgesInto(targetIds: string[]): string[] {
+    const files = new Set<string>();
+    for (let i = 0; i < targetIds.length; i += SQLITE_PARAM_CHUNK_SIZE) {
+      const chunk = targetIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
+      const placeholders = chunk.map(() => '?').join(',');
+      const rows = this.db
+        .prepare(
+          `SELECT DISTINCT src.file_path AS file_path
+             FROM edges e
+             JOIN nodes src ON src.id = e.source
+            WHERE e.target IN (${placeholders})
+              AND e.kind != 'contains'
+              AND (e.provenance IS NULL OR e.provenance != 'heuristic')`
+        )
+        .all(...chunk) as Array<{ file_path: string }>;
+      for (const row of rows) files.add(row.file_path);
+    }
+    return [...files];
+  }
+
+  /** The import nodes named one of `names` (an import node is named by the module path it imports). */
+  getImportNodesByNames(names: string[]): Array<{ filePath: string; name: string; language: Language }> {
+    const out: Array<{ filePath: string; name: string; language: Language }> = [];
+    for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) {
+      const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE);
+      const placeholders = chunk.map(() => '?').join(',');
+      const rows = this.db
+        .prepare(`SELECT file_path, name, language FROM nodes WHERE name IN (${placeholders}) AND kind = 'import'`)
+        .all(...chunk) as Array<{ file_path: string; name: string; language: Language }>;
+      for (const row of rows) out.push({ filePath: row.file_path, name: row.name, language: row.language });
+    }
+    return out;
+  }
+
   // ===========================================================================
   // File Operations
   // ===========================================================================