diff --git a/CHANGELOG.md b/CHANGELOG.md index 35ca4b897..1bbf2b498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -571,6 +571,9 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph sync` and the file watcher now link a component to the JSX that renders it, like ``, when the component is added in a file with no JSX of its own, as in `export default function Team() { return null; }`. Before, the link waited for an unrelated change to a file with JSX, or for a re-index, so the new component looked unused. - In JavaScript and TypeScript, a default import now links to what the module's `export default` statement exports, wherever the statement sits in the file. Before, a module that declares its default, like `export default function Vans() {…}` or `export default class Store {…}`, had its default import linked to the first function it exports instead: in a React Router data-router app, where a page exports its `loader` or `action` above the page component, `import Vans, { loader as vansLoader } from './pages/Vans'` made the route `/vans` and every use of `Vans` lead to the loader. A component file that also exports a styled or memoized component no longer hands its default import to that component, and an anonymous default, like `export default function () {…}` or `export default () => …`, is no longer linked to a function exported beside it or nested inside it. A module loaded with `require('./x').default` is read the same way. Re-index JavaScript and TypeScript projects after upgrading. + +- `codegraph sync` and the file watcher now update the files that import through a module that re-exports what it imports, like a package's `__init__.py` or an `index.ts` barrel, when that module is deleted or added, and in Python also the files that reach it through another package re-exporting it. Before, deleting `shop/__init__.py` left every `from shop import Order` linked to the class it re-exported, and deleting `components/index.ts` left every `import { Button } from './components'` linked, until each importing file changed or the project was re-indexed. When such a module came back, the imports stayed linked the way they were while it was missing, and in a large Python project an import of a module whose name is as common as `models` or `test`, like `from django.test import TestCase`, stayed unlinked. Python, JavaScript, TypeScript, Vue, Svelte and Astro. + #### MCP / indexing - Full indexes of large projects no longer sometimes run two to three times slower than usual. A timing clash between two background database maintenance steps could switch reference resolution from several threads to one partway through. diff --git a/README.md b/README.md index 3abfc9339..6da976ad3 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Follow [@getcodegraph](https://x.com/getcodegraph) on X for updates. ## About this fork -This is **bompus/codegraph**, a fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph). Its default branch, `fork/consolidated`, contains upstream `main` through [`a3fdb920`](https://github.com/colbymchenry/codegraph/commit/a3fdb920) (after v1.6.2) plus the fork's own work, and it takes upstream changes as they land. Changes that suit upstream are also offered there as pull requests. +This is **bompus/codegraph**, a fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph). Its default branch, `fork/consolidated`, contains upstream `main` through [`7ba8eee4`](https://github.com/colbymchenry/codegraph/commit/7ba8eee4) (after v1.6.2) plus the fork's own work, and it takes upstream changes as they land. Changes that suit upstream are also offered there as pull requests. The fork publishes no releases. The install scripts, npm package, badges and `codegraph upgrade` further down this page install **upstream's** releases. To run the fork, build it from source (below). diff --git a/__tests__/sync-module-importers.test.ts b/__tests__/sync-module-importers.test.ts new file mode 100644 index 000000000..2a8c27236 --- /dev/null +++ b/__tests__/sync-module-importers.test.ts @@ -0,0 +1,154 @@ +/** + * A sync re-resolves what an import statement binds when the module it names + * appears or goes away. + * + * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`, + * which re-exports `Order`; `import { Button } from './components'` reaches + * `components/Button.ts` through `components/index.ts`. Deleting the + * re-exporting file left every importer linked through it: those edges point + * into files that stay, so nothing removed them, while a fresh index of the + * tree without the file links them by name or not at all. And a module that + * appeared after its importers were bound by name in its absence left those + * edges as the name lookup made them, while a fresh index binds them through + * the module. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +type Files = Record; + +/** + * Python: an absolute import, imports from inside and beside the package, a + * renamed re-export, a plain module, a subpackage imported from the package + * around it (`from app import orm`, then `orm.Model`), and an import inside a + * function, which has no import node. Two packages re-export the shop package + * in turn, one with a star import and one by name, and are imported in its + * stead. `vendored.py` makes `shop` as common a name among failed imports as + * `models` or `test` is in a large project: more than sync's failed-import + * retry looks up by name. + */ +const PYTHON: Files = { + 'vendored.py': Array.from({ length: 501 }, (_, i) => `import vendor${i}.shop\n`).join(''), + 'app/store/__init__.py': 'from app.shop import *\n', + 'report.py': 'from app.store import Order\n\n\ndef report():\n return Order()\n', + 'app/front/__init__.py': 'from ..shop import Cart as Basket\n', + 'kiosk.py': 'from app.front import Basket\n\n\ndef sell():\n return Basket()\n', + 'app/__init__.py': '', + 'app/shop/models.py': 'class _Order:\n pass\n\n\nclass Cart:\n pass\n', + 'app/shop/__init__.py': 'from .models import _Order as Order\nfrom .models import Cart\n', + 'app/shop/api/views.py': 'from .. import Order, Cart\n\n\ndef view():\n return Order(), Cart()\n', + 'app/checkout.py': 'from .shop import Cart\n\n\ndef run():\n return Cart()\n', + 'main.py': 'from app.shop import Order\n\n\ndef go():\n return Order()\n', + 'lib/__init__.py': '', + 'lib/impl.py': 'def helper():\n return 1\n', + 'lib/api.py': 'from .impl import helper as public_helper\n', + 'tool.py': 'from lib.api import public_helper\n\n\ndef use():\n return public_helper()\n', + 'app/orm/base.py': 'class _Model:\n pass\n', + 'app/orm/__init__.py': 'from .base import _Model as Model\n', + 'app/blog.py': 'from app import orm\n\n\nclass Post(orm.Model):\n pass\n', + 'test_orders.py': 'def test_order():\n from app.shop import Order\n return Order()\n', +}; +const PYTHON_BARRELS = ['app/shop/__init__.py', 'lib/api.py', 'app/orm/__init__.py']; + +/** ES modules: an `index.ts` re-exporting under the same name and under another. */ +const TS: Files = { + 'package.json': '{"name":"app"}', + 'src/components/Button.ts': 'export function Button() { return 1; }\n', + 'src/components/Icon.ts': 'export function BaseIcon() { return 2; }\n', + 'src/components/index.ts': "export { Button } from './Button';\nexport { BaseIcon as Icon } from './Icon';\n", + 'src/app.ts': "import { Button, Icon } from './components';\nexport function render() { return [Button(), Icon()]; }\n", +}; +const TS_BARRELS = ['src/components/index.ts']; + +let roots: string[] = []; +let graphs: CodeGraph[] = []; + +afterEach(() => { + for (const graph of graphs) graph.close(); + graphs = []; + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); + roots = []; +}); + +function tempRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-importers-')); + roots.push(root); + return root; +} + +function write(root: string, files: Files): void { + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } +} + +async function index(root: string): Promise { + const graph = await CodeGraph.init(root, { index: true }); + graphs.push(graph); + return graph; +} + +/** Every edge, by its ends' natural keys, with where it is written and how it resolved. */ +function edgesOf(graph: CodeGraph): string[] { + const keys = new Map( + graph.getFiles() + .flatMap((file) => graph.getNodesInFile(file.path)) + .map((n) => [n.id, `${n.kind} ${n.filePath}:${n.qualifiedName}:${n.startLine}`] as const) + ); + return graph.getOutgoingEdgesFrom([...keys.keys()]) + .map((e) => `${e.kind} ${keys.get(e.source)} -> ${keys.get(e.target) ?? e.target} @${e.line}:${e.column} ${JSON.stringify(e.metadata ?? {})}`) + .sort(); +} + +/** The tree, indexed from scratch in a folder of its own: what the sync has to match. */ +async function fullIndexOf(files: Files): Promise { + const root = tempRoot(); + write(root, files); + return edgesOf(await index(root)); +} + +const without = (files: Files, gone: string[]): Files => + Object.fromEntries(Object.entries(files).filter(([file]) => !gone.includes(file))); +const only = (files: Files, kept: string[]): Files => + Object.fromEntries(Object.entries(files).filter(([file]) => kept.includes(file))); + +describe.each([ + ['Python package and module', PYTHON, PYTHON_BARRELS], + ['ES module barrel', TS, TS_BARRELS], +])('sync re-resolves the imports through a re-exporting file: %s', (_label, files, barrels) => { + it('when it is deleted', async () => { + const root = tempRoot(); + write(root, files); + const graph = await index(root); + + for (const file of barrels) fs.rmSync(path.join(root, file)); + expect((await graph.sync()).filesRemoved).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(without(files, barrels))); + }, 120_000); + + it('when it comes back after it was deleted', async () => { + const root = tempRoot(); + write(root, files); + const graph = await index(root); + + for (const file of barrels) fs.rmSync(path.join(root, file)); + await graph.sync(); + write(root, only(files, barrels)); + expect((await graph.sync()).filesAdded).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(files)); + }, 120_000); + + it('when it appears after its importers were bound without it', async () => { + const root = tempRoot(); + write(root, without(files, barrels)); + const graph = await index(root); + + write(root, only(files, barrels)); + expect((await graph.sync()).filesAdded).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(files)); + }, 120_000); +}); diff --git a/src/codegraph.ts b/src/codegraph.ts index 545f1fe8e..d4dd581f3 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -66,6 +66,7 @@ import { CodeGraphPackageVersion } from './mcp/version'; import { extractSegmentSearchWords, segmentLookupVariants, splitIdentifierSegments } from './search/identifier-segments'; import { seedLiteralsInQuery, seedRouteNamesInQuery } from './extraction/literal-capture'; import { createYielder } from './resolution/cooperative-yield'; +import { createImportReader, reopenReferencesThroughAddedFiles } from './resolution/module-importers'; import { minRefsForPool } from './resolution/resolver-pool'; /** The extraction, resolution and watcher modules, loaded on first use. */ @@ -1023,6 +1024,9 @@ export class CodeGraph { // per-name ceiling counts only edges up to it (see // resurrectStaleResolutionEdges). let edgesBeforeRetry: number | undefined; + // And the last edge before this sync resolved anything: an importer's + // edge up to it was bound before a module this sync adds was there. + const edgesBeforeResolution = this.queries.getMaxEdgeId(); if (filesChanged) { if (result.changedFilePaths) { // Scope resolution to changed files (git fast path — bounded set) @@ -1088,6 +1092,21 @@ export class CodeGraph { }); } if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-failed-ref-retry: ${Date.now() - tRetry}ms (${retryable.length} refs)`); + + // A module that appeared, such as a package `__init__.py` or an + // `index.ts` re-exporting what it imports: what the files importing + // it bound by name in its absence, or failed, goes back to pending + // for the sweep below, which binds it through the module as a fresh + // index does. (The removal half runs before removed files are deleted.) + if (result.addedFilePaths && result.addedFilePaths.length > 0) { + const tAdded = Date.now(); + const added = result.addedFilePaths.flatMap((filePath) => { + const file = this.queries.getFileByPath(filePath); + return file ? [{ path: filePath, language: file.language }] : []; + }); + const reopened = reopenReferencesThroughAddedFiles(this.queries, createImportReader(this.projectRoot, this.queries), added, edgesBeforeResolution); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-added-module-importers: ${Date.now() - tAdded}ms (${reopened} refs re-opened)`); + } } else { // No git info — use batched resolution to avoid OOM const unresolvedCount = this.queries.getUnresolvedReferencesCount(); diff --git a/src/db/queries.ts b/src/db/queries.ts index 40bb881b7..c8ac08f25 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -76,6 +76,16 @@ const IS_INTERFACE_MEMBER = (alias: string): string => `EXISTS ( WHERE ce.target = ${alias}.id AND ce.kind = 'contains' AND owner.kind = 'interface' )`; +/** + * A SQL predicate: is the reference name in `column` written through one of + * the names in the JSON array bound to the next two parameters, as the name + * itself or as a member read through it (`Order.objects` through `Order`)? + */ +const THROUGH_NAMES = (column: string): string => `( + ${column} IN (SELECT value FROM json_each(?)) + OR (instr(${column}, '.') > 1 AND substr(${column}, 1, instr(${column}, '.') - 1) IN (SELECT value FROM json_each(?))) +)`; + /** * How much of the exact-name bonus a `deprioritize`d path keeps (#982). Damped * rather than zeroed: a query that genuinely targets that tree must still rank @@ -3261,6 +3271,134 @@ export class QueryBuilder { })); } + /** + * Resolution edges out of the nodes of `filePath` whose reference was + * written through one of `names` (see {@link THROUGH_NAMES}), with the source + * file and language a resurrection needs and the file of each target. + * Synthesized (`provenance='heuristic'`) and `contains` edges are left out. + */ + getResolutionEdgesThrough( + filePath: string, + names: readonly string[] + ): Array { + const rows = this.db + .prepare( + `SELECT * FROM ( + SELECT e.*, src.file_path AS source_file_path, src.language AS source_language, tgt.file_path AS target_file_path, + CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name + FROM nodes src + JOIN edges e ON e.source = src.id + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic') + ) WHERE ${THROUGH_NAMES('ref_name')}` + ) + .all(filePath, JSON.stringify(names), JSON.stringify(names)) as Array< + EdgeRow & { source_file_path: string; source_language: Language; target_file_path: string } + >; + return rows.map((row) => ({ + ...rowToEdge(row), + edgeId: row.id, + sourceFilePath: row.source_file_path, + sourceLanguage: row.source_language, + targetFilePath: row.target_file_path, + })); + } + + /** + * The row ids of the failed references of `filePath` that import one of + * `modules` or are written through one of `names` (see {@link THROUGH_NAMES}). + */ + getFailedReferencesThrough(filePath: string, modules: readonly string[], names: readonly string[]): number[] { + return (this.db + .prepare( + `SELECT id FROM unresolved_refs + WHERE file_path = ? AND status = 'failed' + AND ((reference_kind = 'imports' AND reference_name IN (SELECT value FROM json_each(?))) + OR ${THROUGH_NAMES('reference_name')})` + ) + .all(filePath, JSON.stringify(modules), JSON.stringify(names), JSON.stringify(names)) as Array<{ id: number }>) + .map((row) => row.id); + } + + /** The targets, in other files, of the non-synthesized `imports` edges out of the nodes of `filePath`. */ + getImportTargetsFromFile(filePath: string): string[] { + return (this.db + .prepare( + `SELECT DISTINCT e.target AS target + FROM nodes src + JOIN edges e ON e.source = src.id + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind = 'imports' + AND (e.provenance IS NULL OR e.provenance != 'heuristic') + AND tgt.file_path != ?` + ) + .all(filePath, filePath) as Array<{ target: string }>) + .map((row) => row.target); + } + + /** + * The `imports` edges into the node `targetId` from files other than + * `exceptFile`: each importing file, its language, and the module specifier + * it wrote (the edge's `refName`; null when it carries none). + */ + getImportsInto( + targetId: string, + exceptFile: string + ): Array<{ filePath: string; language: Language; refName: string | null }> { + return (this.db + .prepare( + `SELECT src.file_path AS file_path, src.language AS language, + CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE e.target = ? AND e.kind = 'imports' AND src.file_path != ?` + ) + .all(targetId, exceptFile) as Array<{ file_path: string; language: Language; ref_name: unknown }>) + .map((row) => ({ + filePath: row.file_path, + language: row.language, + refName: typeof row.ref_name === 'string' ? row.ref_name : null, + })); + } + + /** The files with a resolution edge into one of `targetIds`. */ + getFilesWithEdgesInto(targetIds: string[]): string[] { + const files = new Set(); + for (let i = 0; i < targetIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = targetIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT DISTINCT src.file_path AS file_path + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE e.target IN (${placeholders}) + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic')` + ) + .all(...chunk) as Array<{ file_path: string }>; + for (const row of rows) files.add(row.file_path); + } + return [...files]; + } + + /** The import nodes named one of `names` (an import node is named by the module path it imports). */ + getImportNodesByNames(names: string[]): Array<{ filePath: string; name: string; language: Language }> { + const out: Array<{ filePath: string; name: string; language: Language }> = []; + for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT file_path, name, language FROM nodes WHERE name IN (${placeholders}) AND kind = 'import'`) + .all(...chunk) as Array<{ file_path: string; name: string; language: Language }>; + for (const row of rows) out.push({ filePath: row.file_path, name: row.name, language: row.language }); + } + return out; + } + // =========================================================================== // File Operations // =========================================================================== diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 3f6842000..7892b1c8b 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -41,6 +41,7 @@ import { extractAnalogRoutes, isAnalogPage } from '../resolution/frameworks/anal import type { ResolutionContext } from '../resolution/types'; import { createYielder, type MaybeYield } from '../resolution/cooperative-yield'; import { resurrectRefFromDroppedEdge } from '../resolution/resurrect-ref'; +import { createImportReader, reopenReferencesThroughRemovedFiles } from '../resolution/module-importers'; import { MAX_SOURCE_FILE_SIZE_BYTES, oversizeStamp, readBoundedSource, readBoundedSourceSync } from '../file-limits'; export { oversizeStamp }; @@ -3343,6 +3344,8 @@ export class ExtractionOrchestrator { // BEFORE their nodes are replaced/deleted. Compared against the post-store // pairs below to derive `definitionDelta` (CG-33). const pairsBefore = new Set(); + // Importers' import statements, read once per sync for the files it removes. + const importReader = createImportReader(this.rootDir, this.queries); onProgress?.({ phase: 'scanning', @@ -3448,14 +3451,31 @@ export class ExtractionOrchestrator { removedFilePaths.push(tracked.path); filesRemoved++; }; + const removed: FileRecord[] = []; for (const tracked of trackedFiles) { if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) { - removeTracked(tracked); + removed.push(tracked); } if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) { await new Promise((resolve) => setImmediate(resolve)); } } + // The references other files make THROUGH the removed ones to files that + // stay (a package `__init__.py` or an `index.ts` re-exporting what it + // imports) go back to pending too: the cascade keeps their edges, and a + // fresh index of the tree without these files binds them differently or + // not at all. Before any of them is deleted, and for all of them at once. + if (removed.length > 0) { + const tImporters = Date.now(); + const reopened = reopenReferencesThroughRemovedFiles(this.queries, importReader, removed); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-removed-module-importers: ${Date.now() - tImporters}ms (${reopened} refs re-opened)`); + } + for (const tracked of removed) { + removeTracked(tracked); + if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + } // Adds / modifications. for (const filePath of currentFiles) { @@ -3507,7 +3527,10 @@ export class ExtractionOrchestrator { // Not source after all — an MPEG transport stream named `.ts` (#1910): // a new one is ignored, a tracked file that became one is removed. if (content === null) { - if (tracked) removeTracked(tracked); + if (tracked) { + reopenReferencesThroughRemovedFiles(this.queries, importReader, [tracked]); + removeTracked(tracked); + } continue; } const contentHash = hashContent(content); diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index e608303b1..da452446a 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -1152,7 +1152,7 @@ function projectFolderSuffixes(context: ResolutionContext): Set { * from `app/sub/views.py`, `.models` → 'app/sub/models', `..` → 'app' and * `..forms` → 'app/forms'. Null when its dots climb above the project. */ -function pythonModulePath(source: string, fromFile: string): string | null { +export function pythonModulePath(source: string, fromFile: string): string | null { const name = source.replace(/^\.+/, ''); const dots = source.length - name.length; if (dots === 0) return name.replace(/\./g, '/'); diff --git a/src/resolution/module-importers.ts b/src/resolution/module-importers.ts new file mode 100644 index 000000000..b2d9aa07a --- /dev/null +++ b/src/resolution/module-importers.ts @@ -0,0 +1,370 @@ +/** + * The references an import statement makes through a module that appears or + * goes away. + * + * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`, + * which re-exports `Order`; `import { Button } from './components'` reaches + * `components/Button.ts` through `components/index.ts`. When the re-exporting + * file goes away, their edges point into files that stay, so the cascade keeps + * them; when it comes back, the edges a name lookup made in its absence stay. + * Nothing else revisits either: the importers did not change. A fresh index + * binds them through the module when it is there, and another way (by name) + * or not at all when it is not. + * + * A Python module that re-exports another one (`from .shop import *`, or a + * package's `__init__.py` importing names from it) is the way in for its own + * importers, so theirs are followed too, as deep as the resolver chases a + * name through such modules. An ES module's `export … from` leaves no import + * edge or node to find it by, so an `index.ts` re-exporting another is not + * followed. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { QueryBuilder } from '../db/queries'; +import type { Language, UnresolvedReference } from '../types'; +import type { ImportMapping } from './types'; +import { importMappingsFromBindings, pythonModulePath } from './import-resolver'; +import { resurrectRefFromDroppedEdge } from './resurrect-ref'; + +/** + * Languages whose import bindings name what their references are written as: + * Python's `from x import Y` and the ES-module family's `import { Y } from`. + * An `#include` or a Go package import binds no reference name of its own. + */ +const BINDING_LANGUAGES = new Set([ + 'python', 'typescript', 'tsx', 'javascript', 'jsx', 'arkts', 'vue', 'svelte', 'astro', +]); + +/** + * How many re-exporting Python modules deep the importers of a file are + * followed: as deep as the resolver chases a name through star imports and + * re-exports (`pythonModuleSymbol`). + */ +const PYTHON_REEXPORT_DEPTH = 3; + +/** A Python star import, read the way the resolver reads one. */ +const PYTHON_STAR_IMPORT = /^\s*from\s+([\w.]+)\s+import\s+\*/gm; + +/** + * A file this sync adds or removes, or a module re-exporting from one, with + * the names it passes on from them (null: whatever it has). + */ +interface Module { + path: string; + language: Language; + passes: Set | null; +} + +/** + * Each importer's language and, per module specifier of a statement naming + * one of the modules, which of its bindings go through it: all of them + * (null), or those importing one of these names — what a re-exporting module + * passes on, or the module the package around a file imports as a name + * (`from django.db import models`, whose `models` is the file's module). + */ +type Importers = Map | null> }>; + +/** A statement naming a module: whole, or as `onlyName` of the package around it. */ +type AddImporter = (importer: string, language: Language, source: string, module: Module, onlyName?: string) => void; + +/** What a file's import statements say, each file read once per sync. */ +export interface ImportReader { + /** Its import mappings. */ + mappings(file: string, language: Language): ImportMapping[]; + /** The modules a Python file star-imports, which have no import mapping. */ + starImports(file: string, language: Language): string[]; +} + +export function createImportReader(rootDir: string, queries: QueryBuilder): ImportReader { + const mappingsMemo = new Map(); + const starsMemo = new Map(); + return { + // The file's `import` binding rows, which the resolver reads its imports + // from (`extractImportMappings` reads no source text). A star import has + // no local name to bind, so it leaves none. + mappings: (file) => { + let mappings = mappingsMemo.get(file); + if (!mappings) { + mappings = (importMappingsFromBindings(queries.getBindingsByFile(file)) ?? []).filter((m) => m.localName !== '*'); + mappingsMemo.set(file, mappings); + } + return mappings; + }, + starImports: (file, language) => { + if (language !== 'python') return []; + let stars = starsMemo.get(file); + if (!stars) { + let content = ''; + try { + content = fs.readFileSync(path.join(rootDir, file), 'utf8'); + } catch { + // Gone or unreadable: it imports nothing. + } + stars = [...content.matchAll(PYTHON_STAR_IMPORT)].map((m) => m[1]!); + starsMemo.set(file, stars); + } + return stars; + }, + }; +} + +/** + * Re-open, as pending references for the sync's sweep, the references the + * files importing any of `removed` make through those imports. The sweep binds + * them against the post-sync graph, or parks them where a sync that brings a + * file back finds them. Called before the files' rows are deleted, for all of + * them at once, so an importer of several is visited once; an edge into a + * removed file is left to the resurrection its removal does. Returns the + * number of references re-opened. + */ +export function reopenReferencesThroughRemovedFiles( + queries: QueryBuilder, + reader: ImportReader, + removed: ReadonlyArray<{ path: string; language: Language }>, +): number { + const gone = new Set(removed.map((file) => file.path)); + const bindings = bindingsOf(importersOf(queries, removed, reader), reader); + return reopenEdges(queries, bindings, (e) => !gone.has(e.targetFilePath)); +} + +/** + * The other half: after a sync's retries, re-open the references the files + * importing any of `added` make through those imports — the edges a name + * lookup made while a module was missing (only those up to + * `resolvedUpToEdgeId`: later ones were just resolved with the modules there) + * and the references that failed, the import of the module itself among + * them. (The failed-import retry looks those up by the module's name, and + * skips a name as common as `models` or `test`.) Returns the number re-opened. + */ +export function reopenReferencesThroughAddedFiles( + queries: QueryBuilder, + reader: ImportReader, + added: ReadonlyArray<{ path: string; language: Language }>, + resolvedUpToEdgeId: number, +): number { + const bindings = bindingsOf(importersOf(queries, added, reader), reader); + const failed = [...bindings].flatMap(([file, { modules, names }]) => queries.getFailedReferencesThrough(file, modules, names)); + return reopenEdges(queries, bindings, (e) => e.edgeId <= resolvedUpToEdgeId) + queries.reopenFailedReferences(failed); +} + +/** + * The files with an import statement naming one of `files`, or one of the + * modules re-exporting from them, other than `files` themselves (going away, + * or just resolved with the others in place). + */ +function importersOf( + queries: QueryBuilder, + files: ReadonlyArray<{ path: string; language: Language }>, + reader: ImportReader, +): Importers { + const importers: Importers = new Map(); + const changed = new Set(files.map((file) => file.path)); + const seen = new Set(changed); + let level: Module[] = files.filter((file) => BINDING_LANGUAGES.has(file.language)).map((file) => ({ ...file, passes: null })); + for (let depth = 1; level.length > 0; depth++) { + // Per importer found here, its statements naming each module of the level. + const naming = new Map }>(); + const add: AddImporter = (importer, language, source, module, onlyName) => { + if (!BINDING_LANGUAGES.has(language) || changed.has(importer)) return; + const only = onlyName !== undefined ? new Set([onlyName]) : module.passes; + const entry = importers.get(importer) ?? { language, sources: new Map | null>() }; + const names = entry.sources.get(source); + entry.sources.set(source, names === null || only === null ? null : new Set([...(names ?? []), ...only])); + importers.set(importer, entry); + const found = naming.get(importer) ?? { language, statements: [] }; + found.statements.push({ source, module, onlyName }); + naming.set(importer, found); + }; + for (const module of level) { + // A module-path import that resolved to the module: an ES module + // specifier, a dotted Python import. + for (const i of queries.getImportsInto(`file:${module.path}`, module.path)) { + if (i.refName !== null) add(i.filePath, i.language, i.refName, module); + } + } + const python = level.filter((module) => module.language === 'python'); + if (python.length > 0) addPythonImporters(queries, python, reader, add); + // The next level: the Python importers that pass on what they import from these. + if (depth > PYTHON_REEXPORT_DEPTH) break; + const next: Module[] = []; + for (const [importer, { language, statements }] of naming) { + if (language !== 'python' || seen.has(importer)) continue; + const passes = pythonPassedOn(importer, statements, reader); + if (passes === undefined) continue; + seen.add(importer); + next.push({ path: importer, language, passes }); + } + level = next; + } + return importers; +} + +/** + * What the Python module `importer` passes on of what its `statements` + * import, or undefined when the resolver reads no name through it: what it + * star-imports, and for a package's `__init__.py`, the names it imports too. + */ +function pythonPassedOn( + importer: string, + statements: ReadonlyArray<{ source: string; module: Module; onlyName?: string }>, + reader: ImportReader, +): Set | null | undefined { + let passes: Set | null | undefined; + const pass = (names: Set | null): void => { + passes = passes === null || names === null ? null : new Set([...(passes ?? []), ...names]); + }; + const stars = new Set(reader.starImports(importer, 'python')); + const isPackage = importer === '__init__.py' || importer.endsWith('/__init__.py'); + for (const { source, module, onlyName } of statements) { + if (onlyName === undefined && stars.has(source)) pass(module.passes); + if (!isPackage) continue; + const names = reader.mappings(importer, 'python') + .filter((m) => m.source === source && (onlyName !== undefined + ? m.exportedName === onlyName + : module.passes === null || module.passes.has(m.exportedName))) + .map((m) => m.localName); + if (names.length > 0) pass(new Set(names)); + } + return passes; +} + +/** + * Python's other ways to name the modules: an import node whose module path + * denotes one, relative to the importer's own package or absolutely (`from . + * import Order`, `from .. import Order`, `from shop import Order`), which + * never resolve to a file, or the package around one with its module among + * the names imported (`from django.db import models`). An import inside a + * function has no import node, so a file that reaches what the modules pass + * on — the targets of their own import edges — counts too when its imports + * name one. + */ +function addPythonImporters( + queries: QueryBuilder, + modules: readonly Module[], + reader: ImportReader, + add: AddImporter, +): void { + // What a module path a statement names stands for: a module whole, or a + // package with the module of `onlyName` among the names imported. + type Naming = { module: Module; onlyName?: string }; + const byPath = new Map(); + const bySuffix = new Map(); + const importNames = new Set(); + const push = (map: Map, key: string, naming: Naming): void => { + const list = map.get(key); + if (list) list.push(naming); + else map.set(key, [naming]); + }; + const index = (modulePath: string, naming: Naming): void => { + push(byPath, modulePath, naming); + const segments = modulePath.split('/'); + for (let i = 0; i < segments.length; i++) push(bySuffix, segments.slice(i).join('/'), naming); + for (const name of pythonImportNamesFor(modulePath)) importNames.add(name); + }; + for (const module of modules) { + const modulePath = pythonModuleOfFile(module.path); + if (!modulePath) continue; + index(modulePath, { module }); + const slash = modulePath.lastIndexOf('/'); + if (slash > 0) index(modulePath.slice(0, slash), { module, onlyName: modulePath.slice(slash + 1) }); + } + if (importNames.size === 0) return; + // A relative import names a module by where the importer sits, an absolute + // one by the tail of its path (the project's import root is not known). + const named = (source: string, importer: string): Naming[] => + source.startsWith('.') + ? byPath.get(pythonModulePath(source, importer)?.replace(/\/+$/, '') ?? '') ?? [] + : bySuffix.get(source.replace(/\./g, '/')) ?? []; + + for (const node of queries.getImportNodesByNames([...importNames])) { + if (node.language !== 'python') continue; + for (const { module, onlyName } of named(node.name, node.filePath)) add(node.filePath, 'python', node.name, module, onlyName); + } + const paths = new Set(modules.map((module) => module.path)); + const passedOnTargets = new Set([...paths].flatMap((filePath) => queries.getImportTargetsFromFile(filePath))); + if (passedOnTargets.size === 0) return; + for (const candidate of queries.getFilesWithEdgesInto([...passedOnTargets])) { + if (!candidate.endsWith('.py') || paths.has(candidate)) continue; + for (const m of reader.mappings(candidate, 'python')) { + for (const { module, onlyName } of named(m.source, candidate)) { + if (onlyName === undefined || onlyName === m.exportedName) add(candidate, 'python', m.source, module, onlyName); + } + } + } +} + +/** + * Per importer, what its statements naming the modules import: the modules + * named whole (not the package around one), and the local names bound. + */ +function bindingsOf(importers: Importers, reader: ImportReader): Map { + const bindings = new Map(); + for (const [file, { language, sources }] of importers) { + const names = new Set( + reader.mappings(file, language) + .filter((m) => { + const only = sources.get(m.source); + return only === null || (only !== undefined && only.has(m.exportedName)); + }) + .map((m) => m.localName) + ); + const modules = [...sources].filter(([, only]) => only === null).map(([source]) => source); + if (modules.length > 0 || names.size > 0) bindings.set(file, { modules, names: [...names] }); + } + return bindings; +} + +/** + * Swap for their references the edges of each importer's references through + * its bound names — the binding itself or a member read through it + * (`Order.objects`, `ns.fn`) — that pass `keep`. Returns how many. + */ +function reopenEdges( + queries: QueryBuilder, + bindings: Map, + keep: (e: { edgeId: number; targetFilePath: string }) => boolean, +): number { + const edgeIds: number[] = []; + const refs: UnresolvedReference[] = []; + for (const [file, { names }] of bindings) { + if (names.length === 0) continue; + for (const e of queries.getResolutionEdgesThrough(file, names)) { + if (!keep(e)) continue; + const ref = resurrectRefFromDroppedEdge(e); + if (!ref) continue; + edgeIds.push(e.edgeId); + refs.push(ref); + } + } + if (refs.length > 0) queries.replaceResolutionEdgesWithUnresolvedRefs(edgeIds, refs); + return refs.length; +} + +/** A Python file's module as a path: `shop/models.py` → 'shop/models', `shop/__init__.py` → 'shop'. Null for a top-level `__init__.py`. */ +function pythonModuleOfFile(filePath: string): string | null { + if (!filePath.endsWith('.py') || filePath === '__init__.py') return null; + if (filePath.endsWith('/__init__.py')) return filePath.slice(0, -'/__init__.py'.length); + return filePath.slice(0, -'.py'.length); +} + +/** Deepest relative import a lookup spells out: `from ........ import x`. */ +const MAX_RELATIVE_DOTS = 8; + +/** + * The import names that can denote the module at `modulePath`: its trailing + * dotted paths (`models`, `shop.models`), each also behind one dot or more + * (`.models`, `..shop.models`), and the dots alone (`.`, `..`), which name a + * package by where the importer sits. + */ +function pythonImportNamesFor(modulePath: string): string[] { + const segments = modulePath.split('/'); + const absolute = segments.map((_, i) => segments.slice(i).join('.')); + const names = new Set(absolute); + for (let dots = 1; dots <= MAX_RELATIVE_DOTS; dots++) { + const prefix = '.'.repeat(dots); + names.add(prefix); + for (const name of absolute) names.add(prefix + name); + } + return [...names]; +}