From d30c40251d9bc0dabd1ee5365e864c239e3e2b1c Mon Sep 17 00:00:00 2001 From: Aaron Queen Date: Mon, 5 Oct 2026 02:38:48 -0600 Subject: [PATCH 1/2] fix(build): invalidate kernel stamps for untracked source edits --- .github/workflows/kernel-stamp.yml | 42 ++++++++ CHANGELOG.md | 2 + __tests__/ensure-kernel-stamp.test.ts | 140 ++++++++++++++++++++++---- scripts/kernel-stamp.mjs | 37 +++++-- 4 files changed, 195 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/kernel-stamp.yml diff --git a/.github/workflows/kernel-stamp.yml b/.github/workflows/kernel-stamp.yml new file mode 100644 index 0000000000..583393cf53 --- /dev/null +++ b/.github/workflows/kernel-stamp.yml @@ -0,0 +1,42 @@ +name: Kernel source stamps +on: + pull_request: + branches: [fork/consolidated] + paths: + - 'scripts/kernel-stamp.mjs' + - 'scripts/ensure-kernel.mjs' + - '__tests__/ensure-kernel-stamp.test.ts' + - '.github/workflows/kernel-stamp.yml' +permissions: + contents: read +jobs: + stamps: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@v7.0.0 + with: + node-version: 24 + cache: npm + - run: npm ci --ignore-scripts + - name: Run source stamp fixtures without a native kernel build + shell: bash + run: | + node --input-type=module <<'JS' + import { writeFileSync } from 'node:fs'; + const config = { test: { + include: ['__tests__/ensure-kernel-stamp.test.ts'], + setupFiles: ['__tests__/file-tmpdir.setup.ts', '__tests__/setup-home-sandbox.ts'], + environment: 'node', maxWorkers: 1, minWorkers: 1, + }}; + writeFileSync('.kernel-stamp-config.mjs', `export default ${JSON.stringify(config)};`); + writeFileSync('.kernel-stamp-workspace.mjs', `export default [${JSON.stringify(config)}];`); + JS + npx vitest run --config .kernel-stamp-config.mjs --workspace .kernel-stamp-workspace.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index cd86f37161..b57b6c2b24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- Source builds now rebuild the native kernel after edits to new, untracked source files instead of using an older prebuild. + - `codegraph_explore` keeps local callee source ahead of unrelated declarations when a query names functions and pins several files. - `codegraph_explore` now returns matching callbacks from pinned JavaScript and TypeScript test files and includes caller-linked test source when asked which tests cover named symbols. - The time `codegraph init`, `codegraph index` and `codegraph sync` print now covers the whole run. It used to cover only file parsing, so a slow linking step never showed in the reported time. diff --git a/__tests__/ensure-kernel-stamp.test.ts b/__tests__/ensure-kernel-stamp.test.ts index c8861723cf..e9b153e7fd 100644 --- a/__tests__/ensure-kernel-stamp.test.ts +++ b/__tests__/ensure-kernel-stamp.test.ts @@ -37,26 +37,26 @@ function ensure(): { status: number | null; out: string } { return { status: r.status, out: `${r.stdout}${r.stderr}` }; } -describe.runIf(process.platform !== 'win32')('ensure-kernel source stamp', () => { - beforeEach(() => { - root = fs.mkdtempSync(path.join(os.tmpdir(), 'ensure-kernel-')); - for (const f of ['ensure-kernel.mjs', 'kernel-stamp.mjs']) { - write(`scripts/${f}`, fs.readFileSync(path.join(SCRIPTS, f), 'utf8')); - } - write('codegraph-kernel/.gitignore', 'prebuilds/\n'); - write('codegraph-kernel/src/lib.rs', 'pub fn one() -> u32 { 1 }\n'); - write('fake-bin/cargo', '#!/bin/sh\nexit 1\n'); - fs.chmodSync(path.join(root, 'fake-bin', 'cargo'), 0o755); - git('init', '-q'); - git('-c', 'user.name=t', '-c', 'user.email=t@t', 'add', '.'); - git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '-m', 'kernel'); - write(path.relative(root, prebuild()), 'binary'); - }); +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'ensure-kernel-')); + for (const f of ['ensure-kernel.mjs', 'kernel-stamp.mjs']) { + write(`scripts/${f}`, fs.readFileSync(path.join(SCRIPTS, f), 'utf8')); + } + write('codegraph-kernel/.gitignore', 'target/\ntarget-*/\nprebuilds/\n*.node\n'); + write('codegraph-kernel/src/lib.rs', 'pub fn one() -> u32 { 1 }\n'); + write('fake-bin/cargo', '#!/bin/sh\nexit 1\n'); + fs.chmodSync(path.join(root, 'fake-bin', 'cargo'), 0o755); + git('init', '-q'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'add', '.'); + git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '-m', 'kernel'); + write(path.relative(root, prebuild()), 'binary'); +}); - afterEach(() => { - fs.rmSync(root, { recursive: true, force: true }); - }); +afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); +}); +describe.runIf(process.platform !== 'win32')('ensure-kernel source stamp', () => { it('accepts a prebuild stamped with the current kernel sources', async () => { fs.writeFileSync(`${prebuild()}.stamp`, `${await stamp()}\n`); const r = ensure(); @@ -87,4 +87,108 @@ describe.runIf(process.platform !== 'win32')('ensure-kernel source stamp', () => expect(await stamp()).not.toBe(clean); expect(await stamp()).not.toBe(dirty); }); + + it('flags a prebuild after an untracked source changes at the same path', async () => { + write('codegraph-kernel/src/pending.rs', 'pub fn pending() -> u32 { 1 }\n'); + fs.writeFileSync(`${prebuild()}.stamp`, `${await stamp()}\n`); + write('codegraph-kernel/src/pending.rs', 'pub fn pending() -> u32 { 2 }\n'); + const r = ensure(); + expect(r.status).toBe(0); + expect(r.out).toContain('was not built from this checkout'); + }); + + it('fails visibly when Git cannot enumerate untracked inputs', async () => { + fs.writeFileSync(`${prebuild()}.stamp`, `${await stamp()}\n`); + const realGit = spawnSync('which', ['git'], { encoding: 'utf8' }); + expect(realGit.status).toBe(0); + write('fake-bin/git', `#!/bin/sh\nif [ "$1" = ls-files ]; then exit 1; fi\nexec "${realGit.stdout.trim()}" "$@"\n`); + fs.chmodSync(path.join(root, 'fake-bin/git'), 0o755); + const r = ensure(); + expect(r.status).not.toBe(0); + expect(r.out).toContain('Cannot enumerate untracked kernel sources'); + }); +}); + +describe('kernel source stamp inputs', () => { + it.each([false, true])('includes nested untracked bytes and additions with hidden status=%s', async (hidden) => { + if (hidden) git('config', 'status.showUntrackedFiles', 'no'); + const clean = await stamp(); + write('codegraph-kernel/src/pending/one.rs', 'one'); + const added = await stamp(); + expect(added).not.toBe(clean); + expect(await stamp()).toBe(added); + write('codegraph-kernel/src/pending/one.rs', 'two'); + const edited = await stamp(); + expect(edited).not.toBe(added); + write('codegraph-kernel/src/pending/two.rs', 'two'); + expect(await stamp()).not.toBe(edited); + }); + + it('preserves spaces and non-ASCII untracked filenames', async () => { + write('codegraph-kernel/src/pending space é.rs', 'one'); + const first = await stamp(); + write('codegraph-kernel/src/pending space é.rs', 'two'); + expect(await stamp()).not.toBe(first); + const edited = await stamp(); + fs.renameSync(path.join(root, 'codegraph-kernel/src/pending space é.rs'), path.join(root, 'codegraph-kernel/src/renamed é.rs')); + const renamed = await stamp(); + expect(renamed).not.toBe(edited); + expect(await stamp()).toBe(renamed); + }); + + it.runIf(process.platform !== 'win32')('preserves newlines in untracked filenames', async () => { + write('codegraph-kernel/src/pending\nfile.rs', 'one'); + const first = await stamp(); + write('codegraph-kernel/src/pending\nfile.rs', 'two'); + expect(await stamp()).not.toBe(first); + }); + + it('excludes ignored build outputs and files outside the kernel', async () => { + write('codegraph-kernel/src/pending.rs', 'one'); + const first = await stamp(); + for (const rel of ['target/cache', 'target-debug/cache', 'prebuilds/cache', 'native.node']) { + write(`codegraph-kernel/${rel}`, 'output'); + } + write('outside.rs', 'outside'); + expect(await stamp()).toBe(first); + }); + + it.runIf(process.platform !== 'win32')('hashes symlink text without following its target', async () => { + write('outside.rs', 'one'); + const link = path.join(root, 'codegraph-kernel/src/pending.rs'); + fs.symlinkSync('../../outside.rs', link); + const first = await stamp(); + write('outside.rs', 'two'); + expect(await stamp()).toBe(first); + fs.unlinkSync(link); + fs.symlinkSync('../../missing.rs', link); + expect(await stamp()).not.toBe(first); + }); + + it.runIf(process.platform !== 'win32')('rejects an unreadable untracked input', async () => { + write('codegraph-kernel/src/pending.rs', 'one'); + const file = path.join(root, 'codegraph-kernel/src/pending.rs'); + fs.chmodSync(file, 0o000); + try { + await expect(stamp()).rejects.toThrow(/EACCES/); + } finally { + fs.chmodSync(file, 0o644); + } + }); + + it.runIf(process.platform !== 'win32')('rejects an unreadable untracked directory even when Git exits zero', async () => { + write('codegraph-kernel/src/pending/one.rs', 'one'); + const dir = path.join(root, 'codegraph-kernel/src/pending'); + fs.chmodSync(dir, 0o000); + try { + await expect(stamp()).rejects.toThrow('Cannot enumerate untracked kernel sources'); + } finally { + fs.chmodSync(dir, 0o755); + } + }); + + it('keeps the no-Git stamp absent', async () => { + fs.rmSync(path.join(root, '.git'), { recursive: true }); + expect(await stamp()).toBeNull(); + }); }); diff --git a/scripts/kernel-stamp.mjs b/scripts/kernel-stamp.mjs index 7db5a7e4ec..4281b5d041 100644 --- a/scripts/kernel-stamp.mjs +++ b/scripts/kernel-stamp.mjs @@ -12,24 +12,45 @@ */ import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; +import { lstatSync, readFileSync, readlinkSync } from 'node:fs'; +import { sep } from 'node:path'; -function git(root, args) { +function git(root, args, rejectStderr = false) { const r = spawnSync('git', args, { cwd: root, encoding: 'buffer', maxBuffer: 256 * 1024 * 1024 }); - return r.status === 0 ? r.stdout : null; + return r.status === 0 && (!rejectStderr || r.stderr.length === 0) ? r.stdout : null; } -/** The stamp for the kernel sources under `root`, or null when git cannot tell. */ +/** The kernel source stamp, or null outside a committed Git checkout. */ export function kernelSourceStamp(root) { const tree = git(root, ['rev-parse', 'HEAD:codegraph-kernel']); if (!tree) return null; const stamp = tree.toString().trim(); - // Uncommitted edits: the diff against HEAD, plus the names of untracked files. + // Git can exit zero after skipping unreadable directories; reject its warnings. + const untracked = git(root, ['ls-files', '--others', '--exclude-standard', '-z', '--', 'codegraph-kernel'], true); + if (untracked === null) throw new Error('Cannot enumerate untracked kernel sources'); const status = git(root, ['status', '--porcelain', '--', 'codegraph-kernel']); - if (status === null) return null; - if (status.length === 0) return stamp; + if (status === null) throw new Error('Cannot read kernel source status'); + if (status.length === 0 && untracked.length === 0) return stamp; const diff = git(root, ['diff', 'HEAD', '--binary', '--', 'codegraph-kernel']); - if (diff === null) return null; - return `${stamp}+${createHash('sha256').update(status).update(diff).digest('hex').slice(0, 16)}`; + if (diff === null) throw new Error('Cannot read kernel source diff'); + const hash = createHash('sha256').update(status).update(diff); + // Keep Git's raw, NUL-separated paths, including newlines and non-ASCII bytes. + const paths = []; + for (let start = 0, end; (end = untracked.indexOf(0, start)) !== -1; start = end + 1) { + paths.push(untracked.subarray(start, end)); + } + paths.sort(Buffer.compare); + for (const name of paths) { + const file = Buffer.concat([Buffer.from(`${root}${sep}`), name]); + const stat = lstatSync(file); + const link = stat.isSymbolicLink(); + if (!link && !stat.isFile()) throw new Error(`Unsupported kernel source input: ${name}`); + // Git stores link text, not external target bytes; dangling links are valid. + const content = link ? readlinkSync(file, { encoding: 'buffer' }) : readFileSync(file); + hash.update('\0').update(name).update('\0').update(link ? 'l' : 'f'); + hash.update(createHash('sha256').update(content).digest()); + } + return `${stamp}+${hash.digest('hex').slice(0, 16)}`; } /** Where the stamp for a staged prebuild lives. */ From a8095d5648de63b6063059574e9574fed214b8d4 Mon Sep 17 00:00:00 2001 From: Aaron Queen Date: Mon, 5 Oct 2026 02:47:29 -0600 Subject: [PATCH 2/2] fix(ci): reuse local checkout and Node setup for stamp checks --- .github/workflows/kernel-stamp.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/kernel-stamp.yml b/.github/workflows/kernel-stamp.yml index 583393cf53..6a9552928d 100644 --- a/.github/workflows/kernel-stamp.yml +++ b/.github/workflows/kernel-stamp.yml @@ -18,13 +18,27 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 10 steps: - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: actions/setup-node@v7.0.0 - with: - node-version: 24 - cache: npm + - name: Check out the tested commit + shell: bash + run: | + git init . + git config core.autocrlf false + git remote add origin "https://github.com/$GITHUB_REPOSITORY.git" + git fetch --no-tags --depth=1 origin "$GITHUB_SHA" + git checkout --detach FETCH_HEAD + - name: Use the runner's Node 24 installation + shell: pwsh + run: | + $versions = Get-ChildItem "$env:RUNNER_TOOL_CACHE/node" -Directory + $version = $versions | Where-Object Name -like '24.*' | Sort-Object { [version]$_.Name } -Descending | Select-Object -First 1 + if (!$version) { throw 'Node 24 is missing from the runner tool cache' } + $arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant() + $nodeDir = Join-Path $version.FullName $arch + if (!$IsWindows) { $nodeDir = Join-Path $nodeDir 'bin' } + $nodeExe = if ($IsWindows) { 'node.exe' } else { 'node' } + if (!(Test-Path (Join-Path $nodeDir $nodeExe))) { throw 'Node 24 executable is missing' } + Add-Content $env:GITHUB_PATH $nodeDir + - run: node -e "if (process.versions.node.split('.')[0] !== '24') process.exit(1)" - run: npm ci --ignore-scripts - name: Run source stamp fixtures without a native kernel build shell: bash