fix(build): invalidate kernel stamps for untracked source edits

.github/workflows/kernel-stamp.yml

@@ -0,0 +1,56 @@
+name: Kernel source stamps
+on:
+  pull_request:
+    branches: [fork/consolidated]
+    paths:
+      - 'scripts/kernel-stamp.mjs'
+      - 'scripts/ensure-kernel.mjs'
+      - '__tests__/ensure-kernel-stamp.test.ts'
+      - '.github/workflows/kernel-stamp.yml'
+permissions:
+  contents: read
+jobs:
+  stamps:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    steps:
+      - name: Check out the tested commit
+        shell: bash
+        run: |
+          git init .
+          git config core.autocrlf false
+          git remote add origin "https://github.com/$GITHUB_REPOSITORY.git"
+          git fetch --no-tags --depth=1 origin "$GITHUB_SHA"
+          git checkout --detach FETCH_HEAD
+      - name: Use the runner's Node 24 installation
+        shell: pwsh
+        run: |
+          $versions = Get-ChildItem "$env:RUNNER_TOOL_CACHE/node" -Directory
+          $version = $versions | Where-Object Name -like '24.*' | Sort-Object { [version]$_.Name } -Descending | Select-Object -First 1
+          if (!$version) { throw 'Node 24 is missing from the runner tool cache' }
+          $arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant()
+          $nodeDir = Join-Path $version.FullName $arch
+          if (!$IsWindows) { $nodeDir = Join-Path $nodeDir 'bin' }
+          $nodeExe = if ($IsWindows) { 'node.exe' } else { 'node' }
+          if (!(Test-Path (Join-Path $nodeDir $nodeExe))) { throw 'Node 24 executable is missing' }
+          Add-Content $env:GITHUB_PATH $nodeDir
+      - run: node -e "if (process.versions.node.split('.')[0] !== '24') process.exit(1)"
+      - run: npm ci --ignore-scripts
+      - name: Run source stamp fixtures without a native kernel build
+        shell: bash
+        run: |
+          node --input-type=module <<'JS'
+          import { writeFileSync } from 'node:fs';
+          const config = { test: {
+            include: ['__tests__/ensure-kernel-stamp.test.ts'],
+            setupFiles: ['__tests__/file-tmpdir.setup.ts', '__tests__/setup-home-sandbox.ts'],
+            environment: 'node', maxWorkers: 1, minWorkers: 1,
+          }};
+          writeFileSync('.kernel-stamp-config.mjs', `export default ${JSON.stringify(config)};`);
+          writeFileSync('.kernel-stamp-workspace.mjs', `export default [${JSON.stringify(config)}];`);
+          JS
+          npx vitest run --config .kernel-stamp-config.mjs --workspace .kernel-stamp-workspace.mjs

CHANGELOG.md

@@ -87,6 +87,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- Source builds now rebuild the native kernel after edits to new, untracked source files instead of using an older prebuild.
+
 - `codegraph_explore` keeps local callee source ahead of unrelated declarations when a query names functions and pins several files.
 - `codegraph_explore` now returns matching callbacks from pinned JavaScript and TypeScript test files and includes caller-linked test source when asked which tests cover named symbols.
 - The time `codegraph init`, `codegraph index` and `codegraph sync` print now covers the whole run. It used to cover only file parsing, so a slow linking step never showed in the reported time.

__tests__/ensure-kernel-stamp.test.ts

@@ -37,26 +37,26 @@ function ensure(): { status: number | null; out: string } {
   return { status: r.status, out: `${r.stdout}${r.stderr}` };
 }
 
-describe.runIf(process.platform !== 'win32')('ensure-kernel source stamp', () => {
-  beforeEach(() => {
-    root = fs.mkdtempSync(path.join(os.tmpdir(), 'ensure-kernel-'));
-    for (const f of ['ensure-kernel.mjs', 'kernel-stamp.mjs']) {
-      write(`scripts/${f}`, fs.readFileSync(path.join(SCRIPTS, f), 'utf8'));
-    }
-    write('codegraph-kernel/.gitignore', 'prebuilds/\n');
-    write('codegraph-kernel/src/lib.rs', 'pub fn one() -> u32 { 1 }\n');
-    write('fake-bin/cargo', '#!/bin/sh\nexit 1\n');
-    fs.chmodSync(path.join(root, 'fake-bin', 'cargo'), 0o755);
-    git('init', '-q');
-    git('-c', 'user.name=t', '-c', 'user.email=t@t', 'add', '.');
-    git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '-m', 'kernel');
-    write(path.relative(root, prebuild()), 'binary');
-  });
+beforeEach(() => {
+  root = fs.mkdtempSync(path.join(os.tmpdir(), 'ensure-kernel-'));
+  for (const f of ['ensure-kernel.mjs', 'kernel-stamp.mjs']) {
+    write(`scripts/${f}`, fs.readFileSync(path.join(SCRIPTS, f), 'utf8'));
+  }
+  write('codegraph-kernel/.gitignore', 'target/\ntarget-*/\nprebuilds/\n*.node\n');
+  write('codegraph-kernel/src/lib.rs', 'pub fn one() -> u32 { 1 }\n');
+  write('fake-bin/cargo', '#!/bin/sh\nexit 1\n');
+  fs.chmodSync(path.join(root, 'fake-bin', 'cargo'), 0o755);
+  git('init', '-q');
+  git('-c', 'user.name=t', '-c', 'user.email=t@t', 'add', '.');
+  git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '-m', 'kernel');
+  write(path.relative(root, prebuild()), 'binary');
+});
 
-  afterEach(() => {
-    fs.rmSync(root, { recursive: true, force: true });
-  });
+afterEach(() => {
+  fs.rmSync(root, { recursive: true, force: true });
+});
 
+describe.runIf(process.platform !== 'win32')('ensure-kernel source stamp', () => {
   it('accepts a prebuild stamped with the current kernel sources', async () => {
     fs.writeFileSync(`${prebuild()}.stamp`, `${await stamp()}\n`);
     const r = ensure();
@@ -87,4 +87,108 @@ describe.runIf(process.platform !== 'win32')('ensure-kernel source stamp', () =>
     expect(await stamp()).not.toBe(clean);
     expect(await stamp()).not.toBe(dirty);
   });
+
+  it('flags a prebuild after an untracked source changes at the same path', async () => {
+    write('codegraph-kernel/src/pending.rs', 'pub fn pending() -> u32 { 1 }\n');
+    fs.writeFileSync(`${prebuild()}.stamp`, `${await stamp()}\n`);
+    write('codegraph-kernel/src/pending.rs', 'pub fn pending() -> u32 { 2 }\n');
+    const r = ensure();
+    expect(r.status).toBe(0);
+    expect(r.out).toContain('was not built from this checkout');
+  });
+
+  it('fails visibly when Git cannot enumerate untracked inputs', async () => {
+    fs.writeFileSync(`${prebuild()}.stamp`, `${await stamp()}\n`);
+    const realGit = spawnSync('which', ['git'], { encoding: 'utf8' });
+    expect(realGit.status).toBe(0);
+    write('fake-bin/git', `#!/bin/sh\nif [ "$1" = ls-files ]; then exit 1; fi\nexec "${realGit.stdout.trim()}" "$@"\n`);
+    fs.chmodSync(path.join(root, 'fake-bin/git'), 0o755);
+    const r = ensure();
+    expect(r.status).not.toBe(0);
+    expect(r.out).toContain('Cannot enumerate untracked kernel sources');
+  });
+});
+
+describe('kernel source stamp inputs', () => {
+  it.each([false, true])('includes nested untracked bytes and additions with hidden status=%s', async (hidden) => {
+    if (hidden) git('config', 'status.showUntrackedFiles', 'no');
+    const clean = await stamp();
+    write('codegraph-kernel/src/pending/one.rs', 'one');
+    const added = await stamp();
+    expect(added).not.toBe(clean);
+    expect(await stamp()).toBe(added);
+    write('codegraph-kernel/src/pending/one.rs', 'two');
+    const edited = await stamp();
+    expect(edited).not.toBe(added);
+    write('codegraph-kernel/src/pending/two.rs', 'two');
+    expect(await stamp()).not.toBe(edited);
+  });
+
+  it('preserves spaces and non-ASCII untracked filenames', async () => {
+    write('codegraph-kernel/src/pending space é.rs', 'one');
+    const first = await stamp();
+    write('codegraph-kernel/src/pending space é.rs', 'two');
+    expect(await stamp()).not.toBe(first);
+    const edited = await stamp();
+    fs.renameSync(path.join(root, 'codegraph-kernel/src/pending space é.rs'), path.join(root, 'codegraph-kernel/src/renamed é.rs'));
+    const renamed = await stamp();
+    expect(renamed).not.toBe(edited);
+    expect(await stamp()).toBe(renamed);
+  });
+
+  it.runIf(process.platform !== 'win32')('preserves newlines in untracked filenames', async () => {
+    write('codegraph-kernel/src/pending\nfile.rs', 'one');
+    const first = await stamp();
+    write('codegraph-kernel/src/pending\nfile.rs', 'two');
+    expect(await stamp()).not.toBe(first);
+  });
+
+  it('excludes ignored build outputs and files outside the kernel', async () => {
+    write('codegraph-kernel/src/pending.rs', 'one');
+    const first = await stamp();
+    for (const rel of ['target/cache', 'target-debug/cache', 'prebuilds/cache', 'native.node']) {
+      write(`codegraph-kernel/${rel}`, 'output');
+    }
+    write('outside.rs', 'outside');
+    expect(await stamp()).toBe(first);
+  });
+
+  it.runIf(process.platform !== 'win32')('hashes symlink text without following its target', async () => {
+    write('outside.rs', 'one');
+    const link = path.join(root, 'codegraph-kernel/src/pending.rs');
+    fs.symlinkSync('../../outside.rs', link);
+    const first = await stamp();
+    write('outside.rs', 'two');
+    expect(await stamp()).toBe(first);
+    fs.unlinkSync(link);
+    fs.symlinkSync('../../missing.rs', link);
+    expect(await stamp()).not.toBe(first);
+  });
+
+  it.runIf(process.platform !== 'win32')('rejects an unreadable untracked input', async () => {
+    write('codegraph-kernel/src/pending.rs', 'one');
+    const file = path.join(root, 'codegraph-kernel/src/pending.rs');
+    fs.chmodSync(file, 0o000);
+    try {
+      await expect(stamp()).rejects.toThrow(/EACCES/);
+    } finally {
+      fs.chmodSync(file, 0o644);
+    }
+  });
+
+  it.runIf(process.platform !== 'win32')('rejects an unreadable untracked directory even when Git exits zero', async () => {
+    write('codegraph-kernel/src/pending/one.rs', 'one');
+    const dir = path.join(root, 'codegraph-kernel/src/pending');
+    fs.chmodSync(dir, 0o000);
+    try {
+      await expect(stamp()).rejects.toThrow('Cannot enumerate untracked kernel sources');
+    } finally {
+      fs.chmodSync(dir, 0o755);
+    }
+  });
+
+  it('keeps the no-Git stamp absent', async () => {
+    fs.rmSync(path.join(root, '.git'), { recursive: true });
+    expect(await stamp()).toBeNull();
+  });
 });

scripts/kernel-stamp.mjs

@@ -12,24 +12,45 @@
  */
 import { createHash } from 'node:crypto';
 import { spawnSync } from 'node:child_process';
+import { lstatSync, readFileSync, readlinkSync } from 'node:fs';
+import { sep } from 'node:path';
 
-function git(root, args) {
+function git(root, args, rejectStderr = false) {
   const r = spawnSync('git', args, { cwd: root, encoding: 'buffer', maxBuffer: 256 * 1024 * 1024 });
-  return r.status === 0 ? r.stdout : null;
+  return r.status === 0 && (!rejectStderr || r.stderr.length === 0) ? r.stdout : null;
 }
 
-/** The stamp for the kernel sources under `root`, or null when git cannot tell. */
+/** The kernel source stamp, or null outside a committed Git checkout. */
 export function kernelSourceStamp(root) {
   const tree = git(root, ['rev-parse', 'HEAD:codegraph-kernel']);
   if (!tree) return null;
   const stamp = tree.toString().trim();
-  // Uncommitted edits: the diff against HEAD, plus the names of untracked files.
+  // Git can exit zero after skipping unreadable directories; reject its warnings.
+  const untracked = git(root, ['ls-files', '--others', '--exclude-standard', '-z', '--', 'codegraph-kernel'], true);
+  if (untracked === null) throw new Error('Cannot enumerate untracked kernel sources');
   const status = git(root, ['status', '--porcelain', '--', 'codegraph-kernel']);
-  if (status === null) return null;
-  if (status.length === 0) return stamp;
+  if (status === null) throw new Error('Cannot read kernel source status');
+  if (status.length === 0 && untracked.length === 0) return stamp;
   const diff = git(root, ['diff', 'HEAD', '--binary', '--', 'codegraph-kernel']);
-  if (diff === null) return null;
-  return `${stamp}+${createHash('sha256').update(status).update(diff).digest('hex').slice(0, 16)}`;
+  if (diff === null) throw new Error('Cannot read kernel source diff');
+  const hash = createHash('sha256').update(status).update(diff);
+  // Keep Git's raw, NUL-separated paths, including newlines and non-ASCII bytes.
+  const paths = [];
+  for (let start = 0, end; (end = untracked.indexOf(0, start)) !== -1; start = end + 1) {
+    paths.push(untracked.subarray(start, end));
+  }
+  paths.sort(Buffer.compare);
+  for (const name of paths) {
+    const file = Buffer.concat([Buffer.from(`${root}${sep}`), name]);
+    const stat = lstatSync(file);
+    const link = stat.isSymbolicLink();
+    if (!link && !stat.isFile()) throw new Error(`Unsupported kernel source input: ${name}`);
+    // Git stores link text, not external target bytes; dangling links are valid.
+    const content = link ? readlinkSync(file, { encoding: 'buffer' }) : readFileSync(file);
+    hash.update('\0').update(name).update('\0').update(link ? 'l' : 'f');
+    hash.update(createHash('sha256').update(content).digest());
+  }
+  return `${stamp}+${hash.digest('hex').slice(0, 16)}`;
 }
 
 /** Where the stamp for a staged prebuild lives. */