diff --git a/bench/browser/micro.ts b/bench/browser/micro.ts index 8694098287c..cb5c6b4edd2 100644 --- a/bench/browser/micro.ts +++ b/bench/browser/micro.ts @@ -15,16 +15,17 @@ const { values, positionals } = parseArgs({ }, }); const benchmark = values.bench; -// The rows each benchmark prints per run. -const ROWS: Record = { - frames: 26, - drift: 36, - ledger: 12, - bandwidth: 44, - replay: 72, - "resume-history": 72, +// Each entry is named in full, so sh/gh/test-inventory.py sees this harness run it. +const benchmarks: Record = { + frames: { entry: "js/net/bench/frames.ts", rows: 26 }, + drift: { entry: "js/net/bench/drift.ts", rows: 36 }, + ledger: { entry: "js/net/bench/ledger.ts", rows: 12 }, + bandwidth: { entry: "js/net/bench/bandwidth.ts", rows: 44 }, + replay: { entry: "js/net/bench/replay.ts", rows: 72 }, + "resume-history": { entry: "js/net/bench/resume-history.ts", rows: 72 }, }; -if (!Object.hasOwn(ROWS, benchmark)) throw new Error(`--bench must be one of ${Object.keys(ROWS).join(", ")}`); +const selected = Object.hasOwn(benchmarks, benchmark) ? benchmarks[benchmark] : undefined; +if (!selected) throw new Error(`--bench must be one of: ${Object.keys(benchmarks).join(", ")}`); const rounds = Number(values.rounds); if (!Number.isSafeInteger(rounds) || rounds < 1) throw new Error("--rounds must be a positive integer"); const roots = new Map(); @@ -36,10 +37,8 @@ for (const argument of positionals.length ? positionals : ["current=.", "aa=."]) } const scripts = new Map(); for (const [name, root] of roots) { - const entry = - benchmark === "frames" - ? resolve(root, "js/net/bench/frames.ts") - : resolve(import.meta.dir, `../../js/net/bench/${benchmark}.ts`); + // Only frames is bundled from each variant; the rest import each variant's `../src/` below. + const entry = resolve(benchmark === "frames" ? root : resolve(import.meta.dir, "../.."), selected.entry); const result = await Bun.build({ entrypoints: [entry], target: "browser", @@ -144,7 +143,7 @@ try { rows.set(variant, []); await pages.get(variant)?.evaluate("globalThis.measure()"); const measured = rows.get(variant) ?? []; - if (measured.length !== ROWS[benchmark]) throw new Error(`incomplete rows: ${variant}`); + if (measured.length !== selected.rows) throw new Error(`incomplete rows: ${variant}`); return measured; }; for (let round = 0; round < rounds; round++) { diff --git a/js/net/bench/resume-history.ts b/js/net/bench/resume-history.ts index e154687603a..10097dc44ca 100644 --- a/js/net/bench/resume-history.ts +++ b/js/net/bench/resume-history.ts @@ -38,7 +38,7 @@ type Access = "lookup" | "fetch" | "subscribe"; /** Full collections with no stack heap pointers, so dead stack slots cannot pin a target. */ async function collect(): Promise { const gc = (globalThis as { gc?: (options: { type: "major"; execution: "async" }) => Promise }).gc; - if (!gc) throw new Error("resume-history needs a precise collector: run it with bench/browser/micro.ts"); + if (!gc) throw new Error("resume-history needs a precise collector: run it in Chromium through bench/browser"); for (let pass = 0; pass < 3; pass++) { // V8's synchronous gc scans the embedder stack conservatively even from a fresh task. // Async gc runs with StackState::kNoHeapPointers, including while tracing is enabled. diff --git a/quest/m2/README.md b/quest/m2/README.md index bb6524f6b79..3d85df4f1ec 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -230,7 +230,6 @@ lite-07; bitrate enforcement is not part of this release. - [fMP4 decode times from the declared reorder depth](/quest/m2/fmp4-declared-clock.md) - `DeclaredClock` replaces the sub-GOP cadence cap, as in TS, FLV, and MKV - [An ended HLS playlist has no trailing gap](/quest/m2/hls-trailing-gap.md) - no zero-duration `EXT-X-GAP` before `EXT-X-ENDLIST` - [moq-video's Apple Sink encodes on its own thread](/quest/m2/apple-sink-thread.md) - macOS and iOS ladders never encode on a tokio worker -- [The test inventory sees JS bench scripts](/quest/m2/js-bench-inventory.md) - JS bench entry points are resolved by the inventory gate - [/fetch answers from the announce sync](/quest/m2/fetch-synced.md) - a missing broadcast gets a 404 once announcements are synced, not after 30 s - [moq auth sign mints capped tokens](/quest/m2/auth-sign-caps.md) - CLI flags for the bitrate caps - [QUIC-LB forwarding reads the cluster's peers and reports its counts](/quest/m2/quic-lb-forward-extras.md) - a cluster-API peer map and forward counters diff --git a/quest/m2/js-bench-inventory.md b/quest/m2/js-bench-inventory.md deleted file mode 100644 index 20d80f4ddf0..00000000000 --- a/quest/m2/js-bench-inventory.md +++ /dev/null @@ -1,13 +0,0 @@ -# [XS] The test inventory sees JS bench scripts - -## Goal - -The test-inventory gate resolves the JS bench scripts Nightly runs, so a JS bench entry point no workflow runs is flagged like any other opt-in target. - -## Plan - -Found by the agent on [moq-dev/moq#7316](https://github.com/moq-dev/moq/pull/7316): Nightly runs every JS bench through the `javascript` helper as `"$@"` (`sh/gh/nightly.sh:66-72`, the loop at `:138-155`), which the checker can't resolve, and `test-inventory.py` skips scripts under `/bench/`. Every JS bench entry point has a caller today, so this closes a coverage gap in the checker, not in CI. - -Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). Make the bench list something the checker can read (a static list Nightly loops over, or a recipe per bench) rather than teaching the checker shell indirection, and stop skipping `/bench/`. - -Public API: none. Wire: none. diff --git a/sh/gh/test-inventory.py b/sh/gh/test-inventory.py index ec99fc3f056..c7edff90c56 100644 --- a/sh/gh/test-inventory.py +++ b/sh/gh/test-inventory.py @@ -7,7 +7,9 @@ - a cargo-fuzz target or a Cargo bench target, named by some reachable run; - a `just` recipe that runs tests, reachable from a workflow; - a standalone test script (`*.test.py`, `*.test.sh`, `*.browser.ts`, or a - `*.test.ts` outside a Bun package that runs `bun test`), reachable from one. + `*.test.ts` outside a Bun package that runs `bun test`), reachable from one; +- a JS bench script under a `bench/` directory that no other source imports + or names, so it is an entry point rather than a module, reachable from one. Reachable means some workflow `run:` step leads to it through `just` recipes, their dependencies, and the scripts they name. The walk is static: a `just` @@ -37,6 +39,9 @@ import yaml SCRIPT = re.compile(r"[\w.$/{}-]*[\w-]\.(?:sh|py|ts|mts|js|mjs)\b") +JS = (".ts", ".mts", ".js", ".mjs") +# A relative module specifier in a static or dynamic import, or a re-export. +IMPORT = re.compile(r"""\b(?:from|import)\s*\(?\s*["'](\.\.?/[^"']+)["']""") # Commands that run a test suite. Python sources spell them as argument lists, # so quotes and commas are dropped before matching. RUNNER = re.compile( @@ -936,10 +941,11 @@ def recipe_line(root: Path, recipe: Recipe) -> str: return str(source.relative_to(root)) -def runs_tests(graph: Graph, recipe: Recipe, found: set[Path]) -> bool: +def runs_tests(graph: Graph, recipe: Recipe, found: set[Path], benches: set[Path]) -> bool: """A recipe runs tests when its body runs a suite, or names a script no workflow reaches - that runs one, is a test script, or drives a browser. A wrapper around a reached script, - or a bare `bun test` in a directory a reached run already tests, adds no test of its own.""" + that runs one, is a test script or a bench entry point, or drives a browser. A wrapper + around a reached script, or a bare `bun test` in a directory a reached run already tests, + adds no test of its own.""" text = recipe.text if any(recipe.cwd.is_relative_to(directory) for directory in found): text = re.sub(r"(?m)^\s*bun test\s*$", "", text) @@ -954,7 +960,7 @@ def runs_tests(graph: Graph, recipe: Recipe, found: set[Path]) -> bool: continue seen.add(path) relative = str(path.relative_to(graph.root)) - if is_test_script(relative) or drives_browser(path, relative): + if is_test_script(relative) or path in benches or drives_browser(path): return True texts.append((path.read_text(errors="replace"), path)) return False @@ -964,13 +970,48 @@ def is_test_script(relative: str) -> bool: return bool(re.search(r"\.test\.(?:py|sh|ts|js|mjs)$|\.browser\.ts$", relative)) -def drives_browser(path: Path, relative: str) -> bool: - """A script that launches a browser is a check, unless it is a benchmark.""" - if "/bench/" in f"/{relative}" or path.suffix not in (".ts", ".js", ".mjs"): +def drives_browser(path: Path) -> bool: + """A script that launches a browser is a check.""" + if path.suffix not in JS: return False return re.search(r"""from ["']playwright["']""", path.read_text(errors="replace")) is not None +def bench_entries(root: Path, graph: Graph) -> set[Path]: + """The JS bench scripts no other source imports or names: the ones something runs. + A module, or a page a bench bundles, is named by its importer; a test importing the + script it tests does not make that script a module.""" + sources = [ + path + for path in tracked(root, *(f"*{suffix}" for suffix in JS)) + if not is_test_script(str(path.relative_to(root))) + ] + named = set() + for source in sources: + # Block comments go first, since a doc comment may name the script that loads this one; + # then line comments, so a commented-out import does not make its target a module. + text = re.sub(r"(?s)(?:^|(?<=\s))/\*.*?\*/", "", source.read_text(errors="replace")) + text = re.sub(r"(?m)(?:^|\s)//.*$", "", text) + named.update(path for path in graph.scripts(text, source.parent, source) if path != source.resolve()) + for specifier in IMPORT.findall(text): + base = (source.parent / specifier).resolve() + named.update( + path + for path in ( + base, + *(base.with_name(base.name + suffix) for suffix in JS), + *(base.with_suffix(suffix) for suffix in JS), + *(base / f"index{suffix}" for suffix in JS), + ) + if path.is_file() + ) + return { + path.resolve() + for path in sources + if "/bench/" in f"/{path.relative_to(root)}" and not path.name.endswith(".d.ts") and path.resolve() not in named + } + + def discovered(root: Path, graph: Graph) -> set[Path]: """What a reached `bun test` discovers: the paths a root workspace's `check`, `test`, or `build` script names, or the workspace, once sh/js/workspace.py runs those scripts, and @@ -1058,11 +1099,12 @@ def inventory(root: Path) -> tuple[list[tuple[str, str]], list[str]]: orphans.append((f"bench {target.package}::{target.name}", where)) found = discovered(root, graph) + benches = bench_entries(root, graph) # Recipes that run tests. recipes = {recipe.path: recipe for module in justfiles.values() for recipe in module.recipes.values()} for _, recipe in sorted(recipes.items()): - if recipe.path not in graph.reached and runs_tests(graph, recipe, found): + if recipe.path not in graph.reached and runs_tests(graph, recipe, found, benches): orphans.append((f"recipe {recipe.path}", recipe_line(root, recipe))) # Standalone test scripts. @@ -1076,6 +1118,12 @@ def inventory(root: Path) -> tuple[list[tuple[str, str]], list[str]]: continue if not graph.reaches(path.resolve()): orphans.append((f"script {relative}", relative)) + + # Bench entry points. + for path in sorted(benches): + if not graph.reaches(path): + relative = str(path.relative_to(root)) + orphans.append((f"script {relative}", relative)) return orphans, errors diff --git a/sh/gh/test-inventory.test.py b/sh/gh/test-inventory.test.py index e5e437b03e9..dce5a05e7ec 100644 --- a/sh/gh/test-inventory.test.py +++ b/sh/gh/test-inventory.test.py @@ -350,6 +350,77 @@ def test_a_bun_workspace_covers_its_tests_once_its_runner_is_reached(self): self.write("js/net/package.json", '{"scripts": {"manual-test": "bun test src"}}') self.assertIn("script js/net/src/codec.test.ts", self.orphans()) + def test_a_js_bench_entry_point_needs_a_caller(self): + # Nightly's shape: a loop's literal names reach a recipe parameter through a helper's "$@". + self.write( + "sh/nightly.sh", + """ + #!/usr/bin/env bash + javascript() { + local label=$1 + shift + "$@" | tee "$label.csv" + } + for benchmark in fills; do + javascript "$benchmark" just js bench "$benchmark" + done + javascript page just js page-bench + """, + ) + self.write( + "js/justfile", + """ + bench name: + bun net/bench/{{ name }}.ts + + page-bench: + bun watch/bench/page.ts + + gc-bench: + bun net/bench/gc.ts + """, + ) + self.write("justfile", JUSTFILE.replace(" mod rs\n", " mod js\n mod rs\n") + " sh/nightly.sh\n") + # A module, imported with or without its extension, and a page a bench bundles are not + # entry points; a doc comment naming its loader, a commented-out import, or a test + # importing it changes nothing. + self.write( + "js/net/bench/fills.ts", + 'import { sweep } from "./sweep";\nimport "./shared.ts"; // import "./forward";\n// import "./gc";\n', + ) + self.write("js/net/bench/sweep.ts", "/** Shared by `fills.ts` and `forward.ts`. */\n") + self.write("js/net/bench/shared.ts", "") + self.write("js/net/bench/forward.ts", 'import { sweep } from "./sweep.ts";\n') + self.write("js/net/bench/forward.test.ts", 'await import("./forward");\n') + self.write("js/net/bench/gc.ts", "") + self.write( + "js/watch/bench/page.ts", + 'import { chromium } from "playwright";\n' + "await Bun.build({ entrypoints: [`${import.meta.dir}/page-entry.ts`] });\n", + ) + self.write("js/watch/bench/page-entry.ts", "") + orphans = self.orphans() + self.assertIn("script js/net/bench/forward.ts", orphans) + self.assertIn("script js/net/bench/gc.ts", orphans) + self.assertIn("recipe js::gc-bench", orphans) + for reached in ("fills", "sweep", "shared"): + self.assertNotIn(f"script js/net/bench/{reached}.ts", orphans) + self.assertNotIn("script js/watch/bench/page.ts", orphans) + self.assertNotIn("script js/watch/bench/page-entry.ts", orphans) + + # Dropping a name from the loop, or the browser bench's call, orphans its script. + nightly = (self.root / "sh/nightly.sh").read_text() + self.write( + "sh/nightly.sh", + nightly.replace("in fills;", "in forward;").replace("javascript page just js page-bench\n", ""), + ) + orphans = self.orphans() + self.assertIn("script js/net/bench/fills.ts", orphans) + self.assertNotIn("script js/net/bench/forward.ts", orphans) + self.assertIn("script js/watch/bench/page.ts", orphans) + self.assertIn("recipe js::page-bench", orphans) + self.assertNotIn("script js/watch/bench/page-entry.ts", orphans) + def test_a_selection_outside_the_ignored_set_covers_nothing(self): # Same filter, but without --run-ignored nextest skips the ignored tests. self.write("justfile", JUSTFILE.replace("--run-ignored only ", ""))