Skip to content
27 changes: 13 additions & 14 deletions bench/browser/micro.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,17 @@ const { values, positionals } = parseArgs({
},
});
const benchmark = values.bench;
// The rows each benchmark prints per run.
const ROWS: Record<string, number> = {
frames: 26,
drift: 36,
ledger: 12,
bandwidth: 44,
replay: 72,
"resume-history": 72,
// Each entry is named in full, so sh/gh/test-inventory.py sees this harness run it.
const benchmarks: Record<string, { entry: string; rows: number }> = {
frames: { entry: "js/net/bench/frames.ts", rows: 26 },
drift: { entry: "js/net/bench/drift.ts", rows: 36 },
ledger: { entry: "js/net/bench/ledger.ts", rows: 12 },
bandwidth: { entry: "js/net/bench/bandwidth.ts", rows: 44 },
replay: { entry: "js/net/bench/replay.ts", rows: 72 },
"resume-history": { entry: "js/net/bench/resume-history.ts", rows: 72 },
};
if (!Object.hasOwn(ROWS, benchmark)) throw new Error(`--bench must be one of ${Object.keys(ROWS).join(", ")}`);
const selected = Object.hasOwn(benchmarks, benchmark) ? benchmarks[benchmark] : undefined;
if (!selected) throw new Error(`--bench must be one of: ${Object.keys(benchmarks).join(", ")}`);
const rounds = Number(values.rounds);
if (!Number.isSafeInteger(rounds) || rounds < 1) throw new Error("--rounds must be a positive integer");
const roots = new Map<string, string>();
Expand All @@ -36,10 +37,8 @@ for (const argument of positionals.length ? positionals : ["current=.", "aa=."])
}
const scripts = new Map<string, string>();
for (const [name, root] of roots) {
const entry =
benchmark === "frames"
? resolve(root, "js/net/bench/frames.ts")
: resolve(import.meta.dir, `../../js/net/bench/${benchmark}.ts`);
// Only frames is bundled from each variant; the rest import each variant's `../src/` below.
const entry = resolve(benchmark === "frames" ? root : resolve(import.meta.dir, "../.."), selected.entry);
const result = await Bun.build({
entrypoints: [entry],
target: "browser",
Expand Down Expand Up @@ -144,7 +143,7 @@ try {
rows.set(variant, []);
await pages.get(variant)?.evaluate("globalThis.measure()");
const measured = rows.get(variant) ?? [];
if (measured.length !== ROWS[benchmark]) throw new Error(`incomplete rows: ${variant}`);
if (measured.length !== selected.rows) throw new Error(`incomplete rows: ${variant}`);
return measured;
};
for (let round = 0; round < rounds; round++) {
Expand Down
2 changes: 1 addition & 1 deletion js/net/bench/resume-history.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ type Access = "lookup" | "fetch" | "subscribe";
/** Full collections with no stack heap pointers, so dead stack slots cannot pin a target. */
async function collect(): Promise<void> {
const gc = (globalThis as { gc?: (options: { type: "major"; execution: "async" }) => Promise<void> }).gc;
if (!gc) throw new Error("resume-history needs a precise collector: run it with bench/browser/micro.ts");
if (!gc) throw new Error("resume-history needs a precise collector: run it in Chromium through bench/browser");
for (let pass = 0; pass < 3; pass++) {
// V8's synchronous gc scans the embedder stack conservatively even from a fresh task.
// Async gc runs with StackState::kNoHeapPointers, including while tracing is enabled.
Expand Down
1 change: 0 additions & 1 deletion quest/m2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,6 @@ lite-07; bitrate enforcement is not part of this release.
- [fMP4 decode times from the declared reorder depth](/quest/m2/fmp4-declared-clock.md) - `DeclaredClock` replaces the sub-GOP cadence cap, as in TS, FLV, and MKV
- [An ended HLS playlist has no trailing gap](/quest/m2/hls-trailing-gap.md) - no zero-duration `EXT-X-GAP` before `EXT-X-ENDLIST`
- [moq-video's Apple Sink encodes on its own thread](/quest/m2/apple-sink-thread.md) - macOS and iOS ladders never encode on a tokio worker
- [The test inventory sees JS bench scripts](/quest/m2/js-bench-inventory.md) - JS bench entry points are resolved by the inventory gate
- [/fetch answers from the announce sync](/quest/m2/fetch-synced.md) - a missing broadcast gets a 404 once announcements are synced, not after 30 s
- [moq auth sign mints capped tokens](/quest/m2/auth-sign-caps.md) - CLI flags for the bitrate caps
- [QUIC-LB forwarding reads the cluster's peers and reports its counts](/quest/m2/quic-lb-forward-extras.md) - a cluster-API peer map and forward counters
Expand Down
13 changes: 0 additions & 13 deletions quest/m2/js-bench-inventory.md

This file was deleted.

66 changes: 57 additions & 9 deletions sh/gh/test-inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
- a cargo-fuzz target or a Cargo bench target, named by some reachable run;
- a `just` recipe that runs tests, reachable from a workflow;
- a standalone test script (`*.test.py`, `*.test.sh`, `*.browser.ts`, or a
`*.test.ts` outside a Bun package that runs `bun test`), reachable from one.
`*.test.ts` outside a Bun package that runs `bun test`), reachable from one;
- a JS bench script under a `bench/` directory that no other source imports
or names, so it is an entry point rather than a module, reachable from one.

Reachable means some workflow `run:` step leads to it through `just` recipes,
their dependencies, and the scripts they name. The walk is static: a `just`
Expand Down Expand Up @@ -37,6 +39,9 @@
import yaml

SCRIPT = re.compile(r"[\w.$/{}-]*[\w-]\.(?:sh|py|ts|mts|js|mjs)\b")
JS = (".ts", ".mts", ".js", ".mjs")
# A relative module specifier in a static or dynamic import, or a re-export.
IMPORT = re.compile(r"""\b(?:from|import)\s*\(?\s*["'](\.\.?/[^"']+)["']""")
# Commands that run a test suite. Python sources spell them as argument lists,
# so quotes and commas are dropped before matching.
RUNNER = re.compile(
Expand Down Expand Up @@ -936,10 +941,11 @@ def recipe_line(root: Path, recipe: Recipe) -> str:
return str(source.relative_to(root))


def runs_tests(graph: Graph, recipe: Recipe, found: set[Path]) -> bool:
def runs_tests(graph: Graph, recipe: Recipe, found: set[Path], benches: set[Path]) -> bool:
"""A recipe runs tests when its body runs a suite, or names a script no workflow reaches
that runs one, is a test script, or drives a browser. A wrapper around a reached script,
or a bare `bun test` in a directory a reached run already tests, adds no test of its own."""
that runs one, is a test script or a bench entry point, or drives a browser. A wrapper
around a reached script, or a bare `bun test` in a directory a reached run already tests,
adds no test of its own."""
text = recipe.text
if any(recipe.cwd.is_relative_to(directory) for directory in found):
text = re.sub(r"(?m)^\s*bun test\s*$", "", text)
Expand All @@ -954,7 +960,7 @@ def runs_tests(graph: Graph, recipe: Recipe, found: set[Path]) -> bool:
continue
seen.add(path)
relative = str(path.relative_to(graph.root))
if is_test_script(relative) or drives_browser(path, relative):
if is_test_script(relative) or path in benches or drives_browser(path):
return True
texts.append((path.read_text(errors="replace"), path))
return False
Expand All @@ -964,13 +970,48 @@ def is_test_script(relative: str) -> bool:
return bool(re.search(r"\.test\.(?:py|sh|ts|js|mjs)$|\.browser\.ts$", relative))


def drives_browser(path: Path, relative: str) -> bool:
"""A script that launches a browser is a check, unless it is a benchmark."""
if "/bench/" in f"/{relative}" or path.suffix not in (".ts", ".js", ".mjs"):
def drives_browser(path: Path) -> bool:
"""A script that launches a browser is a check."""
if path.suffix not in JS:
return False
return re.search(r"""from ["']playwright["']""", path.read_text(errors="replace")) is not None


def bench_entries(root: Path, graph: Graph) -> set[Path]:
"""The JS bench scripts no other source imports or names: the ones something runs.
A module, or a page a bench bundles, is named by its importer; a test importing the
script it tests does not make that script a module."""
sources = [
path
for path in tracked(root, *(f"*{suffix}" for suffix in JS))
if not is_test_script(str(path.relative_to(root)))
]
named = set()
for source in sources:
# Block comments go first, since a doc comment may name the script that loads this one;
# then line comments, so a commented-out import does not make its target a module.
text = re.sub(r"(?s)(?:^|(?<=\s))/\*.*?\*/", "", source.read_text(errors="replace"))
text = re.sub(r"(?m)(?:^|\s)//.*$", "", text)
named.update(path for path in graph.scripts(text, source.parent, source) if path != source.resolve())
for specifier in IMPORT.findall(text):
base = (source.parent / specifier).resolve()
named.update(
path
for path in (
base,
*(base.with_name(base.name + suffix) for suffix in JS),
*(base.with_suffix(suffix) for suffix in JS),
*(base / f"index{suffix}" for suffix in JS),
)
if path.is_file()
)
return {
path.resolve()
for path in sources
if "/bench/" in f"/{path.relative_to(root)}" and not path.name.endswith(".d.ts") and path.resolve() not in named
}


def discovered(root: Path, graph: Graph) -> set[Path]:
"""What a reached `bun test` discovers: the paths a root workspace's `check`, `test`, or
`build` script names, or the workspace, once sh/js/workspace.py runs those scripts, and
Expand Down Expand Up @@ -1058,11 +1099,12 @@ def inventory(root: Path) -> tuple[list[tuple[str, str]], list[str]]:
orphans.append((f"bench {target.package}::{target.name}", where))

found = discovered(root, graph)
benches = bench_entries(root, graph)

# Recipes that run tests.
recipes = {recipe.path: recipe for module in justfiles.values() for recipe in module.recipes.values()}
for _, recipe in sorted(recipes.items()):
if recipe.path not in graph.reached and runs_tests(graph, recipe, found):
if recipe.path not in graph.reached and runs_tests(graph, recipe, found, benches):
orphans.append((f"recipe {recipe.path}", recipe_line(root, recipe)))

# Standalone test scripts.
Expand All @@ -1076,6 +1118,12 @@ def inventory(root: Path) -> tuple[list[tuple[str, str]], list[str]]:
continue
if not graph.reaches(path.resolve()):
orphans.append((f"script {relative}", relative))

# Bench entry points.
for path in sorted(benches):
if not graph.reaches(path):
relative = str(path.relative_to(root))
orphans.append((f"script {relative}", relative))
return orphans, errors


Expand Down
71 changes: 71 additions & 0 deletions sh/gh/test-inventory.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,77 @@ def test_a_bun_workspace_covers_its_tests_once_its_runner_is_reached(self):
self.write("js/net/package.json", '{"scripts": {"manual-test": "bun test src"}}')
self.assertIn("script js/net/src/codec.test.ts", self.orphans())

def test_a_js_bench_entry_point_needs_a_caller(self):
# Nightly's shape: a loop's literal names reach a recipe parameter through a helper's "$@".
self.write(
"sh/nightly.sh",
"""
#!/usr/bin/env bash
javascript() {
local label=$1
shift
"$@" | tee "$label.csv"
}
for benchmark in fills; do
javascript "$benchmark" just js bench "$benchmark"
done
javascript page just js page-bench
""",
)
self.write(
"js/justfile",
"""
bench name:
bun net/bench/{{ name }}.ts

page-bench:
bun watch/bench/page.ts

gc-bench:
bun net/bench/gc.ts
""",
)
self.write("justfile", JUSTFILE.replace(" mod rs\n", " mod js\n mod rs\n") + " sh/nightly.sh\n")
# A module, imported with or without its extension, and a page a bench bundles are not
# entry points; a doc comment naming its loader, a commented-out import, or a test
# importing it changes nothing.
self.write(
"js/net/bench/fills.ts",
'import { sweep } from "./sweep";\nimport "./shared.ts"; // import "./forward";\n// import "./gc";\n',
)
self.write("js/net/bench/sweep.ts", "/** Shared by `fills.ts` and `forward.ts`. */\n")
self.write("js/net/bench/shared.ts", "")
self.write("js/net/bench/forward.ts", 'import { sweep } from "./sweep.ts";\n')
self.write("js/net/bench/forward.test.ts", 'await import("./forward");\n')
self.write("js/net/bench/gc.ts", "")
self.write(
"js/watch/bench/page.ts",
'import { chromium } from "playwright";\n'
"await Bun.build({ entrypoints: [`${import.meta.dir}/page-entry.ts`] });\n",
)
self.write("js/watch/bench/page-entry.ts", "")
orphans = self.orphans()
self.assertIn("script js/net/bench/forward.ts", orphans)
self.assertIn("script js/net/bench/gc.ts", orphans)
self.assertIn("recipe js::gc-bench", orphans)
for reached in ("fills", "sweep", "shared"):
self.assertNotIn(f"script js/net/bench/{reached}.ts", orphans)
self.assertNotIn("script js/watch/bench/page.ts", orphans)
self.assertNotIn("script js/watch/bench/page-entry.ts", orphans)

# Dropping a name from the loop, or the browser bench's call, orphans its script.
nightly = (self.root / "sh/nightly.sh").read_text()
self.write(
"sh/nightly.sh",
nightly.replace("in fills;", "in forward;").replace("javascript page just js page-bench\n", ""),
)
orphans = self.orphans()
self.assertIn("script js/net/bench/fills.ts", orphans)
self.assertNotIn("script js/net/bench/forward.ts", orphans)
self.assertIn("script js/watch/bench/page.ts", orphans)
self.assertIn("recipe js::page-bench", orphans)
self.assertNotIn("script js/watch/bench/page-entry.ts", orphans)

def test_a_selection_outside_the_ignored_set_covers_nothing(self):
# Same filter, but without --run-ignored nextest skips the ignored tests.
self.write("justfile", JUSTFILE.replace("--run-ignored only ", ""))
Expand Down
Loading