diff --git a/.config/nextest.toml b/.config/nextest.toml index ce703f19528..b6e2b141de7 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -72,7 +72,7 @@ slow-timeout = { period = "60s", terminate-after = 3 } io-uring = { max-threads = 4 } v4l2 = { max-threads = 1 } -# Nightly's ignored benchmark tests (`just gh nightly rust-test`) keep each test's +# Nightly's ignored benchmark tests (`just gh nightly rust-test|rust-test-apps`) keep each test's # printed tables in target/nextest/bench/junit.xml for the benchmark report. [profile.bench.junit] path = "junit.xml" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 796496f576d..b768dd61b0b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -93,6 +93,7 @@ jobs: # build environment, so a benchmark reuses the dependencies its group already # compiled. The Rust benchmarks outgrew one hour, so they split by harness: # nextest's ignored benchmark tests and cargo's bench targets share few builds. + # The benchmark tests split again by layer, since fat LTO links each test binary. # A paired group also builds and runs the previous nightly's commit beside this # one on the same runner, which is what fails the job on a regression; the # cross-night history on the benchmark site stays informational. @@ -117,6 +118,11 @@ jobs: artifact: bench-rust-test artifact-path: target/bench/nightly/rust-test benchmark: true + - name: Rust application benchmark tests + recipe: gh nightly rust-test-apps + artifact: bench-rust-test-apps + artifact-path: target/bench/nightly/rust-test-apps + benchmark: true - name: Binding benchmark runners recipe: gh nightly bindings - name: Rust bench targets diff --git a/bench/README.md b/bench/README.md index 32260c5f47f..8087c820612 100644 --- a/bench/README.md +++ b/bench/README.md @@ -842,7 +842,7 @@ The table includes these measurements: `size_bytes` and `peak_bytes`, `mib_per_s`, and `*_packets` and `*_messages` columns are measurements, and every other column is a workload dimension in each series name, so print outcomes only in measured columns. The Rust benchmark tests - (`just gh nightly rust-test`) run in release builds, one at a time, under + (`just gh nightly rust-test` and `rust-test-apps`) run in release builds, one at a time, under nextest's `bench` profile, which keeps each test's output in a JUnit report; their series are named by package and test path, such as `rust-test/moq-congestion/gcc/bench/benchmark_per_packet`. A test with no runnable diff --git a/quest/m0/ci/hakari-followups.md b/quest/m0/ci/hakari-followups.md index 25489d7a58c..60e21250209 100644 --- a/quest/m0/ci/hakari-followups.md +++ b/quest/m0/ci/hakari-followups.md @@ -83,7 +83,8 @@ Public API: none. Wire: none. cost is material, present the measured tradeoff to the maintainer; recommend keeping the narrow final exclusions unless the measurements justify another design. That job timed out on five of the six scheduled runs from - 2026-11-05 to 11-10 and on every one since 11-07, so a complete run waits on - [it fitting its hour](/quest/m0/ci/nightly-rust-test-hour.md). That change - regroups the job's builds, so compare only the mbx summaries of builds whose - crates, features, and profile match the baseline's. + 2026-11-05 to 11-10 and on every one since 11-07. It is now two jobs, + `Nightly (Rust benchmark tests)` and `Nightly (Rust application benchmark + tests)`, each one release build grouping several crates, so none of their + builds match a baseline build's crates and features; compare only builds + that do. diff --git a/quest/m0/ci/nightly-relay-session-hour.md b/quest/m0/ci/nightly-relay-session-hour.md index 57688d5f489..89e56632527 100644 --- a/quest/m0/ci/nightly-relay-session-hour.md +++ b/quest/m0/ci/nightly-relay-session-hour.md @@ -22,10 +22,5 @@ with `Evicted`. The completed the sweep in about 25 and 23 minutes (their jobs failed later, at `Fail on paired regressions`). Find what grew between those sweeps and the 60-minute cancellations from 11-08, then trim it or split the group across -matrix entries, as -[the Rust benchmark tests quest](/quest/m0/ci/nightly-rust-test-hour.md) does -for its job. The sweep still covers both fan-out axes. - -## Related - -- [Rust benchmark tests fit their hour](/quest/m0/ci/nightly-rust-test-hour.md) - the same timeout, caused by builds +matrix entries, as the Rust benchmark tests did. The sweep still covers both +fan-out axes. diff --git a/quest/m0/ci/nightly-rust-test-hour.md b/quest/m0/ci/nightly-rust-test-hour.md deleted file mode 100644 index 8bddcba84c9..00000000000 --- a/quest/m0/ci/nightly-rust-test-hour.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] Nightly's Rust benchmark tests fit their hour - -## Goal - -`Nightly (Rust benchmark tests)` (`just gh nightly rust-test`) finishes well -inside its 60-minute `timeout-minutes` on a scheduled run, with every benchmark -it runs today still measured in a release build, one at a time, and reported. -Splitting it across jobs and cutting what it builds are both in scope. - -## Plan - -Decided 2026-11-10 (maintainer, /quest-unblock): ✅ split the job and reduce -its size (rejected: finish the hakari measurement with one run; raise the -timeout). - -Evidence: the job was cancelled at the timeout on the 2026-11-03, 11-05, -11-07, 11-08, 11-09, and 11-10 scheduled runs; the 11-04 and 11-06 runs -finished in 57 to 59 minutes, and the last to pass, on 11-02, took 34. In -[the 11-10 run](https://github.com/moq-dev/moq/actions/runs/42842827718/job/129922278502) -the benchmarks themselves ran for about two minutes in all. The hour went to -`sh/gh/nightly.sh`'s `rust-test` group compiling a separate release build per -`nextest` invocation (3 to 5.5 minutes each, about ten of them), and the job -was still compiling `moq-ffi` when it was cancelled. - -Start from the builds: fewer `nextest` invocations sharing one release build -where the feature sets allow it, then split what remains across matrix entries -so no entry nears the hour. Keep `sh/gh/nightly.test.py`'s guarantees (release -builds, one JUnit report per run, the report publish step) or update them with -the change. - -## Related - -- [hakari follow-ups](/quest/m0/ci/hakari-followups.md) - its mbx measurement needs a complete run of this job -- [Relay session benchmarks fit their hour](/quest/m0/ci/nightly-relay-session-hour.md) - the same timeout in another Nightly job diff --git a/quest/m1/audio-jitter-target/calibrate.md b/quest/m1/audio-jitter-target/calibrate.md index f99ed24f488..43a10e789c4 100644 --- a/quest/m1/audio-jitter-target/calibrate.md +++ b/quest/m1/audio-jitter-target/calibrate.md @@ -26,7 +26,7 @@ Carried decisions: [moq-dev/moq#8308](https://github.com/moq-dev/moq/pull/8308#issuecomment-6730322286)): ✅ budget `trimmed_ms_per_min` and `player_skips_per_min` (rejected: budgeting accelerates as discarded audio), and ✅ the nightly runner's - `bench-playback` counts as the quiet-host run (rejected: naming another + playback benchmarks count as the quiet-host run (rejected: naming another host). The rule: the worst of seven clean nights times 1.25, plus the documented diff --git a/rs/justfile b/rs/justfile index 4e7ed79622a..a211ef3fdc3 100644 --- a/rs/justfile +++ b/rs/justfile @@ -584,14 +584,10 @@ codspeed: # replaces only its own packages' benches, so `cargo codspeed run` runs both. cd rs/moq-auth && VALGRIND_BUG_494162=1 cargo codspeed build --locked -m simulation --bench jwt -# Held feed FETCH readiness across independent request and track counts. +# Run the tests the arguments select in a release build, printing their output; benchmarks add `--run-ignored only`. [positional-arguments] -bench-feed-fetch *args: - bash sh/rs/nextest.sh cargo nextest run --locked --release -p moq-net --lib --run-ignored only -E 'test(benchmark_feed_fetch_wakes)' --success-output immediate "$@" - -# Device-free mixer and resampler cost, swept over fanout, packetization, and layout. -bench-playback: - bash sh/rs/nextest.sh cargo nextest run --locked --release -p moq-audio --features playback --run-ignored only -E 'test(benchmark_mixer_callbacks) | test(benchmark_resampler) | test(benchmark_raw_audio)' --success-output immediate +bench-tests *args: + bash sh/rs/nextest.sh cargo nextest run --locked --release --success-output immediate "$@" # Sweep retained RTMP chunk-stream IDs independently of message and chunk sizes. [positional-arguments] @@ -608,10 +604,6 @@ bench-relay-auth *args: bench-stats *args: bash sh/local.sh cargo bench --locked -p moq-stats --features bench --bench producer_tick --bench decode --bench aggregate -- "$@" -# Sweep latency observations against writer count and report density. -bench-latency: - bash sh/rs/nextest.sh cargo nextest run --locked --release -p moq-bench --run-ignored only -E 'test(benchmark_observation_contention)' --success-output immediate - # Exact TS receiver scheduling across PID and queued-unit fanout. [positional-arguments] bench-ts-schedule *args: diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 8770f5f94bd..dbda8f6fd56 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -849,7 +849,7 @@ mod tests { } #[tokio::test] - #[ignore = "measures real codec CPU; run with just rs bench-playback"] + #[ignore = "measures real codec CPU; release benchmark, run by nightly"] async fn benchmark_raw_audio() { println!("tracks,retention_secs,always_on,idle_ms,catchup_ms"); for tracks in [1, 8] { diff --git a/rs/moq-audio/src/playback/mixer.rs b/rs/moq-audio/src/playback/mixer.rs index 920e6264845..42cd3a5f81c 100644 --- a/rs/moq-audio/src/playback/mixer.rs +++ b/rs/moq-audio/src/playback/mixer.rs @@ -550,7 +550,7 @@ mod tests { /// Measurement-only harness: no device and no timing acceptance threshold. #[test] - #[ignore = "run with just rs bench-playback"] + #[ignore = "release benchmark, run by nightly"] #[expect( clippy::disallowed_methods, clippy::disallowed_types, diff --git a/rs/moq-audio/src/resample.rs b/rs/moq-audio/src/resample.rs index 8c8a9b0c4af..da6120245da 100644 --- a/rs/moq-audio/src/resample.rs +++ b/rs/moq-audio/src/resample.rs @@ -552,7 +552,7 @@ mod tests { /// Measurement-only: sweep caller packetization and layout without timing gates. #[test] - #[ignore = "run with just rs bench-playback"] + #[ignore = "release benchmark, run by nightly"] fn benchmark_resampler() { println!("input_rate,output_rate,channels,chunk,packet,ns_per_input_frame"); for (input_rate, output_rate) in [(44_100, 48_000), (48_000, 44_100), (96_000, 48_000)] { diff --git a/rs/moq-bench/README.md b/rs/moq-bench/README.md index 18269da657a..85232722e2f 100644 --- a/rs/moq-bench/README.md +++ b/rs/moq-bench/README.md @@ -295,7 +295,7 @@ to the results. ### Observation cost -`just rs bench-latency` measures latency recording with 1, 4 and 16 concurrent +`just rs bench-tests -p moq-bench --run-ignored only -E 'test(benchmark_observation_contention)'` measures latency recording with 1, 4 and 16 concurrent writers and 0, 10 and 100 snapshots per run. It checks final sample conservation and prints the cost per sample for each case. The nightly workflow runs the same matrix and records it on the benchmark table. diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 6a554c61a3e..32c017db22b 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -2,7 +2,7 @@ # Keep benchmark builds together while unrelated Nightly workloads run independently. set -euo pipefail -usage='usage: just gh nightly rust-test|rust-bench|rust-shared|rust-relay|rust-extra|rust-uring|js|bindings' +usage='usage: just gh nightly rust-test|rust-test-apps|rust-bench|rust-shared|rust-relay|rust-extra|rust-uring|js|bindings' if [[ $# != 1 ]]; then echo "$usage" >&2 exit 2 @@ -12,7 +12,7 @@ fi case "$1" in bindings) report= ;; rust-bench | rust-shared | rust-relay | rust-extra | rust-uring) report=rust ;; - rust-test) report=rust-test ;; + rust-test | rust-test-apps) report=rust-test ;; js) report=js ;; *) echo "$usage" >&2 @@ -73,43 +73,43 @@ javascript() { # The bench nextest profile stores each test's printed tables in its JUnit report, # which every run overwrites; keep each run's copy. nextest() { - local junit=${CARGO_TARGET_DIR:-$root/target}/nextest/bench/junit.xml + local junit=${CARGO_TARGET_DIR:-$root/target}/nextest/bench/junit.xml report name rm -f "$junit" run "$@" - if [[ -s $junit ]]; then - mv "$junit" "$out/raw/nextest-$((++nextest_runs)).xml" - else + if [[ ! -s $junit ]]; then failed+=("no JUnit report from: $*") + return fi + report=$out/raw/nextest-$((++nextest_runs)).xml + mv "$junit" "$report" + # A filter that matches nothing passes, as when its test needs a feature the build lacks. + while read -r name; do + if ! grep -qE "> "$NIGHTLY_CALLS"\n' 'printf "%s\n" "${BENCH_BASE:-}" >> "$NIGHTLY_CALLS.base"\n' - # Nextest's bench profile writes a JUnit report, unless the test withholds it. + # Nextest's bench profile writes a JUnit report with a case per filtered test, + # unless the test withholds the report or one test. 'if [[ ${NEXTEST_PROFILE:-} == bench && "$*" != "$NIGHTLY_NO_JUNIT" ]]; then\n' ' mkdir -p "$CARGO_TARGET_DIR/nextest/bench"\n' - ' printf "" > "$CARGO_TARGET_DIR/nextest/bench/junit.xml"\n' + " names=$(grep -oE 'test\\(=?[[:alnum:]_:]+\\)' <<<\"$*\" | sed -E 's/^test\\(=?//; s/\\)$//')\n" + " for name in $names; do\n" + ' [[ $name == "${NIGHTLY_UNMATCHED:-}" ]] ||\n' + ' printf \'\' "$name"\n' + ' done > "$CARGO_TARGET_DIR/nextest/bench/junit.xml"\n' + ' printf "" >> "$CARGO_TARGET_DIR/nextest/bench/junit.xml"\n' "fi\n" '[[ "$*" != "$NIGHTLY_FAIL" ]]\n' ) @@ -167,7 +176,7 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): BASH_ENV="/dev/null", CARGO_TARGET_DIR=str(scratch / "target"), ) - for name in ("BENCH_BASE", "NEXTEST_PROFILE", "NIGHTLY_NO_JUNIT"): + for name in ("BENCH_BASE", "NEXTEST_PROFILE", "NIGHTLY_NO_JUNIT", "NIGHTLY_UNMATCHED"): env.pop(name, None) env.update(environment) result = subprocess.run( @@ -222,7 +231,7 @@ def test_paired_base_is_the_previous_scheduled_nightly(self): self.assertEqual(calls, []) def test_paired_base_applies_only_to_rust_bench_groups(self): - for group in ("js", "rust-test", "bindings"): + for group in ("js", "rust-test", "rust-test-apps", "bindings"): result, calls = self.run_group(group, BENCH_BASE="HEAD") self.assertEqual(result.returncode, 2, group) self.assertEqual(calls, []) @@ -349,32 +358,40 @@ def test_dev_shell_jobs_trust_the_nix_cache(self): self.assertIn("uses: ./.github/actions/nix", job, job.splitlines()[0]) def test_benchmark_tests_stay_out_of_cargo_bench(self): - result, calls = self.run_group("rust-test") - self.assertEqual(result.returncode, 0, result.stderr) - *runs, publish = calls - self.assertFalse(any(call.startswith(("cargo bench", "just gh")) for call in runs)) - # Every benchmark measures a release build, and each run keeps its JUnit report. - self.assertTrue(all("--release" in call or " bench-" in call for call in runs), runs) - self.assertIn("just rs bench-feed-fetch", runs) - self.assertEqual(set(self.raw), {f"nextest-{index}.xml" for index in range(1, len(runs) + 1)}) - self.assertTrue(publish.startswith("just gh benchmark-report rust-test "), publish) + workflow = (ROOT / ".github/workflows/nightly.yml").read_text() + for group in RUST_TEST_GROUPS: + self.assertIn(f"recipe: gh nightly {group}\n", workflow) + result, calls = self.run_group(group) + self.assertEqual(result.returncode, 0, result.stderr) + *runs, publish = calls + # One release build per group, keeping its JUnit report, so the benchmarks + # share the build and the report the publish step reads. + self.assertEqual(runs, [call for call in runs if call.startswith("just rs bench-tests ")]) + self.assertEqual(len(runs), 1) + self.assertEqual(self.raw, ["nextest-1.xml"]) + self.assertTrue(publish.startswith("just gh benchmark-report rust-test "), publish) def test_benchmark_test_without_its_report_fails(self): - result, calls = self.run_group("rust-test", NIGHTLY_NO_JUNIT="rs bench-latency") + _, (run, _) = self.run_group("rust-test") + result, calls = self.run_group("rust-test", NIGHTLY_NO_JUNIT=run.removeprefix("just ")) + self.assertEqual(result.returncode, 1) + self.assertIn("no JUnit report from: just rs bench-tests", result.stderr) + self.assertFalse(any(call.startswith("just gh") for call in calls)) + + def test_benchmark_filter_that_runs_nothing_fails(self): + # A test behind a feature the build lacks matches nothing, and nextest still passes. + result, calls = self.run_group("rust-test", NIGHTLY_UNMATCHED="benchmark_feed_fetch_wakes") self.assertEqual(result.returncode, 1) - self.assertIn("no JUnit report from: just rs bench-latency", result.stderr) - self.assertIn("just rs bench-feed-fetch", calls) + self.assertIn("no test ran for benchmark_feed_fetch_wakes", result.stderr) self.assertFalse(any(call.startswith("just gh") for call in calls)) def test_every_nightly_benchmark_test_exists(self): - # A filter naming a removed test matches nothing, and nextest still passes. - recipes = (ROOT / "rs/justfile").read_text() - _, calls = self.run_group("rust-test") + # A filter naming a removed test matches nothing; this catches it without a build. named = set() - for call in calls: - if match := re.fullmatch(r"just rs (bench-[\w-]+)", call): - call = re.search(rf"^{match[1]}\b.*:\n (.+)$", recipes, re.M)[1] - named |= set(re.findall(r"test\((\w+)\)", call)) + for group in RUST_TEST_GROUPS: + _, calls = self.run_group(group) + for call in calls: + named |= set(re.findall(r"test\(=?(?:\w+::)*(\w+)\)", call)) self.assertTrue(named) sources = "".join(path.read_text() for path in (ROOT / "rs").glob("*/**/*.rs") if "/target/" not in str(path)) # A filter can name one test or a module containing several benchmark cases. diff --git a/sh/gh/test-inventory.py b/sh/gh/test-inventory.py index 82ee10aa55f..59d524e072d 100644 --- a/sh/gh/test-inventory.py +++ b/sh/gh/test-inventory.py @@ -626,8 +626,10 @@ def packages(root: Path) -> list[Package]: entry["name"]: directory / entry.get("path", f"{auto}/{entry['name']}.rs") for entry in manifest.get(kind, []) } - if kind == "bin" and (directory / "src/main.rs").is_file(): - explicit.setdefault(package.name, directory / "src/main.rs") + # Cargo infers the package-named binary only when no `[[bin]]` claims its file. + main = directory / "src/main.rs" + if kind == "bin" and main.is_file() and main not in explicit.values(): + explicit.setdefault(package.name, main) if package_settings.get(f"auto{kind}s" if kind != "test" else "autotests", True): for file in sorted((directory / auto).glob("*.rs")): explicit.setdefault(file.stem, file) diff --git a/sh/gh/test-inventory.test.py b/sh/gh/test-inventory.test.py index ae890ad8924..adb1716ceed 100644 --- a/sh/gh/test-inventory.test.py +++ b/sh/gh/test-inventory.test.py @@ -207,6 +207,14 @@ def test_an_action_script_input_is_a_caller(self): self.assertNotIn("recipe rs::sweep", orphans) self.assertNotIn("ignore media::probe::long_sweep", orphans) + def test_a_renamed_binary_is_selected_by_its_own_name(self): + # A `[[bin]]` naming src/main.rs replaces the package-named binary Cargo would infer. + self.write("rs/tool/Cargo.toml", '[package]\nname = "tool"\n\n[[bin]]\nname = "t"\npath = "src/main.rs"\n') + self.write("rs/tool/src/main.rs", '#[test]\n#[ignore = "slow"]\nfn sweep() {}\n') + self.assertIn("ignore tool::sweep", self.orphans()) + self.write("justfile", JUSTFILE + " cargo nextest run -p tool --bin t --run-ignored only\n") + self.assertNotIn("ignore tool::sweep", self.orphans()) + def test_cargo_ndk_options_are_not_test_filters(self): caller = "just rs test -p media --lib --run-ignored only -E 'test(/^gpu::tests::(renders|exports)$/)'" ndk = 'cargo ndk -t "$abi" --platform 26 test -p media --lib -- --ignored gpu::tests::renders'