From 40fb1f9414e0945d42360cdfcc2a4e5f11939f98 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 08:53:09 -0800 Subject: [PATCH] test(gst): quarantine a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times It failed the merge queue for #8551 on unchanged main, its third CI failure since 2026-10-16 and the first after #8160 and #8526. Its assertion now prints the pads and streams the generation saw, and the child quest holds the evidence and the replay paths left to check. Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-2/README.md | 8 +++ .../test-flakes-2/gst-moqsink-restart-pads.md | 69 +++++++++++++++++++ rs/moq-gst/src/source/imp.rs | 7 +- sh/gh/test-inventory.allow | 1 + 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 quest/m0/test-flakes-2/gst-moqsink-restart-pads.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 9b43b806b1..d840e6596d 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -278,4 +278,12 @@ timeout aborted it in the merge queue for [moq-dev/moq#8525](https://github.com/moq-dev/moq/pull/8525). It is quarantined with `#[ignore]` until its child fixes it. +Added 2026-11-12 (maintainer's standing rule that flakes join this epic): +moq-gst's moqsink restart pad test, which failed the merge queue for +[moq-dev/moq#8551](https://github.com/moq-dev/moq/pull/8551) on unchanged main, +its third failure since 2026-10-16 and its first after +[moq-dev/moq#8160](https://github.com/moq-dev/moq/pull/8160) and +[moq-dev/moq#8526](https://github.com/moq-dev/moq/pull/8526). It is +quarantined with `#[ignore]` until its child fixes it. + Public API: none. Wire: none. diff --git a/quest/m0/test-flakes-2/gst-moqsink-restart-pads.md b/quest/m0/test-flakes-2/gst-moqsink-restart-pads.md new file mode 100644 index 0000000000..f76700e848 --- /dev/null +++ b/quest/m0/test-flakes-2/gst-moqsink-restart-pads.md @@ -0,0 +1,69 @@ +# [S] moq-gst's moqsink restart pad test holds up + +## Goal + +moq-gst's +`source::imp::session_tests::a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times` +passes reliably, fixed at its cause in moqsrc (or the test, if the test is +wrong), and its `#[ignore]` is removed. + +## Plan + +It failed three times in CI, always on the generation 2 assertion +`playing.iter().all(|pad| held.contains(pad) && pad.starts_with("video_"))`, +about 1 s into the test: + +- 2026-10-16, `Check` on [moq-dev/moq#5715](https://github.com/moq-dev/moq/pull/5715) + ([run 38971716370](https://github.com/moq-dev/moq/actions/runs/38971716370)). +- 2026-11-07 05:07Z, `Check` on [moq-dev/moq#8122](https://github.com/moq-dev/moq/pull/8122) + ([run 42341390803](https://github.com/moq-dev/moq/actions/runs/42341390803)). +- 2026-11-12, the merge queue for [moq-dev/moq#8551](https://github.com/moq-dev/moq/pull/8551) + ([run 43250788147](https://github.com/moq-dev/moq/actions/runs/43250788147)), on + main at 8c8f0effaa. #8551 only changes moq-net's group error caching, and + the previous queue run passed this test. + +The first two predate [moq-dev/moq#8160](https://github.com/moq-dev/moq/pull/8160) +(a failed track is not replayed on its pad) and +[moq-dev/moq#8526](https://github.com/moq-dev/moq/pull/8526) (a track's end is +recorded even when its instance closed first). The third came after both, so +some replay path is still open, or the earlier failures had another cause. +None of the 1,181 other failed-run logs from 2026-10-12 to 2026-11-12 that +were scanned show this test failing. + +Not reproduced locally on 8c8f0effaa, with or without #8551 merged, on a +24-core host: 200 serial runs; 400 runs at 48 in parallel under +`quest gate`; 360 runs of the whole `session_tests` module at 12 and at 48 +copies in parallel; and 200 runs at 4 copies pinned to one core. + +What the assertion means: each pad's probe counts stream-starts after +`pad-added`. moqsrc's `begin` pushes the first stream-start before `add_pad`, +so a pad's count is the generation it plays. In generation 2 only video is +chained, and audio must end with EOS. The loop ends once one pad shows +count-2 buffers and audio's EOS arrives. A pad added late would need two +further stream-starts to reach count 2, so the likely failure is the audio pad +showing count-2 buffers. That means a +third `begin` on the audio pad: a second play of generation 1's audio, whose +replayed group reads as generation 2. With a single serial run of the test, +each of generations 1 and 2 reconciles two catalog snapshots, which differ +only in the media clock and `join`, so neither reshapes a rendition. That +leaves these ways for a reader to play its rendition again on one instance: + +- `Follower::exited` on `Exit::Unlisted` while the rendition is listed, + which calls `play()`. +- `Follower::start` with the same instance announced again, which plays + every idle rendition. +- A reader that exited `Done` without a `Finished` record (subscription + failed while the instance was up, a push error, an invalid buffer), then a + later non-final catalog snapshot's `play()`. + +Steps: + +- The test's assertion now prints `held`, `playing`, and every `(pad, stream, + eos)` the generation saw. Reproduce with that, or force each candidate path + above by delaying it, such as holding the generation 1 catalog's second + snapshot until audio's track has ended or failed. +- Fix the path at its cause in moqsrc, so a rendition never replays a group + on its held pad, then remove the `#[ignore]` and the + `sh/gh/test-inventory.allow` entry. + +Public API: none expected. Wire: none. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index c80a83cf9e..2b9f087862 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -6165,6 +6165,7 @@ mod session_tests { /// A real moqsink restart negotiates audio after video, keeping both held moqsrc pads. #[test] + #[ignore = "flaky: the dropped audio pad sometimes plays a third stream; quest/m0/test-flakes-2/gst-moqsink-restart-pads.md"] fn a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times() { let _pad_ids = pad_ids(); let relay = Relay::new(); @@ -6244,12 +6245,15 @@ mod session_tests { } } let mut playing = std::collections::BTreeSet::new(); + // Every (pad, stream, eos) this generation saw, for a failure to show which pad replayed. + let mut seen = Vec::new(); let mut dropped = false; let returning = if generation == 2 { 1 } else { 2 }; while playing.len() < returning || (generation == 2 && !dropped) { let (pad, run, eos) = received .recv_timeout(Duration::from_secs(10)) .expect("both renditions should resume"); + seen.push((pad.clone(), run, eos)); if eos { assert_eq!(generation, 2, "a returning rendition received EOS"); assert!(pad.starts_with("audio_"), "the returning video received EOS"); @@ -6267,7 +6271,8 @@ mod session_tests { assert!( playing .iter() - .all(|pad| held.contains(pad) && pad.starts_with("video_")) + .all(|pad| held.contains(pad) && pad.starts_with("video_")), + "held {held:?}, playing {playing:?}, seen {seen:?}" ); } sink.set_state(gst::State::Ready).unwrap(); diff --git a/sh/gh/test-inventory.allow b/sh/gh/test-inventory.allow index eded70c95e..97cedd4e5c 100644 --- a/sh/gh/test-inventory.allow +++ b/sh/gh/test-inventory.allow @@ -37,3 +37,4 @@ ignore moq-video::render::export::dx12::tests::media_foundation_encodes_the_expo # Flaked under load; disabled until fixed at its cause. ignore moq-ffi::test::raw_audio_decode_delay /quest/m0/test-flakes-2/ffi-raw-audio-decode-delay.md +ignore moq-gst::source::imp::session_tests::a_moqsink_restart_keeps_pads_when_caps_arrive_at_different_times /quest/m0/test-flakes-2/gst-moqsink-restart-pads.md