diff --git a/quest/m1/README.md b/quest/m1/README.md index fab8bc3fc05..55660210155 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -166,7 +166,6 @@ per-frame CPU growth since #5926. - [Native rung disable](/quest/m1/simulcast-rung-disable.md) - OBS stops sending the top renditions its grant cannot fund, as the Rust/FFI publishers do, and a shaped uplink through moq-ffi proves the resume - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles -- [Decoding from a track's first packet keeps its start](/quest/m1/opus-preskip.md) - moq-audio drops only OpusHead's pre-skip from a track's first packet, keeping the 80 ms warmup for mid-stream joins - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor - [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - consumers hold a disabled rendition, and an unmute encodes at once with the mic still open diff --git a/quest/m1/opus-preskip.md b/quest/m1/opus-preskip.md deleted file mode 100644 index 55fe1a2316b..00000000000 --- a/quest/m1/opus-preskip.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] Decoding from a track's first packet keeps its start - -## Goal - -moq-audio decoding an Opus track from its first packet outputs everything but -the codec's pre-skip, so recordings keep their first 80 ms and a track shorter -than that decodes to something, in every binding. - -## Plan - -Decided 2026-10-30 (maintainer, `/quest-plan`): from the track's first packet, -drop only the OpusHead pre-skip, as the codec defines. A decoder that joins -mid-stream keeps today's 80 ms warmup, since it has no history to warm up -from. Found in [moq-dev/moq#7033](https://github.com/moq-dev/moq/pull/7033). - -Test both cases: a short finished track decodes to its length minus -pre-skip, and a mid-stream join still drops its warmup. diff --git a/rs/moq-audio/src/decode/conformance.rs b/rs/moq-audio/src/decode/conformance.rs index efa28af0fde..541b0f00f18 100644 --- a/rs/moq-audio/src/decode/conformance.rs +++ b/rs/moq-audio/src/decode/conformance.rs @@ -242,12 +242,7 @@ async fn consumer_preserves_reference_samples_and_timeline() { pcm.extend(samples(&frame.data)); } } - let warmup = config.warmup.unwrap_or_default(); - let skip = (warmup.as_nanos() * vector.rate as u128 / 1_000_000_000) as usize * vector.channels as usize; - let reference = Vector { - reference: &vector.reference[skip * 4..], - ..*vector - }; - compare(&reference, "consumer", &pcm); + // Decoded from the track's first packet, so no warmup is trimmed. + compare(vector, "consumer", &pcm); } } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index d40957e9004..9323e8eb52b 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -125,9 +125,12 @@ pub struct Consumer { remix: Option, options: Options, max_delay: std::time::Duration, - /// Prime after a join or loss, never after a declared endpoint. + /// Prime after a join or loss, never after a declared endpoint or at the track's + /// first packet, which the codec's own delay already primes. warmup: Duration, warmup_pending: bool, + /// Whether a packet was read or lost yet, so the track's first packet can be told apart. + started: bool, warmup_until: Option, resolved_sample_rate: u32, resolved_layout: Layout, @@ -248,6 +251,7 @@ impl Consumer { warmup, warmup_pending: !warmup.is_zero(), warmup_until: None, + started: false, resolved_sample_rate: sample_rate, resolved_layout: layout, next_start: None, @@ -319,17 +323,22 @@ impl Consumer { return Ok(Some(frame)); } - let mux_frame = match self.track.read().await? { + let (mux_frame, track_start) = match self.track.read().await? { Some(Encoded::Frame { group, frame }) => { + let first = !std::mem::replace(&mut self.started, true); // A later group after a declared end comes from a fresh encoder; the end's // own group carries the stopped encoder's final packets, trimmed below. if self.end.is_some() && group != self.end_group { self.restart()?; } - frame + // Nothing precedes group 0, and a group delivers from its frame 0 or not at + // all, so reading it first means decoding from the encoder's first packet. + // A finished group's end marker may come first; it carries no media. + (frame, first && group == 0) } // Behind whatever the resampler held from before the loss. Some(Encoded::Gap { timestamp, duration }) => { + self.started = true; self.restart()?; self.warmup_pending = !self.warmup.is_zero(); self.ready.push_back(Read::Gap { timestamp, duration }); @@ -384,11 +393,12 @@ impl Consumer { let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); let decoded = self.decoder.decode(&mux_frame.payload)?; - // Anchored only once a packet decodes: a refused one primed nothing. - if self.warmup_pending { + // Anchored only once a packet decodes: a refused one primed nothing. The track's + // first packet has no history to warm up from: the codec's pre-skip, trimmed by + // the decoder, is all the priming its encoder declared. + if std::mem::take(&mut self.warmup_pending) && !track_start { let warmup = moq_net::Timestamp::try_from(self.warmup)?.convert(timestamp.scale())?; self.warmup_until = Some(timestamp.checked_add(warmup)?); - self.warmup_pending = false; } // Codec delay trimmed off the front is media this packet covered even // though no samples came out, so it still moves the packet after it along. @@ -820,6 +830,8 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), ); + // Joined mid-stream: the track's first packet needs no warmup. + producer.seek(1).unwrap(); for epoch in 0..2 { if epoch != 0 && declared { producer.discontinuity().unwrap(); @@ -991,6 +1003,8 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), ); + // Joined mid-stream: the track's first packet needs no warmup. + producer.seek(1).unwrap(); // One 20 ms packet per group. let mut write = |packet: u64| { producer @@ -1135,6 +1149,8 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), ); + // Joined mid-stream: the track's first packet needs no warmup. + producer.seek(1).unwrap(); producer .write(moq_mux::container::Frame { timestamp: Timestamp::from_micros(1_000_000).unwrap(), @@ -1171,6 +1187,72 @@ mod tests { ); } + /// Decodes a finished track of `packets` 20 ms Opus packets starting at group `group`, + /// keeping the encoder's declared warmup, and returns the first frame's stamp and the + /// frames decoded. + async fn opus_track_output(group: u64, packets: u64) -> (Option, usize) { + let mut encoder = Encoder::new(&EncodeConfig::new(48_000, Layout::Mono)).unwrap(); + let catalog = encoder.catalog(); + assert_eq!(catalog.warmup, Some(Duration::from_millis(80))); + + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + producer.seek(group).unwrap(); + let pcm = vec![0.25f32; encoder.frame_size()]; + for packet in 0..packets { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_scale(packet * encoder.frame_size() as u64, 48_000).unwrap(), + duration: None, + payload: encoder.encode(&pcm).unwrap().payload, + keyframe: true, + }) + .unwrap(); + } + producer.finish().unwrap(); + + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "audio", + Options { + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + let (mut first, mut frames) = (None, 0); + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { + first.get_or_insert(frame.timestamp.as_micros()); + frames += frame.data.len() / size_of::(); + } + (first, frames) + } + + /// Decoding from the track's first packet drops only the codec's pre-skip: the encoder + /// started there, so there is no history to warm up from. A track shorter than the + /// warmup still decodes. + #[tokio::test(start_paused = true)] + async fn the_tracks_first_packet_keeps_all_but_pre_skip() { + assert_eq!(opus_track_output(0, 2).await, (Some(0), 2 * 960 - 312)); + } + + /// A join past the track's first group still drops its warmup after the pre-skip. + #[tokio::test(start_paused = true)] + async fn a_mid_stream_join_drops_its_warmup() { + assert_eq!(opus_track_output(1, 6).await, (Some(80_000), 6 * 960 - 312 - 3_840)); + } + /// A refused packet primed nothing, so the window starts at the first one that decodes. #[tokio::test(start_paused = true)] async fn warmup_starts_at_the_first_decoded_packet() { diff --git a/rs/moq-audio/src/decode/test_data/README.md b/rs/moq-audio/src/decode/test_data/README.md index d5f9c5f1fb6..cd2f39e6ec6 100644 --- a/rs/moq-audio/src/decode/test_data/README.md +++ b/rs/moq-audio/src/decode/test_data/README.md @@ -44,8 +44,8 @@ the same. This tests the supported ingest without expanding the container API. Reference PCM includes all decoded AAC padding and excludes the Opus `dOps` pre-skip. Backend tests remove exactly the backend's declared delay. Consumer -tests additionally remove the catalog's 80 ms Opus warmup from the reference; -AAC declares no warmup. No correlation search, gain fitting, channel permutation, +tests decode from each track's first packet, which trims no warmup, so they +compare against the whole reference. No correlation search, gain fitting, channel permutation, or per-run tolerance adjustment is allowed. The following maxima were measured on x86-64 Linux with the pinned Nix toolchain, diff --git a/rs/moq-audio/tests/integration/roundtrip.rs b/rs/moq-audio/tests/integration/roundtrip.rs index ff4fcbbf9b3..706f6d246ba 100644 --- a/rs/moq-audio/tests/integration/roundtrip.rs +++ b/rs/moq-audio/tests/integration/roundtrip.rs @@ -95,12 +95,10 @@ async fn opus_roundtrip(case: Case) { .await .unwrap(); producer.finish().unwrap(); - let warmup = rendition.warmup.unwrap_or_default(); - let skipped = (warmup.as_nanos() * case.rate as u128 / 1_000_000_000) as usize; + // Decoded from the track's first packet, so only the codec's pre-skip is dropped. let mut pcm = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - let expected = - 123_000 + warmup.as_micros() + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128; + let expected = 123_000 + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128; // Microsecond stamping rounds each chunk independently. assert!( frame.timestamp.as_micros().abs_diff(expected) <= 1, @@ -111,12 +109,12 @@ async fn opus_roundtrip(case: Case) { } assert_eq!( pcm.len(), - (expected_frames - skipped) * channels as usize, + expected_frames * channels as usize, "pre-skip, resampling and final padding preserve duration" ); for step in 0..2 { - let start = (step * expected_frames / 2).max(skipped) - skipped; - let end = (step + 1) * expected_frames / 2 - skipped; + let start = step * expected_frames / 2; + let end = (step + 1) * expected_frames / 2; let half = &pcm[start * channels as usize..end * channels as usize]; for channel in 0..channels as usize { let samples: Vec = half.iter().skip(channel).step_by(channels as usize).copied().collect(); diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 90edcc5228a..ab0a0ce1129 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -4972,12 +4972,14 @@ fn gray_rgba(width: u32, height: u32) -> Vec { /// A finished encoder reaches a C decoder as an end read in microseconds rather /// than vanishing: Opus pads a partial last frame, so the finish publishes where /// the source really stopped. The default start decodes the finished track from -/// its first group rather than waiting at its end. +/// its first group rather than waiting at its end, even when it is shorter than +/// the warmup a mid-stream join trims. #[test] fn audio_raw_decode_reads_the_encoder_end() { const START_US: u64 = 123_000; - // Past the 80 ms Opus warmup the decoder trims, ending on a partial frame. - const SAMPLES: usize = 4_700; + // One partial frame, shorter than the 80 ms warmup a mid-stream join trims: decoded + // from the track's first packet, only the codec's pre-skip is dropped. + const SAMPLES: usize = 860; let origin = id(moq_origin_create()); let path = b"audio-raw-end-test";