Skip to content
Merged
1 change: 0 additions & 1 deletion quest/m1/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ per-frame CPU growth since #5926.
- [Native rung disable](/quest/m1/simulcast-rung-disable.md) - OBS stops sending the top renditions its grant cannot fund, as the Rust/FFI publishers do, and a shaped uplink through moq-ffi proves the resume
- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain
- [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles
- [Decoding from a track's first packet keeps its start](/quest/m1/opus-preskip.md) - moq-audio drops only OpusHead's pre-skip from a track's first packet, keeping the 80 ms warmup for mid-stream joins
- [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust
- [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `<moq-watch>` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor
- [Push-to-talk loses no speech](/quest/m1/push-to-talk/README.md) - consumers hold a disabled rendition, and an unmute encodes at once with the mic still open
Expand Down
17 changes: 0 additions & 17 deletions quest/m1/opus-preskip.md

This file was deleted.

9 changes: 2 additions & 7 deletions rs/moq-audio/src/decode/conformance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,12 +242,7 @@ async fn consumer_preserves_reference_samples_and_timeline() {
pcm.extend(samples(&frame.data));
}
}
let warmup = config.warmup.unwrap_or_default();
let skip = (warmup.as_nanos() * vector.rate as u128 / 1_000_000_000) as usize * vector.channels as usize;
let reference = Vector {
reference: &vector.reference[skip * 4..],
..*vector
};
compare(&reference, "consumer", &pcm);
// Decoded from the track's first packet, so no warmup is trimmed.
compare(vector, "consumer", &pcm);
}
}
94 changes: 88 additions & 6 deletions rs/moq-audio/src/decode/consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,12 @@ pub struct Consumer {
remix: Option<Remix>,
options: Options,
max_delay: std::time::Duration,
/// Prime after a join or loss, never after a declared endpoint.
/// Prime after a join or loss, never after a declared endpoint or at the track's
/// first packet, which the codec's own delay already primes.
warmup: Duration,
warmup_pending: bool,
/// Whether a packet was read or lost yet, so the track's first packet can be told apart.
started: bool,
warmup_until: Option<moq_net::Timestamp>,
resolved_sample_rate: u32,
resolved_layout: Layout,
Expand Down Expand Up @@ -248,6 +251,7 @@ impl Consumer {
warmup,
warmup_pending: !warmup.is_zero(),
warmup_until: None,
started: false,
resolved_sample_rate: sample_rate,
resolved_layout: layout,
next_start: None,
Expand Down Expand Up @@ -319,17 +323,22 @@ impl Consumer {
return Ok(Some(frame));
}

let mux_frame = match self.track.read().await? {
let (mux_frame, track_start) = match self.track.read().await? {
Some(Encoded::Frame { group, frame }) => {
let first = !std::mem::replace(&mut self.started, true);
// A later group after a declared end comes from a fresh encoder; the end's
// own group carries the stopped encoder's final packets, trimmed below.
if self.end.is_some() && group != self.end_group {
self.restart()?;
}
frame
// Nothing precedes group 0, and a group delivers from its frame 0 or not at
// all, so reading it first means decoding from the encoder's first packet.
// A finished group's end marker may come first; it carries no media.
(frame, first && group == 0)
}
// Behind whatever the resampler held from before the loss.
Some(Encoded::Gap { timestamp, duration }) => {
self.started = true;
self.restart()?;
self.warmup_pending = !self.warmup.is_zero();
self.ready.push_back(Read::Gap { timestamp, duration });
Expand Down Expand Up @@ -384,11 +393,12 @@ impl Consumer {
let epoch = *self.epoch.get_or_insert(mux_frame.timestamp);
let delay = self.decoder.delay_remaining();
let decoded = self.decoder.decode(&mux_frame.payload)?;
// Anchored only once a packet decodes: a refused one primed nothing.
if self.warmup_pending {
// Anchored only once a packet decodes: a refused one primed nothing. The track's
// first packet has no history to warm up from: the codec's pre-skip, trimmed by
// the decoder, is all the priming its encoder declared.
if std::mem::take(&mut self.warmup_pending) && !track_start {
let warmup = moq_net::Timestamp::try_from(self.warmup)?.convert(timestamp.scale())?;
self.warmup_until = Some(timestamp.checked_add(warmup)?);
self.warmup_pending = false;
}
// Codec delay trimmed off the front is media this packet covered even
// though no samples came out, so it still moves the packet after it along.
Expand Down Expand Up @@ -820,6 +830,8 @@ mod tests {
track,
moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio),
);
// Joined mid-stream: the track's first packet needs no warmup.
producer.seek(1).unwrap();
for epoch in 0..2 {
if epoch != 0 && declared {
producer.discontinuity().unwrap();
Expand Down Expand Up @@ -991,6 +1003,8 @@ mod tests {
track,
moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio),
);
// Joined mid-stream: the track's first packet needs no warmup.
producer.seek(1).unwrap();
// One 20 ms packet per group.
let mut write = |packet: u64| {
producer
Expand Down Expand Up @@ -1135,6 +1149,8 @@ mod tests {
track,
moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio),
);
// Joined mid-stream: the track's first packet needs no warmup.
producer.seek(1).unwrap();
producer
.write(moq_mux::container::Frame {
timestamp: Timestamp::from_micros(1_000_000).unwrap(),
Expand Down Expand Up @@ -1171,6 +1187,72 @@ mod tests {
);
}

/// Decodes a finished track of `packets` 20 ms Opus packets starting at group `group`,
/// keeping the encoder's declared warmup, and returns the first frame's stamp and the
/// frames decoded.
async fn opus_track_output(group: u64, packets: u64) -> (Option<u128>, usize) {
let mut encoder = Encoder::new(&EncodeConfig::new(48_000, Layout::Mono)).unwrap();
let catalog = encoder.catalog();
assert_eq!(catalog.warmup, Some(Duration::from_millis(80)));

let broadcast = moq_net::broadcast::Info::new().produce();
let track = broadcast
.create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio))
.unwrap();
let mut producer = moq_mux::container::Producer::new(
track,
moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio),
);
producer.seek(group).unwrap();
let pcm = vec![0.25f32; encoder.frame_size()];
for packet in 0..packets {
producer
.write(moq_mux::container::Frame {
timestamp: Timestamp::from_scale(packet * encoder.frame_size() as u64, 48_000).unwrap(),
duration: None,
payload: encoder.encode(&pcm).unwrap().payload,
keyframe: true,
})
.unwrap();
}
producer.finish().unwrap();

let mut consumer = Consumer::new(
&broadcast.consume(),
&catalog,
"audio",
Options {
delay: Delay {
max: Some(Duration::from_secs(10)),
..Delay::default()
},
..Options::new()
},
)
.await
.unwrap();
let (mut first, mut frames) = (None, 0);
while let Some(frame) = next_frame(&mut consumer).await.unwrap() {
first.get_or_insert(frame.timestamp.as_micros());
frames += frame.data.len() / size_of::<f32>();
}
(first, frames)
}

/// Decoding from the track's first packet drops only the codec's pre-skip: the encoder
/// started there, so there is no history to warm up from. A track shorter than the
/// warmup still decodes.
#[tokio::test(start_paused = true)]
async fn the_tracks_first_packet_keeps_all_but_pre_skip() {
assert_eq!(opus_track_output(0, 2).await, (Some(0), 2 * 960 - 312));
}

/// A join past the track's first group still drops its warmup after the pre-skip.
#[tokio::test(start_paused = true)]
async fn a_mid_stream_join_drops_its_warmup() {
assert_eq!(opus_track_output(1, 6).await, (Some(80_000), 6 * 960 - 312 - 3_840));
}

/// A refused packet primed nothing, so the window starts at the first one that decodes.
#[tokio::test(start_paused = true)]
async fn warmup_starts_at_the_first_decoded_packet() {
Expand Down
4 changes: 2 additions & 2 deletions rs/moq-audio/src/decode/test_data/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ the same. This tests the supported ingest without expanding the container API.

Reference PCM includes all decoded AAC padding and excludes the Opus `dOps`
pre-skip. Backend tests remove exactly the backend's declared delay. Consumer
tests additionally remove the catalog's 80 ms Opus warmup from the reference;
AAC declares no warmup. No correlation search, gain fitting, channel permutation,
tests decode from each track's first packet, which trims no warmup, so they
compare against the whole reference. No correlation search, gain fitting, channel permutation,
or per-run tolerance adjustment is allowed.

The following maxima were measured on x86-64 Linux with the pinned Nix toolchain,
Expand Down
12 changes: 5 additions & 7 deletions rs/moq-audio/tests/integration/roundtrip.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,10 @@ async fn opus_roundtrip(case: Case) {
.await
.unwrap();
producer.finish().unwrap();
let warmup = rendition.warmup.unwrap_or_default();
let skipped = (warmup.as_nanos() * case.rate as u128 / 1_000_000_000) as usize;
// Decoded from the track's first packet, so only the codec's pre-skip is dropped.
let mut pcm = Vec::new();
while let Some(frame) = next_frame(&mut consumer).await.unwrap() {
let expected =
123_000 + warmup.as_micros() + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128;
let expected = 123_000 + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128;
// Microsecond stamping rounds each chunk independently.
assert!(
frame.timestamp.as_micros().abs_diff(expected) <= 1,
Expand All @@ -111,12 +109,12 @@ async fn opus_roundtrip(case: Case) {
}
assert_eq!(
pcm.len(),
(expected_frames - skipped) * channels as usize,
expected_frames * channels as usize,
"pre-skip, resampling and final padding preserve duration"
);
for step in 0..2 {
let start = (step * expected_frames / 2).max(skipped) - skipped;
let end = (step + 1) * expected_frames / 2 - skipped;
let start = step * expected_frames / 2;
let end = (step + 1) * expected_frames / 2;
let half = &pcm[start * channels as usize..end * channels as usize];
for channel in 0..channels as usize {
let samples: Vec<f32> = half.iter().skip(channel).step_by(channels as usize).copied().collect();
Expand Down
8 changes: 5 additions & 3 deletions rs/moq-c/src/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4972,12 +4972,14 @@ fn gray_rgba(width: u32, height: u32) -> Vec<u8> {
/// A finished encoder reaches a C decoder as an end read in microseconds rather
/// than vanishing: Opus pads a partial last frame, so the finish publishes where
/// the source really stopped. The default start decodes the finished track from
/// its first group rather than waiting at its end.
/// its first group rather than waiting at its end, even when it is shorter than
/// the warmup a mid-stream join trims.
#[test]
fn audio_raw_decode_reads_the_encoder_end() {
const START_US: u64 = 123_000;
// Past the 80 ms Opus warmup the decoder trims, ending on a partial frame.
const SAMPLES: usize = 4_700;
// One partial frame, shorter than the 80 ms warmup a mid-stream join trims: decoded
// from the track's first packet, only the codec's pre-skip is dropped.
const SAMPLES: usize = 860;

let origin = id(moq_origin_create());
let path = b"audio-raw-end-test";
Expand Down