diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 31b6a6f9643..f39a9dc012b 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -499,11 +499,48 @@ them. The cost is over-buffering by the smallest subscribed `delay` when the broadcast's earliest rendition is not subscribed. The playout is recomputed when a rendition is subscribed, unsubscribed, or its -catalog entry rises, so dropping a slow rendition lowers latency. Each track's -own arrivals only ever move its part of the shared reference earlier. A track +catalog entry rises, so dropping a slow rendition lowers latency. A track's +arrivals move its part of the shared reference earlier at once, and later only +after [a lasting step](#a-lasting-step). A track that leaves, or whose timeline restarts, releases its part: the earliest remaining track's anchor takes over, or the next frame re-anchors. +### A lasting step + +The estimator forgets a lasting increase in path delay within `WINDOW`, since +it measures each frame against the fastest of the last two seconds. An anchor +that only moved earlier would not: every frame after the step would be late for +it by the step, and once the target fell back, audio would underrun on it and +video would show each frame late. + +So each track's anchor follows its arrivals later on sustained evidence. A +track's arrivals fall into back-to-back windows of `WINDOW` of arrival time, the +arrival that closes one counting in it and opening the next. When a window +closes and every arrival in it trailed the track's anchor by more than **half +the delay**, the anchor moves later to the window's earliest arrival. The +reference is still the earliest anchor, so it moves fully once every arriving +track has followed; a track that stops arriving keeps its anchor and holds the +reference where it was. + +Half the delay is the deadband because a smaller trail is not worth a move: it +spends at most half the buffer, and a noisy path's fastest frame per window +wanders by far less than the target that same spread sizes. Replayed through the +Nightly's captured arrivals, a 10 or 20 ms deadband moved the reference two to +seven times a minute on `wide`, and a deadband of the whole delay never moved it +on the Opus `step` row, whose 55 ms step matched its 60 ms target. An outage and +its backlog span no whole window, so they never move it; a congestion that +holds every frame past half the delay for a whole window is a step for as long +as it lasts, and the first frame through after it moves the anchor back. + +A move is lag like any other: audio slows down within its stretch threshold or +pads past it, and video holds a frame. No latency is added to absorb a step +before it is evidence, so the frames between the step and the move are late. + +The `clock/corpus.json` cases hold `js/watch` `Sync` and `moq_mux::sync` to the +same reference series. Each case fixes the delay and plays runs of arrivals, +`frames` of them `every` milliseconds from `start`, cycling through `tracks` and +the arrival-minus-media `offsets`, and gives the `reference` offset after each run. + A track that replaces another in the same slot and on the same broadcast path, such as the new name a codec change publishes under, continues the outgoing track's histogram, prior weight included, instead of starting over. The path's diff --git a/doc/concept/audio-jitter/clock/corpus.json b/doc/concept/audio-jitter/clock/corpus.json new file mode 100644 index 00000000000..78c30cef9d3 --- /dev/null +++ b/doc/concept/audio-jitter/clock/corpus.json @@ -0,0 +1,52 @@ +[ + { + "name": "step", + "about": "A lasting step later moves the reference once a whole window trails it by more than half the delay, and a faster frame moves it back at once", + "delay": 60, + "runs": [ + { "tracks": ["audio"], "start": 0, "every": 20, "frames": 150, "offsets": [5], "reference": 5 }, + { "tracks": ["audio"], "start": 3000, "every": 20, "frames": 150, "offsets": [60], "reference": 5 }, + { "tracks": ["audio"], "start": 6000, "every": 20, "frames": 1, "offsets": [60], "reference": 60 }, + { "tracks": ["audio"], "start": 6020, "every": 20, "frames": 50, "offsets": [60], "reference": 60 }, + { "tracks": ["audio"], "start": 7020, "every": 20, "frames": 1, "offsets": [40], "reference": 40 } + ] + }, + { + "name": "shallow", + "about": "A trail within half the delay leaves the reference where it is", + "delay": 60, + "runs": [ + { "tracks": ["audio"], "start": 0, "every": 20, "frames": 100, "offsets": [5], "reference": 5 }, + { "tracks": ["audio"], "start": 2000, "every": 20, "frames": 400, "offsets": [35, 30, 50], "reference": 5 } + ] + }, + { + "name": "spike", + "about": "An outage and the backlog behind it span no whole window, so the reference stays", + "delay": 60, + "runs": [ + { "tracks": ["audio"], "start": 0, "every": 20, "frames": 150, "offsets": [5], "reference": 5 }, + { "tracks": ["audio"], "start": 4500, "every": 5, "frames": 300, "offsets": [600], "reference": 5 }, + { "tracks": ["audio"], "start": 6000, "every": 20, "frames": 150, "offsets": [5], "reference": 5 } + ] + }, + { + "name": "tracks", + "about": "Each track's anchor follows its own window, and the reference is the earliest anchor, so it moves fully only once every track has followed", + "delay": 60, + "runs": [ + { "tracks": ["video", "audio"], "start": 0, "every": 10, "frames": 300, "offsets": [0, 10], "reference": 0 }, + { "tracks": ["video", "audio"], "start": 3000, "every": 10, "frames": 300, "offsets": [60, 70], "reference": 0 }, + { "tracks": ["video", "audio"], "start": 6000, "every": 10, "frames": 100, "offsets": [60, 70], "reference": 60 } + ] + }, + { + "name": "silent", + "about": "A track that stops arriving keeps its anchor, and the reference with it", + "delay": 60, + "runs": [ + { "tracks": ["video", "audio"], "start": 0, "every": 10, "frames": 300, "offsets": [0, 10], "reference": 0 }, + { "tracks": ["audio"], "start": 3000, "every": 20, "frames": 300, "offsets": [70], "reference": 0 } + ] + } +] diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 9e5cf9f09ef..6bce223ed22 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -1,4 +1,7 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { Duration, type Installed, Instant, now, yieldNow } from "@moq/clock"; import { Manual } from "@moq/clock/test"; @@ -658,3 +661,43 @@ describe("tracks", () => { sync.close(); }); }); + +describe("clock corpus", () => { + type Run = { + tracks: ("audio" | "video")[]; + start: number; + every: number; + frames: number; + offsets: number[]; + reference: number; + }; + type Case = { name: string; delay: number; runs: Run[] }; + + // Shared with moq-mux, so neither implementation is graded against the other. + const corpus = join(dirname(fileURLToPath(import.meta.url)), "../../../doc/concept/audio-jitter/clock/corpus.json"); + const cases = JSON.parse(readFileSync(corpus, "utf8")) as Case[]; + + // Arrivals start this far after the clock origin and media this far into the track, so neither is negative. + const ARRIVAL = 5000; + const MEDIA = 100_000; + + for (const { name, delay, runs } of cases) { + it(`follows the corpus reference: ${name}`, async () => { + const sync = new Sync({ delay: Duration.millis(delay) }); + const held = () => ({ target: new Signal(undefined) }); + const tracks = { audio: sync.track(held()), video: sync.track(held()) }; + await flush(); + for (const [index, run] of runs.entries()) { + for (let i = 0; i < run.frames; i++) { + const arrival = run.start + i * run.every; + const offset = run.offsets[i % run.offsets.length]; + const track = tracks[run.tracks[i % run.tracks.length]]; + track.received(Duration.millis(MEDIA + arrival - offset), at(ARRIVAL + arrival)); + } + const reference = (sync.out.reference.peek() ?? Number.NaN) - ARRIVAL + MEDIA; + expect([index, reference]).toEqual([index, run.reference]); + } + sync.close(); + }); + } +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index a162d5a3304..213b4d6881c 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -22,6 +22,10 @@ export const STRETCH = Duration.millis(200); /** The most an `"auto"` delay holds, and how far catalog `warmup` reaches back. @internal */ export const DELAY_MAX = Container.DELAY_MAX; +// How long a track's arrivals must all trail its anchor before the anchor follows them later: the +// estimator's window, so the clock and the target measure lateness against the same fastest frame. +const FOLLOW = 2000; + /** Signals the sync reads. */ export type SyncInput = { /** How far playback trails the live edge. See {@link Delay}. */ @@ -66,7 +70,8 @@ export type SyncOutput = { // The earliest time we've received a frame, relative to its timestamp: arrival minus timestamp // in milliseconds, an offset that is negative when the timestamps run ahead of the local clock. - // This will keep being updated as we catch up to the live playhead then will be relatively static. + // This will keep being updated as we catch up to the live playhead then will be relatively static, + // moving later only after a lasting step in path delay (see doc/concept/audio-jitter.md). // Each track holds its own part of it; see `Track.reset()`. reference: Signal; @@ -107,8 +112,13 @@ export type Part = { anchor?: number; // The furthest playhead this track's own skips jumped to, restored when it takes the reference. floor?: number; + // The window of arrivals that decides whether the anchor follows a lasting step later. + window?: Window; }; +/** A window of one track's arrivals: when it opened and the earliest reference among them. @internal */ +export type Window = { start: number; least: number }; + /** * Where the clock's current timeline stands. `idle` has not played and holds nothing back: it is * unanchored, or anchored with no buffered startup. `starting` holds the media consumers until every @@ -171,11 +181,33 @@ export class Clock { return ref + cap - delay; } + // Move a track's anchor later once a whole window of its arrivals trailed it by more than half the + // delay, as after a lasting increase in path delay. Reports whether the anchor moved. + follow(part: Part, at: number, ref: number): boolean { + const window = part.window; + if (!window) { + part.window = { start: at, least: ref }; + return false; + } + window.least = Math.min(window.least, ref); + if (at - window.start < FOLLOW) return false; + + // The arrival that closes a window counts in it and opens the next. + part.window = { start: at, least: ref }; + if (part.anchor === undefined || window.least - part.anchor <= this.out.delay.peek().asMillis() / 2) { + return false; + } + part.anchor = window.least; + return true; + } + received(part: Part, timestamp: Duration, arrival: Instant): boolean { this.out.timestamp.update((current) => (current === undefined || timestamp.gt(current) ? timestamp : current)); const ref = millis(arrival) - timestamp.asMillis(); const current = this.out.reference.peek(); part.anchor = this.fold(part.anchor, ref); + // Startup holds the opening still, so only a playing clock follows a step. + const followed = this.playing.peek() && this.follow(part, millis(arrival), ref); // First frame anchors the reference. if (current === undefined) { @@ -195,7 +227,12 @@ export class Clock { const late = current - ref + delay < 0; const reference = this.fold(current, ref); - if (reference === current) return late; + if (reference === current) { + // The owner's anchor moving later moves the reference to the earliest anchor left, which only + // a whole window of every track's arrivals trailing it moves. + if (followed && this.owner === part) this.handoff(current); + return late; + } this.out.reference.set(reference); this.owner = part; @@ -210,19 +247,35 @@ export class Clock { return late; } + // The track with the earliest anchor, which holds the reference. + earliest(): Part | undefined { + let earliest: Part | undefined; + for (const other of this.parts.peek()) { + if (other.anchor === undefined) continue; + if (earliest?.anchor === undefined || other.anchor < earliest.anchor) earliest = other; + } + return earliest; + } + + // The owner's anchor moved later: the earliest anchor now holds the reference, never earlier than + // `current`, since every other track's arrivals already folded into it. + handoff(current: number): void { + const earliest = this.earliest(); + if (earliest?.anchor === undefined) return; + this.owner = earliest; + if (earliest.anchor > current) this.out.reference.set(earliest.anchor); + } + // End one track's timeline: its parked frames drop and it releases its part of the reference. end(part: Part): void { part.epoch.update((epoch) => epoch + 1); part.anchor = undefined; part.floor = undefined; + part.window = undefined; if (this.owner !== part) return; // The earliest remaining track holds the reference, else the next frame anchors a new one. - let earliest: Part | undefined; - for (const other of this.parts.peek()) { - if (other.anchor === undefined) continue; - if (earliest?.anchor === undefined || other.anchor < earliest.anchor) earliest = other; - } + const earliest = this.earliest(); this.owner = earliest; if (earliest?.anchor === undefined) { this.unanchor(); @@ -244,6 +297,7 @@ export class Clock { // Clear the reference so the next frame anchors a new timeline. Every parked frame belongs to // the old one. unanchor(): void { + for (const part of this.parts.peek()) part.window = undefined; this.epoch.update((epoch) => epoch + 1); this.floor = undefined; this.owner = undefined; diff --git a/quest/m2/audio-playout/README.md b/quest/m2/audio-playout/README.md index 13321dddbd4..ad9cb4a7c1f 100644 --- a/quest/m2/audio-playout/README.md +++ b/quest/m2/audio-playout/README.md @@ -94,8 +94,9 @@ shared vectors for both players. Order: the 200 ms snap both ways and the [lag rule](https://github.com/moq-dev/moq/pull/8722), a continuous stretch -that replaced the NetEq controller and splices under it, are done, and the -[clock step](/quest/m2/audio-playout/clock-step.md) builds on the lag rule. +that replaced the NetEq controller and splices under it, are done, as is the +shared clock following a lasting path-delay step +([moq-dev/moq#8749](https://github.com/moq-dev/moq/pull/8749)). The estimator stops folding a join's intervals ([moq-dev/moq#8730](https://github.com/moq-dev/moq/pull/8730)); [output latency](/quest/m2/audio-playout/output-latency.md) and the diff --git a/quest/m2/audio-playout/clock-step.md b/quest/m2/audio-playout/clock-step.md deleted file mode 100644 index 42be3d1d129..00000000000 --- a/quest/m2/audio-playout/clock-step.md +++ /dev/null @@ -1,38 +0,0 @@ -# [M] The shared clock follows a lasting path-delay step - -## Goal - -After a lasting increase in path delay, audio and video settle on the shared -clock at the playout delay again, in the browser and natively, instead of -playing against a reference that never moves later. The `step` rows of the -audio-quality harness rest within a frame of the clock after the step, with -no more underruns than before it. - -## Plan - -Decided 2026-11-11 (maintainer, /quest-merge pick of -[moq-dev/moq#8507](https://github.com/moq-dev/moq/pull/8507)): keep. -Re-scoped 2026-11-13 (maintainer, /quest-plan) to the -[audio playout](/quest/m2/audio-playout/README.md) rule, under which a move -of the clock is lag like any other. - -The reference is the earliest lateness any track has shown, and arrivals only -move it earlier (`js/watch/src/sync.ts`, `rs/moq-mux/src/sync.rs`), while the -estimator measures each frame against the fastest of the last 2 s. After a -lasting step the estimate forgets the step within its window, but the clock -keeps the old reference, so every frame is late for it. On the 2026-11-09 -rule audio rested behind: at #8507's head the `step` rows rested 35.2 and -41.8 ms behind in the browser (AAC, Opus) and 5.7 and 17.3 ms natively; a -simulated +200 ms step left native audio 150 to 160 ms behind. Under the -[lag rule](https://github.com/moq-dev/moq/pull/8722) audio follows the clock -instead, so the same stale reference shows as underruns. - -Recommendations: - -- Let the reference move later only on sustained evidence, such as the - least lateness over a window well past the target staying past it. The - move is lag like any other: audio slows down within the threshold or pads - past it, and video holds a frame. -- Keep one rule in `js/watch` `Sync` and `moq_mux::sync`, with a shared test - vector, as the estimator's corpus does. -- Do not add latency to reach zero underruns on the step. diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs index 6781e853d51..a311453d443 100644 --- a/rs/moq-mux/src/sync.rs +++ b/rs/moq-mux/src/sync.rs @@ -3,7 +3,8 @@ //! Each [`Track`] folds its arrivals into the clock with [`Track::received`], //! and [`Producer::due`] says when media is due. The clock's reference is the earliest //! arrival any live track has seen, measured against its timestamp, so it sits -//! on the wall clock and follows the live edge. Every track presents against +//! on the wall clock and follows the live edge, moving later only after a lasting +//! step in path delay (see `doc/concept/audio-jitter.md`). Every track presents against //! it, which holds them together in the steady state. One that falls behind, //! such as audio stalled by congestion, only arrives late: it never holds or //! moves the others. Audio converges on the clock by stretching toward its @@ -37,6 +38,11 @@ use crate::container::Kind; /// One slot per [`Kind`]. const KINDS: usize = 3; +/// How long a track's arrivals must all trail its anchor before the anchor +/// follows them later: the estimator's window, so the clock and the target +/// measure lateness against the same fastest frame. +const FOLLOW: Duration = Duration::from_secs(2); + struct Clock { state: Mutex, changed: kio::Shared<()>, @@ -55,6 +61,39 @@ impl Clock { } } +impl Part { + /// Fold `arrival` into the open window, and once it closes move the anchor + /// later to the window's earliest arrival if every arrival in it trailed the + /// anchor by more than `half`, as after a lasting increase in path delay. + fn follow(&mut self, arrival: Anchor, half: Duration) { + let Some(mut window) = self.window else { + self.window = Some(Window { + start: arrival.at, + least: arrival, + }); + return; + }; + if arrival.earlier(&window.least) { + window.least = arrival; + } + // The arrival that closes a window counts in it and opens the next. + if arrival.at.saturating_duration_since(window.start) < FOLLOW { + self.window = Some(window); + return; + } + self.window = Some(Window { + start: arrival.at, + least: arrival, + }); + let Some(anchor) = self.anchor else { return }; + let Some(at) = anchor.at.checked_add(half) else { return }; + let bound = Anchor { at, ..anchor }; + if bound.earlier(&window.least) { + self.anchor = Some(window.least); + } + } +} + /// A schedule pinning media time to the wall clock: `media` arrived at `at`. #[derive(Clone, Copy, Debug)] struct Anchor { @@ -95,10 +134,20 @@ fn moved(before: Option, after: Option) -> bool { /// One track's hold on the clock. #[derive(Clone, Copy, Default)] struct Part { - /// The earliest arrival on this track's current timeline. + /// The earliest arrival on this track's current timeline, or the earliest + /// of the last window once a whole window trailed it. anchor: Option, /// The delay this track asks playback to hold. target: Option, + /// The window of arrivals that decides whether the anchor follows a lasting step later. + window: Option, +} + +/// A window of one track's arrivals: when it opened and the earliest schedule among them. +#[derive(Clone, Copy)] +struct Window { + start: Instant, + least: Anchor, } /// The playout clock every track of one broadcast presents against. @@ -106,8 +155,10 @@ struct Part { /// Each track keeps the earliest arrival on its own timeline, and the reference /// is the earliest of those. That is what pulls playback toward live: a frame /// arriving earlier than the reference predicted re-pins it, while a late -/// frame only reports that it is late. A track that resets or goes away gives -/// up its arrivals, so the reference falls back to the earliest track left. +/// frame only reports that it is late. A lasting step later in the path's +/// delay moves a track's anchor later once a whole window of its arrivals +/// trailed it by more than half the delay. A track that resets or goes away +/// gives up its arrivals, so the reference falls back to the earliest track left. struct State { /// How far behind the reference playback runs while no track holds a target. delay: Duration, @@ -196,10 +247,12 @@ impl State { media: Duration::from(timestamp), }; let before = self.reference(); + let half = self.delay() / 2; let part = &mut self.parts[kind as usize]; if part.anchor.is_none_or(|anchor| arrival.earlier(&anchor)) { part.anchor = Some(arrival); } + part.follow(arrival, half); self.left = None; moved(before, self.reference()) } @@ -207,7 +260,9 @@ impl State { /// End `kind`'s timeline, reporting whether that moved the schedule. fn reset(&mut self, kind: Kind) -> bool { let before = self.reference(); - self.parts[kind as usize].anchor = None; + let part = &mut self.parts[kind as usize]; + part.anchor = None; + part.window = None; moved(before, self.reference()) } @@ -871,6 +926,62 @@ mod tests { (clock, audio, video) } + #[derive(serde::Deserialize)] + struct Case { + name: String, + delay: u64, + runs: Vec, + } + + #[derive(serde::Deserialize)] + struct Run { + tracks: Vec, + start: u64, + every: u64, + frames: u64, + offsets: Vec, + reference: u64, + } + + /// Each case plays runs of arrivals and checks where the reference sits after each. + #[test] + fn the_reference_conforms_to_the_corpus() { + // Media starts this far into the track, so it is never negative. + const MEDIA: u64 = 100_000; + let cases: Vec = + serde_json::from_str(include_str!("../../../doc/concept/audio-jitter/clock/corpus.json")).unwrap(); + let base = Instant::ZERO.saturating_add(Duration::from_secs(5)); + for case in cases { + let delay = Duration::from_millis(case.delay); + let clock = Producer::new(delay); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + for (index, run) in case.runs.iter().enumerate() { + for i in 0..run.frames { + let arrival = run.start + i * run.every; + let offset = run.offsets[(i as usize) % run.offsets.len()]; + let track = match run.tracks[(i as usize) % run.tracks.len()].as_str() { + "audio" => &audio, + "video" => &video, + other => panic!("unknown track {other}"), + }; + track.received( + ms(MEDIA + arrival - offset), + base.saturating_add(Duration::from_millis(arrival)), + ); + } + // The reference's offset is when media `MEDIA` is due, less the delay, after `base`. + let due = clock.due(ms(MEDIA)).unwrap().saturating_sub(delay); + assert_eq!( + due.duration_since(base), + Duration::from_millis(run.reference), + "{} run {index}", + case.name + ); + } + } + } + /// Earlier arrivals pull the reference forward so playback converges on the /// live edge without giving up the delay; late ones leave it alone. #[test]