diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 59cb816f9af..21f79ac572a 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -49,7 +49,9 @@ group is observed when the stall clears. A missing group, one that has delivered no frame, is waited for only the playout delay plus any lookahead, since past that it would play late: a reordered backlog's holes waited out to the whole budget would delay the frames behind them, which the estimator would measure as -jitter. A caller that estimates keeps the budget past the target, which is why +jitter. The browser player waits less still, only until its clock reaches the +group past the hole: a viewer resuming behind a replayed backlog, which arrives +newest first, would otherwise hold the live audio for the backlog's oldest group. A caller that estimates keeps the budget past the target, which is why `moq play` waits on a stalled audio group for the delay plus the estimate's 2 s ceiling. The observation is also only as honest as the caller is prompt: `read()` is timed when it resolves, so a player drains it diff --git a/js/hang/api/container.api.md b/js/hang/api/container.api.md index 675e857224d..352ed965642 100644 --- a/js/hang/api/container.api.md +++ b/js/hang/api/container.api.md @@ -26,6 +26,7 @@ export class Consumer { export interface ConsumerProps { delay?: Signals.GetterInit; format: Format; + playhead?: Playhead; predecessor?: Consumer; wait?: Signals.GetterInit; warmup?: Net.Clock.Duration; @@ -79,6 +80,11 @@ export * as Loc from "@moq/loc"; export const MAX_AGE: Net.Clock.Duration; +export interface Playhead { + readonly anchor: Signals.Getter; + now(): Net.Clock.Duration | undefined; +} + export type Read = { frame: ReceivedFrame; group: bigint; diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index a159c0c9f9d..293892e4005 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -1,7 +1,8 @@ import { expect, test } from "bun:test"; -import { Duration } from "@moq/clock"; +import { Duration, now } from "@moq/clock"; import { Manual } from "@moq/clock/test"; import { Time, Track } from "@moq/net"; +import { Signal } from "@moq/signals"; import { Consumer, type Read } from "./consumer.ts"; import { encodeFrame, Format } from "./legacy.ts"; @@ -475,3 +476,114 @@ test("a hole is waited for the wait while an arrived group plays up to the max d } } }); + +test.each([ + ["the playout clock reaching the group past it", 900, 100], + ["the wait, while the clock trails", 0, 500], + ["the wait, without a clock", undefined, 500], +])("a hole ends at %s", async (_, start, after) => { + const clock = new Manual(); + const installed = clock.install(); + const origin = now(); + // A playout clock presenting `start` ms of media now, advancing with the manual clock. + const playhead = + start === undefined + ? undefined + : { now: () => Duration.millis(start).add(now().since(origin)), anchor: new Signal(0) }; + const track = new Track.Producer("audio"); + const consumer = new Consumer(track.subscribe({ delay: { max: Duration.millis(30_000) } }), { + format: new Format("audio"), + delay: { max: Duration.millis(2_000) }, + wait: Duration.millis(500), + playhead, + }); + const results: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + results.push(read); + } + })(); + const write = (sequence: bigint, millis: number) => { + const group = track.createGroup(sequence); + group.writeFrame({ + payload: encodeFrame(new Uint8Array([1]), Duration.millis(millis)), + timestamp: Time.Timestamp.fromMicros(millis * 1000), + }); + group.finish(); + }; + try { + write(0n, 0); + await flush(); + expect(results).toHaveLength(1); + // A resume: the groups past the hole start a second of media later, inside the max delay. + write(50n, 1_000); + await flush(); + await clock.advance(Duration.millis(after - 1)); + expect(results).toHaveLength(1); + await clock.advance(Duration.millis(1)); + expect(results.slice(1).map((read) => read.kind)).toEqual(["gap", "frame"]); + } finally { + consumer.close(); + track.finish(); + track.close(); + try { + await reading; + } finally { + installed.close(); + } + } +}); + +test("a hole ends once the playout clock jumps past the group past it", async () => { + const clock = new Manual(); + const installed = clock.install(); + // A clock trailing the group past the hole until it re-anchors a second later. + let position = Duration.ZERO; + const anchor = new Signal(0); + const track = new Track.Producer("audio"); + const consumer = new Consumer(track.subscribe({ delay: { max: Duration.millis(30_000) } }), { + format: new Format("audio"), + delay: { max: Duration.millis(2_000) }, + wait: Duration.millis(500), + playhead: { now: () => position, anchor }, + }); + const results: Read[] = []; + const reading = (async () => { + for (;;) { + const read = await consumer.next(); + if (!read) return; + results.push(read); + } + })(); + const write = (sequence: bigint, millis: number) => { + const group = track.createGroup(sequence); + group.writeFrame({ + payload: encodeFrame(new Uint8Array([1]), Duration.millis(millis)), + timestamp: Time.Timestamp.fromMicros(millis * 1000), + }); + group.finish(); + }; + try { + write(0n, 0); + await flush(); + write(50n, 1_000); + await flush(); + await clock.advance(Duration.millis(10)); + expect(results).toHaveLength(1); + position = Duration.millis(1_000); + anchor.set(1); + await flush(); + expect(results.slice(1).map((read) => read.kind)).toEqual(["gap", "frame"]); + } finally { + consumer.close(); + track.finish(); + track.close(); + try { + await reading; + } finally { + installed.close(); + } + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 870d7e4c781..de02b76c9fe 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -81,6 +81,21 @@ export interface ConsumerProps { * switch neither lowers nor raises {@link Consumer.spread} on its own. */ predecessor?: Consumer; + /** + * The playout clock this reader feeds, such as the player's `Sync` (default: none). A missing + * group whose successor starts at or before the media time it presents now could only arrive + * too late to play, so it is skipped then rather than waited for: a reader resuming behind the + * live edge moves on once the clock passes the hole, not a whole {@link ConsumerProps.wait} later. + */ + playhead?: Playhead; +} + +/** A playout clock, which a {@link Consumer} reads to tell when a missing group could no longer play. */ +export interface Playhead { + /** The media time presenting now, or undefined while nothing presents. */ + now(): Duration | undefined; + /** Changes whenever {@link Playhead.now} jumps rather than advancing with the wall clock: a re-anchor, a new delay, or a pause. */ + readonly anchor: Getter; } /** One item of a {@link Consumer.next} read, in delivery order. Mirrors Rust's `container::Read`. */ @@ -189,6 +204,7 @@ export class Consumer { #format: Format; #delay: Getter; #wait: Getter; + #playhead?: Playhead; #maxBuffer: number; // How much further than the min delay the start reaches, and the age skip tolerates while warming. #warmup: Duration; @@ -279,6 +295,7 @@ export class Consumer { this.#format = props.format; this.#delay = getter(props.delay ?? {}); this.#wait = getter(props.wait); + this.#playhead = props.playhead; this.#maxBuffer = props.maxBuffer ?? DEFAULT_MAX_BUFFER; this.#warmup = (props.warmup ?? Duration.ZERO).min(DELAY_MAX); this.#jitter = props.predecessor ? props.predecessor.#jitter.fork() : new Jitter(); @@ -908,7 +925,8 @@ export class Consumer { // A missing sequence also expires on elapsed monotonic time, so timestamp-free markers // behind a hole can end a paused track without more arrivals. It is late since a group // past it arrived, not since the cursor got here: once the edge stops moving, a backlog - // with every other group missing would otherwise drain one budget per hole. + // with every other group missing would otherwise drain one budget per hole. And once the + // playout clock reaches where the first group past it begins, nothing missing can still play. if (starting === undefined && this.#active !== undefined && this.#groups.length > 0) { const head = this.#groups[0]; if (head.consumer.sequence > this.#active) { @@ -918,19 +936,30 @@ export class Consumer { const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp.asMicros()); const empty = head.empty && head.consumer.done; const wait = this.#holeWait(this.#delay.peek(), this.#wait.peek()); - const bound = this.#groups.find((group) => group.start !== undefined)?.start; + const bound = this.#bound(); const edge = this.#liveEdge(); // Measured against the max delay, not the wait: a reader starved of CPU takes in a burst of // groups at once, and the missing one may land in the same burst. const aged = bound !== undefined && edge !== undefined && edge - bound >= maxMicros(this.#delay.peek()); const elapsed = now().since(this.#hole).asMicros() >= wait; + const passed = this.#passed(bound) === 0; // The ranges ask for nothing in between, so nothing there is owed. const next = this.#track.nextInRange(this.#active); const excluded = next === undefined || next >= head.consumer.sequence; // The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it. const full = this.#full(); - if (startup || empty || contiguous || aged || elapsed || excluded || full || ended !== undefined) { + if ( + startup || + empty || + contiguous || + aged || + elapsed || + passed || + excluded || + full || + ended !== undefined + ) { if (!contiguous && !empty) { this.#lost = true; if (!startup) { @@ -1040,11 +1069,19 @@ export class Consumer { timer = new Effect(); timer.run((effect) => { const delay = effect.get(this.#delay); + // A clock that jumps moves when it reaches the group past a hole. + if (this.#playhead) effect.get(this.#playhead.anchor); // The start waits out its reach, and a hole the wait, which an unlimited one never does. const budget = starting === undefined ? this.#holeWait(delay, effect.get(this.#wait)) : this.#reach(delay); - if (!Number.isFinite(budget)) return; - effect.timer(() => this.#notify?.(), Duration.micros(budget).saturatingSub(now().since(since))); + let left = Number.isFinite(budget) + ? Duration.micros(budget).saturatingSub(now().since(since)) + : undefined; + // A hole also ends once the playout clock reaches the first group past it. + const passing = starting === undefined ? this.#passed(this.#bound()) : undefined; + if (passing !== undefined) left = Duration.micros(Math.min(passing, left?.asMicros() ?? passing)); + if (left === undefined) return; + effect.timer(() => this.#notify?.(), left); }); } @@ -1063,6 +1100,19 @@ export class Consumer { } } + // Where the first stamped buffered group begins, which bounds everything a missing group before it presents. + #bound(): number | undefined { + return this.#groups.find((group) => group.start !== undefined)?.start; + } + + // How long until the playout clock reaches `bound`, in microseconds: zero once it has, undefined + // without a clock that is presenting or without a bound. + #passed(bound: number | undefined): number | undefined { + const playhead = this.#playhead?.now()?.asMicros(); + if (bound === undefined || playhead === undefined) return undefined; + return Math.max(0, bound - playhead); + } + // How long a missing group is waited for, in microseconds: the wait, at most the max delay. #holeWait(delay: Moq.Track.Delay, wait: Duration | undefined): number { const max = maxMicros(delay); diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts index cac63feafac..c98ed9c15e8 100644 --- a/js/hang/src/container/index.ts +++ b/js/hang/src/container/index.ts @@ -7,7 +7,7 @@ /** Re-export of {@link https://www.npmjs.com/package/@moq/loc | @moq/loc}, the LOC container. */ export * as Loc from "@moq/loc"; -export { Consumer, type ConsumerProps, DELAY_MAX, type Read } from "./consumer"; +export { Consumer, type ConsumerProps, DELAY_MAX, type Playhead, type Read } from "./consumer"; export type { Format } from "./format"; /** The legacy varint container. */ export * as Legacy from "./legacy"; diff --git a/js/watch/api/index.api.md b/js/watch/api/index.api.md index f2d29364708..31edc128716 100644 --- a/js/watch/api/index.api.md +++ b/js/watch/api/index.api.md @@ -181,6 +181,7 @@ export * as Signals from "@moq/signals"; export class Sync { constructor(props?: Signals.Inputs); + readonly anchor: Signals.Getter; close(): void; readonly in: Signals.Readonlys; now(): Clock.Duration | undefined; diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index f38fa967a86..cff00137535 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -921,6 +921,7 @@ export class Decoder { wait: this.sync.out.maxDelay, warmup: Duration.millis(config.warmup ?? 0).min(DELAY_MAX), predecessor: continues ? outgoing.consumer : undefined, + playhead: this.sync, }); effect.cleanup(() => consumer.close()); this.#phase.set({ ...phase, consumer: { path, codec: config.codec, consumer } }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index e237929bb7e..457d8304408 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -467,6 +467,8 @@ export class Sync { readonly #clock: Clock; /** Whether buffered startup has released the media consumers. @internal */ readonly playing: Getter; + /** Changes whenever {@link Sync.now} jumps rather than advancing with the wall clock: a re-anchor, a new delay, or a pause. */ + readonly anchor: Getter; #signals = new Effect(); @@ -475,6 +477,10 @@ export class Sync { this.in = this.#clock.in; this.out = readonlys(this.#clock.out); this.playing = this.#clock.playing; + this.anchor = new Derived( + [this.#clock.out.reference, this.#clock.out.delay, this.#clock.playing], + (reference, delay, playing) => ({ reference, delay, playing }), + ); this.#signals.run(this.#runInstant.bind(this)); this.#signals.run(this.#runDelay.bind(this)); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index f4d7d485c28..9f18c74017c 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -725,6 +725,9 @@ class DecoderTrack { format, delay: this.#delay, predecessor: this.consumer ?? this.#predecessor, + // The max-delay decoder reads older groups on purpose, to rebuild references for the clock's + // lane, so only the lane driving the clock skips what the clock passed. + playhead: this.#drives ? this.sync : undefined, }); effect.cleanup(() => consumer.close()); this.consumer = consumer; diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 270764f7085..17ee2df78ac 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -60,5 +60,5 @@ and the chromium `av_lag` rows After #8635 landed, run [43601796176](https://github.com/moq-dev/moq/actions/runs/43601796176) still failed the firefox resume, now on audio arriving late rather than trailing: -[Firefox resume audio](/quest/m0/nightly/firefox-resume-audio.md) owns it. +[moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799) fixed that. diff --git a/quest/m0/nightly/firefox-resume-audio.md b/quest/m0/nightly/firefox-resume-audio.md deleted file mode 100644 index b1a28a68b6a..00000000000 --- a/quest/m0/nightly/firefox-resume-audio.md +++ /dev/null @@ -1,36 +0,0 @@ -# [M] Firefox audio resumes with the picture - -## Goal - -Nightly's `Media (firefox player)` job passes its `after resume` window: after -a pause, audio is audible within the window's first samples, as in Chromium, -instead of staying silent for about half a second after the picture resumes. - -## Plan - -[moq-dev/moq#8635](https://github.com/moq-dev/moq/pull/8635) removed the -1.7 s trail behind the picture, but Nightly -[43601796176](https://github.com/moq-dev/moq/actions/runs/43601796176) (at -4fec611d4, after it and the [lag rule](https://github.com/moq-dev/moq/pull/8722)) -still failed with `68/79 samples carried the fixture tone` (11-13: 60/78). The -window's first 11 samples are silent with the audio byte counter flat -(`audio=94041B`) while video presents live frames; then about 1.8 KB of -audio lands in one burst, `[player] skipping missing groups: track=audio -1165 -> 1258` fires, and audio plays in sync. So the audio subscription's -first groups reach the Firefox player about 0.4 s after video's. - -Not reproduced locally: `just test media --media-engines firefox ---media-cases rename,pause` passed 4 of 4 on an idle host (2026-11-14). The -CI host was 14 to 60% idle during the window, and Firefox dials the relay -over WebSocket, so the difference is likely in that path under load. - -Recommended: reproduce under load first (the job's own -`busiest host processes` table shows the shape), with the relay's and -player's logs kept (`HARNESS_KEEP=1`), and find which hop holds audio's -first groups after the resubscribe: the relay's upstream re-subscribe and -gap backfill (the fixture logs a fetch of audio group 1156 that the relay -aborts), the WebSocket session's stream scheduling, or the consumer's hole -wait from its pre-pause cursor (1165). Fix that hop; do not widen the test's -window or agreement. - -Kept in m0 by the maintainer (2026-11-15): ✅ m0 (rejected: m1, drop). diff --git a/quest/m2/audio-playout/rust-playhead-holes.md b/quest/m2/audio-playout/rust-playhead-holes.md new file mode 100644 index 00000000000..a17023ed1dd --- /dev/null +++ b/quest/m2/audio-playout/rust-playhead-holes.md @@ -0,0 +1,23 @@ +# [M] Native players stop waiting on a hole their clock passed + +## Goal + +`moq play` skips a missing group once its presentation clock reaches the +group past it, as the browser player does since +[moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799), instead of +holding the frames behind the hole for the whole hole wait. + +## Plan + +The browser's container consumer takes the player's clock as +`ConsumerProps.playhead` (`js/hang/src/container/consumer.ts`): a hole whose +successor starts at or before the media time presenting now ends then, and the +hole's timer also wakes when the clock gets there. Mirror it in +`rs/moq-mux/src/container/consumer.rs` (`poll_event_once`'s missing-group +branch and its `hole_timer`), reading `moq_mux::sync::Consumer::position`, and +pass the clock from `moq play`'s presentation through +`moq_audio::decode::Consumer` and the video reader in +`rs/moq-cli/src/play/media.rs`. + +Exercise it with the same shape the browser hit: a reader that delivered a +pre-pause tail, then a replayed backlog arriving newest first behind a hole.