From 1c53c4b843719a6077b1936d63b7b161ce78ce9e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:59:38 -0800 Subject: [PATCH 01/12] chore(quest): claim quest/m1/audio-jitter-target/stretch-setting.md Quest-Claim: 7b6221766e739af0 From de73f8d746959d5c69934672b3f546a39ab91a95 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:09:37 -0800 Subject: [PATCH 02/12] wip: stretch setting --- doc/bin/cli.md | 13 ++- doc/concept/audio-jitter.md | 35 +++--- doc/lib/js/watch.md | 31 ++++-- js/watch/api/element.api.md | 7 +- js/watch/api/index.api.md | 2 + js/watch/src/audio/decoder.test.ts | 34 +++++- js/watch/src/audio/decoder.ts | 37 ++++--- js/watch/src/audio/latency.ts | 17 ++- js/watch/src/audio/playout.test.ts | 14 ++- js/watch/src/audio/render.ts | 2 +- js/watch/src/audio/replay.test.ts | 38 +++++-- js/watch/src/audio/replay.ts | 18 ++- js/watch/src/audio/ring-buffer.test.ts | 62 ++++++++++- js/watch/src/audio/ring-buffer.ts | 24 +++- js/watch/src/element.ts | 38 ++++++- js/watch/src/player.ts | 7 +- js/watch/src/sync.ts | 21 +++- rs/moq-audio/src/decode/consumer.rs | 52 +++++++++ rs/moq-audio/src/playback/sink.rs | 11 +- rs/moq-audio/src/ring.rs | 109 +++++++++++++++++-- rs/moq-cli/src/play/args.rs | 54 +++++---- rs/moq-cli/src/play/fake.rs | 2 +- rs/moq-cli/src/play/headless.rs | 3 + rs/moq-cli/src/play/media.rs | 31 +++++- rs/moq-cli/src/play/output.rs | 11 +- rs/moq-cli/src/play/video.rs | 1 + test/audio-quality/clients/js/src/capture.ts | 33 +++--- test/audio-quality/clients/js/src/probe.ts | 3 +- 28 files changed, 559 insertions(+), 151 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index e8b271e7efe..3436bb7fffa 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -174,6 +174,7 @@ MPEG-TS export is covered in the [MPEG-TS guide](/bin/ts#export). ```bash moq --connect https://relay.example.com/anon --broadcast my-stream.hang play moq ... play --delay 500ms # fix the delay instead of measuring it +moq ... play --stretch 200ms # let audio trail the clock by at most 200 ms moq ... play --video off # audio only, without a window moq ... play --video off --metrics play.ndjson # log what the speaker played ``` @@ -188,10 +189,14 @@ names the decoder each track opened. Playback starts at the newest group and trails the live edge by `--delay`. The default, `auto`, sizes the buffer to how unevenly audio arrives, with the same [algorithm](/concept/audio-jitter) as the browser player. A duration fixes the -delay and is also how long a stalled group is waited on before it is skipped. -Audio and video play from one clock and stay in sync while both arrive on time; -a track that stalls plays late and catches up by playing faster, rather than -holding the other one back. +delay instead. Audio and video play from one clock and stay in sync while both +arrive on time. After a stall, video shows each late frame as it arrives, while +audio slows down and falls behind the clock rather than dropping what arrives +late, then speeds up until it is back on it. `--stretch` (default 2 s) bounds +how far audio may trail before it skips ahead, and a stalled group is waited on +for the delay plus the stretch before it is skipped. `--stretch 0` plays +received audio unaltered: late audio is skipped, an underrun is silence, and +drift against the publisher's clock is corrected by a small skip or pad. Each role follows the catalog, switching rendition when the publisher retires the one playing. With `--video off`, `--metrics ` writes a JSON line every 250 ms with the playhead, the delay, every gap and stall the speaker hit, and diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 443e0877bc0..cf95f3057d7 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -43,8 +43,8 @@ Natively that holds only as far as the caller lets it. The container consumer reads groups in sequence order and applies its age budget inside `read()`, so a group skipped for age is never observed, and a frame queued behind a stalled group is observed when the stall clears. A caller that estimates keeps the budget -at least as long as the estimate's ceiling, which is why `moq play` waits 2 s on -a stalled group under `--delay auto`. The observation is also only as honest as +past the target, which is why `moq play` waits on a stalled group for the delay +plus its `--stretch`, 2 s by default. The observation is also only as honest as the caller is prompt: `read()` is timed when it resolves, so a player drains it as packets come and lets the playback buffer hold the target, rather than pacing reads to the speaker. @@ -54,10 +54,12 @@ drop is not the receiver's own doing. The subscription's max delay drops groups before any container sees them, at the relay and in the receiver's own transport subscriber, so a subscription cut to the target hides every frame later than the target: the estimate creeps up one bucket at a time while the frames it -should have measured go unplayed. In automatic mode both implementations -subscribe with at least the ceiling, 2 s: `moq play --delay auto` through its -age budget, and the browser through the audio subscription alone, since its -container consumer observes each frame before applying the local budget. +should have measured go unplayed. Both implementations subscribe with the delay +plus the player's `stretch` setting, 2 s by default, past the estimate's own +ceiling: `moq play` through its age budget, and the browser through the audio +subscription alone, since its container consumer observes each frame before +applying the local budget. A zero stretch cuts the subscription to the target +and accepts that creep. Browser video keeps the shared budget, because a longer one fetches whole stale groups, so a broadcast with no audio measures only the lateness that budget lets through. @@ -672,9 +674,9 @@ At each window boundary a player decides on five inputs: - `lag`: how far the playhead trails the shared clock, negative when it plays ahead of it. - `bound`: how far audio may trail the clock, which is the subscription's max - delay less the playout delay. It is zero at a fixed live delay, the lookahead - for a buffered player, and at `auto` the shared max delay raised to the - estimator's 2 s ceiling, less the delay. + delay less the playout delay: the player's `stretch` setting (2 s by + default), or for a buffered player the larger of it and the lookahead. A zero + stretch turns the stretch off; see [below](#skips-underruns-and-silence). `level` passes through NetEq's `BufferLevelFilter` once per window: three of its 10 ms steps at the factor the target selects, 251, 252, 253, or 254 in 256ths @@ -713,10 +715,9 @@ and slowing down lets audio fall behind by at most `bound`, or brings a playhead ahead of the clock back onto it. These are NetEq's thresholds and rates, untuned. A moved target moves the clock, so a rise leaves audio ahead of it and a fall leaves audio behind, and both become ordinary slowing down and -speeding up: nothing parks in silence on a rise, and in `auto` nothing skips -buffered audio on a fall, since the bound, the max delay less the target, -always covers a fall of the target. A fixed delay lowered at runtime by more -than its bound and one window skips to the bound, as any lag past it does. +speeding up: nothing parks in silence on a rise, and nothing skips buffered +audio on a fall shorter than the bound and one window. A fall past that skips +to the bound, as any lag past it does. Without a clock reading the target is where the playhead belongs, so `lag = level - target`. The browser then never skips; native playback bounds @@ -759,6 +760,14 @@ next frame while what it was given, landed or still decoding, reaches less than its floor and one window past the playhead, so the ring always holds a full window to stretch, and a hole in the media plays as silence. +A zero `stretch` turns the stretch off, for content where a pitch-period splice +is audible, such as music. Every window holds, so received audio plays +unaltered, and the bound is zero: a playhead trailing the clock by more than one +window skips onto it, and one more than a window ahead of it pads silence until +the clock reaches it. Late audio is skipped, an underrun is silence, and drift +between the publisher's clock and the device's is corrected by a skip or a pad +of about one window. + ### Pitch search and admission Use channel zero as NetEq does. All channels share the selected seam and lag, diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index cfb5510b6f0..1993c1067a8 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -34,6 +34,7 @@ in sync at the latency you ask for. | `paused`, `muted`, `volume` | The usual player controls. | | `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames arrive; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint video as it decodes, with no audio. | | `lookahead` | How far ahead of the live edge media may run; see [buffered playback](#buffered-playback). | +| `stretch` | How far audio may trail the clock before it skips ahead: a duration like `"2s"` (the default), or `"0"` to turn the time stretch off; see [catching up](#catching-up). | | `captions` | The caption track to show. | | `visible` | Only download video while the element is on screen (or near it). | | `backend` | How the canvas is drawn; see [@moq/video](/lib/js/video). `auto` (default), `webgpu`, or `2d`. | @@ -218,16 +219,31 @@ leave the connection's estimate unclaimed. Without a claim, video selects the be `video.target` allows. To stop the relay probing while still adapting, keep the claim and call `connection.viewers.setTarget(0)`. +## Catching up + +After a stall, video shows each late frame as it arrives and is live again at +once. Audio cannot skip without losing what arrives late, so it slows down +slightly when the audio received ahead of it runs low, falls behind the clock, +and speeds back up once it recovers until it is back on it. `stretch` bounds +how far it may trail: only audio later than that is skipped, and the audio +subscription waits on a stalled group for `delay + stretch`, so the late audio +still arrives. A voice app might ask for `stretch="200ms"`, a sports stream for +the default 2 s, whether `delay` is `auto` or fixed. + +`stretch="0"` turns the time stretch off, for content where a pitch-period +splice is audible, such as music: received audio plays unaltered, late audio +is skipped, an underrun is silence, and drift between the publisher's clock +and the device's is corrected by a small skip or pad. + ## Buffered playback -By default the player minimizes latency: audio plays at the delay, slows down -slightly when the audio received ahead of it runs low, speeds back up once it -recovers, and skips ahead only audio trailing by more than the subscription -still delivers. Content produced faster than real time, such as a TTS +By default the player minimizes latency: playback trails the live edge by the +delay, and audio catches up after a stall as [above](#catching-up). Content produced faster than real time, such as a TTS response emitted in one burst with future timestamps, wants the opposite. Set `lookahead` to how far ahead it may run and it plays through at the encoded pace. If a stall leaves audio behind, it catches up by playing faster, as live -playback does, and skips only what trails by more than the lookahead: +playback does, and skips only what trails by more than the larger of the +lookahead and `stretch`; a zero `stretch` still turns the stretch off: ```html @@ -237,8 +253,9 @@ playback does, and skips only what trails by more than the lookahead: `delay`, since playback only skips ahead once media sits more than `delay + lookahead` past the playhead. -Durations need a unit; a bare number is rejected. From script, the `delay` and -`lookahead` properties take a `Duration` from `@moq/clock`, such as +Durations need a unit; a bare number other than zero is rejected. From script, +the `delay`, `lookahead`, and `stretch` properties take a `Duration` from +`@moq/clock`, such as `el.delay = Duration.millis(300)`. Audio holds the lookahead as encoded frames, so a large one is cheap; video waits as decoded pictures, which hold decoder memory, so a long video lookahead is not. The first video diff --git a/js/watch/api/element.api.md b/js/watch/api/element.api.md index fa7c30e272a..2455e4f1353 100644 --- a/js/watch/api/element.api.md +++ b/js/watch/api/element.api.md @@ -15,7 +15,7 @@ import { Video } from "@moq/watch"; export default class extends HTMLElement { constructor(); - attributeChangedCallback(name: "delay" | "lookahead" | "url" | "echo" | "name" | "follow" | "volume" | "muted" | "paused" | "visible" | "backend" | "versions" | "announced" | "buffer" | "catalog-format" | "captions", oldValue: string | null, newValue: string | null): void; + attributeChangedCallback(name: "delay" | "lookahead" | "stretch" | "url" | "echo" | "name" | "follow" | "volume" | "muted" | "paused" | "visible" | "backend" | "versions" | "announced" | "buffer" | "catalog-format" | "captions", oldValue: string | null, newValue: string | null): void; readonly audio: Audio.Decoder; get backend(): Video.Backend; set backend(value: Video.Backend); @@ -37,6 +37,7 @@ export default class extends HTMLElement { delay: Signals.Signal; lookahead: Signals.Signal; paused: Signals.Signal; + stretch: Signals.Signal; text: { target: Signals.Signal; }; @@ -63,12 +64,14 @@ export default class extends HTMLElement { set muted(value: boolean); get name(): Net.Path.Valid; set name(value: string | Net.Path.Valid); - static observedAttributes: readonly ["url", "versions", "name", "paused", "volume", "muted", "visible", "backend", "follow", "announced", "buffer", "delay", "lookahead", "catalog-format", "captions", "echo"]; + static observedAttributes: readonly ["url", "versions", "name", "paused", "volume", "muted", "visible", "backend", "follow", "announced", "buffer", "delay", "lookahead", "stretch", "catalog-format", "captions", "echo"]; get paused(): boolean; set paused(value: boolean); readonly renderer: Video.Renderer; reset(): void; readonly signals: Signals.Effect; + get stretch(): Clock.Duration; + set stretch(value: Clock.Duration); readonly sync: Sync; readonly text: Text.Source; readonly textRenderer: Text.Renderer; diff --git a/js/watch/api/index.api.md b/js/watch/api/index.api.md index ee14bdeca90..1ef4310d93c 100644 --- a/js/watch/api/index.api.md +++ b/js/watch/api/index.api.md @@ -156,6 +156,7 @@ export type PlayerInput = { name: Signals.Getter; origin: Signals.Getter; paused: Signals.Getter; + stretch: Signals.Getter; text: { container: Signals.Getter; target: Signals.Getter; @@ -188,6 +189,7 @@ export class Sync { export type SyncInput = { delay: Signals.Getter; lookahead: Signals.Getter; + stretch: Signals.Getter; }; export type SyncOutput = { diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 3392338a718..04ce8de8471 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -7,7 +7,7 @@ import * as Moq from "@moq/net"; import { Effect, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; import { supportCacheKey } from "../support-probe"; -import { type Delay, Sync } from "../sync"; +import { type Delay, STRETCH, Sync } from "../sync"; import { Decoder as VideoDecoder } from "../video/decoder"; import { Source as VideoSource } from "../video/source"; import { decoderConfigKey } from "./config"; @@ -341,7 +341,8 @@ async function play(initial: Delay, buffered = false, cmaf = false) { Object.assign(supportProbes, { [supportCacheKey]: decoderConfigKey }); const source = new Source({ broadcast, supported: supportProbes }); const lookahead = new Signal(Duration.millis(buffered ? 1000 : 0)); - const sync = new Sync({ delay, lookahead }); + const stretch = new Signal(STRETCH); + const sync = new Sync({ delay, lookahead, stretch }); // The decoder's own switch, which a mute or pause turns off. const decoding = new Signal(true); const decoder = new Decoder({ source, sync, enabled: decoding }); @@ -363,6 +364,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { supportProbes, delay, lookahead, + stretch, sync, push, queued, @@ -1711,6 +1713,28 @@ it.each([false, true])("releases shared startup across a hole in queued audio (C } }); +// Audio may trail the clock by the stretch, or a buffered player's whole lookahead when that is longer, +// and a zero stretch turns the time stretch off in either mode. +for (const buffered of [false, true]) { + it(`bounds ${buffered ? "buffered" : "live"} audio's lag by the stretch`, async () => { + const playback = await play(Duration.millis(100), buffered); + try { + await playback.play(12); + const bound = () => worklets.at(-1)?.terms?.bound; + expect(bound()).toBe(2000); + playback.stretch.set(Duration.millis(300)); + await microtasks(); + expect(bound()).toBe(buffered ? 1000 : 300); + playback.stretch.set(Duration.ZERO); + await microtasks(); + expect(bound()).toBe(0); + } finally { + playback.close(); + await microtasks(); + } + }); +} + for (const cmaf of [false, true]) { for (const buffered of [false, true]) { it(`switches a running ${cmaf ? "CMAF" : "legacy"} player to ${buffered ? "buffered" : "live"} audio`, async () => { @@ -1730,9 +1754,9 @@ for (const cmaf of [false, true]) { (buffered ? [0, 20, 40, 60, 80] : [140, 160]).map((ms) => start + ms * 1000), ); - // Either ring keeps what it receives and plays onto the clock. Buffered audio may trail it - // by the lookahead; live audio at a fixed delay by nothing the subscription still delivers. - expect(worklets.at(-1)?.terms).toEqual({ latency: 100, target: 100, bound: buffered ? 1000 : 0 }); + // Either ring keeps what it receives and plays onto the clock, which audio may trail by the + // larger of the lookahead and the default stretch, even at a fixed delay. + expect(worklets.at(-1)?.terms).toEqual({ latency: 100, target: 100, bound: 2000 }); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 1b9b6c276be..18bcabd6a1b 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -22,7 +22,7 @@ import type { Sync, Track } from "../sync"; import { AudioBuffer } from "./buffer"; import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; import { Handover } from "./handover"; -import { AUTO_MAX_DELAY, ringHold, target } from "./latency"; +import { bound, maxDelay, ringHold, target } from "./latency"; import { Order } from "./order"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; @@ -188,7 +188,7 @@ export class Decoder { // The container consumer's arrival estimate, unset while nothing is subscribed. #measured = new Signal(undefined); - // The subscription's max delay: the shared budget, raised to the estimator's ceiling in "auto". + // The subscription's max delay: the shared budget, raised to the delay plus the stretch. #subscribeMaxDelay = new Signal(Duration.ZERO); // The container consumer's delay: the playout delay to start at and the shared budget. Its warmup // reaches the start further back, to the media the decoder primes on. @@ -321,25 +321,28 @@ export class Decoder { }); } - // A group the relay expires is never observed, so a subscription cut to the target would cap the - // estimate at the target it already holds. The container consumer skips at the same max delay, - // since it observes each frame before applying it, so audio the ring could still catch up on is - // delivered, and carves out the warmup the subscription adds once it has warmed, as Rust's does. - // See doc/concept/audio-jitter.md. + // The subscription reaches the stretch past the delay, so audio the ring could still catch up on + // arrives. A group the relay expires is never observed, so a zero stretch, which cuts the + // subscription to the target, leaves the estimate rising one bucket at a time. The container + // consumer skips at the same max delay, since it observes each frame before applying it, and + // carves out the warmup the subscription adds once it has warmed, as Rust's does. See + // doc/concept/audio-jitter.md. #runSubscribeMaxDelay(effect: Effect): void { - const delay = effect.get(this.sync.out.delay); - const max = this.#maxDelay(effect); + const budget = this.#budget(effect); + const delay = budget.delay; + const max = maxDelay(budget); const warmup = Duration.millis(effect.get(this.source.out.config)?.warmup ?? 0); this.#delay.set({ min: delay, max }); this.#subscribeMaxDelay.set(max.add(warmup)); } - // The subscription's max delay, less warmup: the shared max delay, at least the estimator's ceiling - // for "auto". - #maxDelay(effect: Effect): Duration { - const maxDelay = effect.get(this.sync.out.maxDelay); - const auto = effect.get(this.sync.in.delay) === "auto"; - return auto ? maxDelay.max(AUTO_MAX_DELAY) : maxDelay; + // The terms of the subscription's max delay: the shared delay and max delay, and the stretch. + #budget(effect: Effect): { delay: Duration; maxDelay: Duration; stretch: Duration } { + return { + delay: effect.get(this.sync.out.delay), + maxDelay: effect.get(this.sync.out.maxDelay), + stretch: effect.get(this.sync.in.stretch), + }; } #runShape(effect: Effect): void { @@ -507,8 +510,8 @@ export class Decoder { // The ring plays onto the shared clock, slowing down onto a rise and speeding up onto a fall, // and skips only audio trailing it past what the subscription still delivers. - const delay = effect.get(this.sync.out.delay); - ring.setLatency(delay, this.#maxDelay(effect).saturatingSub(delay)); + const budget = this.#budget(effect); + ring.setLatency(budget.delay, bound(budget)); } #runDecoder(effect: Effect): void { diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 42903973f43..8ee53995323 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -27,11 +27,20 @@ export function target(props: Target): Duration { } /** - * The least subscription max delay an "auto" track asks for: the estimator's ceiling. - * - * The measured term saturates at 100 buckets of 20 ms, so a frame later than this adds nothing. + * The audio subscription's max delay, less warmup: the shared `maxDelay`, raised to `delay + stretch` + * so audio trailing the clock by up to the stretch still arrives for the ring to catch up on. + */ +export function maxDelay(props: { delay: Duration; maxDelay: Duration; stretch: Duration }): Duration { + return props.maxDelay.max(props.delay.add(props.stretch)); +} + +/** + * How far the ring lets audio trail the clock before it skips ahead: the audio max delay less the + * playout delay, or zero, which turns the time stretch off, when `stretch` is zero. */ -export const AUTO_MAX_DELAY = Duration.millis(2000); +export function bound(props: { delay: Duration; maxDelay: Duration; stretch: Duration }): Duration { + return props.stretch.isZero() ? Duration.ZERO : maxDelay(props).saturatingSub(props.delay); +} // An AudioWorkletProcessor renders in fixed 128-sample quanta, so a ring shallower than one can // never be read from. diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index f447c4faaed..2b0afd8e4c8 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -2,7 +2,8 @@ import { describe, expect, it } from "bun:test"; import { Duration, Instant, Manual } from "@moq/clock"; import * as Container from "@moq/hang/container"; import { Time, Track, Varint } from "@moq/net"; -import { AUTO_MAX_DELAY, target } from "./latency"; +import { STRETCH } from "../sync"; +import { maxDelay, target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; // Replay recorded-shape arrival traces through the ring and count what a listener would hear: @@ -80,7 +81,7 @@ async function measure(t: Arrival[], budget: (target: Duration) => Duration): Pr const subscriber = track.subscribe({ delay: { max: budget(Duration.ZERO) } }); const consumer = new Container.Consumer(subscriber, { format: new Container.Legacy.Format("audio"), - delay: { max: AUTO_MAX_DELAY }, + delay: { max: budget(Duration.ZERO) }, }); const result: Measured = { targets: [], delivered: 0 }; @@ -122,15 +123,18 @@ async function measure(t: Arrival[], budget: (target: Duration) => Duration): Pr return result; } +/** The subscription "auto" asks for at the default stretch: the target plus the stretch. */ +const stretched = (target: Duration) => maxDelay({ delay: target, maxDelay: target, stretch: STRETCH }); + /** The target the estimator settles on by the end of `t`, given the subscription "auto" asks for. */ async function settled(t: Arrival[]): Promise { - const { targets } = await measure(t, (target) => target.max(AUTO_MAX_DELAY)); + const { targets } = await measure(t, stretched); return targets[targets.length - 1]; } /** The worklet's ring at `latencyMs`. */ function build(latencyMs: number): AudioRingBuffer { - return new AudioRingBuffer({ rate: RATE, channels: 1, latency: latencyMs }); + return new AudioRingBuffer({ rate: RATE, channels: 1, latency: latencyMs, bound: STRETCH.asMillis() }); } interface Result { @@ -236,7 +240,7 @@ describe("subscription budget", () => { } it("reaches a flush that starts after the target settled", async () => { - const { targets, delivered } = await measure(t, (target) => target.max(AUTO_MAX_DELAY)); + const { targets, delivered } = await measure(t, stretched); expect(targets[PACED - 1]).toBe(2 * CHUNK_MS); expect(delivered).toBe(t.length); // Every flush lands whole, so its 100ms reaches the 95th percentile within a few intervals. diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 688d0bd9c1d..cc3861dfd66 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -22,7 +22,7 @@ export interface Terms { // The playout delay the shared clock plays at: the depth the stretch keeps, decoded or not. target: number; // How far audio may trail the shared clock before the ring skips ahead: the subscription's max - // delay less the playout delay. + // delay less the playout delay. Zero turns the time stretch off. bound: number; } diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index f6b0ea660bd..0ebc9615708 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -145,14 +145,18 @@ describe("replay", () => { async function lag(trace: Arrival[], options: Options, settle: number) { const lags: number[] = []; let underruns = 0; + let trimmed = 0; + let accelerated = 0; for await (const quantum of replay(trace, options)) { if (quantum.at >= settle && !quantum.stalled && quantum.clock !== undefined) { lags.push(quantum.clock - quantum.timestamp); } underruns = quantum.underruns; + trimmed = quantum.trimmed; + accelerated = quantum.accelerated; } lags.sort((a, b) => a - b); - return { lags, median: lags[lags.length >> 1], underruns }; + return { lags, median: lags[lags.length >> 1], underruns, trimmed, accelerated }; } /** The shortest pitch period a seam moves by: 2.5 ms. A lag shorter than that cannot be corrected. */ @@ -160,8 +164,8 @@ const PERIOD = 2.5; // Audio and video keep separate clocks that align once the delay covers the arrivals. The ring plays // onto the shared clock: in steady state the playhead rests within a pitch period of it, either way. -// A buffer that runs low slows playback and leaves it behind the clock, only as far as the -// subscription's max delay still delivers, which a fixed delay leaves none of. +// A buffer that runs low slows playback and leaves it behind the clock, only as far as the stretch +// allows, which a zero stretch leaves none of. describe("replay against the shared clock", () => { let warn: Mock; beforeEach(() => { @@ -183,16 +187,28 @@ describe("replay against the shared clock", () => { }); } - it("underruns a fixed target short of the spread rather than trailing the clock", async () => { - // 80 ms of spread against a fixed 40 ms target: every flush runs the ring dry, and a fixed delay - // has no max delay past it to fall behind into. - const trace = paced(60, (i) => 30 + (4 - (i % 5)) * 20); - const options: Options = { rate: RATE, config: CONFIG, delay: Duration.millis(40), duration: observed(60) }; - const { lags, median, underruns } = await lag(trace, options, 30_000); - expect(underruns).toBeGreaterThanOrEqual(trace.length / 5); - // Each refill catches back up within the frame after it, never by a whole flush. + // 80 ms of spread against a fixed 40 ms target: every flush runs a ring dry that cannot fall behind. + const flushed = paced(60, (i) => 30 + (4 - (i % 5)) * 20); + const fixed = { rate: RATE, config: CONFIG, delay: Duration.millis(40), duration: observed(60) }; + + it("underruns a fixed target short of the spread with the stretch off, rather than trailing the clock", async () => { + const { lags, median, underruns, accelerated } = await lag(flushed, { ...fixed, stretch: 0 }, 30_000); + expect(underruns).toBeGreaterThanOrEqual(flushed.length / 5); + // Each refill skips back onto the clock within the frame after it, never by a whole flush. expect(median).toBeLessThanOrEqual(20); expect(lags.at(-1)).toBeLessThan(40); + expect(accelerated).toBe(0); + }); + + // The stretch raises no target, so the flushes still underrun, but what arrives late plays and is + // caught up by speeding up, where a zero stretch skips it. + it("keeps the audio a fixed target short of the spread receives late, where a zero stretch skips it", async () => { + const off = await lag(flushed, { ...fixed, stretch: 0 }, 30_000); + const on = await lag(flushed, fixed, 30_000); + // A zero stretch skips most of every flush; the stretch skips only the backlog it joins on. + expect(off.trimmed).toBeGreaterThan(20_000); + expect(on.trimmed).toBeLessThanOrEqual(40); + expect(on.accelerated).toBeGreaterThan(0); }); }); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 3ff4f73f19a..c59a9748c34 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -21,10 +21,10 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { type Group, Time, Track } from "@moq/net"; import { Derived, Effect, Signal } from "@moq/signals"; -import { type Delay, Sync } from "../sync"; +import { type Delay, STRETCH, Sync } from "../sync"; import { Backpressure } from "./buffer"; import { frameDuration } from "./config"; -import { AUTO_MAX_DELAY, ringHold, target } from "./latency"; +import { maxDelay as audioMaxDelay, bound, ringHold, target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; /** An AudioWorklet render quantum, in frames. */ @@ -62,6 +62,8 @@ export interface Options { delay: Exclude; /** The configured lookahead, in ms: nonzero plays in buffered mode, as the decoder does. */ lookahead?: number; + /** The configured stretch, in ms, as the element takes it (default: the element's). */ + stretch?: number; /** The rendition as the catalog described it: its advertised jitter, delay, and codec. */ config: Catalog.AudioConfig; /** @@ -143,11 +145,12 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const lookahead = Duration.millis(options.lookahead ?? 0); const buffered = !lookahead.isZero(); - const sync = new Sync({ delay: options.delay, lookahead }); + const stretch = options.stretch !== undefined ? Duration.millis(options.stretch) : STRETCH; + const sync = new Sync({ delay: options.delay, lookahead, stretch }); const track = new Track.Producer("audio"); // The subscription's max delay, as the decoder sets it, which the consumer skips at too. - const maxDelay = new Derived([sync.out.maxDelay], (max) => - options.delay === "auto" ? max.max(AUTO_MAX_DELAY) : max, + const maxDelay = new Derived([sync.out.delay, sync.out.maxDelay], (delay, max) => + audioMaxDelay({ delay, maxDelay: max, stretch }), ); const consumer = new Container.Consumer(track.subscribe({ delay: { max: WIRE_MAX_DELAY } }), { format: new Container.Legacy.Format("audio"), @@ -182,7 +185,10 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato await settle(); let delay = sync.out.delay.peek(); - const terms = () => ({ target: delay.asMillis(), bound: maxDelay.peek().saturatingSub(delay).asMillis() }); + const terms = () => ({ + target: delay.asMillis(), + bound: bound({ delay, maxDelay: sync.out.maxDelay.peek(), stretch }).asMillis(), + }); // The decode gate, and the worklet's ring on the floor it sets, the way `AudioBuffer` drives both. const gate = new Backpressure(rate, buffered, delay); const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, gate.floor), ...terms() }); diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index 3e1a9baa04f..cf99146f462 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -440,7 +440,7 @@ describe("live anchoring", () => { describe("re-buffer", () => { it("re-stalls after running dry and resumes on a cushion, slowing down to rebuild", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 }); + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100, bound: 2000 }); write(buffer, 0, 100, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); @@ -822,11 +822,13 @@ describe("time stretch", () => { const live = (latency: number, terms: { bound?: number } = {}) => new AudioRingBuffer({ rate: RATE, channels: 1, latency, ...terms }); const frames = (ms: number) => (RATE * ms) / 1000; + // A bound turns the stretch on; without a clock nothing trails it far enough to skip. + const STRETCHING = { bound: 2000 }; // Without a clock, the target is where the playhead belongs, so audio landing 40 ms short of it // plays slower: the ring stretches what it holds rather than run dry on time. it("slows down once arrivals land short of its target", () => { - const buffer = live(100); + const buffer = live(100, STRETCHING); write(buffer, 0, frames(100), { value: 0.5 }); read(buffer, frames(60), 1); write(buffer, 100, frames(20), { value: 0.5 }); @@ -886,7 +888,7 @@ describe("time stretch", () => { }; it("accelerates a ring above its floor back onto it, dropping nothing", async () => { - const buffer = live(100); + const buffer = live(100, STRETCHING); write(buffer, 0, frames(300), { value: 0.5 }); const { timestamps, peaks, written } = await converge(buffer, frames(300), 8); @@ -909,7 +911,7 @@ describe("time stretch", () => { // below its floor, which is arrivals running late rather than audio ahead of the clock, so steering // on that depth would settle a burst behind it. it("rests on its floor right after each burst, never a burst above it", async () => { - const buffer = live(120); + const buffer = live(120, STRETCHING); write(buffer, 0, frames(120), { value: 0.5 }); const { peaks } = await converge(buffer, frames(120), 8, 100); settled(peaks, 120); @@ -919,7 +921,7 @@ describe("time stretch", () => { // A floor short of the arrival spread underruns, and that is the cost of the floor: each burst // lands above the target, which leaves nothing to slow down for. it("underruns every burst that outruns its target", async () => { - const buffer = live(60); + const buffer = live(60, STRETCHING); write(buffer, 0, frames(60), { value: 0.5 }); const { peaks } = await converge(buffer, frames(60), 8, 100); // Each burst lands on an empty ring, never on audio left from the one before. @@ -928,7 +930,7 @@ describe("time stretch", () => { }); it("fast-accelerates a ring four times past its high limit, even straight after a stretch", async () => { - const buffer = live(100); + const buffer = live(100, STRETCHING); write(buffer, 0, frames(600), { value: 0.5 }); await converge(buffer, frames(600), 1); // Fast accelerate removes up to 15 ms of every 30 ms window until the ring is under 400 ms. @@ -1160,6 +1162,54 @@ describe("time stretch", () => { expect(buffer.skips).toBe(2); }); + // A zero bound turns the stretch off: a lag under a window that the stretch would speed through + // stays, more skips onto the clock, and a playhead more than a window ahead of it pads silence until + // the clock reaches it. + it("with a zero bound, plays unaltered and skips or pads onto the clock", () => { + const buffer = live(100, { bound: 0 }); + write(buffer, 0, frames(800), { value: 0.5 }); + buffer.clock(0); + read(buffer, QUANTUM, 1); + const quantum = (QUANTUM * 1_000_000) / RATE; + + // 20 ms behind the clock, with far more than the target buffered. + let clock = buffer.timestamp + 20_000; + const start = buffer.timestamp; + for (let i = 0; i < 20; i++) { + buffer.clock(clock); + expect(read(buffer, QUANTUM, 1)[0].every((sample) => sample === 0.5)).toBe(true); + clock += quantum; + } + expect(buffer.timestamp - start).toBeCloseTo(20 * quantum, 6); + expect(buffer.accelerated).toBe(0); + expect(buffer.skips).toBe(0); + + // 100 ms more skips onto the clock. + clock += 100_000; + buffer.clock(clock); + read(buffer, QUANTUM, 1); + expect(buffer.skips).toBe(1); + expect(buffer.timestamp).toBeCloseTo(clock + quantum, -1); + clock += quantum; + + // 50 ms ahead of it pads silence until the clock reaches the playhead. + clock -= 50_000; + const playhead = buffer.timestamp; + let padded = 0; + for (;;) { + buffer.clock(clock); + const [output] = read(buffer, QUANTUM, 1); + clock += quantum; + if (output.some((sample) => sample !== 0)) break; + padded++; + } + expect(padded).toBe(Math.floor(50_000 / quantum)); + expect(buffer.timestamp).toBeGreaterThan(playhead); + expect(buffer.accelerated).toBe(0); + expect(buffer.skips).toBe(1); + expect(buffer.underruns).toBe(0); + }); + // Audio the stretch could still catch up on plays; only what trails the clock past the bound and // one window skips, back to the bound. for (const bound of [0, 200]) { diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 5680df17a71..c1bd09e2aa3 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -12,7 +12,8 @@ const QUANTUM = 128; * Playback follows the decision rule in doc/concept/audio-jitter.md: each window of received PCM may * play faster to catch up with the shared clock once the buffer holds more than the target, or slower * while the buffer runs low or the playhead runs ahead of the clock. Only audio trailing the clock past - * the subscription's max delay is skipped. + * the bound is skipped. A zero bound turns the stretch off: received audio plays unaltered, and a + * playhead more than a window off the clock skips ahead onto it or waits for it in silence. */ export class AudioRingBuffer { #buffer: Float32Array[]; @@ -29,6 +30,9 @@ export class AudioRingBuffer { #skips = 0; // Whether the last read skipped, so a run of reads skipping late audio counts once. #skipping = false; + // Whether the last read padded silence toward the clock, which continues until the clock reaches the + // playhead. + #padding = false; // Whether the ring has read on the clock since the last reset. Until it has, nothing trailing the // clock was heard, so a join lands on it rather than at the bound. #joined = false; @@ -79,7 +83,8 @@ export class AudioRingBuffer { // Un-stall threshold in samples (how much to buffer before playback starts). #latencySamples: number; - // The playout delay and how far audio may trail the clock, in samples. + // The playout delay and how far audio may trail the clock, in samples. A zero bound turns the + // stretch off. #target: number; #bound: number; // Whether the read/write indices have been anchored to the first inserted sample. @@ -458,6 +463,7 @@ export class AudioRingBuffer { this.#incoming = 0; this.#jumped = 0; this.#skipping = false; + this.#padding = false; } /** @@ -482,8 +488,10 @@ export class AudioRingBuffer { // Skip what trails the clock past its bound and one window, back to the bound, or the jumps of // its reference past one window. Less than that is the stretch's to catch up. A join has the // bound of nothing heard yet, so it lands on the clock past whatever backlog it was handed, and a - // jump before it was never heard. A finite tail plays as received. + // jump before it was never heard. A finite tail plays as received. With the stretch off, a + // playhead more than a window ahead of the clock pads silence until the clock reaches it. let skip = 0; + let pad = 0; if (this.#clock !== undefined) { const inputs = this.#inputs(0); const bound = this.#joined ? inputs.bound : 0; @@ -496,6 +504,11 @@ export class AudioRingBuffer { if (!this.#ending()) { if (inputs.lag > bound + this.#stretch.window) skip = inputs.lag - bound; if (this.#jumped > this.#stretch.window) skip = Math.max(skip, this.#jumped); + if (this.#bound === 0) { + const lead = Math.round(-inputs.lag); + this.#padding = lead > (this.#padding ? 0 : this.#stretch.window); + if (this.#padding) pad = Math.min(lead, output[0].length); + } } } if (skip > 0) { @@ -515,8 +528,9 @@ export class AudioRingBuffer { this.#skipping = false; } + for (const dst of output) dst.fill(0, 0, pad); const stretch = this.#stretch; - let samples = 0; + let samples = pad; while (samples < output[0].length) { if (stretch.available === 0 && !this.#stage(output[0].length - samples, samples)) break; const from = samples; @@ -584,7 +598,7 @@ export class AudioRingBuffer { const buffered = this.#writeIndex - this.#readIndex; if (buffered <= 0) return false; const quiet = pause !== undefined && pause.start <= this.#readIndex; - const enabled = !quiet && !this.#ending(); + const enabled = !quiet && !this.#ending() && this.#bound > 0; const window = this.#stretch.window; let frames = Math.min(buffered, window, enabled ? buffered : need); if (pause) frames = Math.min(frames, (quiet ? pause.end : pause.start) - this.#readIndex); diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index fb2edb77f80..6691768f068 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -15,7 +15,7 @@ import { type Broadcast, CATALOG_FORMATS, type CatalogFormat } from "./broadcast import { formatDuration, parseDuration } from "./duration"; import { checkName, Echo } from "./echo"; import { Player } from "./player"; -import type { Delay, Sync } from "./sync"; +import { type Delay, STRETCH, type Sync } from "./sync"; import type * as Text from "./text"; import type * as Video from "./video"; @@ -66,6 +66,7 @@ const OBSERVED = [ "buffer", "delay", "lookahead", + "stretch", "catalog-format", "captions", "echo", @@ -107,6 +108,14 @@ function parseLookahead(value: string | null): Duration { throw invalidAttribute("lookahead", value, 'a duration like "30s"'); } +// Parse the `stretch` attribute: a duration, or the default when absent. +function parseStretch(value: string | null): Duration { + if (value === null) return STRETCH; + const parsed = parseDuration(value); + if (parsed !== undefined) return parsed; + throw invalidAttribute("stretch", value, 'a duration like "2s", or "0" to turn it off'); +} + // Parse the `echo` attribute: the viewer's feedback name, one visible path segment, or undefined when absent. function parseEcho(value: string | null): string | undefined { if (value === null) return undefined; @@ -209,6 +218,8 @@ export default class MoqWatch extends HTMLElement { delay: new Signal("auto"), // Future-dated media held beyond the live edge before playback skips ahead. lookahead: new Signal(Duration.ZERO), + // How far audio may trail the clock before it skips ahead; zero turns the time stretch off. + stretch: new Signal(STRETCH), video: { // The desired video rendition (resolution/bitrate cap). target: new Signal(undefined), @@ -373,6 +384,9 @@ export default class MoqWatch extends HTMLElement { this.signals.run((effect) => this.#reflect(effect, "lookahead", formatDuration(effect.get(this.controls.lookahead))), ); + this.signals.run((effect) => + this.#reflect(effect, "stretch", formatDuration(effect.get(this.controls.stretch))), + ); // Track the element's rendered size and feed it into the rendition picker, // scaled by devicePixelRatio so high-DPI screens still get sharp renditions. @@ -451,6 +465,7 @@ export default class MoqWatch extends HTMLElement { else if (name === "buffer") this.#accept("buffer"); else if (name === "delay") this.delay = parseDelay(newValue); else if (name === "lookahead") this.lookahead = parseLookahead(newValue); + else if (name === "stretch") this.stretch = parseStretch(newValue); else if (name === "catalog-format") this.catalogFormat = parseCatalogFormat(newValue); else if (name === "captions") this.captions = newValue || undefined; else if (name === "echo") this.echo = parseEcho(newValue); @@ -608,7 +623,7 @@ export default class MoqWatch extends HTMLElement { * future-dated frames (e.g. TTS written faster than real-time) and only skip ahead * once they would sit further than `delay + lookahead` past the playhead. Audio that falls * behind, as after a stall, catches up by playing faster, and skips only what trails by more - * than the lookahead. Call `reset()` at each utterance boundary. Not a jitter buffer: live + * than the larger of the lookahead and the {@link stretch}. Call `reset()` at each utterance boundary. Not a jitter buffer: live * streams leave it at zero and tune `delay`. */ get lookahead(): Duration { @@ -621,6 +636,25 @@ export default class MoqWatch extends HTMLElement { this.#accept("lookahead"); } + /** + * How far audio may trail the clock before it skips ahead (default: 2 s), mirrored by the + * `stretch` attribute. + * + * After a stall, audio slows down and falls behind the clock rather than dropping what arrives + * late, then speeds up until it is back on it. A voice app might ask for 200 ms, a sports stream + * for 2 s. Zero turns the time stretch off: received audio plays unaltered and late audio is + * skipped. + */ + get stretch(): Duration { + return this.controls.stretch.peek(); + } + + set stretch(value: Duration) { + if (!(value instanceof Duration)) throw invalidAttribute("stretch", value, "a Duration"); + this.controls.stretch.set(parseStretch(formatDuration(value))); + this.#accept("stretch"); + } + /** * Re-anchor playback at an utterance boundary in buffered mode: reset the sync reference * and flush the audio buffer so the next utterance plays from its own first frame. diff --git a/js/watch/src/player.ts b/js/watch/src/player.ts index 613f0505f94..e238b491032 100644 --- a/js/watch/src/player.ts +++ b/js/watch/src/player.ts @@ -5,7 +5,7 @@ import { Path } from "@moq/net"; import { Effect, type Getter, getter, type Inputs, type Readonlys, Signal } from "@moq/signals"; import * as Audio from "./audio"; import { Broadcast, type CatalogFormat } from "./broadcast"; -import { type Delay, Sync } from "./sync"; +import { type Delay, STRETCH, Sync } from "./sync"; import * as Text from "./text"; import * as Video from "./video"; import { holdViewer } from "./viewer"; @@ -30,6 +30,8 @@ export type PlayerInput = { delay: Getter; /** Future-dated media held beyond the live edge. */ lookahead: Getter; + /** How far audio may trail the clock before it skips ahead; zero turns the time stretch off. */ + stretch: Getter; /** Video rendition and canvas output. */ video: { /** Desired video rendition or quality limits. */ @@ -125,6 +127,7 @@ export class Player { paused: getter(props.paused ?? false), delay: getter(props.delay ?? "auto"), lookahead: getter(props.lookahead ?? Duration.ZERO), + stretch: getter(props.stretch ?? STRETCH), video: { target: getter(props.video?.target), canvas: getter(props.video?.canvas), @@ -168,7 +171,7 @@ export class Player { this.text = new Text.Source({ broadcast: this.broadcast, target: this.in.text.target }); this.#signals.cleanup(() => this.text.close()); - this.sync = new Sync({ delay: this.in.delay, lookahead: this.in.lookahead }); + this.sync = new Sync({ delay: this.in.delay, lookahead: this.in.lookahead, stretch: this.in.stretch }); this.#signals.cleanup(() => this.sync.close()); this.video = new Video.Decoder({ source: videoSource, sync: this.sync, enabled: this.#videoEnabled }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 6da306434ff..87f6e378795 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -15,6 +15,9 @@ import { Derived, Effect, type Getter, getter, type Inputs, type Readonlys, read */ export type Delay = "instant" | "auto" | Duration; +/** The default {@link SyncInput.stretch}: how far live audio may trail the clock before it skips. @internal */ +export const STRETCH = Duration.secs(2); + /** Signals the sync reads. */ export type SyncInput = { /** How far playback trails the live edge. See {@link Delay}. */ @@ -26,13 +29,28 @@ export type SyncInput = { * Zero minimizes latency, the live default: playback re-anchors as soon as anything arrives * early. A larger value lets faster-than-real-time frames (a TTS response with future * timestamps) build up instead of being skipped. Audio that falls behind, as after a stall, - * catches up by playing faster, and skips only what trails by more than the lookahead. Always + * catches up by playing faster, and skips only what trails by more than the larger of the + * lookahead and {@link SyncInput.stretch}. Always * finite, so worst case the audio ring drops its oldest samples rather than exhausting memory. * With a nonzero lookahead, the first video * frame is a poster until every enabled media track holds the resolved delay; `lookahead` is * the most held ahead, not the amount required to start. */ lookahead: Getter; + + /** + * How far audio may trail the clock before it skips ahead (default: 2 s). + * + * After a stall, audio slows down and falls behind the clock rather than dropping what arrives + * late, then speeds up until it is back on it; only what trails by more than this is skipped. The + * audio subscription asks for `delay + stretch`, so that late audio still arrives. A buffered + * player already keeps its whole lookahead, so audio skips only past the larger of the two. + * + * Zero turns the time stretch off: received audio plays unaltered, late audio is skipped, an + * underrun is silence, and drift against the publisher's clock is corrected by a small skip or + * pad. Video is unaffected either way: it stays on the clock. + */ + stretch: Getter; }; /** Signals the sync derives. */ @@ -133,6 +151,7 @@ export class Clock { this.in = { delay: getter(props?.delay ?? ("auto" as Delay)), lookahead: getter(props?.lookahead ?? Duration.ZERO), + stretch: getter(props?.stretch ?? STRETCH), }; } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 352c3aba245..808461a53a6 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -127,6 +127,8 @@ pub struct Consumer { remix: Option, options: Options, max_delay: std::time::Duration, + /// The track's max age, which clamps the max delay. + max_age: Option, /// Prime after a join or loss, never after a declared endpoint or at the track's /// first packet, which the codec's own delay already primes. warmup: Duration, @@ -273,6 +275,7 @@ impl Consumer { remix, options, max_delay, + max_age, warmup, warmup_pending: !warmup.is_zero(), warmup_until: None, @@ -316,6 +319,16 @@ impl Consumer { self.max_delay } + /// Move [`Delay::max`] mid-stream, clamped to the track's max age like [`Options::delay`]. + /// + /// A player whose playout delay moves calls this to keep the subscription reaching as far + /// behind it, as `moq play` keeps it at the delay plus its time stretch. + pub fn set_max_delay(&mut self, max: std::time::Duration) { + self.options.delay = self.options.delay.with_max(max); + self.track.set_delay(self.options.delay); + self.max_delay = max.min(self.max_age.unwrap_or(std::time::Duration::MAX)); + } + /// The playout delay to hold ahead of the playhead right now: the fixed /// [`Delay::min`], or with [`Options::estimate`] the estimate from every frame read so far. /// @@ -959,6 +972,45 @@ mod tests { ); } + /// A moved max delay reaches the subscription, plus the warmup, clamped to the max age. + #[tokio::test(start_paused = true)] + async fn a_moved_max_delay_reaches_the_subscription() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track( + "audio", + hang::container::track_info(hang::catalog::PRIORITY.audio).with_max_age(Duration::from_secs(5)), + ) + .unwrap(); + let mut catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); + catalog.warmup = Some(Duration::from_millis(100)); + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "audio", + Options { + delay: Delay { + max: Some(Duration::from_millis(500)), + ..Delay::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + assert_eq!(consumer.max_delay(), Duration::from_millis(500)); + + consumer.set_max_delay(Duration::from_secs(2)); + assert_eq!(consumer.max_delay(), Duration::from_secs(2)); + assert_eq!( + track.subscription().unwrap().delay.max, + Some(Duration::from_millis(2_100)) + ); + + consumer.set_max_delay(Duration::from_secs(9)); + assert_eq!(consumer.max_delay(), Duration::from_secs(5)); + } + #[tokio::test(start_paused = true)] async fn conversion_waits_for_decoded_samples() { for (rate, channels) in [(44_100, 1), (4_000_000_000, 8), (48_000, u32::MAX)] { diff --git a/rs/moq-audio/src/playback/sink.rs b/rs/moq-audio/src/playback/sink.rs index 2d3a4ab7c34..9874a4dd1ab 100644 --- a/rs/moq-audio/src/playback/sink.rs +++ b/rs/moq-audio/src/playback/sink.rs @@ -285,14 +285,17 @@ impl Sink { /// The sink then plays onto that clock rather than at its latency alone. /// Behind it, it speeds up once it holds more than its latency; ahead of /// it, it slows down. Its buffer running low slows it down too, falling - /// behind the clock by at most `late`, and audio trailing the clock by more - /// than `late` plus one 30 ms window is skipped. `jumped` is how far the + /// behind the clock by at most `stretch`, and audio trailing the clock by + /// more than `stretch` plus one 30 ms window is skipped. A zero `stretch` + /// turns the time stretch off: written audio plays unaltered, and a sink + /// more than a window off the clock skips onto it or pads silence until + /// the clock reaches it. `jumped` is how far the /// clock's reference moved earlier since the last call, which skips as much /// audio as trails the clock, where a lower latency moves the clock ahead /// by speeding up. Call it after each write and whenever the clock moves; /// until the first call, or after the device changes, the latency alone is /// the clock. - pub fn due(&mut self, at: Instant, late: Duration, jumped: Duration) { + pub fn due(&mut self, at: Instant, stretch: Duration, jumped: Duration) { // The device thread reads std's clock, so carry the distance from now // over to it rather than the instant. let (now, host) = (Instant::now(), crate::ring::host()); @@ -300,7 +303,7 @@ impl Sink { Some(ahead) => host + ahead, None => host.checked_sub(now.duration_since(at)).unwrap_or(host), }; - self.channel.lock().unwrap().prod.due(at, late, jumped); + self.channel.lock().unwrap().prod.due(at, stretch, jumped); } /// The PCM layout this sink was built with. diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index 936653aab43..5bb2004a22e 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -7,7 +7,10 @@ //! Native playback instead enables received-audio stretch, which plays the //! queue onto the writer's clock (see [`Producer::due`]) and skips only audio //! that trails it too far; an actual underrun refills with emergency silence, -//! which never enters the pitch search, until one window is queued again. +//! which never enters the pitch search, until one window is queued again. A +//! clock with a zero stretch turns the stretch off: the queue plays as written, +//! and a reader more than a window off the clock skips onto it or pads silence +//! until the clock reaches it. //! A silence the writer declared is not an underrun: the reader fills it with //! comfort noise, and the next utterance plays one latency after it resumes. //! Neither end allocates, locks, or blocks after construction, so either may @@ -110,7 +113,7 @@ struct Shared { /// The writer's clock, once [`clocked`](Self::clocked): the writer's end /// less the frames from [`Producer::base`] to when it is due, as an `i32` /// in the low half, and how far the reader may trail it before it skips - /// ahead in the high half. One word, so the reader never pairs one anchor's + /// ahead in the high half, zero turning the stretch off. One word, so the reader never pairs one anchor's /// clock with another's bound. Adding the frames to now less `consumed` to /// the clock gives how far the reader trails it. anchor: AtomicU64, @@ -203,6 +206,7 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config measure: config.comfort, trim: config.trim, jumped: 0, + padding: false, joined: false, skips: 0, trimmed: 0, @@ -365,11 +369,12 @@ impl Producer { /// Anchor what was pushed so far to the writer's clock: its end is due to /// play at `at`. The stretch then plays the queue onto that clock, and the - /// reader skips audio trailing it by more than `late` plus one window. + /// reader skips audio trailing it by more than `stretch` plus one window. A + /// zero `stretch` turns the stretch off. /// `jumped` is how far the clock's reference moved earlier since the last /// anchor, which the reader skips too, as far as it trails the clock. /// Without an anchor, the latency is the clock. - pub(crate) fn due(&mut self, at: Instant, late: Duration, jumped: Duration) { + pub(crate) fn due(&mut self, at: Instant, stretch: Duration, jumped: Duration) { let frames = |duration: Duration| (duration.as_secs_f64() * self.rate as f64).round() as usize; let at = match at.checked_duration_since(self.base) { Some(since) => frames(since), @@ -378,11 +383,11 @@ impl Producer { // Within a few seconds of the writer's end, unless the caller's clock is // off by hours, which saturates rather than wraps. let clock = (self.head().wrapping_sub(at) as isize).clamp(i32::MIN as isize, i32::MAX as isize) as i32; - let late = u32::try_from(frames(late)).unwrap_or(u32::MAX); + let stretch = u32::try_from(frames(stretch)).unwrap_or(u32::MAX); // Released, so a reader that sees it also sees the target set before it. self.shared .anchor - .store(u64::from(late) << 32 | u64::from(clock as u32), Ordering::Release); + .store(u64::from(stretch) << 32 | u64::from(clock as u32), Ordering::Release); self.shared.clocked.store(true, Ordering::Release); // After the anchor, so a reader that sees the jump also sees the clock it moved. self.shared.jumped.fetch_add(frames(jumped), Ordering::Release); @@ -610,6 +615,9 @@ pub(crate) struct Consumer { /// the early media that moved it leaves, and the reader still trails the /// clock. jumped: usize, + /// Whether the last read padded silence toward the clock, which continues + /// until the clock reaches the reader. + padding: bool, /// Whether received audio has played on the clock. Until it has, nothing /// trailing the clock was heard, so a join lands on it rather than at the /// bound. @@ -656,7 +664,8 @@ impl Consumer { } let skipped = self.skip(); - let read = self.fill(&mut output[..whole]); + let padded = self.pad(&mut output[..whole]); + let read = padded + self.fill(&mut output[padded..whole]); if read < whole { if self.declared() { self.comfort.fill(&mut output[read..whole]); @@ -780,6 +789,31 @@ impl Consumer { skipped } + /// Whether the stretch may alter what plays: always, until a clock with a + /// zero stretch turns it off. + fn stretching(&self) -> bool { + !self.shared.clocked.load(Ordering::Acquire) || self.shared.anchor.load(Ordering::Acquire) >> 32 != 0 + } + + /// With the stretch off, pad silence ahead of the queue once the reader is + /// more than a window ahead of the clock, until the clock reaches it, + /// returning the samples padded. The stretch would slow down instead. + fn pad(&mut self, output: &mut [f32]) -> usize { + let Some(window) = self.stretch.as_ref().map(|stretch| stretch.window) else { + return 0; + }; + let lead = -self.inputs(0).lag; + let off = + !self.stretching() && !self.refill && !self.shared.finished.load(Ordering::Relaxed) && !self.declared(); + self.padding = off && lead > if self.padding { 0 } else { window as i64 }; + if !self.padding { + return 0; + } + let samples = (lead as usize).min(output.len() / self.channels) * self.channels; + output[..samples].fill(0.0); + samples + } + /// Whether running dry now is a silence the writer declared: an end it has /// not resumed from, on a queue that is not playing out its last samples. fn declared(&self) -> bool { @@ -886,8 +920,9 @@ impl Consumer { if frames == 0 { break; } - // Never transform startup padding or a finite stream's tail. - let enabled = self.prime == 0 && !self.shared.finished.load(Ordering::Relaxed) && !declared; + // Never transform startup padding, a finite stream's tail, or anything with the stretch off. + let enabled = + self.prime == 0 && !self.shared.finished.load(Ordering::Relaxed) && !declared && self.stretching(); let inputs = (enabled && frames == window).then(|| self.inputs(done / channels)); let stretch = self.stretch.as_mut().expect("a window comes from the stretch"); let frames = if self.prime > 0 { frames.min(self.prime) } else { frames }; @@ -1373,7 +1408,7 @@ mod tests { let read = pull(&mut cons, &mut out, base + Duration::from_millis(30)); assert_eq!(read, Read::Trimmed { frames: 1920 }); assert_eq!(cons.trimmed(), 1920, "back to the bound"); - // A fixed delay has no bound: whatever trails the clock goes, here + // A zero stretch has no bound: whatever trails the clock goes, here // what is due before the second the end is due at. prod.due(base + Duration::from_secs(1), Duration::ZERO, Duration::ZERO); let read = pull(&mut cons, &mut out, base + Duration::from_millis(40)); @@ -1384,6 +1419,60 @@ mod tests { assert!((940.0..960.0).contains(&left), "{left} ms left"); } + /// A zero stretch plays what was written unaltered: a lag under a window + /// that the stretch would speed through stays, more skips onto the clock, + /// and a reader more than a window ahead of it pads silence until the clock + /// reaches it. + #[test] + fn a_zero_stretch_skips_or_pads_onto_the_clock() { + let rate = 48_000; + let (mut prod, mut cons, pcm) = tone(rate, Duration::from_millis(10)); + let base = prod.base; + let mut out = vec![0.0; 480 * 2]; + for _ in 0..100 { + assert_eq!(prod.push(&pcm), Push::Ok); + } + // Two seconds queued, trailing the clock by 20 ms: deep enough for the + // stretch to speed through, short of a window. + prod.due(base + Duration::from_millis(1990), Duration::ZERO, Duration::ZERO); + let mut heard = Vec::new(); + let mut now = base; + for _ in 0..50 { + assert_eq!(pull(&mut cons, &mut out, now), Read::Ok); + heard.extend_from_slice(&out); + now += Duration::from_millis(10); + } + assert_eq!(cons.accelerated(), 0); + assert_eq!(cons.skips(), 0); + // The 10 ms prime, then the tone exactly as written. + assert!(heard[..480 * 2].iter().all(|sample| *sample == 0.0)); + let written = pcm.iter().cycle(); + assert!( + heard[480 * 2..] + .iter() + .zip(written) + .all(|(heard, written)| heard == written) + ); + + // 100 ms more behind the clock skips onto it, the 20 ms included. + prod.due(base + Duration::from_millis(1890), Duration::ZERO, Duration::ZERO); + assert_eq!(pull(&mut cons, &mut out, now), Read::Trimmed { frames: 5760 }); + now += Duration::from_millis(10); + + // 50 ms ahead of it pads silence until the clock reaches the reader. + prod.due(base + Duration::from_millis(1940), Duration::ZERO, Duration::ZERO); + let available = cons.available(); + for _ in 0..5 { + assert_eq!(pull(&mut cons, &mut out, now), Read::Ok); + assert!(out.iter().all(|sample| *sample == 0.0)); + now += Duration::from_millis(10); + } + assert_eq!(cons.available(), available, "padding plays nothing written"); + assert_eq!(pull(&mut cons, &mut out, now), Read::Ok); + assert!(out.iter().any(|sample| *sample != 0.0)); + assert_eq!((cons.accelerated(), cons.skips()), (0, 1)); + } + /// A clock whose reference moved earlier skips as much, as far as the /// reader trails it, where the same lag from a fallen target is caught up /// by speeding up. diff --git a/rs/moq-cli/src/play/args.rs b/rs/moq-cli/src/play/args.rs index f7108d6cc1f..e3de1852f87 100644 --- a/rs/moq-cli/src/play/args.rs +++ b/rs/moq-cli/src/play/args.rs @@ -16,14 +16,6 @@ use crate::subscribe::{CatalogFormatArg, Mode, SelectArgs}; /// below pins the two together in a build that has both. const DELAY_MAX: Duration = Duration::from_secs(10); -/// How long `auto` waits on a stalled group before skipping it. -/// -/// The estimate only sees what the container hands over, and a skipped group is -/// never handed over, so a budget under the estimate's ceiling would cap the -/// target at the budget it was cut to. Two seconds is that ceiling, the most -/// lateness the estimate can represent. -const AUTO_MAX_DELAY: Duration = Duration::from_secs(2); - /// How far playback trails the live edge under `auto` while audio holds no /// estimate, as when there is none: video has no estimator of its own. const AUTO_VIDEO_DELAY: Duration = Duration::from_millis(100); @@ -77,6 +69,17 @@ pub struct Args { #[usage(long, default = "auto")] pub delay: Delay, + /// How far audio may trail the clock before it skips ahead, or `0` to turn the time stretch off. + /// + /// After a stall, audio slows down and falls behind the clock rather than dropping what + /// arrives late, then speeds up until it is back on it; only what trails by more than this + /// is skipped. Every subscription waits on a stalled group for the delay plus this, so the + /// late audio still arrives. Zero plays received audio unaltered: late audio is skipped, an + /// underrun is silence, and drift against the publisher's clock is corrected by a small + /// skip or pad. Video stays on the clock either way. + #[usage(long, default = "2s")] + pub stretch: moq_tokio::cli::Duration, + /// Rendition selection by track name or codec. #[usage(flatten)] pub select: SelectArgs, @@ -120,9 +123,16 @@ impl Args { } } - /// How long a stalled group is waited on before it is skipped. + /// How long a stalled audio group is waited on before it is skipped, while + /// the shared delay is still the configured one: the delay plus the stretch. + /// A shared delay past it grows the budget by the excess. pub(super) fn max_delay(&self) -> Duration { - self.fixed_delay().unwrap_or(AUTO_MAX_DELAY) + self.fixed_delay().unwrap_or_default() + self.stretch() + } + + /// How far audio may trail the clock before it skips ahead; zero turns the time stretch off. + pub(super) fn stretch(&self) -> Duration { + self.stretch.into() } /// How far playback trails the live edge while no audio estimate sets it. @@ -130,13 +140,14 @@ impl Args { self.fixed_delay().unwrap_or(AUTO_VIDEO_DELAY) } - /// How long a stalled video group is waited on, given the catalog's audio. + /// How long a stalled video group is waited on, given the catalog's audio: + /// video's delay plus the stretch, as audio's. /// /// With no enabled speaker to follow, the playhead is video's own, so waiting - /// on the audio estimate's budget would only freeze the picture. + /// on audio's budget would only freeze the picture. pub(super) fn video_max_delay(&self, audio: &hang::catalog::Audio) -> Duration { match audio.enabled().next() { - Some(_) => self.max_delay(), + Some(_) => self.video_delay() + self.stretch(), None => self.video_delay(), } } @@ -236,23 +247,28 @@ mod tests { } } - /// A fixed delay is the playout offset and the staleness budget at once. + /// A fixed delay is the playout offset, and with the stretch the staleness budget. #[test] fn a_fixed_delay_sets_the_staleness_budget() { let args = parse(&["--delay", "500ms"]); assert_eq!(args.fixed_delay(), Some(Duration::from_millis(500))); - assert_eq!(args.max_delay(), Duration::from_millis(500)); + assert_eq!(args.max_delay(), Duration::from_millis(2500)); assert_eq!(args.video_delay(), Duration::from_millis(500)); + + let args = parse(&["--delay", "500ms", "--stretch", "0"]); + assert_eq!(args.max_delay(), Duration::from_millis(500)); + let args = parse(&["--delay", "500ms", "--stretch", "200ms"]); + assert_eq!(args.max_delay(), Duration::from_millis(700)); } - /// The default measures the delay, and waits on a stalled group as long as the - /// estimate can see, so the budget never caps what it measures. + /// The default measures the delay, and waits on a stalled group for the stretch past it. #[test] fn the_delay_defaults_to_the_estimate() { let args = parse(&[]); assert_eq!(args.delay, Delay::Auto); assert_eq!(args.fixed_delay(), None); - assert_eq!(args.max_delay(), AUTO_MAX_DELAY); + assert_eq!(args.stretch(), Duration::from_secs(2)); + assert_eq!(args.max_delay(), Duration::from_secs(2)); assert_eq!(parse(&["--delay", "auto"]).delay, Delay::Auto); assert_eq!(Delay::Auto.to_string(), "auto"); assert_eq!(Delay::Fixed(Duration::from_millis(250)).to_string(), "250ms"); @@ -267,7 +283,7 @@ mod tests { let config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); audio.renditions.insert("audio".into(), config); - assert_eq!(args.video_max_delay(&audio), AUTO_MAX_DELAY); + assert_eq!(args.video_max_delay(&audio), AUTO_VIDEO_DELAY + args.stretch()); audio.renditions.get_mut("audio").unwrap().enabled = false; assert_eq!(args.video_max_delay(&audio), AUTO_VIDEO_DELAY); diff --git a/rs/moq-cli/src/play/fake.rs b/rs/moq-cli/src/play/fake.rs index 6243f2f546a..b9c09e31df5 100644 --- a/rs/moq-cli/src/play/fake.rs +++ b/rs/moq-cli/src/play/fake.rs @@ -163,7 +163,7 @@ impl Sink for FakeSink { } /// Playing onto the clock is the real ring's stretch, which this fake leaves out. - fn due(&mut self, _at: Instant, _late: Duration, _jumped: Duration) {} + fn due(&mut self, _at: Instant, _stretch: Duration, _jumped: Duration) {} async fn finish(mut self) { // A finite short tail drains even when it cannot refill the target. diff --git a/rs/moq-cli/src/play/headless.rs b/rs/moq-cli/src/play/headless.rs index ecc6b9d02b9..534a0d4889b 100644 --- a/rs/moq-cli/src/play/headless.rs +++ b/rs/moq-cli/src/play/headless.rs @@ -149,6 +149,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: super::super::args::Delay::Auto, + stretch: std::time::Duration::from_secs(2).into(), select: crate::subscribe::SelectArgs { video: Some(crate::subscribe::Mode::Off), ..Default::default() @@ -176,6 +177,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: super::super::args::Delay::Auto, + stretch: std::time::Duration::from_secs(2).into(), select: crate::subscribe::SelectArgs { video: Some(crate::subscribe::Mode::Off), ..Default::default() @@ -206,6 +208,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: super::super::args::Delay::Auto, + stretch: std::time::Duration::from_secs(2).into(), select: crate::subscribe::SelectArgs { video: Some(crate::subscribe::Mode::Off), ..Default::default() diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 7e50b7a5b61..ef83303f4e9 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -436,6 +436,7 @@ impl Media { latency: self.args.fixed_delay().unwrap_or_default().max(AUDIO_BUFFER_MIN), changed: self.drained.clone(), output: self.output.clone(), + stretch: self.args.stretch(), }; let (stop, stopped) = tokio::sync::oneshot::channel(); let (retire, retired) = tokio::sync::oneshot::channel(); @@ -569,6 +570,8 @@ struct AudioPlayback { output: O, /// The rendition's catalog `delay`, added to the jitter target this track holds. delay: tokio::sync::watch::Receiver>, + /// How far audio may trail the clock before it skips ahead; zero turns the time stretch off. + stretch: Duration, } /// Play a track until it ends, or until `retired` resolves, which ends it where @@ -586,6 +589,7 @@ async fn play_audio( latency, output, delay: stated, + stretch, } = playback; // The playout delay is the consumer's to size, from how unevenly packets arrive, @@ -660,8 +664,8 @@ async fn play_audio( let target = || schedule.delay().clamp(AUDIO_BUFFER_MIN, cap); // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches - // onto that clock, and skips only what trails it past the age budget the - // subscription still delivers, which a fixed delay leaves none of. + // onto that clock, and skips only what trails it past the stretch, which + // the subscription still delivers. // The last anchor, as when its end was due, the end, and the delay. The // clock's reference is when media zero arrived, `at - end - delay`: one that // moved earlier jumped the clock ahead, which the sink skips rather than @@ -686,7 +690,7 @@ async fn play_audio( earlier + nanos(end) - nanos(from) + nanos(delay) - nanos(before) }); let jumped = Duration::from_nanos(jumped.clamp(0, u64::MAX as i128) as u64); - sink.due(at, max_delay.saturating_sub(delay), jumped); + sink.due(at, stretch, jumped); } }; @@ -714,7 +718,17 @@ async fn play_audio( // Media went missing since the last frame, so the next one does not continue it. let mut lost = false; let mut ends = VecDeque::new(); + // The subscription reaches the stretch past the shared delay wherever that moves, never + // below what it opened with, so audio the sink may still catch up on arrives. + let floor = consumer.options().delay.max.unwrap_or_default(); + let mut asked = floor; loop { + let seen = schedule.delay(); + let max = seen.saturating_add(stretch).max(floor); + if max != asked { + consumer.set_max_delay(max); + asked = max; + } // Stop reading while play is held behind a resume with more than the age // budget queued, so the subscription evicts the excess instead of this // queue growing without bound. @@ -736,6 +750,7 @@ async fn play_audio( retarget(*stated.borrow_and_update()); continue; } + _ = role.delay_changed(seen) => continue, }; let event = match event { Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), @@ -1104,6 +1119,7 @@ mod tests { playhead: Default::default(), latency: AUDIO_BUFFER_MIN, output: recorder, + stretch: Duration::ZERO, }; let _sink = play_audio(consumer, playback, std::future::ready(())).await.unwrap(); } @@ -1141,6 +1157,7 @@ mod tests { playhead: Default::default(), latency: AUDIO_BUFFER_MIN, output: recorder.clone(), + stretch: Duration::ZERO, }; let task = tokio::spawn(async move { play_audio(consumer, playback, std::future::pending()) @@ -1240,6 +1257,7 @@ mod tests { // What `--delay auto` opens on, well short of the lead the held media needs. latency: AUDIO_BUFFER_MIN, output: recorder.clone(), + stretch: Duration::ZERO, }; let task = tokio::spawn(async move { play_audio(consumer, playback, std::future::pending()) @@ -1314,6 +1332,7 @@ mod tests { playhead: Default::default(), latency: delay, output: recorder.clone(), + stretch: Duration::ZERO, }; let task = tokio::spawn(async move { play_audio(consumer, playback, std::future::pending()) @@ -1458,6 +1477,7 @@ mod tests { playhead: Default::default(), latency: delay, output: recorder.clone(), + stretch: Duration::ZERO, }; let video_play = Video { presentation: media.presentation.clone(), @@ -1592,6 +1612,7 @@ mod tests { playhead: Default::default(), latency: audio_latency, output: recorder.clone(), + stretch: Duration::ZERO, }; let video_play = Video { presentation: media.presentation.clone(), @@ -1750,6 +1771,7 @@ mod tests { playhead: Default::default(), latency: delay, output: recorder.clone(), + stretch: Duration::ZERO, }; let task = tokio::spawn(async move { play_audio(consumer, playback, std::future::pending()) @@ -1832,6 +1854,7 @@ mod tests { playhead: Default::default(), latency: delay, output: recorder.clone(), + stretch: Duration::ZERO, }; let task = tokio::spawn(async move { play_audio(consumer, playback, std::future::pending()) @@ -1890,6 +1913,7 @@ mod tests { playhead: Default::default(), latency: delay, output: recorder.clone(), + stretch: Duration::ZERO, }; let task = tokio::spawn(async move { play_audio(consumer, playback, std::future::pending()) @@ -2051,6 +2075,7 @@ mod tests { #[cfg(all(target_os = "linux", feature = "v4l2"))] v4l2_decoder: None, delay: Delay::Fixed(delay), + stretch: Duration::ZERO.into(), select: Default::default(), }, video: Default::default(), diff --git a/rs/moq-cli/src/play/output.rs b/rs/moq-cli/src/play/output.rs index 6b5440378db..823e88abfe9 100644 --- a/rs/moq-cli/src/play/output.rs +++ b/rs/moq-cli/src/play/output.rs @@ -42,9 +42,10 @@ pub(super) trait Sink: Send + Sync + 'static { fn set_latency(&mut self, latency: Duration) -> anyhow::Result<()>; /// The audio written so far is due to finish sounding at `at`; trailing that - /// by more than `late` is skipped, and by as much as the clock's reference - /// `jumped` earlier since the last call. - fn due(&mut self, at: Instant, late: Duration, jumped: Duration); + /// by more than `stretch` is skipped, and by as much as the clock's reference + /// `jumped` earlier since the last call. A zero `stretch` turns the time + /// stretch off. + fn due(&mut self, at: Instant, stretch: Duration, jumped: Duration); /// How much audio is queued ahead of the speaker. fn buffered(&self) -> Duration; @@ -72,8 +73,8 @@ impl Sink for moq_audio::playback::Sink { Ok(moq_audio::playback::Sink::set_latency(self, latency)?) } - fn due(&mut self, at: Instant, late: Duration, jumped: Duration) { - moq_audio::playback::Sink::due(self, at, late, jumped); + fn due(&mut self, at: Instant, stretch: Duration, jumped: Duration) { + moq_audio::playback::Sink::due(self, at, stretch, jumped); } fn write(&mut self, samples: &[u8]) -> anyhow::Result { diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 8f132680df7..2fa4ac56271 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -827,6 +827,7 @@ mod tests { broadcast: "room".into(), args: Args { delay: Delay::Fixed(delay), + stretch: Duration::ZERO.into(), catalog_format: None, metrics: None, #[cfg(all(target_os = "linux", feature = "vaapi"))] diff --git a/test/audio-quality/clients/js/src/capture.ts b/test/audio-quality/clients/js/src/capture.ts index 06d9e6e105e..6a0a9ccb8cd 100644 --- a/test/audio-quality/clients/js/src/capture.ts +++ b/test/audio-quality/clients/js/src/capture.ts @@ -1,37 +1,30 @@ /** Capture arrivals before the player's container orders or skips them. @module */ -import { Duration, Instant, now } from "@moq/clock"; +import { type Duration, Instant, now } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Cmaf from "@moq/hang/cmaf"; import * as Container from "@moq/hang/container"; import { type Group, Error as NetError } from "@moq/net"; import { Effect, type Getter } from "@moq/signals"; -import type { Broadcast, Delay } from "@moq/watch"; +import type { Broadcast } from "@moq/watch"; import { frameDuration, packetDuration } from "../../../../../js/watch/src/audio/config.ts"; +import { maxDelay } from "../../../../../js/watch/src/audio/latency.ts"; import type { Arrival } from "./schema.ts"; -/** - * The least max delay the decoder subscribes with at "auto": the estimator's 2 s ceiling, so a frame - * later than the target is still observed (doc/concept/audio-jitter.md). The observer asks for the - * same, or it would drop exactly the late frames the estimator measures. - */ -const AUTO_MAX_DELAY = Duration.millis(2000); - /** The public player inputs needed to observe its existing subscription. */ export type Input = { broadcast: Getter; track: Getter; config: Getter; - /** The configured delay, which decides whether the decoder subscribes past `maxDelay`. */ - delay: Getter; + /** + * The terms of the max delay the decoder subscribes with, which the observer matches, or it would + * drop exactly the late frames the estimator measures (doc/concept/audio-jitter.md). + */ + delay: Getter; maxDelay: Getter; + stretch: Getter; }; -/** The max delay the decoder subscribes with, which the observer matches. */ -function subscribed(delay: Delay, maxDelay: Duration): Duration { - return delay === "auto" ? maxDelay.max(AUTO_MAX_DELAY) : maxDelay; -} - /** Record frames on the player's connection with its subscription settings. */ export class Capture { readonly #signals = new Effect(); @@ -52,7 +45,13 @@ export class Capture { if (!active) return; const track = active.track(name).subscribe({ priority: Catalog.PRIORITY.audio, - delay: { max: subscribed(input.delay.peek(), input.maxDelay.peek()) }, + delay: { + max: maxDelay({ + delay: input.delay.peek(), + maxDelay: input.maxDelay.peek(), + stretch: input.stretch.peek(), + }), + }, }); const groups = new Set(); effect.cleanup(() => { diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 22f6e113708..60d416e3074 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -56,8 +56,9 @@ export function probe(watch: MoqWatch, capture = false): Probe { broadcast: audio.source.in.broadcast, track: audio.source.out.track, config: audio.source.out.config, - delay: sync.in.delay, + delay: sync.out.delay, maxDelay: sync.out.maxDelay, + stretch: sync.in.stretch, }) : undefined; From 38fe8a8dc3c3330b7b20b7e390c31d325b65432a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:10:39 -0800 Subject: [PATCH 03/12] wip: docs and replay budgets --- js/watch/README.md | 1 + test/audio-quality/README.md | 4 ++-- test/audio-quality/budgets.json | 12 ++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/js/watch/README.md b/js/watch/README.md index 8225f2cac34..3def3a2dba6 100644 --- a/js/watch/README.md +++ b/js/watch/README.md @@ -74,6 +74,7 @@ The simplest way to watch a stream: | `follow` | boolean | true | Follow the broadcast's announcements: wait for one before subscribing, and move to each restart. Ignored when the relay does not support broadcast discovery. | | `delay` | `auto`, duration, `instant` | `auto` | Distance from the live edge. `instant` paints frames as they decode and disables audio. | | `lookahead` | duration | `0ms` | How far future-dated media may run ahead of the live edge. Not a jitter buffer: live streams tune `delay`. | +| `stretch` | duration | `2000ms` | How far audio may trail the clock after a stall before it skips ahead. `0` turns the time stretch off, so received audio plays unaltered. | | `captions` | string | off | Text rendition to render. | | `catalog-format` | hang, msf, manual | auto-detected | The catalog format; detected from the name suffix unless set. | | `echo` | path segment | - | The viewer's feedback name: once the catalog solicits feedback, publish `/.echo` reporting what each rendition received and played. | diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 2cfc4c6f887..8f0ec14d2c4 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -113,8 +113,8 @@ configuration. `--capture` also writes `.arrivals.ndjson`: `[at, timestamp, group, duration]` on the same viewer clock as `.ndjson`, before the container orders or skips groups. The observer shares the player's -connection and track, priority, and subscription max delay (at least 2 s at "auto", as the decoder -asks, not the target), so it adds no upstream subscription demand and still sees a frame later than +connection and track, priority, and subscription max delay (the delay plus the stretch as capture starts, as the +decoder asks, not the target), so it adds no upstream subscription demand and still sees a frame later than the target. Groups drain concurrently, so an earlier stalled group cannot hide a later arrival. Capture errors void the row. Capture adds local decoding and recording work; it is off by default and diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index d80f8797bf1..0a38bcf0bbf 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002.", + "note": "The browser rows are measured on the nightly runner (ubuntu-24.04-arm, headless Chromium, seed 7, 60 s a row), from the audio-quality-run artifacts of the seven scheduled Nightly runs on main from 2026-10-29 to 2026-11-04 (40885347517, 41065371094, 41204380716, 41313466339, 41483654426, 41656613559, 41852517544), the first after a fresh subscriber stopped receiving the relay's cached history. The six AAC rows other than congestion are measured the same way from the seven runs from 2026-11-05 to 2026-11-11 (42050683248, 42236107954, 42377867523, 42487375545, 42659882487, 42842827718, 43041909115), the first after the TS importer published each PES as one group; the first four predate the browser's time stretch, and the six rows keep the four provisional ceilings set below from 2026-11-09 and 2026-11-10. Each ceiling is the worst of its seven nights times 1.25, plus one runner hiccup, rounded up to two significant figures. The hiccup is sized from the single gaps the 2026-10-08 night put on otherwise clean rows, the fixed-250 control among them: 250 underruns/min, 700 gap ms/min, 3.3 skips/min, 280 dropped ms/min (both sized on the playhead's skip-ahead inference, and now applied to player_skips_per_min and trimmed_ms_per_min), 0.01 stalled, 0.015 silence, and one 20 ms bucket of target. Budgeted this way, every night passes when it is held out of its row's measurement except one, and that row's own ceiling includes it: the 2026-11-03 Opus bursty row raised its target to 300 ms 2.6 s in and held it for one 20 s half-life before settling at 180. Such startup holds, Opus near-zero's 100 ms on 2026-10-30 and 10-31 among them, are budgeted as measured. These ceilings record today's player as the runner hears it, not target audio quality. To re-measure, download the same artifact from the latest scheduled nightlies (`gh run download -n audio-quality-run`), which keep every run, and apply the same rule. On 2026-11-10 the inferred skip_aheads and discarded_ms keys were dropped, since they read a time stretch's catch-up as dropped audio, and every browser row gained player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, by the same rule from the only two scheduled nights on main with today's browser player and its trim counter, 2026-11-09 and 2026-11-10 (42659882487, 42842827718). A lag is signed, so its ceiling is the worst night plus a quarter of its size plus one 20 ms bucket. These four keys are provisional until seven nights with the stretch model re-measure every row. The grade follows the model: audio rests on the shared clock, trails it through a stall, and catches up after. When a change moves a row, a steady-state lag (av_lag_ms p50) that rises, or audio the player trimmed, is a regression to fix rather than budget; a p95 lag that rises through a stall, or an underrun count that moves, is re-measured in. No ceiling is fixed at zero underruns or zero stalls; a replay row's is exactly what it measured. The replay rows are deterministic, so their ceilings are exactly what they measured, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. They were last re-measured on 2026-10-22 using version 3 traces with encoded frame durations: AAC-LC uses its fixed 1024 samples, and relay-mic retains Opus packet TOC durations. relay-mic was re-recorded on 2026-11-03 from a real microphone, Chrome 156 on a MacBook Air publishing its built-in microphone through the public relay, recorded by headless Chromium on a Linux host (35 s, minimum RTT 2 ms), which raised its target ceiling from 60 to 80 ms and its silence from 0 to 0.001 as measured. The recording path itself has no added RTT. relay-mic-rtt40 is the 2026-11-03 Chrome row at 40 ms of added RTT on that laptop (180 s, minimum RTT 47 ms), kept as the baseline time stretch improves on; like every replay row, its player_underruns_per_min ceiling is what it measures, not a fixed zero. player_underruns_per_min, the player's own counter that its stats panel shows, is budgeted on the replay rows only; it counts the run-dry re-stalls the tap files as stalled time. On 2026-11-07, after the ordered read stopped delivering groups below its cursor (#8071) and took Rust's hole and age bounds (#8218), relay-bbb was re-measured at 16.1 player underruns/min, 0.025 stalled and 0.02 silence, and relay-mic-rtt40 at 8.2, its discarded audio at 70.9 ms/min and its target p95 at 320 ms rather than 780, since the stale backlog it never played no longer reaches the estimator; neither row lost audio the previous player heard. On 2026-11-08 the player began time-stretching live playback with NetEq's accelerate, deciding on the ring's depth right after each arrival and never expanding, and every replay row was re-measured with it, the browser rows left to the nightly runner. Each row's av_lag_ms, how far the playhead trails the shared clock video plays from, is what it measured: the ring rests one frame ahead of the newest arrival's playout, as the trim it replaces left it, so audio leads the clock by 19 to 23 ms at p50. Against the trim, the A/V lag p50 fell from 0.7, -17.5, -21.5, -18.2 and -15.6 ms to -22.4 on bursty-aac, -22.2 on local-aac, -22.7 on relay-bbb, -19 on relay-mic and -20.1 on relay-mic-rtt40, and its p95 fell on every row, most on relay-bbb (10.5 to -4.2) and relay-mic-rtt40 (-0.9 to -13.4). Underruns stayed where the trim had them, as the latency they would cost is not spent: player_underruns_per_min held at 4.4, 12.1, 16.1 and 6.1 and rose from 8.2 to 10.6 on relay-mic-rtt40, and relay-mic's tap now files 2 short quanta a minute, 2 ms of missing audio, where it filed none. What the stretch buys is the audio the trim dropped: discarded_ms_per_min fell from 87.8 to 0 on relay-bbb and from 127.5 to 100.6 on local-aac, whose remainder drops nothing: two receive stalls 14 and 15.3 s in leave audio 159 and 132 ms behind the clock, and the fast accelerate brings it back within half a second. player_skips_per_min and trimmed_ms_per_min, the ring's own count of re-anchors that dropped buffered audio and the media they dropped, are budgeted beside it, at 0 on every recorded row. relay-mic-stall, added 2026-11-10, is relay-mic with nothing arriving from 15 s to 16.5 s and the held frames then trickling in at twice real time until 18 s; today's player trims 2152.5 ms/min of it and its target p95 rises to 1540 ms. Once durations became whole nanoseconds, relay-bbb's delay, 200 ms plus one 1024-sample AAC frame, rounds up by a fraction of a sample, so its ring holds one more sample (23 µs) and its A/V lag p50 measures -22.6 rather than -22.7. Neither replay calibration changed the browser or native ceilings. bursty-aac is the 2026-10-18 AAC bursty isolated capture, recorded with the observer subscribed at the decoder's 2 s budget, so it holds every group the player saw. The native rows are provisional: measured on a loaded developer host (x86_64 Linux, the private PulseAudio null sink, seed 7, 60 s a row) on 2026-10-17, three runs of every row from the debug build nightly uses plus three more of the AAC rows from a release build, which played alike. Each ceiling is the worst clean run times 1.25, plus one native hiccup sized from the unimpaired Opus row's worst run (20 underruns/min, 50 gap ms/min, 2.2 skips/min, 110 discarded ms/min, 0.01 stalled, 0.005 silence, one 20 ms bucket of target), rounded up to two significant figures. On 2026-11-10, after native playback began reporting its A/V lag (#8361), the native rows' player_skips_per_min, trimmed_ms_per_min, and av_lag_ms p50 and p95, and every key of the native and browser congestion rows, were measured by the same rules from three runs of each on a loaded developer host (x86_64 Linux, the debug build, seed 7, 60 s a row), with no void; native audio rests 9 to 95 ms behind its clock at p50. These are provisional until the stretch model's nightly re-measure, which also moves the browser congestion rows onto the runner. On the 2026-10-17 host the null sink's clock ran about 150 ppm slow against the monotonic clock, read within 5 ms at p95 per sample, and its output latency sat between 14 and 28 ms. Five of the 36 AAC rows (none of the 18 Opus rows) voided on the clock: the device stopped pulling for about two seconds mid-row, sometimes after a backend error and a stream restart, which is a native playback defect rather than noise, and the ceilings exclude those runs. Re-measure from the nightly runner's artifacts once it has a week of native rows. On 2026-11-11 every player began following one stretch rule on the shared clock: it slows down below its target, speeds up above it, and skips only past its bound, which in the browser is the subscription's max delay. Every replay row was re-measured with it, the browser and native rows left to the nightly runner. Audio now rests on the clock rather than one frame ahead of it, so the A/V lag p50 rose from about -20 ms to 0.4 ms on relay-bbb, 1.1 on relay-mic, 1.2 on relay-mic-rtt40 and 1.8 on local-aac. Its p95 rose on every row, since audio that falls behind is now caught up by speeding up rather than trimmed: the shared clock steps forward as its reference converges through a tune-in backlog, and again when a target falls, and audio follows at the accelerate's pace, which on the replay's constant tone is one 2.5 ms period a window, slower than on speech. bursty-aac rests 5 ms behind its clock, so its p50 of 10.4 ms is mostly its catch-ups, and relay-mic-stall's p50 of 66.1 ms is the catch-up after its target falls from 1540 to 80 ms, which outlasts half the row. In exchange nothing is trimmed: relay-mic-stall's trimmed_ms_per_min fell from 2152.5 to 0, its skips from 6.1 to 0, and its underruns from 24.2 to 0 a minute, and player_underruns_per_min fell from 12.1 to 4 on local-aac, 16.1 to 0 on relay-bbb, 6.1 to 0 on relay-mic, 10.6 to 2.7 on relay-mic-rtt40 and 6.1 to 4 on relay-mic-stall, holding at 4.4 on bursty-aac. The tap now files 3.3 to 6.1 short quanta a minute on the AAC rows, under 0.3 ms of missing audio, where it filed none. To deliver the audio a stall delays, the container consumer now waits and skips at the subscription's max delay, as native already does, rather than at the delay plus lookahead. relay-mic-rtt40's join backlog arrives reordered, so the consumer now waits on its holes and the estimator observes the frames that fill them, up to 775 ms late, and its target p95 returns from 320 to 780 ms, held for one half-life after the join. Later on 2026-11-11 audio began skipping as far as the shared clock's reference moves earlier, once those moves add up to one stretch window and only as far as it holds audio past its target, rather than speeding through them, and every replay row was re-measured. The tune-in backlog's jumps now land before the graded window, so the A/V lag p95 fell from 453.1 to 2.1 ms on relay-bbb (its p50 from 0.4 to -1.3), 743.7 to 1.9 on relay-mic-rtt40, whose p50 fell from 1.2 to 0.4 and its tap's missing audio rose from 0.6 to 0.9 ms a minute, since a jump past what is decoded skips the rest once it lands,, 335.5 to 39.2 on local-aac and 88.6 to 0.3 on relay-mic, and the p50 from 66.1 to 25.3 on relay-mic-stall, with nothing trimmed in any graded window. The tap's underruns fell from 6.1 to 2 a minute on local-aac and 6.1 to 0 on relay-bbb; local-aac's silence rose from 0.001 to 0.002. Also on 2026-11-11 the stretch became a player setting, 2 s by default, and the audio subscription's max delay became the delay plus it rather than the 2 s constant, so the container consumer, which waits on a hole for the max delay, waits that much longer, and every replay row was re-measured: relay-bbb's A/V lag p50 moved from -1.3 to -2.2 ms and its p95 from 2.1 to 1.8, relay-mic's p50 and p95 from 0.3 to 1.6, under the 2.5 ms pitch period the stretch can move it by, and relay-mic-rtt40's tap's missing audio from 0.9 to 1.1 ms a minute. Holding the constant max delay with the new bound reproduces the old values exactly.", "rows": [ { "runtime": "chromium", @@ -466,8 +466,8 @@ "stalled_share": 0, "silence_share": 0, "target_ms_p95": 223.2, - "av_lag_ms_p50": -1.3, - "av_lag_ms_p95": 2.1 + "av_lag_ms_p50": -2.2, + "av_lag_ms_p95": 1.8 }, { "runtime": "replay", @@ -482,8 +482,8 @@ "stalled_share": 0, "silence_share": 0, "target_ms_p95": 80, - "av_lag_ms_p50": 0.3, - "av_lag_ms_p95": 0.3 + "av_lag_ms_p50": 1.6, + "av_lag_ms_p95": 1.6 }, { "runtime": "replay", @@ -493,7 +493,7 @@ "player_underruns_per_min": 2.7, "player_skips_per_min": 0, "underruns_per_min": 0.7, - "underrun_ms_per_min": 0.9, + "underrun_ms_per_min": 1.1, "trimmed_ms_per_min": 0, "stalled_share": 0, "silence_share": 0, From fddf593cb2fe42b3481196f9eae705304e465472 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:16:25 -0800 Subject: [PATCH 04/12] wip: capture test --- .../audio-quality/clients/js/src/capture.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/audio-quality/clients/js/src/capture.test.ts b/test/audio-quality/clients/js/src/capture.test.ts index 5ef3ac3c74e..afedb3b79a8 100644 --- a/test/audio-quality/clients/js/src/capture.test.ts +++ b/test/audio-quality/clients/js/src/capture.test.ts @@ -4,7 +4,6 @@ import * as Catalog from "@moq/hang/catalog"; import { Legacy } from "@moq/hang/container"; import { Group, Error as NetError, Origin, Path, Time } from "@moq/net"; import { Signal } from "@moq/signals"; -import type { Delay } from "@moq/watch"; import { Broadcast } from "../../../../../js/watch/src/broadcast.ts"; import { arrivals, Capture, format } from "./capture.ts"; import type { Arrival } from "./schema.ts"; @@ -89,8 +88,9 @@ test("the observer shares playback demand and closes without ending playback", a numberOfChannels: 1, container: { kind: "legacy" }, } as Catalog.AudioConfig), - delay: new Signal(Duration.millis(250)), + delay: new Signal(Duration.millis(250)), maxDelay: new Signal(Duration.millis(250)), + stretch: new Signal(Duration.ZERO), }); try { while (!track.subscription.peek()) await track.subscription.changed(); @@ -117,7 +117,7 @@ test("the observer shares playback demand and closes without ending playback", a } }); -test("at auto the observer asks for the decoder's 2 s budget, not the target", async () => { +test("the observer asks for the decoder's delay plus its stretch, not the target", async () => { // A capture cut to the target would drop the frames later than it, which are what the target // is measured from, so a trace recorded that way can never show the player falling short. const origin = new Origin.Producer(); @@ -135,12 +135,13 @@ test("at auto the observer asks for the decoder's 2 s budget, not the target", a numberOfChannels: 1, container: { kind: "legacy" }, } as Catalog.AudioConfig), - delay: new Signal("auto"), - maxDelay: new Signal(Duration.millis(250)), + delay: new Signal(Duration.millis(100)), + maxDelay: new Signal(Duration.millis(100)), + stretch: new Signal(Duration.secs(2)), }); try { while (!track.subscription.peek()) await track.subscription.changed(); - expect(track.subscription.peek()).toMatchObject({ delay: { max: Duration.millis(2000) } }); + expect(track.subscription.peek()).toMatchObject({ delay: { max: Duration.millis(2100) } }); } finally { await capture.close(); source.close(); @@ -165,8 +166,9 @@ async function captureGroupEnd(end: (group: Group.Producer) => void): Promise(Duration.millis(250)), + delay: new Signal(Duration.millis(250)), maxDelay: new Signal(Duration.millis(250)), + stretch: new Signal(Duration.ZERO), }); try { while (!track.subscription.peek()) await track.subscription.changed(); From 4ac8a85b72c31aaabbb081c69b92d5335e642e37 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:23:17 -0800 Subject: [PATCH 05/12] wip: capture follows the decoder's budget --- test/audio-quality/README.md | 2 +- test/audio-quality/clients/js/src/capture.ts | 8 +++++++- test/audio-quality/clients/js/src/ptt.ts | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 8f0ec14d2c4..b039ce4f0df 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -113,7 +113,7 @@ configuration. `--capture` also writes `.arrivals.ndjson`: `[at, timestamp, group, duration]` on the same viewer clock as `.ndjson`, before the container orders or skips groups. The observer shares the player's -connection and track, priority, and subscription max delay (the delay plus the stretch as capture starts, as the +connection and track, priority, and subscription max delay (the delay plus the stretch, as the decoder asks, not the target), so it adds no upstream subscription demand and still sees a frame later than the target. Groups drain concurrently, so an earlier stalled group cannot hide a later arrival. Capture diff --git a/test/audio-quality/clients/js/src/capture.ts b/test/audio-quality/clients/js/src/capture.ts index 6a0a9ccb8cd..ed7775ab1d4 100644 --- a/test/audio-quality/clients/js/src/capture.ts +++ b/test/audio-quality/clients/js/src/capture.ts @@ -61,7 +61,13 @@ export class Capture { effect.run((inner) => track.update({ priority: Catalog.PRIORITY.audio, - delay: { max: subscribed(inner.get(input.delay), inner.get(input.maxDelay)) }, + delay: { + max: maxDelay({ + delay: inner.get(input.delay), + maxDelay: inner.get(input.maxDelay), + stretch: inner.get(input.stretch), + }), + }, }), ); effect.spawn(async () => { diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index 1eed9ee60f1..21879afd58b 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -138,12 +138,15 @@ const capture = new Capture({ broadcast: audio.source.in.broadcast, track: audio.source.out.track, config: audio.source.out.config, - delay: watch.sync.in.delay, // Decoder subscriptions include codec warmup beyond Sync's budget. The observer must // request the same age so its own subscription cannot discard an otherwise visible frame. + delay: new Derived([watch.sync.out.delay, audio.source.out.config], (delay, config) => + delay.add(Duration.millis(config?.warmup ?? 0)), + ), maxDelay: new Derived([watch.sync.out.maxDelay, audio.source.out.config], (max, config) => max.add(Duration.millis(config?.warmup ?? 0)), ), + stretch: watch.sync.in.stretch, }); const arrivals: Arrival[] = []; let attached: AudioNode | undefined; From 8093635b35834b8dee56940eb6b8323345529ae7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:42:46 -0800 Subject: [PATCH 06/12] chore(quest): finish the stretch setting --- quest/m1/audio-jitter-target/README.md | 5 +- .../m1/audio-jitter-target/stretch-setting.md | 65 ------------------- 2 files changed, 2 insertions(+), 68 deletions(-) delete mode 100644 quest/m1/audio-jitter-target/stretch-setting.md diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index 80682a6374c..0d80fdee610 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -67,9 +67,8 @@ reach zero underruns, and some underruns during congestion are expected. the spec already declined as a second knob; deleting the estimator and letting the lag be the only network term, which underruns a 20 ms Opus target until the lag builds). -- Bound: ✅ configurable, as - [the `stretch` setting](/quest/m1/audio-jitter-target/stretch-setting.md) - records. +- Bound: ✅ configurable: the `stretch` setting (``, + `moq play --stretch`), 2 s by default, where zero turns the stretch off. - Tests: no gate may reward moving away from the rule, so the audio harness grades steady-state lag, the player's own trims, and recovery from a stall before [the model](https://github.com/moq-dev/moq/pull/8507) lands. diff --git a/quest/m1/audio-jitter-target/stretch-setting.md b/quest/m1/audio-jitter-target/stretch-setting.md deleted file mode 100644 index 1627d08054f..00000000000 --- a/quest/m1/audio-jitter-target/stretch-setting.md +++ /dev/null @@ -1,65 +0,0 @@ -# [M] A stretch setting bounds how far audio trails before it skips - -## Goal - -A player chooses how much lateness its audio forgives. `` and `moq play --stretch 2s` set how far live audio may trail -the shared clock before it skips ahead. Zero turns the time stretch off: the -player never alters received audio, late audio is skipped, an underrun is -silence, and drift between the publisher's clock and the device's is -corrected by a small skip or pad. The default is 2 s whether `delay` is -`auto` or fixed, so a fixed-delay live player also keeps late audio and -catches up. -`AUTO_MAX_DELAY` is deleted in both `js/watch/src/audio/latency.ts` and -`rs/moq-cli/src/play/args.rs`: every budget it fed (the subscription max -delay, the sink cap, and video retention) is derived from `delay + stretch`. - -## Plan - -Planned 2026-11-09 (maintainer, /quest-plan). Asked how far audio may trail, -the maintainer answered that it is a product decision and not everybody will -want stretching. - -- Shape: ✅ one duration, where zero turns it off (rejected: an on/off switch - with a fixed bound; no new setting, leaving `auto` at 2 s and a fixed - delay at none; NetEq's four times the target as the bound). A voice app - can ask for 200 ms while a sports stream takes 2 s. -- Name: ✅ `stretch` (maintainer; rejected: `catchup`). -- At zero: ✅ no stretching at all (rejected: only falling behind is off, - with small corrections still stretched). A pitch-period splice is more - audible on music than on speech, so one value gives unaltered audio. -- Default: ✅ 2 s for `auto` and fixed delays alike (rejected: a fixed delay - keeping max delay equal to the delay). -- Milestone: ✅ m1 (rejected: m2). The attribute and flag are additive. -- Decided 2026-11-09 (maintainer, /quest-unblock): ✅ delete - `AUTO_MAX_DELAY` in both players and derive every budget it fed - (subscription max delay, sink cap, video retention) from `delay + stretch` - (rejected: keep the 2 s constant beside the setting). The 2 s is an - arbitrary constant nobody can trace. - -The audio subscription's max delay is what makes late audio arrive at all: -the relay and the container consumer drop media older than it. Today it is 2 s -under `auto` (`AUTO_MAX_DELAY` in `js/watch/src/audio/latency.ts` and -`rs/moq-cli/src/play/args.rs`) and the delay itself when fixed, which leaves a -fixed-delay live player nothing to catch up on. The setting feeds that bound: -audio asks for `delay + stretch`. - -Recommendations: - -- Mirror the name in `js/watch` (`Sync`, the element attribute and property), - `moq play`, and whatever `moq-audio` type carries the bound, so a Rust - application gets it without the CLI. `rs/moq-ffi` already takes the decode - consumer's `delay` min and max and has no playback ring, so no binding - changes. -- The attribute needs a unit, and an invalid value throws, as `delay` and - `lookahead` do. -- Say how it composes with `lookahead`: a buffered player already keeps - everything inside its lookahead, so `stretch` bounds a live player's lag - and its zero still turns the stretch off in both modes. -- The zero path is the player before the stretch existed. Keep it tested, not - only reachable. -- Docs: `doc/lib/js/watch.md`, the element's attribute docs, and - `moq play --help`. - -Public API: an additive attribute, property, and flag. Wire: none; a -fixed-delay player's audio subscription asks for a longer max delay. From 86ee65770a2cfe4737bd917548ee6e891cfd49fe Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:50:27 -0800 Subject: [PATCH 07/12] fix(play): the clock holds the whole measured target past the opening budget --- rs/moq-cli/src/play/media.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index ef83303f4e9..25e9cbe3dad 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -988,9 +988,12 @@ async fn play_audio( // target, which playback holds unless another track asks for more. let mut moved = role.received(mapped, arrived_at); // The publisher's encoder delay is on top of the arrival jitter, so the - // clock holds both while the sink keeps sizing for jitter alone. - *jitter.lock().unwrap() = Some(input.latency); - moved |= role.set_target(input.latency + stated.borrow().unwrap_or_default()); + // clock holds both while the sink keeps sizing for jitter alone. The + // clock holds the whole measured target, which the subscription follows, + // though the sink holds no more of it than its cap: the rest stays encoded. + let held = latency.max(delay.max(AUDIO_BUFFER_MIN)); + *jitter.lock().unwrap() = Some(held); + moved |= role.set_target(held + stated.borrow().unwrap_or_default()); // The consumer releases media due further out than the sink holds only // when its timeline ends before the media behind it covers the hold, so // that little waits here instead of overflowing the sink. @@ -1208,6 +1211,14 @@ mod tests { assert_on_time(sounded, jitter + catalog); } + /// A measured target past the budget the track opened with still sets the + /// clock: the subscription follows the delay, so the budget never caps it. + #[tokio::test] + async fn audio_holds_a_target_past_its_opening_budget() { + let jitter = Duration::from_millis(300); + assert_on_time(first_sound(jitter, Duration::from_millis(100), None, None).await, jitter); + } + fn assert_on_time(sounded: Duration, due: Duration) { assert!( sounded.max(due) - sounded.min(due) <= Duration::from_millis(1), From 7b3ca060a0bbe974ee8d25586f599c51b590df92 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:10:49 -0800 Subject: [PATCH 08/12] style: fmt --- rs/moq-cli/src/play/media.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 25e9cbe3dad..607224e8613 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -1216,7 +1216,10 @@ mod tests { #[tokio::test] async fn audio_holds_a_target_past_its_opening_budget() { let jitter = Duration::from_millis(300); - assert_on_time(first_sound(jitter, Duration::from_millis(100), None, None).await, jitter); + assert_on_time( + first_sound(jitter, Duration::from_millis(100), None, None).await, + jitter, + ); } fn assert_on_time(sounded: Duration, due: Duration) { From ddaa90476f3631c837f04c310b6cb5c7ccfb0f13 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:07:54 -0800 Subject: [PATCH 09/12] fix: settle review threads on the stretch setting - Capture adds the codec warmup to its subscription itself, as the decoder does, so the probe and ptt observers both match playback. - The playout harness shares one reactive budget between the subscriber and the container consumer, as the decoder does. - moq play caps the sink at its latency limit, so a large --stretch keeps the excess encoded instead of failing set_latency. - The jitter doc states what a stretch below 2 s and a buffered player's lookahead do to the subscription. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 14 +++++++++----- js/watch/src/audio/playout.test.ts | 10 +++++++--- rs/moq-cli/src/play/media.rs | 10 ++++++---- test/audio-quality/clients/js/src/capture.test.ts | 5 +++-- test/audio-quality/clients/js/src/capture.ts | 12 +++++++----- test/audio-quality/clients/js/src/ptt.ts | 12 ++---------- 6 files changed, 34 insertions(+), 29 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index cf95f3057d7..7029be0b9c9 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -55,11 +55,15 @@ before any container sees them, at the relay and in the receiver's own transport subscriber, so a subscription cut to the target hides every frame later than the target: the estimate creeps up one bucket at a time while the frames it should have measured go unplayed. Both implementations subscribe with the delay -plus the player's `stretch` setting, 2 s by default, past the estimate's own -ceiling: `moq play` through its age budget, and the browser through the audio -subscription alone, since its container consumer observes each frame before -applying the local budget. A zero stretch cuts the subscription to the target -and accepts that creep. +plus the player's `stretch` setting: `moq play` through its age budget, and the +browser through the audio subscription alone, since its container consumer +observes each frame before applying the local budget. The 2 s default reaches +the estimate's own ceiling, so any lateness the estimator can measure is +observed at once. A smaller stretch hides frames later than the target plus the +stretch, so a jump in lateness larger than the stretch is learned in steps of +about the stretch; a zero stretch cuts a live subscription to the target and +accepts the one-bucket creep. A buffered browser player keeps its subscription +at least the delay plus its lookahead, whatever the stretch. Browser video keeps the shared budget, because a longer one fetches whole stale groups, so a broadcast with no audio measures only the lateness that budget lets through. diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index 2b0afd8e4c8..8477904a5b3 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "bun:test"; import { Duration, Instant, Manual } from "@moq/clock"; import * as Container from "@moq/hang/container"; import { Time, Track, Varint } from "@moq/net"; +import { Signal } from "@moq/signals"; import { STRETCH } from "../sync"; import { maxDelay, target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; @@ -78,10 +79,12 @@ async function measure(t: Arrival[], budget: (target: Duration) => Duration): Pr const clock = new Manual(); const installed = clock.install(); const track = new Track.Producer("audio"); - const subscriber = track.subscribe({ delay: { max: budget(Duration.ZERO) } }); + // One budget for both, as the decoder keeps: the container skips at the max delay it subscribes with. + const delay = new Signal({ max: budget(Duration.ZERO) }); + const subscriber = track.subscribe({ delay: delay.peek() }); const consumer = new Container.Consumer(subscriber, { format: new Container.Legacy.Format("audio"), - delay: { max: budget(Duration.ZERO) }, + delay, }); const result: Measured = { targets: [], delivered: 0 }; @@ -110,7 +113,8 @@ async function measure(t: Arrival[], budget: (target: Duration) => Duration): Pr const current = target({ measured: consumer.spread.peek(), frame: Duration.millis(CHUNK_MS) }); result.targets.push(current.asMillis()); - subscriber.update({ delay: { max: budget(current) } }); + delay.set({ max: budget(current) }); + subscriber.update({ delay: delay.peek() }); } } finally { track.finish(); diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 607224e8613..0e4eebb3629 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -637,10 +637,12 @@ async fn play_audio( // subtracting the incoming frame before it reaches the sink. let lead = |delay: Duration| latency.max(delay.min(max_delay).max(AUDIO_BUFFER_MIN)); - // The most the sink holds. A shared delay past it, such as a slow video - // encoder's, stays encoded in the consumer until it is due instead, so the - // ring never grows with the delay. - let cap = max_delay.max(AUDIO_BUFFER_MIN); + // The most the sink holds, within what it accepts. A shared delay past it, + // such as a slow video encoder's, stays encoded in the consumer until it is + // due instead, so the ring never grows with the delay. + let cap = max_delay + .max(AUDIO_BUFFER_MIN) + .min(moq_audio::playback::Input::LATENCY_MAX); let schedule = presentation.consume(); // The consumer decodes a packet once the sink has room for it. Its playhead is // where the shared clock says the last write sounds, rather than what the sink diff --git a/test/audio-quality/clients/js/src/capture.test.ts b/test/audio-quality/clients/js/src/capture.test.ts index afedb3b79a8..5d53de50f41 100644 --- a/test/audio-quality/clients/js/src/capture.test.ts +++ b/test/audio-quality/clients/js/src/capture.test.ts @@ -117,7 +117,7 @@ test("the observer shares playback demand and closes without ending playback", a } }); -test("the observer asks for the decoder's delay plus its stretch, not the target", async () => { +test("the observer asks for the decoder's delay plus its stretch and warmup, not the target", async () => { // A capture cut to the target would drop the frames later than it, which are what the target // is measured from, so a trace recorded that way can never show the player falling short. const origin = new Origin.Producer(); @@ -134,6 +134,7 @@ test("the observer asks for the decoder's delay plus its stretch, not the target sampleRate: 48000, numberOfChannels: 1, container: { kind: "legacy" }, + warmup: 80, } as Catalog.AudioConfig), delay: new Signal(Duration.millis(100)), maxDelay: new Signal(Duration.millis(100)), @@ -141,7 +142,7 @@ test("the observer asks for the decoder's delay plus its stretch, not the target }); try { while (!track.subscription.peek()) await track.subscription.changed(); - expect(track.subscription.peek()).toMatchObject({ delay: { max: Duration.millis(2100) } }); + expect(track.subscription.peek()).toMatchObject({ delay: { max: Duration.millis(2180) } }); } finally { await capture.close(); source.close(); diff --git a/test/audio-quality/clients/js/src/capture.ts b/test/audio-quality/clients/js/src/capture.ts index ed7775ab1d4..60dca86c128 100644 --- a/test/audio-quality/clients/js/src/capture.ts +++ b/test/audio-quality/clients/js/src/capture.ts @@ -1,6 +1,6 @@ /** Capture arrivals before the player's container orders or skips them. @module */ -import { type Duration, Instant, now } from "@moq/clock"; +import { Duration, Instant, now } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Cmaf from "@moq/hang/cmaf"; import * as Container from "@moq/hang/container"; @@ -17,8 +17,9 @@ export type Input = { track: Getter; config: Getter; /** - * The terms of the max delay the decoder subscribes with, which the observer matches, or it would - * drop exactly the late frames the estimator measures (doc/concept/audio-jitter.md). + * The terms of the max delay the decoder subscribes with, which the observer matches, codec warmup + * included, or it would drop exactly the late frames the estimator measures + * (doc/concept/audio-jitter.md). */ delay: Getter; maxDelay: Getter; @@ -43,6 +44,7 @@ export class Capture { const decoder = format(config); const active = broadcast.relativeBroadcast(effect, config.broadcast); if (!active) return; + const warmup = Duration.millis(config.warmup ?? 0); const track = active.track(name).subscribe({ priority: Catalog.PRIORITY.audio, delay: { @@ -50,7 +52,7 @@ export class Capture { delay: input.delay.peek(), maxDelay: input.maxDelay.peek(), stretch: input.stretch.peek(), - }), + }).add(warmup), }, }); const groups = new Set(); @@ -66,7 +68,7 @@ export class Capture { delay: inner.get(input.delay), maxDelay: inner.get(input.maxDelay), stretch: inner.get(input.stretch), - }), + }).add(warmup), }, }), ); diff --git a/test/audio-quality/clients/js/src/ptt.ts b/test/audio-quality/clients/js/src/ptt.ts index 21879afd58b..488cfdf8ec7 100644 --- a/test/audio-quality/clients/js/src/ptt.ts +++ b/test/audio-quality/clients/js/src/ptt.ts @@ -1,11 +1,9 @@ /** Measure the shipped push-to-talk path on one monotonic page clock. @module */ import "@moq/publish/element"; import "@moq/watch/element"; -import { Duration } from "@moq/clock"; import * as Container from "@moq/hang/container"; import { Track } from "@moq/net"; import type MoqPublish from "@moq/publish/element"; -import { Derived } from "@moq/signals"; import type MoqWatch from "@moq/watch/element"; import { Capture } from "./capture"; import type { Arrival } from "./schema"; @@ -138,14 +136,8 @@ const capture = new Capture({ broadcast: audio.source.in.broadcast, track: audio.source.out.track, config: audio.source.out.config, - // Decoder subscriptions include codec warmup beyond Sync's budget. The observer must - // request the same age so its own subscription cannot discard an otherwise visible frame. - delay: new Derived([watch.sync.out.delay, audio.source.out.config], (delay, config) => - delay.add(Duration.millis(config?.warmup ?? 0)), - ), - maxDelay: new Derived([watch.sync.out.maxDelay, audio.source.out.config], (max, config) => - max.add(Duration.millis(config?.warmup ?? 0)), - ), + delay: watch.sync.out.delay, + maxDelay: watch.sync.out.maxDelay, stretch: watch.sync.in.stretch, }); const arrivals: Arrival[] = []; From 149a47241da5c39423aab3979f83e82622d565a0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 21:38:56 -0800 Subject: [PATCH 10/12] fix(play): cap the decoded target within the sink's ring A stretch past the ring's three seconds of headroom let the measured target exceed what the sink opened with, failing set_latency. The excess now stays encoded, and the fake sink rejects a latency past the real ring's capacity. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/fake.rs | 9 +++++++++ rs/moq-cli/src/play/media.rs | 21 ++++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/src/play/fake.rs b/rs/moq-cli/src/play/fake.rs index b9c09e31df5..d9bd7bd259c 100644 --- a/rs/moq-cli/src/play/fake.rs +++ b/rs/moq-cli/src/play/fake.rs @@ -118,6 +118,7 @@ impl Speaker for Recorder { stride: input.layout.channels() as usize * size_of::(), sample_rate: input.sample_rate, latency: input.latency, + capacity: input.latency + RING_HEADROOM, refill: false, declared: false, next: false, @@ -132,6 +133,11 @@ impl Speaker for Recorder { } } +/// The real ring's room past the latency it opens on, and the processing window +/// a latency must leave free of it (`moq_audio::playback::Sink::set_latency`). +const RING_HEADROOM: Duration = Duration::from_secs(3); +const RING_WINDOW: Duration = Duration::from_millis(30); + /// What an underrun waits for before playing again: the real ring's 30 ms /// stretch window and a 10 ms device period. const CUSHION: Duration = Duration::from_millis(40); @@ -143,6 +149,8 @@ pub(super) struct FakeSink { stride: usize, sample_rate: u32, latency: Duration, + /// How much the real ring holds, which a larger latency fails as it does there. + capacity: Duration, refill: bool, /// Running dry is a declared silence rather than an underrun. declared: bool, @@ -158,6 +166,7 @@ pub(super) struct FakeSink { impl Sink for FakeSink { fn set_latency(&mut self, latency: Duration) -> anyhow::Result<()> { + anyhow::ensure!(latency + RING_WINDOW < self.capacity, "past the ring's capacity"); self.latency = latency; Ok(()) } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 0e4eebb3629..4e569eb5b4c 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -32,6 +32,11 @@ enum Crossing { /// network jitter, and one with no depth at all can never be read from. const AUDIO_BUFFER_MIN: Duration = Duration::from_millis(50); +/// How far past the latency it opened on the sink may be asked to hold. Its ring +/// reaches three seconds past that latency, less a 30 ms processing window +/// (`Sink::set_latency`), so this leaves margin, and it covers the default stretch. +const SINK_ROOM: Duration = Duration::from_secs(2); + /// Maximum sample frames per silence write, independent of the catalog rate. const AUDIO_SILENCE: usize = 960; @@ -637,11 +642,13 @@ async fn play_audio( // subtracting the incoming frame before it reaches the sink. let lead = |delay: Duration| latency.max(delay.min(max_delay).max(AUDIO_BUFFER_MIN)); - // The most the sink holds, within what it accepts. A shared delay past it, - // such as a slow video encoder's, stays encoded in the consumer until it is - // due instead, so the ring never grows with the delay. + // The most the sink holds, within what its ring was sized for. A shared delay + // past it, such as a slow video encoder's or a measured target past a long + // stretch, stays encoded in the consumer until it is due instead, so the + // ring never grows with the delay. let cap = max_delay .max(AUDIO_BUFFER_MIN) + .min(latency + SINK_ROOM) .min(moq_audio::playback::Input::LATENCY_MAX); let schedule = presentation.consume(); // The consumer decodes a packet once the sink has room for it. Its playhead is @@ -1224,6 +1231,14 @@ mod tests { ); } + /// A target past what the sink's ring was sized for, under a budget longer + /// still, holds the excess encoded rather than failing the sink. + #[tokio::test] + async fn audio_holds_a_target_past_its_ring_encoded() { + let jitter = Duration::from_secs(4); + assert_on_time(first_sound(jitter, Duration::from_secs(5), None, None).await, jitter); + } + fn assert_on_time(sounded: Duration, due: Duration) { assert!( sounded.max(due) - sounded.min(due) <= Duration::from_millis(1), From a5beab6451c573f40a9dbb9808c05096fd103c92 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:20:41 -0800 Subject: [PATCH 11/12] fix(play): cap the clock's audio target at the deepest delay The clock now follows the whole measured target, so a hostile advertised jitter could hold playback, and the media it retains, without bound. It is capped at LATENCY_MAX, as the catalog delay already is. The capture observer also follows a warmup change on the same track. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 17 ++++++++++++++++- test/audio-quality/clients/js/src/capture.ts | 5 ++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 4e569eb5b4c..0f360399586 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -1000,7 +1000,11 @@ async fn play_audio( // clock holds both while the sink keeps sizing for jitter alone. The // clock holds the whole measured target, which the subscription follows, // though the sink holds no more of it than its cap: the rest stays encoded. - let held = latency.max(delay.max(AUDIO_BUFFER_MIN)); + // Capped as the catalog delay is, so a hostile advertised jitter cannot + // make playback retain media without bound. + let held = latency + .max(delay.max(AUDIO_BUFFER_MIN)) + .min(moq_audio::playback::Input::LATENCY_MAX); *jitter.lock().unwrap() = Some(held); moved |= role.set_target(held + stated.borrow().unwrap_or_default()); // The consumer releases media due further out than the sink holds only @@ -1231,6 +1235,17 @@ mod tests { ); } + /// A target past the deepest delay a sink holds, such as a hostile advertised + /// jitter's, holds the clock at that depth instead. + #[tokio::test] + async fn audio_caps_a_target_past_the_deepest_delay() { + let latency = moq_audio::playback::Input::LATENCY_MAX; + assert_on_time( + first_sound(Duration::from_secs(20), Duration::from_secs(5), None, None).await, + latency, + ); + } + /// A target past what the sink's ring was sized for, under a budget longer /// still, holds the excess encoded rather than failing the sink. #[tokio::test] diff --git a/test/audio-quality/clients/js/src/capture.ts b/test/audio-quality/clients/js/src/capture.ts index 60dca86c128..77365d7adb2 100644 --- a/test/audio-quality/clients/js/src/capture.ts +++ b/test/audio-quality/clients/js/src/capture.ts @@ -44,7 +44,6 @@ export class Capture { const decoder = format(config); const active = broadcast.relativeBroadcast(effect, config.broadcast); if (!active) return; - const warmup = Duration.millis(config.warmup ?? 0); const track = active.track(name).subscribe({ priority: Catalog.PRIORITY.audio, delay: { @@ -52,7 +51,7 @@ export class Capture { delay: input.delay.peek(), maxDelay: input.maxDelay.peek(), stretch: input.stretch.peek(), - }).add(warmup), + }).add(Duration.millis(config.warmup ?? 0)), }, }); const groups = new Set(); @@ -68,7 +67,7 @@ export class Capture { delay: inner.get(input.delay), maxDelay: inner.get(input.maxDelay), stretch: inner.get(input.stretch), - }).add(warmup), + }).add(Duration.millis(inner.get(input.config)?.warmup ?? 0)), }, }), ); From f459ee1a1fb0cd9dbf61681ff701fa9e284c4f96 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 22:31:56 -0800 Subject: [PATCH 12/12] fix(play): bound the hole fill and opening cushion by the sink cap A --stretch past the ring's headroom let a hole or the opening cushion write more silence than the ring holds, dropping the frame after it. Both are now bounded by the sink cap, which equals the budget at the defaults. The fake sink drops a write past the real ring's capacity. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/fake.rs | 9 +++- rs/moq-cli/src/play/media.rs | 95 ++++++++++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/rs/moq-cli/src/play/fake.rs b/rs/moq-cli/src/play/fake.rs index d9bd7bd259c..86bec2998e2 100644 --- a/rs/moq-cli/src/play/fake.rs +++ b/rs/moq-cli/src/play/fake.rs @@ -149,7 +149,7 @@ pub(super) struct FakeSink { stride: usize, sample_rate: u32, latency: Duration, - /// How much the real ring holds, which a larger latency fails as it does there. + /// How much the real ring holds, past which a latency fails and a write drops as there. capacity: Duration, refill: bool, /// Running dry is a declared silence rather than an underrun. @@ -223,6 +223,12 @@ impl Sink for FakeSink { .first_chunk::<4>() .map(|bytes| f32::from_le_bytes(*bytes)) .unwrap_or_default(); + let mut written = moq_audio::playback::Write::default(); + // A write past the ring's capacity is dropped, as the real ring drops it. + if self.buffered() + duration > self.capacity { + written.dropped_sample_frames = frames as usize; + return Ok(written); + } if self.refill { self.pending.push((sample, duration)); self.queued += duration; @@ -232,7 +238,6 @@ impl Sink for FakeSink { } else { self.play(sample, duration); } - let mut written = moq_audio::playback::Write::default(); written.accepted_sample_frames = frames as usize; Ok(written) } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 0f360399586..e4bc90e4e31 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -628,20 +628,6 @@ async fn play_audio( let samples = |duration: Duration| (duration.as_secs_f64() * sample_rate as f64).round() as u64; let mut silence = Vec::new(); - // The longest hole worth playing through, in samples. A hole this player would - // rather sit through is one it is already willing to buffer, which is what the - // decoder's latency budget says: anything longer is what that budget chose to - // skip, so playing it as silence would hand back the delay the skip avoided. - // Past it the sink skips the hole and the clock re-anchors, as it does today. - let fill_max = samples(max_delay); - - let mut timeline = AudioTimeline::default(); - - // A replacement's opening cushion and its timeline plan must agree. The - // callback now owns target convergence, rather than write-time padding - // subtracting the incoming frame before it reaches the sink. - let lead = |delay: Duration| latency.max(delay.min(max_delay).max(AUDIO_BUFFER_MIN)); - // The most the sink holds, within what its ring was sized for. A shared delay // past it, such as a slow video encoder's or a measured target past a long // stretch, stays encoded in the consumer until it is due instead, so the @@ -650,6 +636,22 @@ async fn play_audio( .max(AUDIO_BUFFER_MIN) .min(latency + SINK_ROOM) .min(moq_audio::playback::Input::LATENCY_MAX); + + // The longest hole worth playing through, in samples. A hole this player would + // rather sit through is one it is already willing to buffer, which is what the + // decoder's latency budget says, within what the sink holds: anything longer + // is what that budget chose to skip, or silence the ring would drop along with + // the frame after it. Past it the sink skips the hole and the clock re-anchors. + let fill_max = samples(cap); + + let mut timeline = AudioTimeline::default(); + + // A replacement's opening cushion and its timeline plan must agree. The + // callback now owns target convergence, rather than write-time padding + // subtracting the incoming frame before it reaches the sink. Like a hole, + // the cushion is silence the sink must hold. + let lead = |delay: Duration| latency.max(delay.min(cap).max(AUDIO_BUFFER_MIN)); + let schedule = presentation.consume(); // The consumer decodes a packet once the sink has room for it. Its playhead is // where the shared clock says the last write sounds, rather than what the sink @@ -1254,6 +1256,71 @@ mod tests { assert_on_time(first_sound(jitter, Duration::from_secs(5), None, None).await, jitter); } + /// A hole longer than the sink's ring holds is skipped rather than filled with + /// silence the ring would drop along with the frame after it. + #[tokio::test] + async fn a_hole_past_the_ring_is_skipped() { + tokio::time::pause(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + let jitter = Duration::from_millis(100); + // A budget past the ring, as a long `--stretch` asks for. + options.delay = moq_audio::decode::Delay { + min: Some(jitter), + max: Some(Duration::from_secs(5)), + }; + options.estimate = false; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let (_catalog, stated) = tokio::sync::watch::channel(None); + let playback = AudioPlayback { + delay: stated, + changed: Default::default(), + speaker: recorder.clone(), + presentation: Presentation::new(jitter), + playhead: Default::default(), + latency: AUDIO_BUFFER_MIN, + output: recorder.clone(), + stretch: Duration::from_secs(5), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback, std::future::pending()) + .await + .unwrap() + .finish() + .await; + }); + for index in 0..30 { + producer.write(packet(index, 0.25)).unwrap(); + kio::clock::sleep(PACKET_DURATION).await; + } + // Three seconds of packets the publisher never sent, then live again. + let after = 30 + 3000 / PACKET_DURATION.as_millis() as u64; + producer.write(packet(after, 0.5)).unwrap(); + for index in after + 1..after + 30 { + kio::clock::sleep(PACKET_DURATION).await; + producer.write(packet(index, 0.25)).unwrap(); + } + let _finished = producer.finish().unwrap(); + task.await.unwrap(); + assert!( + recorder.played().iter().any(|part| part.sample == 0.5), + "the frame after the hole played" + ); + } + fn assert_on_time(sounded: Duration, due: Duration) { assert!( sounded.max(due) - sounded.min(due) <= Duration::from_millis(1),