diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 66388cd34d..29df3d586e 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -423,7 +423,9 @@ The advertised floor is the one term that can exceed that bound, and it is deliberately not clamped to it. The ceiling is a property of the histogram's size; the floor is a number the publisher declared about itself, so clamping the floor to the ceiling would under-buffer precisely the publisher that was -honest about a long flush. +honest about a long flush. Honoring it must not cost memory in proportion to +the declared number, so the browser decodes at most 500 ms of any target and +holds the rest as the encoded frames it actually received. Policy limits sit above the algorithm. A receiver's own configured maximum and the track's retention window both cap what it may hold, and neither is the diff --git a/js/watch/bench/ring-page.ts b/js/watch/bench/ring-page.ts index d4f8944c2f..e392a9ffa6 100644 --- a/js/watch/bench/ring-page.ts +++ b/js/watch/bench/ring-page.ts @@ -1,5 +1,5 @@ /** The page `ring.ts` loads: N players feeding the real render worklet through `AudioBuffer`. */ -import { Micro } from "@moq/clock"; +import { Micro, Milli } from "@moq/clock"; import { AudioBuffer } from "../src/audio/buffer"; const RATE = 48_000; @@ -12,7 +12,7 @@ const players = Array.from({ length: count }, () => { const worklet = new AudioWorkletNode(context, "render", { outputChannelCount: [2] }); worklet.connect(context.destination); // The 200 ms floor, unbuffered: what a live viewer plays at. - const buffer = new AudioBuffer(worklet, { channels: 2, rate: RATE, latency: RATE / 5, buffered: false }); + const buffer = new AudioBuffer(worklet, { channels: 2, rate: RATE, delay: Milli(200), buffered: false }); return { worklet, buffer }; }); await context.resume(); diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 8861a1b0f3..aacabd46e2 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from "bun:test"; -import { type Micro, Milli } from "@moq/clock"; -import { AudioBuffer } from "./buffer"; -import type { Message, State } from "./render"; +import { Micro, Milli } from "@moq/clock"; +import { AudioBuffer, Backpressure } from "./buffer"; +import type { Init, Message, State } from "./render"; +import { AudioRingBuffer } from "./ring-buffer"; + +const HOUR = Milli(60 * 60 * 1000); +const ms = (value: number) => Micro.fromMilli(Milli(value)); // A buffered buffer over a real port, with the worklet's end of it left to the test. function open(): { @@ -13,8 +17,8 @@ function open(): { const channel = new MessageChannel(); const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { channels: 1, - rate: 1000, - latency: 100, + rate: 48_000, + delay: Milli(100), buffered: true, }); return { @@ -49,8 +53,8 @@ describe("AudioBuffer", () => { const channel = new MessageChannel(); const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { channels: 2, - rate: 1000, - latency: 100, + rate: 48_000, + delay: Milli(100), buffered: true, }); const messages: Message[] = []; @@ -66,12 +70,12 @@ describe("AudioBuffer", () => { buffer.insert(500_000 as Micro, [left, right]); expect(left.byteLength).toBe(0); expect(right.byteLength).toBe(0); - buffer.setLatency(200); + buffer.setLatency(Milli(200)); buffer.truncate(502_000 as Micro); buffer.reset(); await received; expect(messages).toEqual([ - { type: "init", channels: 2, rate: 1000, latency: Milli(100), buffered: true }, + { type: "init", channels: 2, rate: 48_000, latency: Milli(100), buffered: true }, { type: "data", timestamp: 500_000 as Micro, @@ -181,18 +185,21 @@ describe("AudioBuffer", () => { } }); - // Decoded audio can land short of the timestamp its frame was released at (Opus pre-skip), so a - // refill that stops at the floor exactly leaves the ring a sample short of un-stalling, with the - // gate waiting on the ring. - it("refills until a frame reaches the floor", async () => { - const { buffer, state, close } = open(); + // Decoded audio can land short of the timestamp its frame was released at (Opus pre-skip), so the + // refill counts the PCM that reaches the ring: stopping at the released frames would leave the + // ring short of un-stalling, with the gate waiting on it. + it("refills until the ring holds the floor", () => { + const { buffer, close } = open(); const signal = new AbortController().signal; try { - // The ring anchored on a first frame at 6 ms, still short of the floor. - await state(6_000, true, 0); - expect(Bun.peek.status(buffer.wait(100_000 as Micro, signal))).toBe("fulfilled"); - expect(Bun.peek.status(buffer.wait(120_000 as Micro, signal))).toBe("fulfilled"); - expect(Bun.peek.status(buffer.wait(140_000 as Micro, signal))).toBe("pending"); + for (let t = 0; t <= 100_000; t += 20_000) { + expect(Bun.peek.status(buffer.wait(t as Micro, signal))).toBe("fulfilled"); + // The first output lost 6 ms to pre-skip. + const start = Math.max(t, 6_000); + buffer.insert(start as Micro, [new Float32Array(((t + 20_000 - start) * 48) / 1000)]); + buffer.decoded((t + 20_000) as Micro); + } + expect(Bun.peek.status(buffer.wait(120_000 as Micro, signal))).toBe("pending"); } finally { close(); } @@ -206,7 +213,9 @@ describe("AudioBuffer", () => { try { await state(100_000, true, 0); expect(Bun.peek.status(buffer.wait(10_000_000 as Micro, signal))).toBe("fulfilled"); - const next = buffer.wait(10_050_000 as Micro, signal); + buffer.insert(10_000_000 as Micro, [new Float32Array(960)]); + buffer.decoded(10_020_000 as Micro); + const next = buffer.wait(10_020_000 as Micro, signal); expect(Bun.peek.status(next)).toBe("pending"); // The frame past the gap un-stalls the ring, which plays the gap out up to it. @@ -256,3 +265,276 @@ describe("AudioBuffer", () => { } }); }); + +describe("Backpressure", () => { + it("releases every live frame when the whole target fits the floor", () => { + const gate = new Backpressure(1000, false, Milli(100)); + gate.observe(ms(0), false); + expect(Bun.peek.status(gate.wait(ms(10_000), new AbortController().signal))).toBe("fulfilled"); + }); + + it("holds buffered playback to the floor ahead of the playhead", () => { + const gate = new Backpressure(1000, true, Milli(200)); + gate.observe(ms(0), false); + const wait = gate.wait(ms(1000), new AbortController().signal); + gate.observe(ms(799), false); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.observe(ms(800), false); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + // Two seconds is a 500 ms floor and 1.5 s held encoded. + it("releases a live frame once the queued media covers the hold behind it", () => { + const gate = new Backpressure(1000, false, Milli(2000)); + gate.observe(ms(0), false); + const wait = gate.wait(ms(1000), new AbortController().signal); + gate.queued(ms(2499)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.queued(ms(2500)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + it("plays buffered media through instead of releasing it ahead of the floor", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.observe(ms(0), false); + const wait = gate.wait(ms(1000), new AbortController().signal); + gate.queued(ms(60_000)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.observe(ms(500), false); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + it("starts a stalled ring only once the queued media covers the hold", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + const wait = gate.wait(ms(0), new AbortController().signal); + gate.queued(ms(1499)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.queued(ms(1500)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + // The ring un-stalls only once the decoder's output reaches it, so covered media released + // outright would overflow the fixed ring before the worklet reports it playing. + it("refills a covered buffered ring only to its floor", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + const signal = new AbortController().signal; + expect(Bun.peek.status(gate.wait(ms(0), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(500), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(520), signal))).toBe("pending"); + }); + + // A refill that released its floor before the ring reported playing holds the next frame back, so + // no write could end the park. The queued media covering the hold is what ends it, once. + it("resumes a park once the queued media covers the hold", () => { + const gate = new Backpressure(1000, true, Milli(1000)); + expect(gate.queued(ms(1000))).toBe(false); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + gate.observe(ms(0), true); + expect(gate.setLatency(Milli(2000))).toBe(true); + expect(gate.queued(ms(1999))).toBe(false); + expect(gate.queued(ms(2000))).toBe(true); + expect(gate.queued(ms(3000))).toBe(false); + }); + + // The ring starts playing a report interval before the gate learns its playhead, so a rise in that + // window parks from the frame the ring anchors on. + it("parks a ring that started before reporting its playhead", () => { + const gate = new Backpressure(1000, false, Milli(1000)); + gate.queued(ms(1000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + expect(gate.setLatency(Milli(2000))).toBe(true); + expect(gate.queued(ms(2000))).toBe(true); + }); + + // A warmup longer than the floor discards everything the refill releases toward it, so the ring + // would never anchor and report the playhead the gate waits on. + it("refills past output discarded before the ring anchors", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + const wait = gate.wait(ms(520), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + for (let t = 0; t <= 500; t += 20) gate.decoded(ms(t + 20)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(1020), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(1040), signal))).toBe("pending"); + }); + + // A replacement rendition keeps the ring and its playhead, and its warmup can outlast the floor. + it("refills a stalled ring past output discarded behind its playhead", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + gate.observe(ms(1000), true); + const signal = new AbortController().signal; + for (let t = 1000; t <= 1500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + const wait = gate.wait(ms(1520), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + // The decoder may answer several frames with one output. + gate.decoded(ms(1500)); + gate.decoded(ms(1520)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2020), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2040), signal))).toBe("pending"); + }); + + // Opus pre-skip stamps the PCM behind the input it was decoded from, so a playing ring compares a + // frame to its playhead where its output lands, or a pre-skip past the floor would drain it. + it("releases a playing frame by where its output lands", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + gate.observe(ms(0), false); + const signal = new AbortController().signal; + const wait = gate.wait(ms(1500), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.decoded(ms(1020), ms(20)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(1520), signal))).toBe("pending"); + }); + + // A replacement rendition keeps the ring, and the decoder it replaced never answers its frames. + it("refills past frames a closed decoder dropped", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + const wait = gate.wait(ms(520), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.dropped(); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + // An utterance that arrived whole before the reset changes nothing queued, so nothing would + // report it again: forgetting it would hold its first frame until the next utterance. + it("keeps what is queued across a reset", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(5000)); + gate.observe(ms(0), false); + gate.reset(); + expect(Bun.peek.status(gate.wait(ms(0), new AbortController().signal))).toBe("fulfilled"); + }); +}); + +// A worklet whose port records what the buffer posts and delivers state reports synchronously. +function fake(delay: Milli, buffered = false) { + const sent: Message[] = []; + const port = Object.assign(new EventTarget(), { + postMessage: (msg: Message) => sent.push(msg), + start() {}, + }); + const buffer = new AudioBuffer({ port } as unknown as AudioWorkletNode, { + channels: 1, + rate: 1000, + delay, + buffered, + }); + const state = (timestamp: Micro, stalled: boolean, epoch: number) => { + const data: State = { type: "state", timestamp, stalled, anchored: true, underruns: 0, epoch }; + port.dispatchEvent(new MessageEvent("message", { data })); + }; + return { buffer, sent, state }; +} + +describe("AudioBuffer past its decoded floor", () => { + // A catalog advertising an hour of jitter once sized the ring to two hours of stereo PCM, 4 GiB. + it("allocates the same ring for an hour of delay as for a tenth of a second", () => { + const bytes = [Milli(100), HOUR].map((delay) => { + const sent: Message[] = []; + const port = Object.assign(new EventTarget(), { + postMessage: (msg: Message) => sent.push(msg), + start() {}, + }); + new AudioBuffer({ port } as unknown as AudioWorkletNode, { + channels: 2, + rate: 48_000, + delay, + buffered: false, + }).close(); + const ring = new AudioRingBuffer(sent[0] as Init); + return ring.capacity * ring.channels * Float32Array.BYTES_PER_ELEMENT; + }); + expect(bytes[1]).toBe(bytes[0]); + expect(bytes[1]).toBeLessThanOrEqual(2 ** 20); + }); + + it("holds a frame encoded until an hour of queued media is behind it", () => { + const { buffer } = fake(HOUR); + try { + // The ring holds 500 ms decoded; the rest of the hour waits upstream. + const hold = HOUR - 500; + const wait = buffer.wait(ms(0), new AbortController().signal); + buffer.queued(ms(hold - 1)); + expect(Bun.peek.status(wait)).toBe("pending"); + buffer.queued(ms(hold)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + } finally { + buffer.close(); + } + }); + + // One second is a 500 ms floor and 500 ms held encoded. Playing 100 ms in, a rise to 1.5 s needs + // queued media through 1600 ms to keep playing. A report of a stall may be stale, so it parks + // regardless: parking a ring that is still stalled changes nothing. + for (const stalled of [false, true]) { + for (const [queued, parked] of [ + [1599, true], + [1600, false], + ] as const) { + it(`${parked ? "parks" : "keeps playing"} on a rise with ${queued} ms queued, reported ${stalled ? "stalled" : "playing"}`, () => { + const { buffer, sent, state } = fake(Milli(1000)); + try { + state(ms(100), stalled, 0); + buffer.queued(ms(queued)); + buffer.setLatency(Milli(1500)); + expect(sent.slice(1)).toEqual([ + { type: "latency", latency: Milli(500), ...(parked ? { park: 1 } : {}) }, + ]); + } finally { + buffer.close(); + } + }); + } + } + + // The ring resumes on any floor update without a park, so a fall that leaves the remaining hold + // uncovered parks again, and the queued media covering it resumes the ring. + it("keeps a park across a fall until the queued media covers the hold", () => { + const { buffer, sent, state } = fake(Milli(1000)); + try { + state(ms(0), false, 0); + buffer.queued(ms(1000)); + buffer.setLatency(Milli(2000)); + buffer.setLatency(Milli(1800)); + buffer.queued(ms(1800)); + buffer.setLatency(Milli(1600)); + expect(sent.slice(1)).toEqual([ + { type: "latency", latency: Milli(500), park: 1 }, + { type: "latency", latency: Milli(500), park: 2 }, + { type: "latency", latency: Milli(500) }, + { type: "latency", latency: Milli(500) }, + ]); + } finally { + buffer.close(); + } + }); + + // A report the worklet posted before it applied the park still says the ring is playing. + it("ignores state from before a park", () => { + const { buffer, state } = fake(Milli(1000)); + try { + state(ms(200), false, 0); + buffer.queued(ms(1599)); + buffer.setLatency(Milli(1500)); + const wait = buffer.wait(ms(700), new AbortController().signal); + state(ms(200), false, 0); + expect(Bun.peek.status(wait)).toBe("pending"); + state(ms(200), false, 1); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + } finally { + buffer.close(); + } + }); +}); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 31bd9cbe4d..7ab8c89ecc 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,42 +1,161 @@ import { Micro, Milli, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; +import { ringHold, ringSamples } from "./latency"; import type { Data, Init, Latency, Reset, State, Truncate } from "./render"; /** - * Timestamp-based backpressure for buffered playback. The decoded PCM ring only holds the latency - * floor; everything above it (the buffered lookahead, up to the ceiling) stays upstream as encoded - * Opus. `wait(timestamp)` stays pending until decoding `timestamp` would land within `headroom` (the - * floor) of the playhead, so the decode loop holds a frame as Opus instead of overflowing the - * floor-sized ring, which drops the oldest unplayed audio. The playhead comes from the worklet's - * state messages. A no-op when not buffered (the ring bounds itself). + * Decides when the decode loop may decode a frame, which is how the target delay is split between + * the decoded ring and the encoded media upstream of it. + * + * The ring holds the floor as PCM, at most a receiver-chosen budget. The rest of the target (the + * hold) and, in buffered mode, the lookahead stay encoded in the container consumer, so memory follows + * the bytes actually received rather than whatever a catalog advertised. A frame at `timestamp` is due: + * + * - while the ring is stalled, once the media queued behind it covers the hold, so playback starts + * or refills with the whole target ahead of the playhead, decoded or not. A buffered refill also + * stops once the PCM in the ring plus the frames still decoding reach the floor, since the ring + * plays through and cannot shed what overflows it; + * - while playing, once the playhead is within the floor of where its output lands, keeping the + * ring on its floor; + * - while playing live, also once the media queued behind it covers the hold, so media past the + * target reaches the ring, whose skip band catches up to the live edge. Buffered mode plays + * through instead. + * + * With no hold, live playback releases every frame at once and the ring bounds itself. The playhead + * comes from the worklet's state messages. + * + * @internal Exported for the replay harness. */ -class Backpressure { - #enabled: boolean; - #headroom: Micro; - #waiters: Array<{ timestamp: Micro; resolve: () => void }> = []; +export class Backpressure { + readonly #rate: number; + readonly #buffered: boolean; + #closed = false; + + // The ring's floor in samples and as a duration, and the part of the target above it, held encoded. + #floor = 0; + #headroom = 0 as Micro; + #hold = 0 as Micro; + + // The newest media timestamp queued ahead of the decoder, unset while none is. + #queued?: Micro; // The ring as last observed. The playhead is undefined until one is known on the current timeline. #playhead?: Micro; #stalled = true; + // The first frame released on the current timeline. Until PCM lands and a report carries a + // playhead, it stands in for where the ring anchors, since the ring may already be playing. + #anchor?: Micro; + // Whether the ring is parked for a hold the queued media does not cover yet. The worklet keeps a + // park until the next floor update, so only this decides when it ends: a write of output decoded + // before the park cannot. + #parked = false; + + // A buffered refill counts what reaches the ring, not what it released: output the decoder + // discards (warmup, Opus pre-skip, a terminal trim) never lands, so counting released frames would + // stop the refill short with the ring stalled and nothing left to decode. Releasing outright would + // instead let the whole lookahead through before the output reaches the ring and un-stalls it. + // + // The frames released and not yet decoded, oldest first, and where the first PCM inserted on this + // timeline starts (the ring anchors there) and the newest ends. + #decoding: Micro[] = []; + #inserted?: { start: Micro; end: Micro }; + // How far output lands in the ring behind the frame it was decoded from (Opus pre-skip). A playing + // ring compares a frame to its playhead where the output lands, or an offset past the floor would + // drain it. + #offset = 0; + + #waiters: Array<{ timestamp: Micro; resolve: () => void }> = []; + + constructor(rate: number, buffered: boolean, delay: Milli) { + this.#rate = rate; + this.#buffered = buffered; + this.setLatency(delay); + } + + /** The ring's floor in samples: the part of the target held as decoded PCM. */ + get floor(): number { + return this.#floor; + } + + /** + * Split `delay` between the floor and the hold. Returns true when the ring must park with the new + * floor: a rise in the hold that the queued media does not cover yet holds playback until it does, + * the way a deeper floor parks the ring until it refills, since video holds the extra delay on its + * own. A floor update ends a park, so a ring already parked stays parked while the remaining hold + * is uncovered. + */ + setLatency(delay: Milli): boolean { + this.#floor = ringSamples(this.#rate, delay); + this.#headroom = Micro.fromSecond((this.#floor / this.#rate) as Second); + const hold = ringHold(delay); + const rise = hold > this.#hold; + this.#hold = hold; + + // Parks whatever the ring was last reported doing: a report of a stall may be stale, and + // parking a ring that is still stalled changes nothing. + const playhead = this.#start(); + this.#parked = (rise || this.#parked) && playhead !== undefined && !this.#covered(playhead + this.#headroom); + if (this.#parked) this.#stalled = true; + this.#release(); + return this.#parked; + } + + /** Note the ring's playhead and whether it is stalled, and release every frame now due. */ + observe(playhead: Micro | undefined, stalled: boolean): void { + this.#playhead = playhead; + this.#stalled = stalled; + this.#release(); + } - // The first and latest frames released since the ring stalled. A stalled ring refills to the - // floor from its parked playhead, but releasing it outright would let the whole lookahead through - // before the decoder's output reaches the ring and un-stalls it. - #first?: Micro; - #last?: Micro; + /** + * Note the newest media timestamp queued ahead of the decoder, or undefined once none is. Returns + * true when that covers the hold of a parked ring, which a floor update then resumes. + */ + queued(newest: Micro | undefined): boolean { + this.#queued = newest; + const playhead = this.#start(); + const resume = this.#parked && playhead !== undefined && this.#covered(playhead + this.#headroom); + if (resume) this.#parked = false; + this.#release(); + return resume; + } - constructor(enabled: boolean, headroom: Micro) { - this.#enabled = enabled; - this.#headroom = headroom; + /** + * Note that the decoder produced output ending at `end`, which lands in the ring up to `landed` + * unless it was discarded. It answers the oldest frame still decoding and every one it reaches + * past: a decoder may emit nothing for a frame. + */ + decoded(end: Micro, landed?: Micro): void { + if (landed !== undefined) this.#offset = end - landed; + let answered = 0; + while (answered < this.#decoding.length && (answered === 0 || this.#decoding[answered] < end)) answered++; + this.#decoding.splice(0, answered); + this.#release(); + } + + /** Note that the decoder closed, dropping every frame still decoding. */ + dropped(): void { + this.#decoding = []; + this.#release(); } - // Move the gate as the floor changes (e.g. "auto" tracking the measured arrival spread). - setHeadroom(headroom: Micro): void { - this.#headroom = headroom; + /** Note that PCM from `start` to `end` reached the ring. */ + inserted(start: Micro, end: Micro): void { + const inserted = this.#inserted; + this.#inserted = { start: inserted?.start ?? start, end: inserted && inserted.end > end ? inserted.end : end }; + this.#release(); + } + + /** Note that the ring dropped what it held at or after `timestamp`, keeping what is already due. */ + truncated(timestamp: Micro): void { + const inserted = this.#inserted; + const start = this.#playhead ?? inserted?.start; + if (!inserted || start === undefined || timestamp >= inserted.end) return; + this.#inserted = { start: inserted.start, end: (timestamp > start ? timestamp : start) as Micro }; } wait(timestamp: Micro, signal: AbortSignal): Promise { - if (!this.#enabled || signal.aborted || this.#admit(timestamp)) return Promise.resolve(); + if (this.#closed || signal.aborted || this.#admit(timestamp)) return Promise.resolve(); return new Promise((resolve) => { // An aborted wait drops its waiter, so a retired decode loop leaves nothing queued here. const abort = () => { @@ -55,35 +174,28 @@ class Backpressure { }); } - // Record the ring's playhead and stall state, and resolve every waiter that now fits. - observe(playhead: Micro | undefined, stalled: boolean): void { - this.#playhead = playhead; - this.#stalled = stalled; - if (!stalled) { - this.#first = undefined; - this.#last = undefined; - } - this.#release(); - } - - // The ring was flushed for a new timeline: its playhead is unknown until the next frame anchors it. + /** The ring was flushed for a new timeline: its playhead is unknown until the next frame anchors it. */ reset(): void { this.#playhead = undefined; + this.#anchor = undefined; this.#stalled = true; - this.#first = undefined; - this.#last = undefined; + this.#parked = false; + this.#decoding = []; + this.#inserted = undefined; + this.#offset = 0; this.#release(); } - // Stop gating for good. Nothing advances a closed buffer, so a later wait would never settle. + /** Stop gating for good. Nothing advances a closed buffer, so a later wait would never settle. */ close(): void { - this.#enabled = false; + this.#closed = true; for (const { resolve } of this.#waiters) resolve(); this.#waiters = []; } - // Thresholds are recomputed live so a changed headroom takes effect on queued waiters too. + // Thresholds are recomputed live, so a moved target takes effect on queued waiters too. #release(): void { + if (this.#waiters.length === 0) return; this.#waiters = this.#waiters.filter(({ timestamp, resolve }) => { if (!this.#admit(timestamp)) return true; resolve(); @@ -91,31 +203,50 @@ class Backpressure { }); } + // Whether the queued media reaches the hold past `timestamp`. + #covered(timestamp: number): boolean { + if (this.#hold === 0) return true; + return this.#queued !== undefined && this.#queued >= timestamp + this.#hold; + } + + // The frame the ring plays from: its reported playhead, else where it anchored or will anchor, + // mapped back to the frame its output came from, which is what the queued media covers. + #start(): Micro | undefined { + const start = this.#playhead ?? this.#inserted?.start; + if (start !== undefined) return (start + this.#offset) as Micro; + return this.#anchor; + } + #admit(timestamp: Micro): boolean { - const playhead = this.#playhead; - if (!this.#stalled && playhead !== undefined) return timestamp - this.#headroom <= playhead; - - // Refill from the parked playhead, or from the first frame while the ring has not anchored. The - // first frame always goes, so a ring that needs it to anchor, or to see past a gap, never waits - // on a playhead that will not move. Frames keep going until one reaches the floor: the ring - // only un-stalls once it holds that much, and decoded audio can land a little short of the - // timestamp it was released at (Opus pre-skip), so stopping at the floor exactly can leave the - // ring stalled with the gate waiting on it. - const first = this.#first; - const last = this.#last; - if (first !== undefined && last !== undefined) { - const from = playhead ?? first; - if (timestamp - this.#headroom > from && last - this.#headroom >= from) return false; + const due = this.#due(timestamp); + if (due) { + this.#anchor ??= timestamp; + if (this.#buffered) this.#decoding.push(timestamp); } - this.#first ??= timestamp; - this.#last = last === undefined ? timestamp : (Math.max(last, timestamp) as Micro); - return true; + return due; } -} -/** Convert a sample count to a Micro duration at the given sample rate. */ -function samplesToMicro(samples: number, rate: number): Micro { - return Micro.fromSecond((samples / rate) as Second); + #due(timestamp: Micro): boolean { + const playhead = this.#playhead; + if (!this.#stalled && playhead !== undefined) { + if (timestamp - this.#offset - this.#headroom <= playhead) return true; + return !this.#buffered && this.#covered(timestamp); + } + + if (!this.#covered(timestamp)) return false; + // A live ring sheds what overflows its floor, so it refills from everything that is due. + if (!this.#buffered) return true; + + // Refill while the PCM in the ring past its playhead, plus the span of frames still decoding, is + // short of the floor. The ring un-stalls once it holds the floor, and output the decoder + // discards never counts, so the refill cannot stop with the ring short of it. + const inserted = this.#inserted; + const start = playhead ?? inserted?.start; + const buffered = inserted && start !== undefined ? Math.max(0, inserted.end - start) : 0; + const decoding = this.#decoding; + const pending = decoding.length > 0 ? (decoding[decoding.length - 1] as number) - (decoding[0] as number) : 0; + return buffered + pending < this.#headroom; + } } /** @@ -141,8 +272,8 @@ export class AudioBuffer { // Backpressure runs off the playhead the worklet reports in its state messages. #backpressure: Backpressure; - // Bumped per reset. A state message posted before the worklet saw the reset reports the old - // timeline's playhead, which would open the gate for the new one. + // Bumped per reset or park. A state message posted before the worklet saw it reports the ring + // playing on the old terms, which would open the gate. #epoch = 0; #signals = new Effect(); @@ -152,8 +283,8 @@ export class AudioBuffer { props: { channels: number; rate: number; - /** The latency floor, in samples. */ - latency: number; + /** The target delay. The ring holds up to a fixed floor of it decoded; the rest stays encoded. */ + delay: Milli; /** Anchor to the first frame and play through, holding the lookahead upstream. */ buffered: boolean; }, @@ -163,10 +294,9 @@ export class AudioBuffer { this.channels = channels; this.rate = rate; - this.#backpressure = new Backpressure(buffered, samplesToMicro(props.latency, rate)); + this.#backpressure = new Backpressure(rate, buffered, props.delay); - const latency = Milli.fromSecond((props.latency / rate) as Second); - const msg: Init = { type: "init", channels, rate, latency, buffered }; + const msg: Init = { type: "init", channels, rate, latency: this.#latency(), buffered }; worklet.port.postMessage(msg); this.#signals.event(worklet.port, "message", (ev: Event) => { @@ -184,8 +314,28 @@ export class AudioBuffer { worklet.port.start(); } + // The floor as the worklet takes it. + #latency(): Milli { + return Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); + } + + /** + * Note that the decoder produced output ending at `end` on its own timeline, which lands in the ring + * up to `landed` unless it was discarded. + */ + decoded(end: Micro, landed?: Micro): void { + this.#backpressure.decoded(end, landed); + } + + /** Note that the decoder closed, dropping every frame still decoding. */ + dropped(): void { + this.#backpressure.dropped(); + } + /** Insert audio samples at the given timestamp, transferring their buffers. Handles out-of-order writes. */ insert(timestamp: Micro, data: Float32Array[]): void { + const end = (timestamp + Micro.fromSecond(((data[0]?.length ?? 0) / this.rate) as Second)) as Micro; + this.#backpressure.inserted(timestamp, end); const msg: Data = { type: "data", data, timestamp }; // Transfer the ArrayBuffers to avoid a copy: the caller loses access to them. this.#worklet.port.postMessage( @@ -194,12 +344,25 @@ export class AudioBuffer { ); } - /** Update the latency floor, in samples. */ - setLatency(samples: number): void { - this.#backpressure.setHeadroom(samplesToMicro(samples, this.rate)); + /** + * Update the target delay. The ring holds up to a fixed floor of it decoded, and the rest stays + * encoded upstream until the media queued behind a frame covers it. + */ + setLatency(delay: Milli): void { + const park = this.#backpressure.setLatency(delay); + const msg: Latency = { type: "latency", latency: this.#latency() }; + // One message, so the worklet never plays a quantum between the new floor and the park. + if (park) msg.park = ++this.#epoch; + this.#worklet.port.postMessage(msg); + } - const latency = Milli.fromSecond((samples / this.rate) as Second); - const msg: Latency = { type: "latency", latency }; + /** + * Note the newest media timestamp queued ahead of the decoder, or undefined once none is. The + * queue is what covers the part of the target held encoded, so it is what resumes a parked ring. + */ + queued(newest: Micro | undefined): void { + if (!this.#backpressure.queued(newest)) return; + const msg: Latency = { type: "latency", latency: this.#latency() }; this.#worklet.port.postMessage(msg); } @@ -211,6 +374,7 @@ export class AudioBuffer { * Unlike `reset()` this keeps playing, so there's no gap at the handover. */ truncate(timestamp: Micro): void { + this.#backpressure.truncated(timestamp); const msg: Truncate = { type: "truncate", timestamp }; this.#worklet.port.postMessage(msg); } @@ -224,11 +388,9 @@ export class AudioBuffer { } /** - * Resolve once the playhead is near enough to decode a frame at `timestamp`. In buffered mode this - * applies backpressure: it stays pending while decoding `timestamp` would run more than the latency - * floor ahead of the playhead, so the caller holds the (encoded) frame instead of decoding it too - * far ahead of the floor-sized ring. Resolves immediately when not buffered (the ring bounds itself). - * Resolves early, releasing the wait, once `signal` aborts. + * Resolve once a frame at `timestamp` may be decoded, so the caller holds it encoded until then + * instead of decoding it too far ahead of the floor-sized ring. Resolves at once for live playback + * whose whole target fits the floor. Resolves early, releasing the wait, once `signal` aborts. */ wait(timestamp: Micro, signal: AbortSignal): Promise { // The worklet-reported playhead lags by a state-message interval, which only holds frames longer. diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 3bb299bc60..344182defb 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -64,8 +64,13 @@ class FakeWorklet { postMessage: (msg: Message) => { if (msg.type === "init") this.ring = new AudioRingBuffer(msg); else if (msg.type === "data") this.ring?.write(msg.timestamp, msg.data); - else if (msg.type === "latency") this.ring?.resize(msg.latency); - else if (msg.type === "truncate") this.ring?.truncate(msg.timestamp); + else if (msg.type === "latency") { + this.ring?.setLatency(msg.latency); + if (msg.park !== undefined) { + this.ring?.park(); + this.#epoch = msg.park; + } + } else if (msg.type === "truncate") this.ring?.truncate(msg.timestamp); else if (msg.type === "reset") { this.ring?.reset(); this.#epoch = msg.epoch; @@ -337,6 +342,8 @@ async function play(initial: Delay, buffered = false, cmaf = false) { consumer = producer.consume(); frameTimestamp = 0; }, + // The resolved playout target. + target: () => sync.out.delay.peek(), // The rings written to, and the timestamps of every frame that reached one. rings: () => worklets.filter((worklet) => worklet.ring && insert.mock.contexts.includes(worklet.ring)), inserted: () => insert.mock.calls.map(([timestamp]) => timestamp as number), @@ -394,17 +401,60 @@ for (const cmaf of [false, true]) { } expect(playback.inserted()).toEqual([]); - // The live edge moves on, so the head is complete and plays from its oldest frame. + // The live edge moves on, so the head is complete and plays from its oldest frame, refilling + // the 100 ms floor. frameTimestamp = 140_000; playback.push({ ...frame(), group: 8 }); await microtasks(); - expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000, 100_000]); + expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000]); } finally { playback.close(); await microtasks(); } }); + // A buffered head above the decoded floor waits in the reorder buffer, not the consumer, until it + // runs past the target. Those frames are what cover the part of the target held encoded. + it(`starts a ${cmaf ? "CMAF" : "legacy"} buffered head held past a target above the floor`, async () => { + const playback = await play(Milli(1000), true, cmaf); + try { + const target = playback.target(); + expect(target).toBeGreaterThan(500); + while (frameTimestamp <= target * 1000 + 20_000) { + playback.push(frame()); + await microtasks(); + } + expect(playback.inserted()[0]).toBe(20_000); + } finally { + playback.close(); + await microtasks(); + } + }); + + // Pre-skip longer than the decoded floor discards every output the refill released toward it, so + // the refill has to count what reaches the ring, not what it released. + if (!cmaf) { + it("starts a buffered Opus head whose pre-skip outlasts the floor", async () => { + const playback = await play(Milli(1000), true); + try { + // OpusHead, version 1, stereo, a 520 ms pre-skip of 24,960 samples: past the 500 ms floor, yet + // within the head the harness covers, since it queues nothing behind a released batch. + playback.restore({ description: "4f707573486561640102" + "8061" + "80bb0000" + "0000" + "00" }); + await microtasks(); + const target = playback.target(); + while (frameTimestamp <= (target + 2000) * 1000) { + playback.push(frame()); + await microtasks(); + } + // The first output to outlast the pre-skip lands at the start of the timeline. + expect(playback.inserted()[0]).toBe(20_000); + } finally { + playback.close(); + await microtasks(); + } + }); + } + // A fresh encoder after a declared end may restart its timeline below the old one, and the decoder // resets for it before anything is judged out of order. it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 4dd5d68f7a..dc0cd40bd0 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -1,4 +1,4 @@ -import { Micro, Milli } from "@moq/clock"; +import { Micro, Milli, type Second } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Util from "@moq/hang/util"; @@ -21,7 +21,7 @@ import type { Sync } from "../sync"; import { AudioBuffer } from "./buffer"; import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; import { Handover } from "./handover"; -import { AUTO_MAX_DELAY, ringSamples, target } from "./latency"; +import { AUTO_MAX_DELAY, target } from "./latency"; import { Order } from "./order"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; @@ -265,16 +265,11 @@ export class Decoder { }); effect.cleanup(() => worklet.disconnect()); - // Initial ring depth in samples. - const delay = this.sync.out.delay.peek(); - const latencySamples = ringSamples(sampleRate, delay); - const buffered = this.sync.out.buffered.peek(); - const ring = new AudioBuffer(worklet, { channels: channelCount, rate: sampleRate, - latency: latencySamples, - buffered, + delay: this.sync.out.delay.peek(), + buffered: this.sync.out.buffered.peek(), }); effect.cleanup(() => ring.close()); effect.set(this.#ring, ring); @@ -335,11 +330,10 @@ export class Decoder { const ring = effect.get(this.#ring); if (!ring) return; - // A rise parks playback until the ring refills to the new floor, which is what keeps audio in - // step with video. The measured target moves a bucket at a time, and a rise the buffered - // slack already covers costs no silence at all. - const delay = effect.get(this.sync.out.delay); - ring.setLatency(ringSamples(ring.rate, delay)); + // A rise parks playback until the queued media covers the new target, which is what keeps + // audio in step with video. The measured target moves a bucket at a time, and a rise the + // buffered slack already covers costs no silence at all. + ring.setLatency(effect.get(this.sync.out.delay)); } #runDecoder(effect: Effect): void { @@ -453,7 +447,7 @@ export class Decoder { config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); // Create consumer with slightly less latency than the render worklet to avoid underflowing. // TODO include JITTER_UNDERHEAD - const consumer = this.#consume(effect, sub, format, { config, path }); + const { consumer, read } = this.#consume(effect, sub, format, { config, path }); spawnFatal(effect, failure, async () => { const loaded = await Util.Libav.polyfill(); @@ -465,6 +459,8 @@ export class Decoder { }); effect.cleanup(() => { if (decoder.state !== "closed") decoder.close(); + // Closing drops what is still decoding, which a refill would otherwise wait on. + this.#ring.peek()?.dropped(); }); // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header. @@ -483,7 +479,10 @@ export class Decoder { decoder.configure(decoderConfig); const order = this.#order(); - const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); + const play = (reads: FrameRead[]) => { + read.set(order.newest(reads) as Micro | undefined); + return this.#play(effect, decoder, decoderConfig, order, reads); + }; for (;;) { const next = await nextMedia(consumer); if (next?.kind !== "frame") { @@ -543,7 +542,7 @@ export class Decoder { ? Util.Hex.toBytes(config.description) : init.description; - const consumer = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); + const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); spawnFatal(effect, failure, async () => { const loaded = await Util.Libav.polyfill(); @@ -555,6 +554,8 @@ export class Decoder { }); effect.cleanup(() => { if (decoder.state !== "closed") decoder.close(); + // Closing drops what is still decoding, which a refill would otherwise wait on. + this.#ring.peek()?.dropped(); }); // Configure decoder with description from catalog @@ -567,7 +568,10 @@ export class Decoder { decoder.configure(decoderConfig); const order = this.#order(); - const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); + const play = (reads: FrameRead[]) => { + read.set(order.newest(reads) as Micro | undefined); + return this.#play(effect, decoder, decoderConfig, order, reads); + }; for (;;) { const next = await nextMedia(consumer); if (next?.kind !== "frame") { @@ -628,8 +632,9 @@ export class Decoder { const ring = await this.#ready(effect); if (!ring) return false; - // Backpressure: in buffered mode this holds the encoded frame until the playhead nears - // it, keeping the lookahead above the floor as Opus instead of decoded PCM. No-op live. + // Backpressure: this holds the encoded frame until the playhead nears it or, for the part of + // the target above the decoded floor, until enough media is queued behind it. A no-op for live + // playback whose whole target fits the floor. await ring.wait(frame.timestamp, effect.abort); if (effect.abort.aborted) return false; @@ -665,7 +670,7 @@ export class Decoder { sub: Moq.Track.Subscriber, format: Container.Format, { config, path }: { config: DecoderConfig; path: Moq.Path.Valid }, - ): Container.Consumer { + ): { consumer: Container.Consumer; read: Signal } { const outgoing = this.#consumer; const continues = outgoing?.path === path; const consumer = new Container.Consumer(sub, { @@ -688,10 +693,26 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // What has arrived but not been decoded is what covers the part of the target held encoded, so + // the decode loop's held frame is released once enough is queued behind it. That includes the + // frames the loop has read but not decoded yet: a buffered head waits in the reorder buffer + // until it spans the whole target, leaving the consumer empty. + const read = new Signal(undefined); + effect.run((inner) => { + const ring = inner.get(this.#ring); + if (!ring) return; + let newest = inner.get(read); + for (const { end } of inner.get(consumer.buffered)) { + const micro = Micro.fromMilli(end); + if (newest === undefined || micro > newest) newest = micro; + } + ring.queued(newest); + }); + // Feed the arrival estimate into the playout target. It outlives this consumer, so the target // holds until the replacement takes over; see #retire. effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); - return consumer; + return { consumer, read }; } // No consumer replaces the outgoing one, so its estimate stops holding the buffer open. @@ -712,6 +733,21 @@ export class Decoder { } #emit(sample: AudioData) { + // Whatever becomes of it, the output answers the frames it was decoded from, so the gate counts + // only what still decodes. Reported after any insert, so the gate always counts the frame as + // either decoding or in the ring. + const decoded = (sample.timestamp + + Micro.fromSecond((sample.numberOfFrames / sample.sampleRate) as Second)) as Micro; + let landed: Micro | undefined; + try { + landed = this.#place(sample); + } finally { + this.#ring.peek()?.decoded(decoded, landed); + } + } + + // Trim the output and write it to the ring, returning where the PCM written ends. + #place(sample: AudioData): Micro | undefined { const { timestamp, frameOffset, frames } = this.#terminal.span(sample); const timestampMilli = Milli.fromMicro(timestamp); if (frames === 0) { @@ -774,6 +810,7 @@ export class Decoder { ring.insert(timestamp, channelData); sample.close(); + return (timestamp + durationMicro) as Micro; } #addDecodeBuffered(start: Milli, end: Milli): void { diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index bd2374f76c..b88e5e31c8 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "bun:test"; -import type { Milli } from "@moq/clock"; -import { ringSamples, target } from "./latency"; +import type { Micro, Milli } from "@moq/clock"; +import { ringCapacity, ringHold, ringSamples, target } from "./latency"; const ms = (value: number) => value as Milli; @@ -25,7 +25,7 @@ describe("target", () => { describe("ringSamples", () => { // `delay="instant"` reports a zero buffer. Passed through, the ring rejects it and the - // worklet is left with no backend that any later resize can revive. + // worklet is left with no backend that any later update can revive. it("floors a zero delay at one render quantum", () => { expect(ringSamples(48_000, ms(0))).toBe(128); }); @@ -38,4 +38,22 @@ describe("ringSamples", () => { it("leaves a delay above the floor alone", () => { expect(ringSamples(48_000, ms(100))).toBe(4_800); }); + + // A catalog's jitter or delay is any u53, so the part held decoded is the receiver's to cap. + it("caps the decoded part at half the fixed capacity", () => { + expect(ringSamples(48_000, ms(60 * 60 * 1000))).toBe(24_000); + expect(ringCapacity(48_000)).toBe(48_000); + }); +}); + +describe("ringHold", () => { + it("holds the target above the cap", () => { + expect(ringHold(ms(1_500))).toBe(1_000_000 as Micro); + }); + + // At 44.1 kHz this target's floor rounds to a sample count a few picoseconds short of it. + it("holds exactly nothing for a target within the cap", () => { + expect(ringHold(ms(203.21995464852608))).toBe(0 as Micro); + expect(ringHold(ms(500))).toBe(0 as Micro); + }); }); diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 8f28a04d51..8510629432 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,4 +1,4 @@ -import { Milli, Second } from "@moq/clock"; +import { Micro, Milli, Second } from "@moq/clock"; /** The terms of the audio playout target, all in milliseconds. */ export interface Target { @@ -37,14 +37,38 @@ export const AUTO_MAX_DELAY = 2000 as Milli; // never be read from. const RENDER_QUANTUM = 128; +// The most of the target the ring holds decoded. The rest stays encoded upstream until the playhead +// nears it, so a catalog advertising an hour of jitter costs the bytes actually received rather than +// an hour of PCM. Comfortably above a main-thread stall or a state-message interval. +const FLOOR_MAX = Milli(500); + +/** + * The ring's capacity in samples: twice the deepest floor, so a ring sitting on its floor has a + * whole floor of headroom for a burst. Fixed by the receiver, never by the target. + */ +export function ringCapacity(rate: number): number { + return Math.ceil(rate * Second.fromMilli(FLOOR_MAX) * 2); +} + /** - * The ring depth for a target delay, floored at one AudioWorklet render quantum. + * The ring's floor for a target delay: the part held as decoded PCM, at least one AudioWorklet render + * quantum and at most 500 ms. * * `delay="instant"` reports a zero buffer, which the ring rejects outright: construction throws, - * the worklet is left with no backend, and every later resize is gated on that backend existing, so + * the worklet is left with no backend, and every later update is gated on that backend existing, so * audio never recovers. Audio keeps its own floor rather than deriving its depth verbatim from a * buffer whose meaning is "video holds nothing". */ export function ringSamples(rate: number, delay: Milli): number { - return Math.max(RENDER_QUANTUM, Math.ceil(rate * Second.fromMilli(delay))); + return Math.max(RENDER_QUANTUM, Math.ceil(rate * Second.fromMilli(Milli(Math.min(delay, FLOOR_MAX))))); +} + +/** + * The part of a target delay above the ring's floor, held encoded upstream rather than decoded. + * + * Measured against the cap rather than the rounded floor, so a target within the cap holds exactly + * nothing instead of the float residue of a sample count. + */ +export function ringHold(delay: Milli): Micro { + return Micro.fromMilli(Milli(Math.max(0, delay - FLOOR_MAX))); } diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts index d8269b7e03..b0ed6d84a6 100644 --- a/js/watch/src/audio/order.test.ts +++ b/js/watch/src/audio/order.test.ts @@ -37,6 +37,16 @@ describe("Order", () => { expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); }); + // The newest frame still to decode covers the hold, wherever it waits: a head arrives newest-first, + // so the frame just read is often the oldest. + it("reports the newest frame held or released", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 60 })); + expect(order.newest([])).toBeUndefined(); + for (const t of [60, 40, 20]) expect(order.newest(order.push(read(t / 20, t)))).toBe(60); + expect(order.newest(order.push(read(0, 0)))).toBe(60); + expect(order.newest([])).toBeUndefined(); + }); + it("releases the head as it stands on a flush", () => { const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); order.push(read(1, 20)); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts index 098ba25e40..95cad335bb 100644 --- a/js/watch/src/audio/order.ts +++ b/js/watch/src/audio/order.ts @@ -48,6 +48,15 @@ export class Order { return []; } + /** The newest timestamp among the held head and `released`, the frames still to decode. */ + newest(released: T[]): number | undefined { + let newest = this.#held.length > 0 ? this.#span?.newest : undefined; + for (const { frame } of released) { + if (newest === undefined || frame.timestamp > newest) newest = frame.timestamp; + } + return newest; + } + /** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */ flush(): T[] { this.#hold = undefined; diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index 6fd19d9fdb..48eef2ecb3 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -82,7 +82,27 @@ describe("render dispatch", () => { ]); }); - it("parks on a latency rise and preserves unread PCM through the resize", () => { + // A covered ring parks and resumes only on the main thread's word: data decoded before the park + // may still land. The echoed epoch is what tells the main thread that state reports no longer + // predate the park. + it("parks until the next floor update and echoes the park's epoch", () => { + const { send, read, sent } = renderer(); + send({ type: "data", timestamp: Micro(0), data: [new Float32Array(200).fill(1)] }); + expect([...read(20)]).toEqual(Array(20).fill(1)); + send({ type: "latency", latency: Milli(100), park: 3 }); + expect([...read(20)]).toEqual(Array(20).fill(0)); + send({ type: "data", timestamp: Micro(200_000), data: [new Float32Array(20).fill(2)] }); + expect([...read(20)]).toEqual(Array(20).fill(0)); + send({ type: "latency", latency: Milli(100) }); + // It fades back in on the parked audio, not the new data. + expect(read(20)[19]).toBe(1); + read(20); + expect(sent).toEqual([ + { type: "state", epoch: 3, timestamp: Micro(60_000), stalled: false, anchored: true, underruns: 0 }, + ]); + }); + + it("parks on a latency rise and preserves unread PCM", () => { const { send, read } = renderer(); send({ type: "data", timestamp: Micro(0), data: [new Float32Array(100).fill(1)] }); expect([...read(20)]).toEqual(Array(20).fill(1)); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index d202f006d7..59475f162b 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -26,7 +26,11 @@ class Render extends AudioWorkletProcessor { } else if (msg.type === "data") { this.#ring?.write(msg.timestamp, msg.data); } else if (msg.type === "latency") { - this.#ring?.resize(msg.latency); + this.#ring?.setLatency(msg.latency); + if (msg.park !== undefined) { + this.#ring?.park(); + this.#epoch = msg.park; + } } else if (msg.type === "truncate") { this.#ring?.truncate(msg.timestamp); } else if (msg.type === "reset") { diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 31002e63a4..77ecb70fa2 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -9,8 +9,8 @@ export interface Init { channels: number; rate: number; latency: Milli; - // Buffered mode: anchor to the first frame and play through; the ring is sized to the floor and - // the lookahead above it is held back upstream (the main thread applies the backpressure). + // Buffered mode: anchor to the first frame and play through. The ring holds the floor; the + // lookahead above it is held back upstream (the main thread applies the backpressure). buffered: boolean; } @@ -34,10 +34,13 @@ export interface Data { timestamp: Micro; } -/** Latency update. */ +/** Latency update: the floor held as decoded PCM. */ export interface Latency { type: "latency"; latency: Milli; + // When set, also hold playback, keeping what is buffered, until the next latency message without + // it. Echoed like a reset's epoch, so state from before the park cannot report the ring playing. + park?: number; } /** State update from the worklet back to the main thread. */ @@ -49,6 +52,6 @@ export interface State { anchored: boolean; // How many times the ring has run dry mid-playback, cumulative. underruns: number; - // The epoch of the last reset the worklet applied. + // The epoch of the last reset or park the worklet applied. epoch: number; } diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index c5f00c8adf..bed45c0b41 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -230,3 +230,35 @@ it("replay refuses missing, non-finite or non-positive recorded durations", asyn await expect(reader.next()).rejects.toThrow("invalid recorded frame duration"); } }); + +// Five seconds advertised is ten times what the ring holds decoded, so the rest waits encoded in the +// consumer until the received media covers it, then plays at the advertised delay. +it("plays an advertised flush span past the decoded floor at its delay without a gap", async () => { + const config = { ...CONFIG, jitter: 5000 } as Catalog.AudioConfig; + let start: number | undefined; + let short = 0; + let lag = 0; + let delay = 0; + for await (const quantum of replay( + paced(12, () => 30), + { + rate: RATE, + config, + delay: "auto", + duration: observed(12), + }, + )) { + const filled = quantum.output.findLastIndex((v) => v !== 0) + 1; + if (start === undefined && filled > 0) start = quantum.at; + if (start !== undefined && filled < QUANTUM) short++; + lag = quantum.at - quantum.timestamp; + delay = quantum.delay; + } + expect(delay).toBe(5020); + // The first frame arrived 30 ms late, so it plays the target after its own capture plus that. + expect(start).toBeGreaterThanOrEqual(5020); + expect(start).toBeLessThan(5040); + expect(short).toBe(0); + expect(lag).toBeGreaterThanOrEqual(5020); + expect(lag).toBeLessThan(5040); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 48576068e2..82a9b91d63 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -6,9 +6,9 @@ * sets. Delivered frames go into the real ring, and a new codec epoch resets it, the way the decoder * does. One render quantum is read every quantum's worth of that same clock, which is what the * AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder - * registers, measured by that same consumer from the recorded catalog config, and the ring follows - * it the way the decoder resizes it, so a change to any of them moves what a replay hears. Decoding - * is taken as instant and sample exact. + * registers, measured by that same consumer from the recorded catalog config, and the ring and the + * decode gate follow it the way the decoder's audio buffer moves them, so a change to any of them + * moves what a replay hears. Decoding is taken as instant and sample exact. * * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test * can assert exact counts. @@ -23,8 +23,9 @@ import { Group, Time, Track } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; import { type Arrival as Diagnostic, gap } from "../arrivals"; import { type Delay, Sync } from "../sync"; +import { Backpressure } from "./buffer"; import { frameDuration } from "./config"; -import { ringSamples, target } from "./latency"; +import { target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; import { Terminal } from "./terminal"; @@ -81,7 +82,7 @@ export interface Quantum { stalled: boolean; /** The playhead after this quantum, on the media clock, in ms. */ timestamp: number; - /** The delay `Sync` resolved as of this quantum, in ms: what sizes the ring. */ + /** The delay `Sync` resolved as of this quantum, in ms: what the ring and the decode gate hold. */ delay: number; } @@ -151,8 +152,19 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato await settle(); let delay = sync.out.delay.peek(); - // The worklet's ring, sized the way `AudioBuffer` asks the worklet to size it. - const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, ringSamples(rate, delay)) }); + // The decode gate, and the worklet's ring on the floor it sets, the way `AudioBuffer` drives both. + const gate = new Backpressure(rate, false, delay); + const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, gate.floor) }); + const observe = () => gate.observe(ring.anchored ? ring.timestamp : undefined, ring.stalled); + + // Arrived but not yet decoded media, read or still in the consumer, is what covers the part of the + // target held encoded, and what resumes a parked ring. + const read = new Signal(undefined); + signals.run((effect) => { + let newest = effect.get(read); + for (const { end } of effect.get(consumer.buffered)) newest = Math.max(newest ?? 0, micros(end)) as Micro; + if (gate.queued(newest)) ring.setLatency(millis(rate, gate.floor)); + }); // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. const terminal = new Terminal(); @@ -160,23 +172,35 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const delivery = Promise.withResolvers(); if (options.decodeAfter === undefined) delivery.resolve(); let failure: { error: unknown } | undefined; + const abort = new AbortController(); const decode = (async () => { await delivery.promise; for (;;) { const next = await consumer.next(); if (!next) return; - if (terminal.update(next)) ring.reset(); + if (terminal.update(next)) { + ring.reset(); + gate.reset(); + } if (next.kind === "gap") arrivals.push(gap(next)); if (next.kind !== "frame") continue; const { timestamp, arrival } = next.frame; + read.set(timestamp); const media = Milli.fromMicro(timestamp); const late = sync.received(media, arrival); arrivals.push({ kind: "frame", timestamp: media, arrival, late }); const entry = recorded.get(timestamp); if (!entry) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); frame.set(Milli(entry.duration)); - ring.write(timestamp, [new Float32Array(Math.round((entry.duration * rate) / 1000)).fill(0.5)]); + await gate.wait(timestamp, abort.signal); + if (abort.signal.aborted) return; + const pcm = new Float32Array(Math.round((entry.duration * rate) / 1000)).fill(0.5); + ring.write(timestamp, [pcm]); + // The decoder reports each output to the gate the way `AudioBuffer` and `Decoder.#emit` do. + const end = (timestamp + micros((pcm.length / rate) * 1000)) as Micro; + gate.inserted(timestamp, end); + gate.decoded(end, end); } })().catch((error: unknown) => { failure = { error }; @@ -236,15 +260,20 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato } if (failure) throw failure.error; - // The decoder resizes the ring as the resolved delay moves, which only an arrival does. + // The decoder moves the ring and the gate as the resolved delay moves, which only an arrival + // does. const resolved = sync.out.delay.peek(); if (resolved !== delay) { delay = resolved; - ring.resize(millis(rate, ringSamples(rate, delay))); + observe(); + const park = gate.setLatency(delay); + ring.setLatency(millis(rate, gate.floor)); + if (park) ring.park(); } output.fill(0); ring.read([output]); + observe(); yield { arrivals, at: now, @@ -256,6 +285,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato } } finally { delivery.resolve(); + abort.abort(); consumer.close(); track.close(); await decode; diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index bb5b22e02a..1d1c4f3d77 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -35,9 +35,9 @@ describe("initialization", () => { it("should initialize with valid parameters", () => { const buffer = new AudioRingBuffer({ rate: 48000, channels: 2, latency: 100 as Milli }); - // Capacity is twice the target: the ring has to be able to hold the slack the overflow band - // tolerates, or every chunk arriving on a full ring would drop the oldest samples. - expect(buffer.capacity).toBe(9600); // 48000 * 0.1 * 2 + // The receiver fixes the capacity at twice the deepest floor, never from the target: whatever + // of the target is above the floor stays encoded upstream. + expect(buffer.capacity).toBe(48000); // 48000 * 0.5 * 2 expect(buffer.length).toBe(0); }); @@ -377,39 +377,16 @@ describe("edge cases", () => { }); }); -describe("resize", () => { - it("should resize to a larger buffer", () => { +describe("setLatency", () => { + it("keeps the receiver's capacity whatever the floor", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.capacity).toBe(200); - - // Write 50 samples + const capacity = buffer.capacity; write(buffer, 0 as Milli, 50, { channels: 1, value: 1.0 }); - expect(buffer.length).toBe(50); - - // Resize to larger buffer (200ms = 200 samples) - buffer.resize(200 as Milli); - - expect(buffer.capacity).toBe(400); - expect(buffer.length).toBe(50); // Samples preserved - expect(buffer.stalled).toBe(true); // Should trigger stall - }); - - it("should resize to a smaller buffer and keep the most recent samples", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.capacity).toBe(200); - // Write 80 samples: first 40 with value 1.0, next 40 with value 2.0 - write(buffer, 0 as Milli, 40, { channels: 1, value: 1.0 }); - write(buffer, 40 as Milli, 40, { channels: 1, value: 2.0 }); - expect(buffer.length).toBe(80); - - // Resize to a 30 sample target (60 samples of capacity). - // Should keep the most recent 60 samples (samples 20-79) - buffer.resize(30 as Milli); - - expect(buffer.capacity).toBe(60); - expect(buffer.length).toBe(60); // Truncated to new capacity - expect(buffer.stalled).toBe(false); // Already covers the new floor + buffer.setLatency(400 as Milli); + expect(buffer.capacity).toBe(capacity); + expect(buffer.length).toBe(50); + expect(buffer.stalled).toBe(true); }); // Buffered playback promises to play through everything queued, so a lower floor must not drop @@ -418,233 +395,43 @@ describe("resize", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli, buffered: true }); write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); write(buffer, 100 as Milli, 80, { channels: 1, value: 2.0 }); - expect(buffer.length).toBe(180); - buffer.resize(30 as Milli); - expect(buffer.capacity).toBe(180); - expect(buffer.length).toBe(180); + buffer.setLatency(30 as Milli); + write(buffer, 180 as Milli, 20, { channels: 1, value: 3.0 }); + expect(buffer.length).toBe(200); expect(Milli.fromMicro(buffer.timestamp)).toBe(0 as Milli); - const [output] = read(buffer, 180, 1); - expect([...output]).toEqual([...Array(100).fill(1), ...Array(80).fill(2)]); - - // Drained, it shrinks back to the new floor's capacity on the next resize. - buffer.resize(30 as Milli); - expect(buffer.capacity).toBe(60); - }); - - it("should be a no-op when capacity is unchanged", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Exit stalled mode - write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - expect(buffer.stalled).toBe(false); - - // Resize to same capacity - buffer.resize(100 as Milli); - - // Should still not be stalled (no-op) - expect(buffer.stalled).toBe(false); - expect(buffer.capacity).toBe(200); - }); - - it("should throw on zero latency", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(() => buffer.resize(0 as Milli)).toThrow(/empty buffer/); - }); - - it("should handle resize with stereo data", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 2, latency: 100 as Milli }); - - // Write stereo data with different values per channel - const data = [new Float32Array(60), new Float32Array(60)]; - for (let i = 0; i < 60; i++) { - data[0][i] = 1.0; // Left channel - data[1][i] = 2.0; // Right channel - } - buffer.write(0 as Micro, data); - expect(buffer.length).toBe(60); - - // Resize to smaller buffer - buffer.resize(50 as Milli); - expect(buffer.capacity).toBe(100); - expect(buffer.length).toBe(60); - expect(buffer.stalled).toBe(false); - }); - - it("should handle resize when buffer is empty", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.length).toBe(0); - - // Resize empty buffer - buffer.resize(200 as Milli); - - expect(buffer.capacity).toBe(400); - expect(buffer.length).toBe(0); - expect(buffer.stalled).toBe(true); - }); - - it("should handle resize after partial read", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Fill and exit stalled mode - write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - expect(buffer.stalled).toBe(false); - - // Read 60 samples (readIndex at 60, writeIndex at 100) - read(buffer, 60, 1); - expect(buffer.length).toBe(40); - - // Write more data - write(buffer, 100 as Milli, 30, { channels: 1, value: 2.0 }); - expect(buffer.length).toBe(70); // 130 - 60 - - // Resize to a 50 sample target - 100 of capacity, so all 70 survive (samples 60-129) - buffer.resize(50 as Milli); - expect(buffer.capacity).toBe(100); - expect(buffer.length).toBe(70); - expect(buffer.stalled).toBe(false); + const [output] = read(buffer, 200, 1); + expect([...output]).toEqual([...Array(100).fill(1), ...Array(80).fill(2), ...Array(20).fill(3)]); }); - it("should resume and then read new data after a shrinking resize", () => { + it("refuses an empty floor or one past the capacity", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Write some initial data - write(buffer, 0 as Milli, 50, { channels: 1, value: 1.0 }); - - // Resize to a floor the buffered data already covers - buffer.resize(50 as Milli); - expect(buffer.stalled).toBe(false); - - // Nothing is dropped: the ring has room for the preserved samples plus a chunk above the - // target. - write(buffer, 50 as Milli, 50, { channels: 1, value: 2.0 }); - expect(buffer.stalled).toBe(false); - expect(buffer.length).toBe(100); - - // Read plays the preserved samples first, then the new ones. - const output = read(buffer, 100, 1); - expect(output[0].length).toBe(100); - for (let i = 0; i < 50; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 50; i < 100; i++) { - expect(output[0][i]).toBe(2.0); - } + expect(() => buffer.setLatency(0 as Milli)).toThrow(/empty buffer/); + expect(() => buffer.setLatency((buffer.capacity + 1) as Milli)).toThrow(/exceeds capacity/); + expect(() => new AudioRingBuffer({ rate: 1000, channels: 1, latency: 60_000 as Milli })).toThrow( + /exceeds capacity/, + ); }); +}); - it("should preserve samples correctly when resizing larger then filling", () => { +describe("park", () => { + it("holds a covered ring until the next floor update, keeping what it buffered", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 50 as Milli }); - expect(buffer.capacity).toBe(100); - - // Write 30 samples - write(buffer, 0 as Milli, 30, { channels: 1, value: 1.0 }); - expect(buffer.length).toBe(30); - - // Resize to larger buffer (100ms = 100 samples) - buffer.resize(100 as Milli); - expect(buffer.capacity).toBe(200); - expect(buffer.length).toBe(30); // All samples preserved - expect(buffer.stalled).toBe(true); - - // Write 70 more samples to fill the buffer and exit stall - write(buffer, 30 as Milli, 70, { channels: 1, value: 2.0 }); - expect(buffer.stalled).toBe(false); - expect(buffer.length).toBe(100); - - // Read all - should have 30 samples of 1.0 then 70 of 2.0 - const output = read(buffer, 100, 1); - expect(output[0].length).toBe(100); - for (let i = 0; i < 30; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 30; i < 100; i++) { - expect(output[0][i]).toBe(2.0); - } - }); - - it("should read preserved samples back correctly after shrinking", () => { - // Regression: the copy loop inside resize() used a relative dst index - // (`i % dst.length`) while read() uses the absolute ring position - // (`readIndex % capacity`). When `copyStart` was not a multiple of the new - // capacity, preserved samples ended up in the wrong slots and read() returned - // mangled data. This test fails under that bug by constructing a scenario - // where `copyStart % newCapacity !== 0`. - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Fill the buffer to exit stall (writeIndex=100, readIndex=0). write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - expect(buffer.stalled).toBe(false); + expect(read(buffer, 10, 1)[0].length).toBe(10); - // Advance readIndex to 20 so there's room to wrap the write pointer. - read(buffer, 20, 1); + buffer.park(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 10, 1)[0].length).toBe(0); - // Write samples that wrap: abs 100-109 → slots 0-9, value 2.0. + // Output decoded before the park can still land, so a write does not resume it. write(buffer, 100 as Milli, 10, { channels: 1, value: 2.0 }); - // abs 110-119 → slots 10-19, value 3.0. - write(buffer, 110 as Milli, 10, { channels: 1, value: 3.0 }); - - // Preserved range after resize: all 100 buffered samples = abs 20..119. - // copyStart = 120 - 100 = 20, and 20 % 100 = 20 (non-zero → triggers the bug). - buffer.resize(50 as Milli); - expect(buffer.capacity).toBe(100); - expect(buffer.length).toBe(100); - expect(buffer.stalled).toBe(false); - - // Read the preserved samples. Expected layout by absolute index: - // abs 20..99 = 1.0 (80 samples, from the initial fill) - // abs 100..109 = 2.0 (10 samples) - // abs 110..119 = 3.0 (10 samples) - const output = read(buffer, 100, 1); - expect(output[0].length).toBe(100); - for (let i = 0; i < 80; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 80; i < 90; i++) { - expect(output[0][i]).toBe(2.0); - } - for (let i = 90; i < 100; i++) { - expect(output[0][i]).toBe(3.0); - } - }); - - it("should continue accepting absolute-timestamp writes after resize", () => { - // resize() must keep readIndex/writeIndex on the same absolute axis that - // write() uses (`round(timestamp * rate)`). If the indices were reset to 0 - // while timestamps stayed absolute, the next write would leave a giant zero - // gap. This test asserts that post-resize writes land contiguously with the - // preserved samples. - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Fill the buffer, then partially drain it. - write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - read(buffer, 40, 1); - // Wrap the writer over slots 0-29 with value 2.0 (abs 100-129). - write(buffer, 100 as Milli, 30, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); - // Resize smaller. All 90 buffered samples fit the new 100 sample capacity: abs 40..129. - buffer.resize(50 as Milli); - expect(buffer.length).toBe(90); + // The main thread knows when the media it holds encoded covers the target. + buffer.setLatency(50 as Milli); expect(buffer.stalled).toBe(false); - - // Write the next 10 samples at their real timestamp. This should append, - // not create a gap or be discarded as "too old". - write(buffer, 130 as Milli, 10, { channels: 1, value: 3.0 }); - - // That put the ring past the band, so it drops back to the 50 sample target. - // Remaining: abs 90..99 = 1.0 (10), abs 100..129 = 2.0 (30), abs 130..139 = 3.0 (10). - expect(buffer.length).toBe(50); - - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); - for (let i = 0; i < 10; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 10; i < 40; i++) { - expect(output[0][i]).toBe(2.0); - } - for (let i = 40; i < 50; i++) { - expect(output[0][i]).toBe(3.0); - } + expect(read(buffer, 10, 1)[0]).toEqual(new Float32Array(10).fill(1.0)); }); }); @@ -674,7 +461,6 @@ describe("live anchoring", () => { // un-stalled on that overflow, and left the playhead a floor before the frame. it("anchors a live ring to the first frame instead of gap-filling from zero", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.capacity).toBe(200); write(buffer, 2000 as Milli, 40, { channels: 1, value: 0.5 }); // Still short of the floor, so nothing plays yet and the ring holds only real samples. @@ -791,16 +577,16 @@ describe("buffered mode", () => { expect(out[0][0]).toBeCloseTo(0.9, 5); }); - it("drops the oldest samples once the ring fills beyond the floor", () => { - // 50ms floor at 1000Hz -> 50-sample floor, 100-sample ring. Backpressure normally keeps the - // decode loop from running this far ahead; if it slips, the ring drops its oldest as a backstop. + it("drops the oldest samples once the ring fills beyond its capacity", () => { + // 50ms floor at 1000Hz in a 1000-sample ring. Backpressure normally keeps the decode loop from + // running this far ahead; if it slips, the ring drops its oldest as a backstop. const buffer = createBuffered(50); - write(buffer, 0 as Milli, 100, { channels: 1, value: 0.1 }); // [0, 100) - write(buffer, 100 as Milli, 100, { channels: 1, value: 0.2 }); // exceeds 100-sample ring; drops [0, 100) + write(buffer, 0 as Milli, 1000, { channels: 1, value: 0.1 }); // [0, 1000) + write(buffer, 1000 as Milli, 100, { channels: 1, value: 0.2 }); // past the ring; drops [0, 100) expect(Milli.fromMicro(buffer.timestamp)).toBe(100 as Milli); - expect(read(buffer, 100, 1)[0][0]).toBeCloseTo(0.2, 5); + expect(buffer.length).toBe(1000); }); it("truncate drops the write-ahead tail a successor supersedes", () => { @@ -846,7 +632,7 @@ describe("latency increase", () => { expect(buffer.stalled).toBe(false); read(buffer, 50, 1); - buffer.resize(110 as Milli); + buffer.setLatency(110 as Milli); expect(buffer.stalled).toBe(true); expect(buffer.length).toBe(50); @@ -866,7 +652,7 @@ describe("latency increase", () => { write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); read(buffer, 50, 1); - buffer.resize(90 as Milli); + buffer.setLatency(90 as Milli); expect(buffer.stalled).toBe(false); }); @@ -875,11 +661,11 @@ describe("latency increase", () => { write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); read(buffer, 50, 1); - buffer.resize(110 as Milli); + buffer.setLatency(110 as Milli); expect(buffer.stalled).toBe(true); // No further frame arrives, as when the source stops, so only the lower floor can resume it. - buffer.resize(50 as Milli); + buffer.setLatency(50 as Milli); expect(buffer.stalled).toBe(false); expect(read(buffer, 1, 1)[0][0]).toBe(1.0); }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 7c167626a7..8706af6968 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -1,4 +1,5 @@ import { Micro, type Milli, Second } from "@moq/clock"; +import { ringCapacity } from "./latency"; export class AudioRingBuffer { #buffer: Float32Array[]; @@ -8,6 +9,8 @@ export class AudioRingBuffer { readonly rate: number; readonly channels: number; #stalled = true; + // Held stalled for the part of the target above the floor, until the next floor update. + #parked = false; #underruns = 0; // Samples in the most recent write, i.e. one decoded chunk. The overflow band tolerates this @@ -40,12 +43,12 @@ export class AudioRingBuffer { this.channels = props.channels; this.#buffered = props.buffered ?? false; - // The ring holds the latency floor as PCM, with headroom above it. Sizing capacity to the - // floor exactly makes the ring physically incapable of holding the slack the overflow band - // wants, so every chunk that arrives while the ring is on target drops the oldest samples. - // In buffered mode the headroom is also what keeps the backpressure-paced decode loop (on - // the main thread) from overflow-dropping; the rest of the lookahead stays encoded upstream. - const capacity = this.#capacityFor(this.#latencySamples); + // The ring holds the latency floor as PCM, with headroom above it, at a capacity the receiver + // fixes rather than the target: whatever of the target is above the floor stays encoded + // upstream (the main thread holds it back). The headroom tolerates a chunk above the floor in + // live mode, and in buffered mode keeps the paced decode loop from overflow-dropping. + const capacity = ringCapacity(this.rate); + if (this.#latencySamples > capacity) throw new Error("latency exceeds capacity"); this.#buffer = []; for (let i = 0; i < this.channels; i++) { @@ -53,10 +56,6 @@ export class AudioRingBuffer { } } - #capacityFor(latencySamples: number): number { - return latencySamples * 2; - } - get stalled(): boolean { return this.#stalled; } @@ -83,8 +82,11 @@ export class AudioRingBuffer { return this.#buffer[0]?.length; } - resize(latency: Milli): void { + /** Move the floor, which never exceeds the fixed capacity, and end a park. */ + setLatency(latency: Milli): void { const latencySamples = Math.ceil(this.rate * Second.fromMilli(latency)); + if (latencySamples <= 0) throw new Error("empty buffer"); + if (latencySamples > this.capacity) throw new Error("latency exceeds capacity"); // A deeper floor parks playback until it refills. Video holds the extra delay on its own, so // audio that kept draining at the old depth would run ahead by the difference. Parking keeps @@ -92,46 +94,23 @@ export class AudioRingBuffer { // already covers the new floor. if (latencySamples > this.#latencySamples) this.#stalled = true; this.#latencySamples = latencySamples; - - // A live ring sheds what a lower floor no longer covers, keeping the newest. Buffered mode - // plays through everything, so its ring keeps the whole unread span; backpressure holds new - // frames until playback drains it below the new floor. - const newCapacity = Math.max(this.#capacityFor(this.#latencySamples), this.#buffered ? this.length : 0); - if (newCapacity !== this.capacity) this.#reallocate(newCapacity); + this.#parked = false; // Resume a refill that the buffer already covers, which `write` would only notice on the // next frame, never for a source that has stopped. - if (latencySamples > 0 && this.length >= latencySamples) this.#stalled = false; + if (this.length >= latencySamples) this.#stalled = false; } - #reallocate(newCapacity: number): void { - if (newCapacity === 0) throw new Error("empty buffer"); - - const newBuffer: Float32Array[] = []; - for (let i = 0; i < this.channels; i++) { - newBuffer[i] = new Float32Array(newCapacity); - } - - // Copy existing data, preserving the most recent samples - const samplesToKeep = Math.min(this.length, newCapacity); - if (samplesToKeep > 0) { - // Copy the most recent samples (closest to writeIndex) - const copyStart = this.#writeIndex - samplesToKeep; - for (let channel = 0; channel < this.channels; channel++) { - const src = this.#buffer[channel]; - const dst = newBuffer[channel]; - for (let i = 0; i < samplesToKeep; i++) { - const srcPos = (copyStart + i) % src.length; - const dstPos = (copyStart + i) % dst.length; - dst[dstPos] = src[srcPos]; - } - } - } - - // Update state for the new buffer, only stall if empty. - this.#buffer = newBuffer; - this.#readIndex = this.#writeIndex - samplesToKeep; - if (samplesToKeep === 0) this.#stalled = true; + /** + * Hold playback until the next floor update, keeping what is buffered. + * + * For a rise in the part of the target held encoded upstream: the ring already covers its floor, + * so only the main thread knows when the received media covers the rest. A write does not resume + * it, since output decoded before the park may still land. + */ + park(): void { + this.#stalled = true; + this.#parked = true; } write(timestamp: Micro, data: Float32Array[]): void { @@ -213,9 +192,9 @@ export class AudioRingBuffer { } // Start playback once we've buffered the latency target. This is the only way out of a - // stall, so an underrun mid-playback refills to the target before resuming rather than - // playing the next chunk on an empty cushion. - if (this.length >= this.#latencySamples) { + // stall but a park, so an underrun mid-playback refills to the target before resuming rather + // than playing the next chunk on an empty cushion. + if (!this.#parked && this.length >= this.#latencySamples) { this.#stalled = false; } } @@ -238,6 +217,7 @@ export class AudioRingBuffer { this.#readIndex = 0; this.#writeIndex = 0; this.#stalled = true; + this.#parked = false; this.#anchored = false; } diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index f966381032..116168613f 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -918,7 +918,7 @@ it("a renamed video track keeps the delay, so audio neither skips nor goes silen const ring = new AudioRingBuffer({ rate: RATE, channels: 1, latency: Milli(STEP) }); const follow = new Effect(); follow.subscribe(playback.sync.out.delay, (delay) => { - ring.resize(Milli.fromSecond((ringSamples(RATE, delay) / RATE) as Second)); + ring.setLatency(Milli.fromSecond((ringSamples(RATE, delay) / RATE) as Second)); }); // Flushed in runs of 12, so each run's first frame lands ~360ms late. diff --git a/quest/m1/README.md b/quest/m1/README.md index f3058fd63e..bafc149946 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -129,7 +129,6 @@ audit's receive-memory, priority, and lease bugs rank ahead of that reshape. - [Native rung disable](/quest/m1/simulcast-rung-disable.md) - Rust/FFI and OBS publishers stop encoding the top renditions their grant cannot fund and resume them once probing raises the estimate - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain - [Play keeps its audio output](/quest/m1/play-audio-handover.md) - `moq play` switches to a replacement run without reopening its audio output, so native-play holds its derived bound -- [Audio playout memory](/quest/m1/audio-playout-memory.md) - a catalog's advertised jitter or delay cannot size decoded PCM; the lookahead stays encoded - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor diff --git a/quest/m1/audio-playout-memory.md b/quest/m1/audio-playout-memory.md deleted file mode 100644 index aa5701dbef..0000000000 --- a/quest/m1/audio-playout-memory.md +++ /dev/null @@ -1,47 +0,0 @@ -# [M] The receiver owns its decoded audio memory - -## Goal - -A catalog's advertised `jitter` or `delay`, however large, cannot make a -viewer allocate decoded PCM in proportion to it. The decoded ring is sized -by the receiver, and a deep playout target is reached by holding encoded -frames upstream, so memory follows bytes actually received. A publisher that -genuinely advertises seconds of flush delay still plays smoothly at that delay. - -## Plan - -Found in a 2026-10-22 media audit at -[main 1f5caa7](https://github.com/moq-dev/moq/commit/1f5caa7026bcfd8a2684da3712b384b5f74a7496) -and confirmed by source at -[main d4cdfe4](https://github.com/moq-dev/moq/commit/d4cdfe4368b17cbdf7d2a0c43d24e364a9626480). -`target()` in `js/watch/src/audio/latency.ts` floors the measured term by the -rendition's advertised `jitter` and adds its catalog `delay`; both are any -u53 in `js/hang/src/catalog/audio.ts`. `ringSamples` turns that target into -`latencySamples`, and the render worklet's `AudioRingBuffer` allocates -`latencySamples * 2` frames per channel of float PCM. The audit computed 512 MiB for a 48 kHz stereo rendition advertising ten minutes -of jitter and 4 GiB for one hour, without allocating them. The estimator's 2 s -ceiling bounds only the measured term. - -Decided 2026-10-22 (maintainer): hold the lookahead encoded rather than refuse -a rendition whose hints exceed a ceiling. The existing buffered mode already -keeps the lookahead above the floor as encoded frames behind `Backpressure`; -generalize that so the ring's capacity comes from the receiver's own budget -(a render-quantum floor plus decode-ahead slack), independent of the target, -and the target delay is enforced by when encoded frames are released to the -decoder. Settle while building: - -- The subscription's max delay must cover the target, or the transport drops - the frames the target waits for (`AUTO_MAX_DELAY` is the current floor). - Encoded retention then sits in the transport's group cache, which is the - receiver's existing flow-controlled store. -- Whether the non-buffered path survives once both paths gate on timestamps, - or folds into the buffered one. Prefer one path. - -Tests: a rendition advertising an hour of jitter allocates a ring of the -receiver's fixed size and plays at the advertised delay on a stubbed clock; -the replay harness rows keep their budgets. No wire change; -any `js/watch` API change says how to migrate in the PR's Impact bullet. - -## Related - -- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target this sizes memory for