From 1f101609cf2f471e5cc37e4b87096c05e50ddec4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:59:52 -0700 Subject: [PATCH 01/10] chore(quest): claim quest/m1/audio-playout-memory.md Quest-Claim: 4b222b068bfbb108 From fcd1dad5daf74178bf5ec87e2dd08cc26421f0d7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:29:05 -0700 Subject: [PATCH 02/10] fix(watch): keep audio above a fixed decoded floor encoded A catalog's advertised jitter or delay is any u53, and the audio ring was sized to twice the playout target, so an hour of advertised jitter allocated 4 GiB of shared PCM. The ring now has a fixed capacity (one second) and decodes at most 500 ms of the target. The rest stays encoded in the container consumer, and the decode gate releases a frame once the media queued behind it covers that hold, or once the playhead nears it, so memory follows the bytes actually received. The gate is the buffered-mode backpressure generalized to both modes and both ring transports. A target within 500 ms holds nothing encoded, so live playback is unchanged there. The rings never grow, so the shared ring's resize and worklet handoff are gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EeyiEy5FWz3z44DYZVo7vU --- doc/concept/audio-jitter.md | 4 +- js/watch/src/audio/buffer.test.ts | 178 +++++++++-- js/watch/src/audio/buffer.ts | 259 ++++++++++------ js/watch/src/audio/decoder.ts | 27 +- js/watch/src/audio/latency.test.ts | 24 +- js/watch/src/audio/latency.ts | 32 +- js/watch/src/audio/playout.test.ts | 6 +- js/watch/src/audio/render-worklet.ts | 8 +- js/watch/src/audio/render.ts | 13 +- js/watch/src/audio/replay.test.ts | 35 +++ js/watch/src/audio/replay.ts | 64 ++-- js/watch/src/audio/ring-buffer.test.ts | 282 +++--------------- js/watch/src/audio/ring-buffer.ts | 64 ++-- js/watch/src/audio/shared-ring-buffer.test.ts | 142 +-------- js/watch/src/audio/shared-ring-buffer.ts | 139 ++------- js/watch/src/video/decoder.test.ts | 2 +- quest/m1/README.md | 1 - quest/m1/audio-backpressure-timestamps.md | 4 - quest/m1/audio-playout-memory.md | 50 ---- 19 files changed, 581 insertions(+), 753 deletions(-) delete mode 100644 quest/m1/audio-playout-memory.md diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 66388cd34d..29df3d586e 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -423,7 +423,9 @@ The advertised floor is the one term that can exceed that bound, and it is deliberately not clamped to it. The ceiling is a property of the histogram's size; the floor is a number the publisher declared about itself, so clamping the floor to the ceiling would under-buffer precisely the publisher that was -honest about a long flush. +honest about a long flush. Honoring it must not cost memory in proportion to +the declared number, so the browser decodes at most 500 ms of any target and +holds the rest as the encoded frames it actually received. Policy limits sit above the algorithm. A receiver's own configured maximum and the track's retention window both cap what it may hold, and neither is the diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 1f40ed2972..fe4f89637c 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -1,36 +1,170 @@ import { describe, expect, it } from "bun:test"; -import type { Micro } from "@moq/clock"; -import { createAudioBuffer } from "./buffer"; -import type { InitShared } from "./render"; +import { Micro, Milli } from "@moq/clock"; +import { Backpressure, createAudioBuffer } from "./buffer"; +import type { InitShared, Message, State } from "./render"; import { SharedRingBuffer } from "./shared-ring-buffer"; +const HOUR = Milli(60 * 60 * 1000); +const ms = (value: number) => Micro.fromMilli(Milli(value)); + +/** A worklet that records what the buffer posts to it. */ +function worklet(): { node: AudioWorkletNode; sent: InitShared[] } { + const sent: InitShared[] = []; + const node = { port: { postMessage: (msg: InitShared) => sent.push(msg) } } as unknown as AudioWorkletNode; + return { node, sent }; +} + +describe("Backpressure", () => { + it("releases every live frame when the whole target fits the floor", () => { + const gate = new Backpressure(1000, false, Milli(100)); + gate.observe(ms(0), false); + expect(Bun.peek.status(gate.wait(ms(10_000), new AbortController().signal))).toBe("fulfilled"); + }); + + it("holds buffered playback to the floor ahead of the playhead", () => { + const gate = new Backpressure(1000, true, Milli(200)); + gate.observe(ms(0), false); + const wait = gate.wait(ms(1000), new AbortController().signal); + gate.observe(ms(799), false); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.observe(ms(800), false); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + // Two seconds is a 500 ms floor and 1.5 s held encoded. + it("releases a live frame once the queued media covers the hold behind it", () => { + const gate = new Backpressure(1000, false, Milli(2000)); + gate.observe(ms(0), false); + const wait = gate.wait(ms(1000), new AbortController().signal); + gate.queued(ms(2499)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.queued(ms(2500)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + it("plays buffered media through instead of releasing it ahead of the floor", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.observe(ms(0), false); + const wait = gate.wait(ms(1000), new AbortController().signal); + gate.queued(ms(60_000)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.observe(ms(500), false); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + it("starts a stalled ring only once the queued media covers the hold", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + const wait = gate.wait(ms(0), new AbortController().signal); + gate.queued(ms(1499)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.queued(ms(1500)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + + // An utterance that arrived whole before the reset changes nothing queued, so nothing would + // report it again: forgetting it would hold its first frame until the next utterance. + it("keeps what is queued across a reset", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(5000)); + gate.observe(ms(0), false); + gate.reset(); + expect(Bun.peek.status(gate.wait(ms(0), new AbortController().signal))).toBe("fulfilled"); + }); +}); + describe("SharedAudioBuffer", () => { - // The worklet keeps reading the old ring until `init-shared` lands, so a rise that grows the ring - // has to park that one too, or audio drains ahead of video in between. - it("parks the ring the worklet still reads when a rise grows it", () => { - const sent: InitShared[] = []; - const worklet = { port: { postMessage: (msg: InitShared) => sent.push(msg) } } as unknown as AudioWorkletNode; - const buffer = createAudioBuffer(worklet, 1, 1000, 100); + // A catalog advertising an hour of jitter once sized the ring to two hours of stereo PCM, 4 GiB. + it("allocates the same ring for an hour of delay as for a tenth of a second", () => { + const bytes = [Milli(100), HOUR].map((delay) => { + const { node, sent } = worklet(); + createAudioBuffer(node, 2, 48_000, delay).close(); + return sent[0].samples.byteLength; + }); + expect(bytes[1]).toBe(bytes[0]); + expect(bytes[1]).toBeLessThanOrEqual(2 ** 20); + }); + + it("holds a frame encoded until an hour of queued media is behind it", () => { + const { node } = worklet(); + const buffer = createAudioBuffer(node, 1, 1000, HOUR); try { - const old = new SharedRingBuffer(sent[0]); - buffer.insert(0 as Micro, [new Float32Array(100).fill(1.0)]); - expect(old.read([new Float32Array(50)])).toBe(50); - - buffer.setLatency(1000); - expect(sent.length).toBe(2); - expect(old.stalled).toBe(true); - expect(old.read([new Float32Array(50)])).toBe(0); - expect(new SharedRingBuffer(sent[1]).stalled).toBe(true); + // The ring holds 500 ms decoded; the rest of the hour waits upstream. + const hold = HOUR - 500; + const wait = buffer.wait(ms(0), new AbortController().signal); + buffer.queued(ms(hold - 1)); + expect(Bun.peek.status(wait)).toBe("pending"); + buffer.queued(ms(hold)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); } finally { buffer.close(); } }); + // One second is a 500 ms floor and 500 ms held encoded. Playing 100 ms in, a rise to 1.5 s needs + // queued media through 1600 ms to keep playing. + for (const [queued, parked] of [ + [1599, true], + [1600, false], + ] as const) { + it(`${parked ? "parks" : "keeps playing"} on a rise with ${queued} ms queued`, () => { + const { node, sent } = worklet(); + const buffer = createAudioBuffer(node, 1, 1000, Milli(1000)); + try { + const reader = new SharedRingBuffer(sent[0]); + buffer.insert(ms(0), [new Float32Array(600).fill(1)]); + expect(reader.read([new Float32Array(100)])).toBe(100); + buffer.queued(ms(queued)); + + buffer.setLatency(Milli(1500)); + expect(reader.stalled).toBe(parked); + expect(sent.length).toBe(1); + } finally { + buffer.close(); + } + }); + } + + // The postMessage ring learns the playhead from the worklet's state messages, and parks the same way. + for (const [queued, parked] of [ + [1599, true], + [1600, false], + ] as const) { + it(`posts ${parked ? "a park" : "no park"} on a rise with ${queued} ms queued`, async () => { + const global = globalThis as { crossOriginIsolated?: boolean }; + global.crossOriginIsolated = false; + const channel = new MessageChannel(); + const sent: Message[] = []; + channel.port2.onmessage = (event: MessageEvent) => sent.push(event.data); + const buffer = createAudioBuffer( + { port: channel.port1 } as unknown as AudioWorkletNode, + 1, + 1000, + Milli(1000), + ); + try { + const state: State = { type: "state", timestamp: ms(100), stalled: false, underruns: 0 }; + channel.port2.postMessage(state); + await new Promise((resolve) => setTimeout(resolve, 0)); + buffer.queued(ms(queued)); + + buffer.setLatency(Milli(1500)); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(sent.map((msg) => msg.type)).toEqual( + parked ? ["init-post", "latency", "park"] : ["init-post", "latency"], + ); + } finally { + buffer.close(); + channel.port1.close(); + delete global.crossOriginIsolated; + } + }); + } + // A decode loop can reach `wait` on a buffer a shape change already closed. Nothing advances a // closed buffer, so a gated wait would never settle and would hold the decoder effect's rerun. it("does not gate a wait on a closed buffer", async () => { - const worklet = { port: { postMessage: () => {} } } as unknown as AudioWorkletNode; - const buffer = createAudioBuffer(worklet, 1, 1000, 100, true); + const { node } = worklet(); + const buffer = createAudioBuffer(node, 1, 1000, Milli(100), true); buffer.insert(0 as Micro, [new Float32Array(200)]); const settles = (wait: Promise) => Promise.race([ @@ -48,8 +182,8 @@ describe("SharedAudioBuffer", () => { // A decode loop retired mid-wait (a rendition change while the playhead is parked) must not leave // its waiter queued, or every such change strands one more for the life of the buffer. it("releases a gated wait when its signal aborts", async () => { - const worklet = { port: { postMessage: () => {} } } as unknown as AudioWorkletNode; - const buffer = createAudioBuffer(worklet, 1, 1000, 100, true); + const { node } = worklet(); + const buffer = createAudioBuffer(node, 1, 1000, Milli(100), true); try { buffer.insert(0 as Micro, [new Float32Array(200)]); const controller = new AbortController(); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 6fd40cc7f1..532c631242 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,34 +1,89 @@ import { Micro, Milli, Nano, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; -import type { Data, InitPost, InitShared, Latency, Reset, State, Truncate } from "./render"; +import { ringCapacity, ringHold, ringSamples } from "./latency"; +import type { Data, InitPost, InitShared, Latency, Park, Reset, State, Truncate } from "./render"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; /** - * Timestamp-based backpressure for buffered playback. The decoded PCM ring only holds the latency - * floor; everything above it (the buffered lookahead, up to the ceiling) stays upstream as encoded - * Opus. `wait(timestamp)` stays pending until the playhead is within `headroom` (the floor) of - * `timestamp`, so the decode loop holds a frame as Opus instead of decoding it too far ahead of the - * floor-sized ring. Both transports share this; they differ only in how they observe the playhead - * (Atomics poll vs worklet state messages). A no-op when not buffered (the ring bounds itself). + * Decides when the decode loop may decode a frame, which is how the target delay is split between + * the decoded ring and the encoded media upstream of it. + * + * The ring holds the floor as PCM, at most a receiver-chosen budget. The rest of the target (the + * hold) and, in buffered mode, the lookahead stay encoded in the container consumer, so memory follows + * the bytes actually received rather than whatever a catalog advertised. A frame at `timestamp` is due: + * + * - while the ring is stalled, once the media queued behind it covers the hold, so playback starts + * or refills with the whole target ahead of the playhead, decoded or not; + * - while playing, once the playhead is within the floor of it, keeping the ring on its floor; + * - while playing live, also once the media queued behind it covers the hold, so media past the + * target reaches the ring, whose skip band catches up to the live edge. Buffered mode plays + * through instead. + * + * With no hold, live playback releases every frame at once and the ring bounds itself. Both transports + * share this; they differ only in how they observe the playhead (Atomics poll vs worklet state messages). + * + * @internal Exported for the replay harness. */ -class Backpressure { - #enabled: boolean; - #headroom: Micro; +export class Backpressure { + readonly #rate: number; + readonly #buffered: boolean; + #closed = false; + + // The ring's floor in samples, and the part of the target above it, held encoded. + #floor = 0; + #hold = 0; + + // The newest media timestamp queued upstream of the decoder, unset while nothing is. + #queued?: number; + + // The ring as last observed. + #playhead = 0; + #stalled = true; + #waiters: Array<{ timestamp: Micro; resolve: () => void }> = []; - constructor(enabled: boolean, headroom: Micro) { - this.#enabled = enabled; - this.#headroom = headroom; + constructor(rate: number, buffered: boolean, delay: Milli) { + this.#rate = rate; + this.#buffered = buffered; + this.setLatency(delay); } - // Move the gate as the floor changes (e.g. "auto" tracking the measured arrival spread). - setHeadroom(headroom: Micro): void { - this.#headroom = headroom; + /** The ring's floor in samples: the part of the target held as decoded PCM. */ + get floor(): number { + return this.#floor; } - wait(timestamp: Micro, playhead: Micro, signal: AbortSignal): Promise { - if (!this.#enabled || signal.aborted) return Promise.resolve(); - if (playhead >= ((timestamp - this.#headroom) | 0)) return Promise.resolve(); + /** + * Split `delay` between the floor and the hold. Returns true when the ring must park: a rise in the + * hold that the queued media does not cover yet holds playback until it does, the way a deeper + * floor parks the ring until it refills, since video holds the extra delay on its own. + */ + setLatency(delay: Milli): boolean { + this.#floor = ringSamples(this.#rate, delay); + const hold = ringHold(delay); + const rise = hold > this.#hold; + this.#hold = hold; + const park = rise && !this.#stalled && !this.#covered(this.#playhead + this.#headroom()); + if (park) this.#stalled = true; + this.#advance(); + return park; + } + + /** Note the ring's playhead and whether it is stalled. */ + observe(playhead: Micro, stalled: boolean): void { + this.#playhead = playhead; + this.#stalled = stalled; + this.#advance(); + } + + /** Note the newest media timestamp queued upstream of the decoder, or undefined once none is. */ + queued(newest: Micro | undefined): void { + this.#queued = newest; + this.#advance(); + } + + wait(timestamp: Micro, signal: AbortSignal): Promise { + if (this.#closed || signal.aborted || this.#due(timestamp)) return Promise.resolve(); return new Promise((resolve) => { // An aborted wait drops its waiter, so a retired decode loop leaves nothing queued here. const abort = () => { @@ -47,27 +102,50 @@ class Backpressure { }); } - // Resolve every waiter the playhead has reached. Thresholds are recomputed live so a changed - // headroom takes effect on queued waiters too. - advance(playhead: Micro): void { + // The floor as a duration. + #headroom(): number { + return samplesToMicro(this.#floor, this.#rate); + } + + // Whether the queued media reaches the hold past `timestamp`. + #covered(timestamp: number): boolean { + if (this.#hold === 0) return true; + return this.#queued !== undefined && this.#queued >= timestamp + this.#hold; + } + + #due(timestamp: Micro): boolean { + if (this.#stalled) return this.#covered(timestamp); + if (this.#playhead >= timestamp - this.#headroom()) return true; + return !this.#buffered && this.#covered(timestamp); + } + + // Resolve every waiter that is now due. Thresholds are recomputed live, so a moved target takes + // effect on queued waiters too. + #advance(): void { if (this.#waiters.length === 0) return; this.#waiters = this.#waiters.filter(({ timestamp, resolve }) => { - if (playhead < ((timestamp - this.#headroom) | 0)) return true; + if (!this.#due(timestamp)) return true; resolve(); return false; }); } - // Resolve everything unconditionally (reset/close): never strand a decode loop. - flush(): void { + // Resolve everything unconditionally: never strand a decode loop. + #flush(): void { for (const { resolve } of this.#waiters) resolve(); this.#waiters = []; } - // Stop gating for good. Nothing advances a closed buffer, so a later wait would never settle. + /** The ring was flushed and re-stalled: the old timeline is gone, so let the decode loop re-anchor. */ + reset(): void { + this.#stalled = true; + this.#flush(); + } + + /** Stop gating for good. Nothing advances a closed buffer, so a later wait would never settle. */ close(): void { - this.#enabled = false; - this.flush(); + this.#closed = true; + this.#flush(); } } @@ -92,8 +170,17 @@ export interface AudioBuffer { /** Insert audio samples at the given timestamp. Handles out-of-order writes. */ insert(timestamp: Micro, data: Float32Array[]): void; - /** Update the target latency in samples. */ - setLatency(samples: number): void; + /** + * Update the target delay. The ring holds up to a fixed floor of it decoded, and the rest stays + * encoded upstream until the playhead nears it. + */ + setLatency(delay: Milli): void; + + /** + * Note the newest media timestamp queued upstream of the decoder, or undefined once none is. The + * queue is what covers the part of the target held encoded. + */ + queued(newest: Micro | undefined): void; /** Flush buffered samples and re-stall, ready to anchor the next utterance (buffered mode). */ reset(): void; @@ -108,11 +195,9 @@ export interface AudioBuffer { truncate(timestamp: Micro): void; /** - * Resolve once the playhead is near enough to decode a frame at `timestamp`. In buffered mode this - * applies backpressure: it stays pending while decoding `timestamp` would run more than the latency - * floor ahead of the playhead, so the caller holds the (encoded) frame instead of decoding it too - * far ahead of the floor-sized ring. Resolves immediately when not buffered (the ring bounds itself). - * Resolves early, releasing the wait, once `signal` aborts. + * Resolve once a frame at `timestamp` may be decoded, so the caller holds it encoded until then + * instead of decoding it too far ahead of the floor-sized ring. Resolves at once for live playback + * whose whole target fits the floor. Resolves early, releasing the wait, once `signal` aborts. */ wait(timestamp: Micro, signal: AbortSignal): Promise; @@ -146,26 +231,25 @@ export function createAudioBuffer( worklet: AudioWorkletNode, channels: number, rate: number, - latencySamples: number, + delay: Milli, buffered = false, ): AudioBuffer { if (supportsSharedArrayBuffer()) { console.log("[audio] using SharedArrayBuffer audio buffer"); - return new SharedAudioBuffer(worklet, channels, rate, latencySamples, buffered); + return new SharedAudioBuffer(worklet, channels, rate, delay, buffered); } console.warn( "[audio] SharedArrayBuffer unavailable, falling back to the higher latency postMessage audio buffer. " + "Serve the page cross-origin isolated (Cross-Origin-Opener-Policy: same-origin, Cross-Origin-Embedder-Policy: require-corp) to avoid this.", ); - return new PostAudioBuffer(worklet, channels, rate, latencySamples, buffered); + return new PostAudioBuffer(worklet, channels, rate, delay, buffered); } /** SharedArrayBuffer-backed implementation. Writes go directly into shared memory. */ class SharedAudioBuffer implements AudioBuffer { readonly rate: number; readonly channels: number; - #worklet: AudioWorkletNode; - #ring: SharedRingBuffer; + readonly #ring: SharedRingBuffer; readonly #timestamp = new Signal(0 as Micro); readonly timestamp: Getter = this.#timestamp; @@ -180,19 +264,18 @@ class SharedAudioBuffer implements AudioBuffer { #signals = new Effect(); - constructor(worklet: AudioWorkletNode, channels: number, rate: number, latencySamples: number, buffered: boolean) { - this.#worklet = worklet; + constructor(worklet: AudioWorkletNode, channels: number, rate: number, delay: Milli, buffered: boolean) { this.channels = channels; this.rate = rate; - // The ring holds the latency floor as decoded PCM (headroom above it for overflow). In - // buffered mode the lookahead above the floor stays encoded upstream, held back by `wait()`. - const capacity = Math.max(rate, latencySamples * 2); - this.#backpressure = new Backpressure(buffered, samplesToMicro(latencySamples, rate)); + // The ring holds the floor as decoded PCM, with headroom above it for overflow, at a capacity + // fixed by the receiver. The rest of the target, and in buffered mode the lookahead, stays + // encoded upstream, held back by `wait()`. + this.#backpressure = new Backpressure(rate, buffered, delay); - const init = allocSharedRingBuffer(channels, capacity, rate, buffered); + const init = allocSharedRingBuffer(channels, ringCapacity(rate), rate, buffered); this.#ring = new SharedRingBuffer(init); - this.#ring.setLatency(latencySamples); + this.#ring.setLatency(this.#backpressure.floor); const msg: InitShared = { type: "init-shared", ...init }; worklet.port.postMessage(msg); @@ -200,14 +283,10 @@ class SharedAudioBuffer implements AudioBuffer { // Poll the shared control array and reflect it into signals. this.#signals.interval( () => { - const stalled = this.#ring.stalled; this.#timestamp.set(this.#ring.timestamp); - this.#stalled.set(stalled); + this.#stalled.set(this.#ring.stalled); this.#underruns.set(this.#ring.underruns); - // While stalled the playhead is parked, so release the decode loop to refill the floor; - // once playing, hold it to ~the floor ahead. - if (stalled) this.#backpressure.flush(); - else this.#backpressure.advance(this.#ring.timestamp); + this.#observe(); }, Nano.fromMilli(Milli(50)), ); @@ -217,21 +296,22 @@ class SharedAudioBuffer implements AudioBuffer { this.#ring.insert(timestamp, data); } - setLatency(samples: number): void { - this.#backpressure.setHeadroom(samplesToMicro(samples, this.rate)); - - // Park the current ring before any resize: the worklet keeps reading it until `init-shared` - // lands, and `resize` carries the floor and the park over to the copy. - this.#ring.setLatency(samples); + setLatency(delay: Milli): void { + this.#observe(); + const park = this.#backpressure.setLatency(delay); + this.#ring.setLatency(this.#backpressure.floor); + if (park) this.#ring.park(); + // A deeper floor the buffer does not cover parked the ring too. + this.#observe(); + } - // Grow the ring (preserving the unread window) if it's too small for the new latency. - if (this.#ring.capacity < samples * 1.5) { - const newCapacity = Math.max(this.rate, samples * 2); - this.#ring = this.#ring.resize(newCapacity); + queued(newest: Micro | undefined): void { + this.#backpressure.queued(newest); + } - const msg: InitShared = { type: "init-shared", ...this.#ring.init }; - this.#worklet.port.postMessage(msg); - } + // Feed the live playhead to the gate. Reading the cursor is cheap and idempotent. + #observe(): void { + this.#backpressure.observe(this.#ring.timestamp, this.#ring.stalled); } truncate(timestamp: Micro): void { @@ -240,16 +320,15 @@ class SharedAudioBuffer implements AudioBuffer { reset(): void { this.#ring.reset(); - this.#backpressure.flush(); // the old timeline is gone; let the decode loop re-anchor + this.#backpressure.reset(); } wait(timestamp: Micro, signal: AbortSignal): Promise { - // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an - // explicit reset): let frames through so the ring refills to the floor. This is the single - // re-buffer path; the ring un-stalls itself on the insert that reaches the floor, so the - // gate closes again within one frame rather than draining the whole lookahead. - if (this.#ring.stalled) return Promise.resolve(); - return this.#backpressure.wait(timestamp, this.#ring.timestamp, signal); + // Stalled (bootstrap, an underrun the reader re-stalled on, a park, or an explicit reset) + // releases frames once the queued media covers the hold, and the ring un-stalls itself on the + // insert that reaches the floor, so the gate closes again within one frame. + this.#observe(); + return this.#backpressure.wait(timestamp, signal); } close(): void { @@ -278,14 +357,14 @@ class PostAudioBuffer implements AudioBuffer { #signals = new Effect(); - constructor(worklet: AudioWorkletNode, channels: number, rate: number, latencySamples: number, buffered: boolean) { + constructor(worklet: AudioWorkletNode, channels: number, rate: number, delay: Milli, buffered: boolean) { this.#worklet = worklet; this.channels = channels; this.rate = rate; - this.#backpressure = new Backpressure(buffered, samplesToMicro(latencySamples, rate)); + this.#backpressure = new Backpressure(rate, buffered, delay); - const latency = Milli.fromSecond((latencySamples / rate) as Second); + const latency = Milli.fromSecond((this.#backpressure.floor / rate) as Second); const msg: InitPost = { type: "init-post", channels, rate, latency, buffered }; worklet.port.postMessage(msg); @@ -296,10 +375,7 @@ class PostAudioBuffer implements AudioBuffer { this.#timestamp.set(data.timestamp); this.#stalled.set(data.stalled); this.#underruns.set(data.underruns); - // While stalled the playhead is parked, so release the decode loop to refill the floor; - // once playing, hold it to ~the floor ahead. - if (data.stalled) this.#backpressure.flush(); - else this.#backpressure.advance(data.timestamp); + this.#backpressure.observe(data.timestamp, data.stalled); } }); // addEventListener on a MessagePort requires start() to begin delivery. @@ -316,12 +392,20 @@ class PostAudioBuffer implements AudioBuffer { ); } - setLatency(samples: number): void { - this.#backpressure.setHeadroom(samplesToMicro(samples, this.rate)); + setLatency(delay: Milli): void { + const park = this.#backpressure.setLatency(delay); - const latency = Milli.fromSecond((samples / this.rate) as Second); + const latency = Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); const msg: Latency = { type: "latency", latency }; this.#worklet.port.postMessage(msg); + if (park) { + const msg: Park = { type: "park" }; + this.#worklet.port.postMessage(msg); + } + } + + queued(newest: Micro | undefined): void { + this.#backpressure.queued(newest); } truncate(timestamp: Micro): void { @@ -332,16 +416,13 @@ class PostAudioBuffer implements AudioBuffer { reset(): void { const msg: Reset = { type: "reset" }; this.#worklet.port.postMessage(msg); - this.#backpressure.flush(); // the old timeline is gone; let the decode loop re-anchor + this.#backpressure.reset(); } wait(timestamp: Micro, signal: AbortSignal): Promise { - // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an - // explicit reset): let frames through so the ring refills to the floor. See SharedAudioBuffer. - if (this.#stalled.peek()) return Promise.resolve(); // Uses the worklet-reported playhead, which lags by a state-message interval; the floor's // headroom covers that. The worklet still drops the oldest if a frame slips through. - return this.#backpressure.wait(timestamp, this.#timestamp.peek(), signal); + return this.#backpressure.wait(timestamp, signal); } close(): void { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 51d4c12694..8df4e8ffb9 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -21,7 +21,7 @@ import type { Sync } from "../sync"; import { type AudioBuffer, createAudioBuffer } from "./buffer"; import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; import { Handover } from "./handover"; -import { AUTO_MAX_DELAY, ringSamples, target } from "./latency"; +import { AUTO_MAX_DELAY, target } from "./latency"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -264,13 +264,11 @@ export class Decoder { }); effect.cleanup(() => worklet.disconnect()); - // Initial ring depth in samples. const delay = this.sync.out.delay.peek(); - const latencySamples = ringSamples(sampleRate, delay); const buffered = this.sync.out.buffered.peek(); // Let the factory pick the best transport (SharedArrayBuffer or postMessage). - const ring = createAudioBuffer(worklet, channelCount, sampleRate, latencySamples, buffered); + const ring = createAudioBuffer(worklet, channelCount, sampleRate, delay, buffered); effect.cleanup(() => ring.close()); effect.set(this.#ring, ring); @@ -321,11 +319,10 @@ export class Decoder { const ring = effect.get(this.#ring); if (!ring) return; - // A rise parks playback until the ring refills to the new floor, which is what keeps audio in - // step with video. The measured target moves a bucket at a time, and a rise the buffered - // slack already covers costs no silence at all. - const delay = effect.get(this.sync.out.delay); - ring.setLatency(ringSamples(ring.rate, delay)); + // A rise parks playback until the queued media covers the new target, which is what keeps + // audio in step with video. The measured target moves a bucket at a time, and a rise the + // buffered slack already covers costs no silence at all. + ring.setLatency(effect.get(this.sync.out.delay)); } #runDecoder(effect: Effect): void { @@ -646,6 +643,18 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // What has arrived but not been decoded is what covers the part of the target held encoded, so + // the decode loop's held frame is released once enough is queued behind it. + effect.run((inner) => { + const ring = inner.get(this.#ring); + if (!ring) return; + let newest: Milli | undefined; + for (const { end } of inner.get(consumer.buffered)) { + if (newest === undefined || end > newest) newest = end; + } + ring.queued(newest === undefined ? undefined : Micro.fromMilli(newest)); + }); + // Feed the arrival estimate into the playout target. It outlives this consumer, so the target // holds until the replacement takes over; see #retire. effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index bd2374f76c..b88e5e31c8 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "bun:test"; -import type { Milli } from "@moq/clock"; -import { ringSamples, target } from "./latency"; +import type { Micro, Milli } from "@moq/clock"; +import { ringCapacity, ringHold, ringSamples, target } from "./latency"; const ms = (value: number) => value as Milli; @@ -25,7 +25,7 @@ describe("target", () => { describe("ringSamples", () => { // `delay="instant"` reports a zero buffer. Passed through, the ring rejects it and the - // worklet is left with no backend that any later resize can revive. + // worklet is left with no backend that any later update can revive. it("floors a zero delay at one render quantum", () => { expect(ringSamples(48_000, ms(0))).toBe(128); }); @@ -38,4 +38,22 @@ describe("ringSamples", () => { it("leaves a delay above the floor alone", () => { expect(ringSamples(48_000, ms(100))).toBe(4_800); }); + + // A catalog's jitter or delay is any u53, so the part held decoded is the receiver's to cap. + it("caps the decoded part at half the fixed capacity", () => { + expect(ringSamples(48_000, ms(60 * 60 * 1000))).toBe(24_000); + expect(ringCapacity(48_000)).toBe(48_000); + }); +}); + +describe("ringHold", () => { + it("holds the target above the cap", () => { + expect(ringHold(ms(1_500))).toBe(1_000_000 as Micro); + }); + + // At 44.1 kHz this target's floor rounds to a sample count a few picoseconds short of it. + it("holds exactly nothing for a target within the cap", () => { + expect(ringHold(ms(203.21995464852608))).toBe(0 as Micro); + expect(ringHold(ms(500))).toBe(0 as Micro); + }); }); diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 8f28a04d51..8510629432 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,4 +1,4 @@ -import { Milli, Second } from "@moq/clock"; +import { Micro, Milli, Second } from "@moq/clock"; /** The terms of the audio playout target, all in milliseconds. */ export interface Target { @@ -37,14 +37,38 @@ export const AUTO_MAX_DELAY = 2000 as Milli; // never be read from. const RENDER_QUANTUM = 128; +// The most of the target the ring holds decoded. The rest stays encoded upstream until the playhead +// nears it, so a catalog advertising an hour of jitter costs the bytes actually received rather than +// an hour of PCM. Comfortably above a main-thread stall or a state-message interval. +const FLOOR_MAX = Milli(500); + +/** + * The ring's capacity in samples: twice the deepest floor, so a ring sitting on its floor has a + * whole floor of headroom for a burst. Fixed by the receiver, never by the target. + */ +export function ringCapacity(rate: number): number { + return Math.ceil(rate * Second.fromMilli(FLOOR_MAX) * 2); +} + /** - * The ring depth for a target delay, floored at one AudioWorklet render quantum. + * The ring's floor for a target delay: the part held as decoded PCM, at least one AudioWorklet render + * quantum and at most 500 ms. * * `delay="instant"` reports a zero buffer, which the ring rejects outright: construction throws, - * the worklet is left with no backend, and every later resize is gated on that backend existing, so + * the worklet is left with no backend, and every later update is gated on that backend existing, so * audio never recovers. Audio keeps its own floor rather than deriving its depth verbatim from a * buffer whose meaning is "video holds nothing". */ export function ringSamples(rate: number, delay: Milli): number { - return Math.max(RENDER_QUANTUM, Math.ceil(rate * Second.fromMilli(delay))); + return Math.max(RENDER_QUANTUM, Math.ceil(rate * Second.fromMilli(Milli(Math.min(delay, FLOOR_MAX))))); +} + +/** + * The part of a target delay above the ring's floor, held encoded upstream rather than decoded. + * + * Measured against the cap rather than the rounded floor, so a target within the cap holds exactly + * nothing instead of the float residue of a sample count. + */ +export function ringHold(delay: Milli): Micro { + return Micro.fromMilli(Milli(Math.max(0, delay - FLOOR_MAX))); } diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index 9d66bdfe82..f5a319871f 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -2,14 +2,14 @@ import { describe, expect, it, spyOn } from "bun:test"; import { Micro, Milli } from "@moq/clock"; import * as Container from "@moq/hang/container"; import { Group, Time, Track, Varint } from "@moq/net"; -import { AUTO_MAX_DELAY, target } from "./latency"; +import { AUTO_MAX_DELAY, ringCapacity, target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; // Replay recorded-shape arrival traces through both rings and count what a listener would hear: // quanta rendered short (an underrun) and samples the ring threw away (a skip). Both rings are // driven through the same harness, since the postMessage fallback is the path every page without -// cross-origin isolation takes. Each ring is sized from the target the estimator settles on, measured +// cross-origin isolation takes. Each ring's floor is the target the estimator settles on, measured // through the transport subscription and container consumer the decoder reads. const RATE = 48000; @@ -138,7 +138,7 @@ interface Ring { function shared(latencyMs: number): Ring { const samples = Math.ceil((RATE * latencyMs) / 1000); - const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(RATE, samples * 2), RATE)); + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, ringCapacity(RATE), RATE)); ring.setLatency(samples); return ring; } diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 51bf02ad02..c86d323522 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -21,8 +21,7 @@ class Render extends AudioWorkletProcessor { const msg = event.data; if (msg.type === "init-shared") { console.log("[audio-worklet] init-shared: using SharedArrayBuffer path"); - const previous = this.#backend instanceof SharedRingBuffer ? this.#backend : undefined; - this.#backend = new SharedRingBuffer(msg, previous); + this.#backend = new SharedRingBuffer(msg); this.#underflow = 0; this.#short = false; } else if (msg.type === "init-post") { @@ -35,7 +34,10 @@ class Render extends AudioWorkletProcessor { if (this.#backend instanceof AudioRingBuffer) this.#backend.write(msg.timestamp, msg.data); } else if (msg.type === "latency") { // Only meaningful in post mode. - if (this.#backend instanceof AudioRingBuffer) this.#backend.resize(msg.latency); + if (this.#backend instanceof AudioRingBuffer) this.#backend.setLatency(msg.latency); + } else if (msg.type === "park") { + // Only meaningful in post mode; shared mode parks via the control array. + if (this.#backend instanceof AudioRingBuffer) this.#backend.park(); } else if (msg.type === "truncate") { // Only meaningful in post mode; shared mode truncates via the control array. if (this.#backend instanceof AudioRingBuffer) this.#backend.truncate(msg.timestamp); diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 045ee2d05c..f3f0608362 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -2,7 +2,7 @@ import type { Micro, Milli } from "@moq/clock"; import type { SharedRingBufferInit } from "./shared-ring-buffer"; /** Everything the main thread sends the render worklet over its port. */ -export type Message = InitShared | InitPost | Data | Latency | Reset | Truncate; +export type Message = InitShared | InitPost | Data | Latency | Park | Reset | Truncate; /** Init message when SharedArrayBuffer is available. */ export interface InitShared extends SharedRingBufferInit { @@ -15,8 +15,8 @@ export interface InitPost { channels: number; rate: number; latency: Milli; - // Buffered mode: anchor to the first frame and play through; the ring is sized to the floor and - // the lookahead above it is held back upstream (the main thread applies the backpressure). + // Buffered mode: anchor to the first frame and play through. The ring holds the floor; the + // lookahead above it is held back upstream (the main thread applies the backpressure). buffered: boolean; } @@ -41,12 +41,17 @@ export interface Data { timestamp: Micro; } -/** Latency update sent via postMessage (fallback path only). */ +/** Latency update sent via postMessage (fallback path only): the floor held as decoded PCM. */ export interface Latency { type: "latency"; latency: Milli; } +/** Hold playback until the next data message (fallback path only; shared path parks via Atomics). */ +export interface Park { + type: "park"; +} + /** State update from the worklet back to main thread (fallback path only). */ export interface State { type: "state"; diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index 0c63c028bf..c5496c2775 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -239,3 +239,38 @@ it("replay refuses missing, non-finite or non-positive recorded durations", asyn await expect(reader.next()).rejects.toThrow("invalid recorded frame duration"); } }); + +describe.each(["shared", "post"] as const)("%s ring past its decoded floor", (ring) => { + // Five seconds advertised is ten times what the ring holds decoded, so the rest waits encoded in + // the consumer until the received media covers it, then plays at the advertised delay. + it("plays an advertised flush span at its delay without a gap", async () => { + const config = { ...CONFIG, jitter: 5000 } as Catalog.AudioConfig; + let start: number | undefined; + let short = 0; + let lag = 0; + let delay = 0; + for await (const quantum of replay( + paced(12, () => 30), + { + ring, + rate: RATE, + config, + delay: "auto", + duration: observed(12), + }, + )) { + const filled = quantum.output.findLastIndex((v) => v !== 0) + 1; + if (start === undefined && filled > 0) start = quantum.at; + if (start !== undefined && filled < QUANTUM) short++; + lag = quantum.at - quantum.timestamp; + delay = quantum.delay; + } + expect(delay).toBe(5020); + // The first frame arrived 30 ms late, so it plays the target after its own capture plus that. + expect(start).toBeGreaterThanOrEqual(5020); + expect(start).toBeLessThan(5040); + expect(short).toBe(0); + expect(lag).toBeGreaterThanOrEqual(5020); + expect(lag).toBeLessThan(5040); + }); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 41a61ab426..5c7a421937 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -6,9 +6,9 @@ * sets. Delivered frames go into the real ring, and a new codec epoch resets it, the way the decoder * does. One render quantum is read every quantum's worth of that same clock, which is what the * AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder - * registers, measured by that same consumer from the recorded catalog config, and the ring follows - * it the way the decoder resizes it, so a change to any of them moves what a replay hears. Decoding - * is taken as instant and sample exact. + * registers, measured by that same consumer from the recorded catalog config, and the ring and the + * decode gate follow it the way the decoder's audio buffer moves them, so a change to any of them + * moves what a replay hears. Decoding is taken as instant and sample exact. * * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test * can assert exact counts. @@ -23,8 +23,9 @@ import { Group, Time, Track } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; import { type Arrival as Diagnostic, gap } from "../arrivals"; import { type Delay, Sync } from "../sync"; +import { Backpressure } from "./buffer"; import { frameDuration } from "./config"; -import { ringSamples, target } from "./latency"; +import { ringCapacity, target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; import { Terminal } from "./terminal"; @@ -84,7 +85,7 @@ export interface Quantum { stalled: boolean; /** The playhead after this quantum, on the media clock, in ms. */ timestamp: number; - /** The delay `Sync` resolved as of this quantum, in ms: what sizes the ring. */ + /** The delay `Sync` resolved as of this quantum, in ms: what the ring and the decode gate hold. */ delay: number; } @@ -94,29 +95,16 @@ interface Ring { read(output: Float32Array[]): number; reset(): void; setLatency(samples: number): void; + park(): void; readonly stalled: boolean; readonly timestamp: Micro; } -/** The shared ring, sized and grown the way `SharedAudioBuffer` sizes and grows it. */ +/** The shared ring, sized the way `SharedAudioBuffer` sizes it. */ function shared(rate: number, latency: number): Ring { - let ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(rate, latency * 2), rate)); + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, ringCapacity(rate), rate)); ring.setLatency(latency); - return { - insert: (timestamp, data) => ring.insert(timestamp, data), - read: (output) => ring.read(output), - reset: () => ring.reset(), - setLatency: (samples) => { - ring.setLatency(samples); - if (ring.capacity < samples * 1.5) ring = ring.resize(Math.max(rate, samples * 2)); - }, - get stalled() { - return ring.stalled; - }, - get timestamp() { - return ring.timestamp; - }, - }; + return ring; } /** Samples as the milliseconds `PostAudioBuffer` posts to the worklet. */ @@ -129,7 +117,8 @@ function post(rate: number, latency: number): Ring { insert: (timestamp, data) => ring.write(timestamp, data), read: (output) => ring.read(output), reset: () => ring.reset(), - setLatency: (samples) => ring.resize(millis(rate, samples)), + setLatency: (samples) => ring.setLatency(millis(rate, samples)), + park: () => ring.park(), get stalled() { return ring.stalled; }, @@ -202,7 +191,16 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato await settle(); let delay = sync.out.delay.peek(); - const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, delay)); + const gate = new Backpressure(rate, false, delay); + const ring = (options.ring === "shared" ? shared : post)(rate, gate.floor); + const observe = () => gate.observe(ring.timestamp, ring.stalled); + + // Arrived but not yet decoded media is what covers the part of the target held encoded. + signals.run((effect) => { + let newest: number | undefined; + for (const { end } of effect.get(consumer.buffered)) newest = Math.max(newest ?? end, end); + gate.queued(newest === undefined ? undefined : micros(newest)); + }); // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. const terminal = new Terminal(); @@ -210,12 +208,16 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const delivery = Promise.withResolvers(); if (options.decodeAfter === undefined) delivery.resolve(); let failure: { error: unknown } | undefined; + const abort = new AbortController(); const decode = (async () => { await delivery.promise; for (;;) { const next = await consumer.next(); if (!next) return; - if (terminal.update(next)) ring.reset(); + if (terminal.update(next)) { + ring.reset(); + gate.reset(); + } if (next.kind === "gap") arrivals.push(gap(next)); if (next.kind !== "frame") continue; @@ -226,6 +228,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const entry = recorded.get(timestamp); if (!entry) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); frame.set(Milli(entry.duration)); + await gate.wait(timestamp, abort.signal); + if (abort.signal.aborted) return; ring.insert(timestamp, [new Float32Array(Math.round((entry.duration * rate) / 1000)).fill(0.5)]); } })().catch((error: unknown) => { @@ -286,15 +290,20 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato } if (failure) throw failure.error; - // The decoder resizes the ring as the resolved delay moves, which only an arrival does. + // The decoder moves the ring and the gate as the resolved delay moves, which only an arrival + // does. const resolved = sync.out.delay.peek(); if (resolved !== delay) { delay = resolved; - ring.setLatency(ringSamples(rate, delay)); + observe(); + const park = gate.setLatency(delay); + ring.setLatency(gate.floor); + if (park) ring.park(); } output.fill(0); ring.read([output]); + observe(); yield { arrivals, at: now, @@ -306,6 +315,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato } } finally { delivery.resolve(); + abort.abort(); consumer.close(); track.close(); await decode; diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index df5aa42e7c..4b699dd85c 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -35,9 +35,9 @@ describe("initialization", () => { it("should initialize with valid parameters", () => { const buffer = new AudioRingBuffer({ rate: 48000, channels: 2, latency: 100 as Milli }); - // Capacity is twice the target: the ring has to be able to hold the slack the overflow band - // tolerates, or every chunk arriving on a full ring would drop the oldest samples. - expect(buffer.capacity).toBe(9600); // 48000 * 0.1 * 2 + // The receiver fixes the capacity at twice the deepest floor, never from the target: whatever + // of the target is above the floor stays encoded upstream. + expect(buffer.capacity).toBe(48000); // 48000 * 0.5 * 2 expect(buffer.length).toBe(0); }); @@ -377,254 +377,43 @@ describe("edge cases", () => { }); }); -describe("resize", () => { - it("should resize to a larger buffer", () => { +describe("setLatency", () => { + it("keeps the receiver's capacity whatever the floor", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.capacity).toBe(200); - - // Write 50 samples + const capacity = buffer.capacity; write(buffer, 0 as Milli, 50, { channels: 1, value: 1.0 }); - expect(buffer.length).toBe(50); - - // Resize to larger buffer (200ms = 200 samples) - buffer.resize(200 as Milli); - - expect(buffer.capacity).toBe(400); - expect(buffer.length).toBe(50); // Samples preserved - expect(buffer.stalled).toBe(true); // Should trigger stall - }); - - it("should resize to a smaller buffer and keep the most recent samples", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.capacity).toBe(200); - - // Write 80 samples: first 40 with value 1.0, next 40 with value 2.0 - write(buffer, 0 as Milli, 40, { channels: 1, value: 1.0 }); - write(buffer, 40 as Milli, 40, { channels: 1, value: 2.0 }); - expect(buffer.length).toBe(80); - - // Resize to a 30 sample target (60 samples of capacity). - // Should keep the most recent 60 samples (samples 20-79) - buffer.resize(30 as Milli); - - expect(buffer.capacity).toBe(60); - expect(buffer.length).toBe(60); // Truncated to new capacity - expect(buffer.stalled).toBe(false); // Already covers the new floor - }); - - it("should be a no-op when capacity is unchanged", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Exit stalled mode - write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - expect(buffer.stalled).toBe(false); - - // Resize to same capacity - buffer.resize(100 as Milli); - - // Should still not be stalled (no-op) - expect(buffer.stalled).toBe(false); - expect(buffer.capacity).toBe(200); - }); - - it("should throw on zero latency", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(() => buffer.resize(0 as Milli)).toThrow(/empty buffer/); - }); - - it("should handle resize with stereo data", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 2, latency: 100 as Milli }); - - // Write stereo data with different values per channel - const data = [new Float32Array(60), new Float32Array(60)]; - for (let i = 0; i < 60; i++) { - data[0][i] = 1.0; // Left channel - data[1][i] = 2.0; // Right channel - } - buffer.write(0 as Micro, data); - expect(buffer.length).toBe(60); - - // Resize to smaller buffer - buffer.resize(50 as Milli); - expect(buffer.capacity).toBe(100); - expect(buffer.length).toBe(60); - expect(buffer.stalled).toBe(false); - }); - - it("should handle resize when buffer is empty", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.length).toBe(0); - // Resize empty buffer - buffer.resize(200 as Milli); - - expect(buffer.capacity).toBe(400); - expect(buffer.length).toBe(0); + buffer.setLatency(400 as Milli); + expect(buffer.capacity).toBe(capacity); + expect(buffer.length).toBe(50); expect(buffer.stalled).toBe(true); }); - it("should handle resize after partial read", () => { + it("refuses an empty floor or one past the capacity", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Fill and exit stalled mode - write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - expect(buffer.stalled).toBe(false); - - // Read 60 samples (readIndex at 60, writeIndex at 100) - read(buffer, 60, 1); - expect(buffer.length).toBe(40); - - // Write more data - write(buffer, 100 as Milli, 30, { channels: 1, value: 2.0 }); - expect(buffer.length).toBe(70); // 130 - 60 - - // Resize to a 50 sample target - 100 of capacity, so all 70 survive (samples 60-129) - buffer.resize(50 as Milli); - expect(buffer.capacity).toBe(100); - expect(buffer.length).toBe(70); - expect(buffer.stalled).toBe(false); - }); - - it("should resume and then read new data after a shrinking resize", () => { - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Write some initial data - write(buffer, 0 as Milli, 50, { channels: 1, value: 1.0 }); - - // Resize to a floor the buffered data already covers - buffer.resize(50 as Milli); - expect(buffer.stalled).toBe(false); - - // Nothing is dropped: the ring has room for the preserved samples plus a chunk above the - // target. - write(buffer, 50 as Milli, 50, { channels: 1, value: 2.0 }); - expect(buffer.stalled).toBe(false); - expect(buffer.length).toBe(100); - - // Read plays the preserved samples first, then the new ones. - const output = read(buffer, 100, 1); - expect(output[0].length).toBe(100); - for (let i = 0; i < 50; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 50; i < 100; i++) { - expect(output[0][i]).toBe(2.0); - } + expect(() => buffer.setLatency(0 as Milli)).toThrow(/empty buffer/); + expect(() => buffer.setLatency((buffer.capacity + 1) as Milli)).toThrow(/exceeds capacity/); + expect(() => new AudioRingBuffer({ rate: 1000, channels: 1, latency: 60_000 as Milli })).toThrow( + /exceeds capacity/, + ); }); +}); - it("should preserve samples correctly when resizing larger then filling", () => { +describe("park", () => { + it("holds a covered ring until the next write, keeping what it buffered", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 50 as Milli }); - expect(buffer.capacity).toBe(100); - - // Write 30 samples - write(buffer, 0 as Milli, 30, { channels: 1, value: 1.0 }); - expect(buffer.length).toBe(30); - - // Resize to larger buffer (100ms = 100 samples) - buffer.resize(100 as Milli); - expect(buffer.capacity).toBe(200); - expect(buffer.length).toBe(30); // All samples preserved - expect(buffer.stalled).toBe(true); - - // Write 70 more samples to fill the buffer and exit stall - write(buffer, 30 as Milli, 70, { channels: 1, value: 2.0 }); - expect(buffer.stalled).toBe(false); - expect(buffer.length).toBe(100); - - // Read all - should have 30 samples of 1.0 then 70 of 2.0 - const output = read(buffer, 100, 1); - expect(output[0].length).toBe(100); - for (let i = 0; i < 30; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 30; i < 100; i++) { - expect(output[0][i]).toBe(2.0); - } - }); - - it("should read preserved samples back correctly after shrinking", () => { - // Regression: the copy loop inside resize() used a relative dst index - // (`i % dst.length`) while read() uses the absolute ring position - // (`readIndex % capacity`). When `copyStart` was not a multiple of the new - // capacity, preserved samples ended up in the wrong slots and read() returned - // mangled data. This test fails under that bug by constructing a scenario - // where `copyStart % newCapacity !== 0`. - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Fill the buffer to exit stall (writeIndex=100, readIndex=0). write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - expect(buffer.stalled).toBe(false); + expect(read(buffer, 10, 1)[0].length).toBe(10); - // Advance readIndex to 20 so there's room to wrap the write pointer. - read(buffer, 20, 1); + buffer.park(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 10, 1)[0].length).toBe(0); - // Write samples that wrap: abs 100-109 → slots 0-9, value 2.0. + // The main thread releases the next frame only once the media it holds encoded covers the + // target, so that write is what resumes playback. write(buffer, 100 as Milli, 10, { channels: 1, value: 2.0 }); - // abs 110-119 → slots 10-19, value 3.0. - write(buffer, 110 as Milli, 10, { channels: 1, value: 3.0 }); - - // Preserved range after resize: all 100 buffered samples = abs 20..119. - // copyStart = 120 - 100 = 20, and 20 % 100 = 20 (non-zero → triggers the bug). - buffer.resize(50 as Milli); - expect(buffer.capacity).toBe(100); - expect(buffer.length).toBe(100); expect(buffer.stalled).toBe(false); - - // Read the preserved samples. Expected layout by absolute index: - // abs 20..99 = 1.0 (80 samples, from the initial fill) - // abs 100..109 = 2.0 (10 samples) - // abs 110..119 = 3.0 (10 samples) - const output = read(buffer, 100, 1); - expect(output[0].length).toBe(100); - for (let i = 0; i < 80; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 80; i < 90; i++) { - expect(output[0][i]).toBe(2.0); - } - for (let i = 90; i < 100; i++) { - expect(output[0][i]).toBe(3.0); - } - }); - - it("should continue accepting absolute-timestamp writes after resize", () => { - // resize() must keep readIndex/writeIndex on the same absolute axis that - // write() uses (`round(timestamp * rate)`). If the indices were reset to 0 - // while timestamps stayed absolute, the next write would leave a giant zero - // gap. This test asserts that post-resize writes land contiguously with the - // preserved samples. - const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - - // Fill the buffer, then partially drain it. - write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); - read(buffer, 40, 1); - // Wrap the writer over slots 0-29 with value 2.0 (abs 100-129). - write(buffer, 100 as Milli, 30, { channels: 1, value: 2.0 }); - - // Resize smaller. All 90 buffered samples fit the new 100 sample capacity: abs 40..129. - buffer.resize(50 as Milli); - expect(buffer.length).toBe(90); - expect(buffer.stalled).toBe(false); - - // Write the next 10 samples at their real timestamp. This should append, - // not create a gap or be discarded as "too old". - write(buffer, 130 as Milli, 10, { channels: 1, value: 3.0 }); - - // That put the ring past the band, so it drops back to the 50 sample target. - // Remaining: abs 90..99 = 1.0 (10), abs 100..129 = 2.0 (30), abs 130..139 = 3.0 (10). - expect(buffer.length).toBe(50); - - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); - for (let i = 0; i < 10; i++) { - expect(output[0][i]).toBe(1.0); - } - for (let i = 10; i < 40; i++) { - expect(output[0][i]).toBe(2.0); - } - for (let i = 40; i < 50; i++) { - expect(output[0][i]).toBe(3.0); - } + expect(read(buffer, 10, 1)[0]).toEqual(new Float32Array(10).fill(1.0)); }); }); @@ -655,7 +444,6 @@ describe("live anchoring", () => { // with zeros, un-stalled on that overflow, and left the playhead a floor before the frame. it("anchors a live ring to the first frame instead of gap-filling from zero", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Milli }); - expect(buffer.capacity).toBe(200); write(buffer, 2000 as Milli, 40, { channels: 1, value: 0.5 }); // Still short of the floor, so nothing plays yet and the ring holds only real samples. @@ -772,16 +560,16 @@ describe("buffered mode", () => { expect(out[0][0]).toBeCloseTo(0.9, 5); }); - it("drops the oldest samples once the ring fills beyond the floor", () => { - // 50ms floor at 1000Hz -> 50-sample floor, 100-sample ring. Backpressure normally keeps the - // decode loop from running this far ahead; if it slips, the ring drops its oldest as a backstop. + it("drops the oldest samples once the ring fills beyond its capacity", () => { + // 50ms floor at 1000Hz in a 1000-sample ring. Backpressure normally keeps the decode loop from + // running this far ahead; if it slips, the ring drops its oldest as a backstop. const buffer = createBuffered(50); - write(buffer, 0 as Milli, 100, { channels: 1, value: 0.1 }); // [0, 100) - write(buffer, 100 as Milli, 100, { channels: 1, value: 0.2 }); // exceeds 100-sample ring; drops [0, 100) + write(buffer, 0 as Milli, 1000, { channels: 1, value: 0.1 }); // [0, 1000) + write(buffer, 1000 as Milli, 100, { channels: 1, value: 0.2 }); // past the ring; drops [0, 100) expect(Milli.fromMicro(buffer.timestamp)).toBe(100 as Milli); - expect(read(buffer, 100, 1)[0][0]).toBeCloseTo(0.2, 5); + expect(buffer.length).toBe(1000); }); it("truncate drops the write-ahead tail a successor supersedes", () => { @@ -827,7 +615,7 @@ describe("latency increase", () => { expect(buffer.stalled).toBe(false); read(buffer, 50, 1); - buffer.resize(110 as Milli); + buffer.setLatency(110 as Milli); expect(buffer.stalled).toBe(true); expect(buffer.length).toBe(50); @@ -847,7 +635,7 @@ describe("latency increase", () => { write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); read(buffer, 50, 1); - buffer.resize(90 as Milli); + buffer.setLatency(90 as Milli); expect(buffer.stalled).toBe(false); }); @@ -856,11 +644,11 @@ describe("latency increase", () => { write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); read(buffer, 50, 1); - buffer.resize(110 as Milli); + buffer.setLatency(110 as Milli); expect(buffer.stalled).toBe(true); // No further frame arrives, as when the source stops, so only the lower floor can resume it. - buffer.resize(50 as Milli); + buffer.setLatency(50 as Milli); expect(buffer.stalled).toBe(false); expect(read(buffer, 1, 1)[0][0]).toBe(1.0); }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 1d2f354eac..8e8959b0e5 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -1,4 +1,5 @@ import { Micro, type Milli, Second } from "@moq/clock"; +import { ringCapacity } from "./latency"; export class AudioRingBuffer { #buffer: Float32Array[]; @@ -40,12 +41,12 @@ export class AudioRingBuffer { this.channels = props.channels; this.#buffered = props.buffered ?? false; - // The ring holds the latency floor as PCM, with headroom above it. Sizing capacity to the - // floor exactly makes the ring physically incapable of holding the slack the overflow band - // wants, so every chunk that arrives while the ring is on target drops the oldest samples. - // In buffered mode the headroom is also what keeps the backpressure-paced decode loop (on - // the main thread) from overflow-dropping; the rest of the lookahead stays encoded upstream. - const capacity = this.#capacityFor(this.#latencySamples); + // The ring holds the latency floor as PCM, with headroom above it, at a capacity the receiver + // fixes rather than the target: whatever of the target is above the floor stays encoded + // upstream (the main thread holds it back). The headroom tolerates a chunk above the floor in + // live mode, and in buffered mode keeps the paced decode loop from overflow-dropping. + const capacity = ringCapacity(this.rate); + if (this.#latencySamples > capacity) throw new Error("latency exceeds capacity"); this.#buffer = []; for (let i = 0; i < this.channels; i++) { @@ -53,10 +54,6 @@ export class AudioRingBuffer { } } - #capacityFor(latencySamples: number): number { - return latencySamples * 2; - } - get stalled(): boolean { return this.#stalled; } @@ -78,8 +75,11 @@ export class AudioRingBuffer { return this.#buffer[0]?.length; } - resize(latency: Milli): void { + /** Move the floor, which never exceeds the fixed capacity. */ + setLatency(latency: Milli): void { const latencySamples = Math.ceil(this.rate * Second.fromMilli(latency)); + if (latencySamples <= 0) throw new Error("empty buffer"); + if (latencySamples > this.capacity) throw new Error("latency exceeds capacity"); // A deeper floor parks playback until it refills. Video holds the extra delay on its own, so // audio that kept draining at the old depth would run ahead by the difference. Parking keeps @@ -88,42 +88,20 @@ export class AudioRingBuffer { if (latencySamples > this.#latencySamples) this.#stalled = true; this.#latencySamples = latencySamples; - const newCapacity = this.#capacityFor(this.#latencySamples); - if (newCapacity !== this.capacity) this.#reallocate(newCapacity); - // Resume a refill that the buffer already covers, which `write` would only notice on the // next frame, never for a source that has stopped. - if (latencySamples > 0 && this.length >= latencySamples) this.#stalled = false; + if (this.length >= latencySamples) this.#stalled = false; } - #reallocate(newCapacity: number): void { - if (newCapacity === 0) throw new Error("empty buffer"); - - const newBuffer: Float32Array[] = []; - for (let i = 0; i < this.channels; i++) { - newBuffer[i] = new Float32Array(newCapacity); - } - - // Copy existing data, preserving the most recent samples - const samplesToKeep = Math.min(this.length, newCapacity); - if (samplesToKeep > 0) { - // Copy the most recent samples (closest to writeIndex) - const copyStart = this.#writeIndex - samplesToKeep; - for (let channel = 0; channel < this.channels; channel++) { - const src = this.#buffer[channel]; - const dst = newBuffer[channel]; - for (let i = 0; i < samplesToKeep; i++) { - const srcPos = (copyStart + i) % src.length; - const dstPos = (copyStart + i) % dst.length; - dst[dstPos] = src[srcPos]; - } - } - } - - // Update state for the new buffer, only stall if empty. - this.#buffer = newBuffer; - this.#readIndex = this.#writeIndex - samplesToKeep; - if (samplesToKeep === 0) this.#stalled = true; + /** + * Hold playback until the next write, keeping what is buffered. + * + * For a rise in the part of the target held encoded upstream: the ring already covers its floor, + * so only the main thread knows when the received media covers the rest, and it releases the next + * frame exactly then. + */ + park(): void { + this.#stalled = true; } write(timestamp: Micro, data: Float32Array[]): void { diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index e4d03ed01c..1fe0ed3efa 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -790,42 +790,21 @@ describe("i32 wrapping", () => { }); }); -describe("SharedRingBuffer.resize", () => { - it("preserves the unread window when growing capacity", () => { - const src = create({ rate: 1000, channels: 2, capacity: 64, latency: 30 }); - insert(src, 0, 30, { value: 3.5 }); - expect(src.stalled).toBe(false); - - const dst = src.resize(256); - expect(dst.capacity).toBe(256); - expect(dst.channels).toBe(2); - expect(dst.rate).toBe(1000); - - // The 30 unread samples should be readable from the new buffer. - const out = read(dst, 30); - expect(out[0].length).toBe(30); - for (let i = 0; i < 30; i++) { - expect(out[0][i]).toBe(3.5); - expect(out[1][i]).toBe(3.5); - } - // Unstalled state carried across. - expect(dst.stalled).toBe(false); - }); - - it("truncates to the newest samples when shrinking below the unread span", () => { - const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 64 }); - // Fill [0, 48) with value 1, then [48, 64) with value 2. - insert(src, 0, 48, { channels: 1, value: 1.0 }); - insert(src, 48, 16, { channels: 1, value: 2.0 }); +describe("park", () => { + it("holds a covered ring until the next insert, keeping what it buffered", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 128, latency: 32 }); + insert(buffer, 0, 64, { channels: 1, value: 1 }); + expect(read(buffer, 16, 1)[0].length).toBe(16); - const dst = src.resize(16); - expect(dst.capacity).toBe(16); + buffer.park(); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 16, 1)[0].length).toBe(0); - // Only the most recent 16 samples fit. - const out = read(dst, 16, 1); - for (let i = 0; i < 16; i++) { - expect(out[0][i]).toBe(2.0); - } + // The main thread releases the next frame only once the media it holds encoded covers the + // target, so that insert is what resumes playback. + insert(buffer, 64, 16, { channels: 1, value: 2 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 16, 1)[0]).toEqual(new Float32Array(16).fill(1)); }); }); @@ -1020,101 +999,6 @@ describe("truncate races the reader", () => { }); describe("packed state", () => { - it("does not replay samples consumed while the replacement message is in flight", () => { - // What the resize handoff exists for: the reader keeps draining the source until the - // message arrives, and the replacement must not start where the snapshot left off. - const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 64 }); - insert(src, 0, 64, { channels: 1, value: 1 }); - - const dst = src.resize(256); - read(src, 16, 1); - - const replacement = new SharedRingBuffer(dst.init, src); - expect(replacement.length).toBe(48); - }); - - it("carries the consumed cursor across a replacement truncate", () => { - const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 64 }); - src.insert(0 as Micro, [Float32Array.from({ length: 64 }, (_, i) => i + 1)]); - const dst = src.resize(256); - expect(read(src, 16, 1)[0]).toEqual(Float32Array.from({ length: 16 }, (_, i) => i + 1)); - dst.truncate(Micro.fromMilli(48 as Milli)); - - const replacement = new SharedRingBuffer(dst.init, src); - expect(replacement.length).toBe(32); - expect(read(replacement, 64, 1)[0]).toEqual(Float32Array.from({ length: 32 }, (_, i) => i + 17)); - expect(read(replacement, 64, 1)[0].length).toBe(0); - }); - - for (const operation of ["truncate", "re-anchor"] as const) { - for (const point of ["state", "timeline", "exchange"] as const) { - it(`preserves the timeline when ${operation} races handoff at ${point}`, () => { - const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 64 }); - insert(src, 0, 64, { channels: 1, value: 1 }); - const dst = src.resize(256); - read(src, 16, 1); - let fired = false; - const mutate = () => { - fired = true; - if (operation === "truncate") { - dst.truncate(Micro.fromMilli(48 as Milli)); - } else { - dst.reset(); - insert(dst, 1000, 64, { channels: 1, value: 2 }); - } - }; - const realLoad = Atomics.load; - const realExchange = Atomics.compareExchange; - const atomics = Atomics as { load: unknown; compareExchange: unknown }; - atomics.load = (arr: Int32Array | BigInt64Array, idx: number) => { - const value = realLoad(arr as Int32Array, idx); - if ( - !fired && - ((point === "state" && arr.buffer === dst.init.state) || - (point === "timeline" && arr.buffer === dst.init.control && idx === 3)) - ) - mutate(); - return value; - }; - atomics.compareExchange = (arr: BigInt64Array, idx: number, expected: bigint, next: bigint) => { - if (!fired && point === "exchange" && arr.buffer === dst.init.state) mutate(); - return realExchange(arr, idx, expected, next); - }; - let replacement: SharedRingBuffer; - try { - replacement = new SharedRingBuffer(dst.init, src); - } finally { - atomics.load = realLoad; - atomics.compareExchange = realExchange; - } - expect(fired).toBe(true); - const count = operation === "truncate" ? 32 : 64; - expect(replacement.length).toBe(count); - expect(read(replacement, 64, 1)[0]).toEqual( - new Float32Array(count).fill(operation === "truncate" ? 1 : 2), - ); - expect(read(replacement, 64, 1)[0].length).toBe(0); - }); - } - } - - it("drops the carried cursor when the replacement re-anchored first", () => { - // A different timeline: the cursor describes audio the replacement has never played. - const src = create({ rate: 1000, channels: 1, capacity: 64, latency: 64 }); - insert(src, 10_000, 64, { channels: 1, value: 1 }); - const worklet = new SharedRingBuffer(src.init); - read(worklet, 64, 1); - - const dst = src.resize(256); - dst.setLatency(64); - dst.reset(); - insert(dst, 30_000, 64, { channels: 1, value: 2 }); - - const replacement = new SharedRingBuffer(dst.init, worklet); - expect(replacement.length).toBe(64); - expect(read(replacement, 64, 1)[0]).toEqual(new Float32Array(64).fill(2)); - }); - // The reader samples the word, copies, then publishes, and the writer can rebase anywhere in // between. Sweeping the re-anchor across every load covers the windows a fixed pause point // misses. Because the publish is one exchange over epoch and cursor together, a rebase makes diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index a85293fa20..7d72194b19 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -14,13 +14,11 @@ const LATENCY = 1; * raise is correct on its face: the samples are there. */ const STALLED = 2; -// Timeline identity changes only on re-anchor, independently of the packed mutation epoch. -const TIMELINE = 3; // The size of the most recent insert, i.e. one decoded chunk. See `read`. -const CHUNK = 4; +const CHUNK = 3; // How many times the reader ran dry mid-playback. Diagnostics only. -const UNDERRUN = 5; -const CONTROL_SLOTS = 6; +const UNDERRUN = 4; +const CONTROL_SLOTS = 5; /** * The playhead and its mutation epoch, packed into one 64-bit word: epoch in the high @@ -128,7 +126,6 @@ export class SharedRingBuffer { readonly capacity: number; readonly rate: number; readonly buffered: boolean; - readonly init: SharedRingBufferInit; #control: Int32Array; #state: BigInt64Array; @@ -151,20 +148,12 @@ export class SharedRingBuffer { #position = 0; #lastRead = 0; - /** - * Wrap the shared memory described by `init`. - * - * Pass `previous` in the worklet when this ring replaces one already being read. `resize` - * snapshots the playhead on the main thread and hands the replacement over by message, so the - * reader keeps draining the old ring in the meantime. Carrying its cursor across means those - * samples are not played twice. - */ - constructor(init: SharedRingBufferInit, previous?: SharedRingBuffer) { + /** Wrap the shared memory described by `init`. */ + constructor(init: SharedRingBufferInit) { this.channels = init.channels; this.capacity = init.capacity; this.rate = init.rate; this.buffered = init.buffered; - this.init = init; this.#control = new Int32Array(init.control); this.#state = new BigInt64Array(init.state); @@ -174,32 +163,6 @@ export class SharedRingBuffer { new Float32Array(init.samples, i * this.capacity * Float32Array.BYTES_PER_ELEMENT, this.capacity), ); } - - if (previous !== undefined) this.#handoff(previous); - } - - /** - * Carry `source`'s playhead across, if the replacement is still the timeline it belongs to. - * - * Truncation changes the mutation epoch but preserves timeline identity. Sample state before - * identity: a re-anchor publishes identity first, then replaces state. Either the identity - * check rejects the old cursor, the exchange fails, or the re-anchor overwrites the carried - * cursor. A truncate only forces a retry, preserving audio consumed since the resize. - */ - #handoff(source: SharedRingBuffer): void { - if (source.channels !== this.channels || source.rate !== this.rate) return; - - const timeline = Atomics.load(source.#control, TIMELINE); - const from = Atomics.load(source.#state, 0); - - for (;;) { - const state = Atomics.load(this.#state, 0); - if (Atomics.load(this.#control, TIMELINE) !== timeline) return; - if (((readOf(from) - readOf(state)) | 0) <= 0) return; - - const next = pack(epochOf(state), readOf(from)); - if (Atomics.compareExchange(this.#state, 0, state, next) === state) return; - } } /** @@ -250,8 +213,6 @@ export class SharedRingBuffer { // One store rebases both halves, so a reader's compare-exchange against the old word // cannot land afterwards. Two, because odd epochs belong to `truncate`. const epoch = epochOf(Atomics.load(this.#state, 0)); - // Publish identity first so a handoff cannot apply an old cursor to the new state. - Atomics.add(this.#control, TIMELINE, 1); Atomics.store(this.#state, 0, pack((epoch + 2) | 0, 0)); Atomics.store(this.#control, WRITE, 0); this.#anchored = true; @@ -424,14 +385,9 @@ export class SharedRingBuffer { * already covers the new floor. */ setLatency(samples: number): void { + if (samples <= 0 || samples > this.capacity) throw new Error("invalid latency"); const previous = Atomics.exchange(this.#control, LATENCY, samples); - if (previous > 0 && samples > previous) { - Atomics.store(this.#control, STALLED, 1); - // A reader already past the STALLED check would still publish its quantum. Moving the - // word fails its exchange, and one that snapshots after this sees STALLED. Two keeps the - // epoch even, since odd belongs to `truncate`. - this.#step(2); - } + if (previous > 0 && samples > previous) this.park(); // Parked, the playhead holds still, so this measures exactly what is buffered. It also // resumes a refill that a shallower floor has already covered, which `insert` would only @@ -439,6 +395,21 @@ export class SharedRingBuffer { this.#resume(); } + /** + * Hold playback until the next insert, keeping what is buffered. Main thread only. + * + * For a rise in the part of the target held encoded upstream: the ring already covers its floor, + * so only the main thread knows when the received media covers the rest, and it releases the next + * frame exactly then. + */ + park(): void { + Atomics.store(this.#control, STALLED, 1); + // A reader already past the STALLED check would still publish its quantum. Moving the word + // fails its exchange, and one that snapshots after this sees STALLED. Two keeps the epoch + // even, since odd belongs to `truncate`. + this.#step(2); + } + /** * Drop buffered samples at or after `timestamp`, keeping whatever is already due. * Main thread only. @@ -480,74 +451,16 @@ export class SharedRingBuffer { } /** - * Allocate a new ring with `newCapacity` samples and copy the unread window - * [READ, WRITE) plus control state into it. Used when growing capacity so - * we don't drop buffered audio. If `newCapacity` is smaller than the unread - * span, the oldest samples are truncated. - * - * Main thread only. `resize()` reads from the source `SharedRingBuffer` and - * writes into a freshly allocated buffer from `allocSharedRingBuffer`, so it - * relies on the same invariant as `insert()`: no concurrent main-thread - * writers. The AudioWorklet reader is tolerated via the CAS discipline used - * by READ/WRITE elsewhere. + * Fold the current READ into `#position`, so the returned offset keeps counting up across the + * i32 wrap. Idempotent: folding an unchanged READ adds a zero delta. */ - resize(newCapacity: number): SharedRingBuffer { - const init = allocSharedRingBuffer(this.channels, newCapacity, this.rate, this.buffered); - const dst = new SharedRingBuffer(init); - dst.#anchored = this.#anchored; - dst.#anchor = this.#anchor; - - const state = Atomics.load(this.#state, 0); - const read = readOf(state); - const write = Atomics.load(this.#control, WRITE); - const latency = Atomics.load(this.#control, LATENCY); - const stalled = Atomics.load(this.#control, STALLED); - - const available = (write - read) | 0; - const copyCount = Math.max(0, Math.min(available, dst.capacity)); - const copyStart = (write - copyCount) | 0; - - for (let channel = 0; channel < this.channels; channel++) { - const src = this.#samples[channel]; - const out = dst.#samples[channel]; - for (let i = 0; i < copyCount; i++) { - const idx = (copyStart + i) | 0; - out[slot(idx, dst.capacity)] = src[slot(idx, this.capacity)]; - } - } - - Atomics.store(dst.#control, TIMELINE, Atomics.load(this.#control, TIMELINE)); - Atomics.store(dst.#state, 0, pack(epochOf(state), copyStart)); - Atomics.store(dst.#control, WRITE, write); - Atomics.store(dst.#control, LATENCY, latency); - Atomics.store(dst.#control, STALLED, stalled); - Atomics.store(dst.#control, CHUNK, Atomics.load(this.#control, CHUNK)); - Atomics.store(dst.#control, UNDERRUN, Atomics.load(this.#control, UNDERRUN)); - - // Carry the unwrapped playhead over, rebased onto dst's READ. Fold the same `read` - // snapshot the copy used so both sides agree on one observation; `copyStart` is at or - // ahead of it whenever the copy dropped the oldest samples. - dst.#position = this.#foldRead(read) + ((copyStart - read) | 0); - dst.#lastRead = copyStart; - - return dst; - } - - /** - * Fold an observed READ into `#position`, so the returned offset keeps counting up across - * the i32 wrap. Idempotent: folding the same value twice adds a zero delta. - */ - #foldRead(read: number): number { + #unwrapRead(): number { + const read = readOf(Atomics.load(this.#state, 0)); this.#position += (read - this.#lastRead) | 0; this.#lastRead = read; return this.#position; } - /** `#foldRead` against the current READ. */ - #unwrapRead(): number { - return this.#foldRead(readOf(Atomics.load(this.#state, 0))); - } - /** * Current playback timestamp derived from READ position. * diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 9e66ac0a2b..18ebe35ce7 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -885,7 +885,7 @@ it("a renamed video track keeps the delay, so audio neither skips nor goes silen const ring = new AudioRingBuffer({ rate: RATE, channels: 1, latency: Milli(STEP) }); const follow = new Effect(); follow.subscribe(playback.sync.out.delay, (delay) => { - ring.resize(Milli.fromSecond((ringSamples(RATE, delay) / RATE) as Second)); + ring.setLatency(Milli.fromSecond((ringSamples(RATE, delay) / RATE) as Second)); }); // Flushed in runs of 12, so each run's first frame lands ~360ms late. diff --git a/quest/m1/README.md b/quest/m1/README.md index 321085966b..d6b1ba3f13 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -115,7 +115,6 @@ tracks, so per-group stats stop costing cluster announces. - [Native rung disable](/quest/m1/simulcast-rung-disable.md) - Rust/FFI and OBS publishers stop encoding the top renditions their grant cannot fund and resume them once probing raises the estimate - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain - [Buffered backpressure timestamps](/quest/m1/audio-backpressure-timestamps.md) - buffered audio keeps holding frames past 2^31 microseconds instead of decoding ahead and dropping audio -- [Audio playout memory](/quest/m1/audio-playout-memory.md) - a catalog's advertised jitter or delay cannot size decoded PCM; the lookahead stays encoded - [AAC alias classification](/quest/m1/aac-he-alias.md) - every accepted AAC spelling gets the right frame duration from one classification - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles - [Signals notify on every structural change](/quest/m1/signals-equality.md) - a renamed `undefined` key or a changed sparse array length notifies, checked by a fast-check model of the notification contract diff --git a/quest/m1/audio-backpressure-timestamps.md b/quest/m1/audio-backpressure-timestamps.md index 3ddb21a940..dae317a770 100644 --- a/quest/m1/audio-backpressure-timestamps.md +++ b/quest/m1/audio-backpressure-timestamps.md @@ -28,7 +28,3 @@ legitimately be an integer; an absolute media timestamp may not. Test: a waiter one second ahead stays pending at a timestamp past 2^31 and 2^32 microseconds and resolves once the playhead reaches it, on a stubbed playhead. No API or wire change. - -## Related - -- [Audio playout memory](/quest/m1/audio-playout-memory.md) - generalizes the same `Backpressure`; land this first, it is independent diff --git a/quest/m1/audio-playout-memory.md b/quest/m1/audio-playout-memory.md deleted file mode 100644 index 66acb1e6e4..0000000000 --- a/quest/m1/audio-playout-memory.md +++ /dev/null @@ -1,50 +0,0 @@ -# [M] The receiver owns its decoded audio memory - -## Goal - -A catalog's advertised `jitter` or `delay`, however large, cannot make a -viewer allocate decoded PCM in proportion to it. The decoded ring is sized -by the receiver, and a deep playout target is reached by holding encoded -frames upstream, so memory follows bytes actually received. A publisher that -genuinely advertises seconds of flush delay still plays smoothly at that delay. - -## Plan - -Found in a 2026-10-22 media audit at -[main 1f5caa7](https://github.com/moq-dev/moq/commit/1f5caa7026bcfd8a2684da3712b384b5f74a7496) -and confirmed by source at -[main d4cdfe4](https://github.com/moq-dev/moq/commit/d4cdfe4368b17cbdf7d2a0c43d24e364a9626480). -`target()` in `js/watch/src/audio/latency.ts` floors the measured term by the -rendition's advertised `jitter` and adds its catalog `delay`; both are any -u53 in `js/hang/src/catalog/audio.ts`. `ringSamples` turns that target into -`latencySamples`, and `createAudioBuffer` allocates -`max(rate, latencySamples * 2)` frames per channel of shared float PCM. The -audit computed 512 MiB for a 48 kHz stereo rendition advertising ten minutes -of jitter and 4 GiB for one hour, without allocating them. The estimator's 2 s -ceiling bounds only the measured term. - -Decided 2026-10-22 (maintainer): hold the lookahead encoded rather than refuse -a rendition whose hints exceed a ceiling. The existing buffered mode already -keeps the lookahead above the floor as encoded frames behind `Backpressure`; -generalize that so the ring's capacity comes from the receiver's own budget -(a render-quantum floor plus decode-ahead slack), independent of the target, -and the target delay is enforced by when encoded frames are released to the -decoder. Settle while building: - -- The subscription's max delay must cover the target, or the transport drops - the frames the target waits for (`AUTO_MAX_DELAY` is the current floor). - Encoded retention then sits in the transport's group cache, which is the - receiver's existing flow-controlled store. -- Whether the non-buffered path survives once both paths gate on timestamps, - or folds into the buffered one. Prefer one path. -- The `postMessage` ring and the shared ring must behave the same. - -Tests: a rendition advertising an hour of jitter allocates a ring of the -receiver's fixed size and plays at the advertised delay on a stubbed clock; -the replay harness rows keep their budgets on both rings. No wire change; -any `js/watch` API change goes in its `MIGRATE.md`. - -## Related - -- [Buffered backpressure timestamps](/quest/m1/audio-backpressure-timestamps.md) - the 32-bit truncation in the same `Backpressure`, landing first -- [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target this sizes memory for From 17f7a9d46199001a6d33846d5a57df9793e76145 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 18:23:53 -0700 Subject: [PATCH 03/10] fix(watch): keep an encoded-hold park, and count read frames as coverage A floor update without a park resumed a ring parked for an uncovered hold, so a fall in the target let audio run ahead of video. The gate now keeps the park while the remaining hold is uncovered, and the park rides on the latency message so the worklet applies both at once. A buffered head waits in the reorder buffer until it spans the target, leaving the consumer empty, so the gate saw no coverage and never released it. The decoder now reports the newest frame it read beside what the consumer still holds. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 48 ++++++++++++++++------- js/watch/src/audio/buffer.ts | 48 ++++++++++++----------- js/watch/src/audio/decoder.test.ts | 34 +++++++++++++--- js/watch/src/audio/decoder.ts | 24 +++++++----- js/watch/src/audio/render-worklet.test.ts | 6 ++- js/watch/src/audio/render-worklet.ts | 7 ++-- js/watch/src/audio/render.ts | 12 ++---- js/watch/src/audio/replay.ts | 10 +++-- 8 files changed, 120 insertions(+), 69 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 1f25f7482a..e76cfdafe8 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -283,9 +283,9 @@ describe("Backpressure", () => { const gate = new Backpressure(1000, false, Milli(2000)); gate.observe(ms(0), false); const wait = gate.wait(ms(1000), new AbortController().signal); - gate.queued(ms(2499)); + gate.received(ms(2499)); expect(Bun.peek.status(wait)).toBe("pending"); - gate.queued(ms(2500)); + gate.received(ms(2500)); expect(Bun.peek.status(wait)).toBe("fulfilled"); }); @@ -293,7 +293,7 @@ describe("Backpressure", () => { const gate = new Backpressure(1000, true, Milli(2000)); gate.observe(ms(0), false); const wait = gate.wait(ms(1000), new AbortController().signal); - gate.queued(ms(60_000)); + gate.received(ms(60_000)); expect(Bun.peek.status(wait)).toBe("pending"); gate.observe(ms(500), false); expect(Bun.peek.status(wait)).toBe("fulfilled"); @@ -302,9 +302,9 @@ describe("Backpressure", () => { it("starts a stalled ring only once the queued media covers the hold", () => { const gate = new Backpressure(1000, true, Milli(2000)); const wait = gate.wait(ms(0), new AbortController().signal); - gate.queued(ms(1499)); + gate.received(ms(1499)); expect(Bun.peek.status(wait)).toBe("pending"); - gate.queued(ms(1500)); + gate.received(ms(1500)); expect(Bun.peek.status(wait)).toBe("fulfilled"); }); @@ -312,7 +312,7 @@ describe("Backpressure", () => { // outright would overflow the fixed ring before the worklet reports it playing. it("refills a covered buffered ring only to its floor", () => { const gate = new Backpressure(1000, true, Milli(2000)); - gate.queued(ms(60_000)); + gate.received(ms(60_000)); const signal = new AbortController().signal; expect(Bun.peek.status(gate.wait(ms(0), signal))).toBe("fulfilled"); expect(Bun.peek.status(gate.wait(ms(500), signal))).toBe("fulfilled"); @@ -323,7 +323,7 @@ describe("Backpressure", () => { // report it again: forgetting it would hold its first frame until the next utterance. it("keeps what is queued across a reset", () => { const gate = new Backpressure(1000, true, Milli(2000)); - gate.queued(ms(5000)); + gate.received(ms(5000)); gate.observe(ms(0), false); gate.reset(); expect(Bun.peek.status(gate.wait(ms(0), new AbortController().signal))).toBe("fulfilled"); @@ -344,7 +344,7 @@ function fake(delay: Milli, buffered = false) { buffered, }); const state = (timestamp: Micro, stalled: boolean, epoch: number) => { - const data: State = { type: "state", timestamp, stalled, underruns: 0, epoch }; + const data: State = { type: "state", timestamp, stalled, anchored: true, underruns: 0, epoch }; port.dispatchEvent(new MessageEvent("message", { data })); }; return { buffer, sent, state }; @@ -378,9 +378,9 @@ describe("AudioBuffer past its decoded floor", () => { // The ring holds 500 ms decoded; the rest of the hour waits upstream. const hold = HOUR - 500; const wait = buffer.wait(ms(0), new AbortController().signal); - buffer.queued(ms(hold - 1)); + buffer.received(ms(hold - 1)); expect(Bun.peek.status(wait)).toBe("pending"); - buffer.queued(ms(hold)); + buffer.received(ms(hold)); expect(Bun.peek.status(wait)).toBe("fulfilled"); } finally { buffer.close(); @@ -399,11 +399,10 @@ describe("AudioBuffer past its decoded floor", () => { const { buffer, sent, state } = fake(Milli(1000)); try { state(ms(100), stalled, 0); - buffer.queued(ms(queued)); + buffer.received(ms(queued)); buffer.setLatency(Milli(1500)); expect(sent.slice(1)).toEqual([ - { type: "latency", latency: Milli(500) }, - ...(parked ? [{ type: "park" as const, epoch: 1 }] : []), + { type: "latency", latency: Milli(500), ...(parked ? { park: 1 } : {}) }, ]); } finally { buffer.close(); @@ -412,12 +411,33 @@ describe("AudioBuffer past its decoded floor", () => { } } + // The ring resumes on any floor update without a park, so a fall that leaves the remaining hold + // uncovered parks again, and one the queued media covers lets the ring resume. + it("keeps a park across a fall until the queued media covers the hold", () => { + const { buffer, sent, state } = fake(Milli(1000)); + try { + state(ms(0), false, 0); + buffer.received(ms(1000)); + buffer.setLatency(Milli(2000)); + buffer.setLatency(Milli(1800)); + buffer.received(ms(1800)); + buffer.setLatency(Milli(1600)); + expect(sent.slice(1)).toEqual([ + { type: "latency", latency: Milli(500), park: 1 }, + { type: "latency", latency: Milli(500), park: 2 }, + { type: "latency", latency: Milli(500) }, + ]); + } finally { + buffer.close(); + } + }); + // A report the worklet posted before it applied the park still says the ring is playing. it("ignores state from before a park", () => { const { buffer, state } = fake(Milli(1000)); try { state(ms(200), false, 0); - buffer.queued(ms(1599)); + buffer.received(ms(1599)); buffer.setLatency(Milli(1500)); const wait = buffer.wait(ms(700), new AbortController().signal); state(ms(200), false, 0); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index fbd4ba4fc5..647e029cb2 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,7 +1,7 @@ import { Micro, Milli, type Second } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; import { ringHold, ringSamples } from "./latency"; -import type { Data, Init, Latency, Park, Reset, State, Truncate } from "./render"; +import type { Data, Init, Latency, Reset, State, Truncate } from "./render"; /** * Decides when the decode loop may decode a frame, which is how the target delay is split between @@ -35,12 +35,15 @@ export class Backpressure { #headroom = 0 as Micro; #hold = 0 as Micro; - // The newest media timestamp queued upstream of the decoder, unset while nothing is. - #queued?: Micro; + // The newest media timestamp received, decoded or not, unset until one is. + #received?: Micro; // The ring as last observed. The playhead is undefined until one is known on the current timeline. #playhead?: Micro; #stalled = true; + // Whether the ring is parked for a hold the queued media does not cover yet. Only a write or a + // floor update releases a park, so a floor update has to carry it while the hold stays uncovered. + #parked = false; // The first and latest frames released since the ring stalled. A stalled ring refills to the // floor from its parked playhead, but releasing it outright would let the whole lookahead through @@ -62,9 +65,10 @@ export class Backpressure { } /** - * Split `delay` between the floor and the hold. Returns true when the ring must park: a rise in the - * hold that the queued media does not cover yet holds playback until it does, the way a deeper - * floor parks the ring until it refills, since video holds the extra delay on its own. + * Split `delay` between the floor and the hold. Returns true when the ring must park with the new + * floor: a rise in the hold that the queued media does not cover yet holds playback until it does, + * the way a deeper floor parks the ring until it refills, since video holds the extra delay on its + * own. A ring already parked stays parked while the remaining hold is uncovered. */ setLatency(delay: Milli): boolean { this.#floor = ringSamples(this.#rate, delay); @@ -76,10 +80,10 @@ export class Backpressure { // Parks whatever the ring was last reported doing: a report of a stall may be stale, and // parking a ring that is still stalled changes nothing. const playhead = this.#playhead; - const park = rise && playhead !== undefined && !this.#covered(playhead + this.#headroom); - if (park) this.#stalled = true; + this.#parked = (rise || this.#parked) && playhead !== undefined && !this.#covered(playhead + this.#headroom); + if (this.#parked) this.#stalled = true; this.#release(); - return park; + return this.#parked; } /** Note the ring's playhead and whether it is stalled, and release every frame now due. */ @@ -87,15 +91,16 @@ export class Backpressure { this.#playhead = playhead; this.#stalled = stalled; if (!stalled) { + this.#parked = false; this.#first = undefined; this.#last = undefined; } this.#release(); } - /** Note the newest media timestamp queued upstream of the decoder, or undefined once none is. */ - queued(newest: Micro | undefined): void { - this.#queued = newest; + /** Note the newest media timestamp received, decoded or not, or undefined until one is. */ + received(newest: Micro | undefined): void { + this.#received = newest; this.#release(); } @@ -123,6 +128,7 @@ export class Backpressure { reset(): void { this.#playhead = undefined; this.#stalled = true; + this.#parked = false; this.#first = undefined; this.#last = undefined; this.#release(); @@ -148,7 +154,7 @@ export class Backpressure { // Whether the queued media reaches the hold past `timestamp`. #covered(timestamp: number): boolean { if (this.#hold === 0) return true; - return this.#queued !== undefined && this.#queued >= timestamp + this.#hold; + return this.#received !== undefined && this.#received >= timestamp + this.#hold; } #admit(timestamp: Micro): boolean { @@ -266,22 +272,18 @@ export class AudioBuffer { */ setLatency(delay: Milli): void { const park = this.#backpressure.setLatency(delay); - const msg: Latency = { type: "latency", latency: this.#latency() }; + // One message, so the worklet never plays a quantum between the new floor and the park. + if (park) msg.park = ++this.#epoch; this.#worklet.port.postMessage(msg); - if (park) { - this.#epoch++; - const msg: Park = { type: "park", epoch: this.#epoch }; - this.#worklet.port.postMessage(msg); - } } /** - * Note the newest media timestamp queued upstream of the decoder, or undefined once none is. The - * queue is what covers the part of the target held encoded. + * Note the newest media timestamp received, decoded or not, or undefined until one is. The + * media is what covers the part of the target held encoded. */ - queued(newest: Micro | undefined): void { - this.#backpressure.queued(newest); + received(newest: Micro | undefined): void { + this.#backpressure.received(newest); } /** diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 611dd3fe38..c1e977fb65 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -64,12 +64,14 @@ class FakeWorklet { postMessage: (msg: Message) => { if (msg.type === "init") this.ring = new AudioRingBuffer(msg); else if (msg.type === "data") this.ring?.write(msg.timestamp, msg.data); - else if (msg.type === "latency") this.ring?.setLatency(msg.latency); - else if (msg.type === "truncate") this.ring?.truncate(msg.timestamp); - else if (msg.type === "park") { - this.ring?.park(); - this.#epoch = msg.epoch; - } else if (msg.type === "reset") { + else if (msg.type === "latency") { + this.ring?.setLatency(msg.latency); + if (msg.park !== undefined) { + this.ring?.park(); + this.#epoch = msg.park; + } + } else if (msg.type === "truncate") this.ring?.truncate(msg.timestamp); + else if (msg.type === "reset") { this.ring?.reset(); this.#epoch = msg.epoch; } @@ -340,6 +342,8 @@ async function play(initial: Delay, buffered = false, cmaf = false) { consumer = producer.consume(); frameTimestamp = 0; }, + // The resolved playout target. + target: () => sync.out.delay.peek(), // The rings written to, and the timestamps of every frame that reached one. rings: () => worklets.filter((worklet) => worklet.ring && insert.mock.contexts.includes(worklet.ring)), inserted: () => insert.mock.calls.map(([timestamp]) => timestamp as number), @@ -408,6 +412,24 @@ for (const cmaf of [false, true]) { } }); + // A buffered head above the decoded floor waits in the reorder buffer, not the consumer, until it + // runs past the target. Those frames are what cover the part of the target held encoded. + it(`starts a ${cmaf ? "CMAF" : "legacy"} buffered head held past a target above the floor`, async () => { + const playback = await play(Milli(1000), true, cmaf); + try { + const target = playback.target(); + expect(target).toBeGreaterThan(500); + while (frameTimestamp <= target * 1000 + 20_000) { + playback.push(frame()); + await microtasks(); + } + expect(playback.inserted()[0]).toBe(20_000); + } finally { + playback.close(); + await microtasks(); + } + }); + // A fresh encoder after a declared end may restart its timeline below the old one, and the decoder // resets for it before anything is judged out of order. it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 2466b6270d..8cf094e0b8 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -447,7 +447,7 @@ export class Decoder { config.container.kind === "loc" ? new Container.Loc.Format("audio") : new Container.Legacy.Format(config); // Create consumer with slightly less latency than the render worklet to avoid underflowing. // TODO include JITTER_UNDERHEAD - const consumer = this.#consume(effect, sub, format, { config, path }); + const { consumer, read } = this.#consume(effect, sub, format, { config, path }); spawnFatal(effect, failure, async () => { const loaded = await Util.Libav.polyfill(); @@ -493,6 +493,7 @@ export class Decoder { continue; } const { frame } = next; + read.set(frame.timestamp as Micro); // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Milli.fromMicro(frame.timestamp as Micro); @@ -537,7 +538,7 @@ export class Decoder { ? Util.Hex.toBytes(config.description) : init.description; - const consumer = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); + const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); spawnFatal(effect, failure, async () => { const loaded = await Util.Libav.polyfill(); @@ -577,6 +578,7 @@ export class Decoder { continue; } const { frame } = next; + read.set(frame.timestamp); const timestamp = Milli.fromMicro(frame.timestamp); const late = this.sync.received(timestamp, frame.arrival); @@ -660,7 +662,7 @@ export class Decoder { sub: Moq.Track.Subscriber, format: Container.Format, { config, path }: { config: DecoderConfig; path: Moq.Path.Valid }, - ): Container.Consumer { + ): { consumer: Container.Consumer; read: Signal } { const outgoing = this.#consumer; const continues = outgoing?.path === path; const consumer = new Container.Consumer(sub, { @@ -683,22 +685,26 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); - // What has arrived but not been decoded is what covers the part of the target held encoded, so - // the decode loop's held frame is released once enough is queued behind it. + // What has arrived is what covers the part of the target held encoded, so the decode loop's + // held frame is released once enough is queued behind it. That includes frames the loop has + // read but not decoded yet: a buffered head waits in the reorder buffer until it spans the + // whole target, leaving the consumer empty. + const read = new Signal(undefined); effect.run((inner) => { const ring = inner.get(this.#ring); if (!ring) return; - let newest: Milli | undefined; + let newest = inner.get(read); for (const { end } of inner.get(consumer.buffered)) { - if (newest === undefined || end > newest) newest = end; + const micro = Micro.fromMilli(end); + if (newest === undefined || micro > newest) newest = micro; } - ring.queued(newest === undefined ? undefined : Micro.fromMilli(newest)); + ring.received(newest); }); // Feed the arrival estimate into the playout target. It outlives this consumer, so the target // holds until the replacement takes over; see #retire. effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); - return consumer; + return { consumer, read }; } // No consumer replaces the outgoing one, so its estimate stops holding the buffer open. diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index 9a5d096eac..ac69d0e4f8 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -88,14 +88,16 @@ describe("render dispatch", () => { const { send, read, sent } = renderer(); send({ type: "data", timestamp: Micro(0), data: [new Float32Array(200).fill(1)] }); expect([...read(20)]).toEqual(Array(20).fill(1)); - send({ type: "park", epoch: 3 }); + send({ type: "latency", latency: Milli(100), park: 3 }); expect([...read(20)]).toEqual(Array(20).fill(0)); send({ type: "data", timestamp: Micro(200_000), data: [new Float32Array(20).fill(2)] }); // It fades back in on the parked audio, not the new data. expect(read(20)[19]).toBe(1); read(20); read(20); - expect(sent).toEqual([{ type: "state", epoch: 3, timestamp: Micro(80_000), stalled: false, underruns: 0 }]); + expect(sent).toEqual([ + { type: "state", epoch: 3, timestamp: Micro(80_000), stalled: false, anchored: true, underruns: 0 }, + ]); }); it("parks on a latency rise and preserves unread PCM", () => { diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 7153810c01..59475f162b 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -27,9 +27,10 @@ class Render extends AudioWorkletProcessor { this.#ring?.write(msg.timestamp, msg.data); } else if (msg.type === "latency") { this.#ring?.setLatency(msg.latency); - } else if (msg.type === "park") { - this.#ring?.park(); - this.#epoch = msg.epoch; + if (msg.park !== undefined) { + this.#ring?.park(); + this.#epoch = msg.park; + } } else if (msg.type === "truncate") { this.#ring?.truncate(msg.timestamp); } else if (msg.type === "reset") { diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 991136f350..24eb8e9942 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -1,7 +1,7 @@ import type { Micro, Milli } from "@moq/clock"; /** Everything the main thread sends the render worklet over its port. */ -export type Message = Init | Data | Latency | Park | Reset | Truncate; +export type Message = Init | Data | Latency | Reset | Truncate; /** Create the worklet's ring. */ export interface Init { @@ -21,13 +21,6 @@ export interface Reset { epoch: number; } -/** Hold playback until the next data message, keeping what is buffered. */ -export interface Park { - type: "park"; - // Echoed like a reset's, so state from before the park cannot report the ring playing. - epoch: number; -} - /** Drop buffered samples at or after `timestamp`, keeping what is already due. */ export interface Truncate { type: "truncate"; @@ -45,6 +38,9 @@ export interface Data { export interface Latency { type: "latency"; latency: Milli; + // When set, also hold playback until the next data message, keeping what is buffered. Echoed like + // a reset's epoch, so state from before the park cannot report the ring playing. + park?: number; } /** State update from the worklet back to the main thread. */ diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index dfbed1f65c..ebe9344d89 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -157,11 +157,12 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, gate.floor) }); const observe = () => gate.observe(ring.timestamp, ring.stalled); - // Arrived but not yet decoded media is what covers the part of the target held encoded. + // Arrived media, read or still in the consumer, is what covers the part of the target held encoded. + const read = new Signal(undefined); signals.run((effect) => { - let newest: number | undefined; - for (const { end } of effect.get(consumer.buffered)) newest = Math.max(newest ?? end, end); - gate.queued(newest === undefined ? undefined : micros(newest)); + let newest = effect.get(read); + for (const { end } of effect.get(consumer.buffered)) newest = Math.max(newest ?? 0, micros(end)) as Micro; + gate.received(newest); }); // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. @@ -184,6 +185,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato if (next.kind !== "frame") continue; const { timestamp, arrival } = next.frame; + read.set(timestamp); const media = Milli.fromMicro(timestamp); const late = sync.received(media, arrival); arrivals.push({ kind: "frame", timestamp: media, arrival, late }); From 1a8c5791b4bdb2339e3f79889a8a119787dd96d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 18:35:02 -0700 Subject: [PATCH 04/10] fix(watch): start a fresh refill when parking for an encoded hold A buffered refill that released frames through the floor before the ring reported playing left its bounds behind. A park then held the next frame for having reached the floor already, while the parked ring waited for that very write, so both sides stalled. A park now starts a fresh refill, the way a reset does. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 16 ++++++++++++++++ js/watch/src/audio/buffer.ts | 8 +++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index e76cfdafe8..2fe1cc755b 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -319,6 +319,22 @@ describe("Backpressure", () => { expect(Bun.peek.status(gate.wait(ms(520), signal))).toBe("pending"); }); + // A park holds the ring until its next write, so it starts a fresh refill: the frames a refill + // released before the ring reported playing must not hold back the write that resumes it. + it("releases the frame that resumes a buffered ring parked after a refill", () => { + const gate = new Backpressure(1000, true, Milli(1000)); + gate.received(ms(1000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + gate.observe(ms(0), true); + expect(gate.setLatency(Milli(2000))).toBe(true); + const wait = gate.wait(ms(520), signal); + gate.received(ms(2019)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.received(ms(2020)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + }); + // An utterance that arrived whole before the reset changes nothing queued, so nothing would // report it again: forgetting it would hold its first frame until the next utterance. it("keeps what is queued across a reset", () => { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 647e029cb2..bc128b26d2 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -81,7 +81,13 @@ export class Backpressure { // parking a ring that is still stalled changes nothing. const playhead = this.#playhead; this.#parked = (rise || this.#parked) && playhead !== undefined && !this.#covered(playhead + this.#headroom); - if (this.#parked) this.#stalled = true; + if (this.#parked) { + // Only a write resumes a parked ring, so the park starts a fresh refill: the frames an + // earlier refill released must not hold back the one that resumes it. + this.#stalled = true; + this.#first = undefined; + this.#last = undefined; + } this.#release(); return this.#parked; } From a8035c6c4ca30504417e43450909242a9050e6c4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 18:47:52 -0700 Subject: [PATCH 05/10] fix(watch): only the main thread ends an encoded-hold park Output decoded before a park could still land after it, and the write resumed the ring while the hold stayed uncovered, leaving audio ahead of video. The worklet now keeps a park until the next floor update, and the gate sends one once the queued media covers the hold. That also ends the refill-bound deadlock without clearing the bounds. A buffered head arrives newest-first, so the frame just read was often the oldest one held. The decoder now reports the newest frame still to decode, held in the reorder buffer or released to it. The replay harness reported a playhead before the ring anchored, which let a startup rise park it; it now mirrors AudioBuffer. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 45 +++++++++++----------- js/watch/src/audio/buffer.ts | 46 ++++++++++++----------- js/watch/src/audio/decoder.ts | 22 ++++++----- js/watch/src/audio/order.test.ts | 10 +++++ js/watch/src/audio/order.ts | 9 +++++ js/watch/src/audio/render-worklet.test.ts | 12 +++--- js/watch/src/audio/render.ts | 4 +- js/watch/src/audio/replay.ts | 7 ++-- js/watch/src/audio/ring-buffer.test.ts | 9 +++-- js/watch/src/audio/ring-buffer.ts | 19 ++++++---- 10 files changed, 110 insertions(+), 73 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 2fe1cc755b..66c2aff2bb 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -283,9 +283,9 @@ describe("Backpressure", () => { const gate = new Backpressure(1000, false, Milli(2000)); gate.observe(ms(0), false); const wait = gate.wait(ms(1000), new AbortController().signal); - gate.received(ms(2499)); + gate.queued(ms(2499)); expect(Bun.peek.status(wait)).toBe("pending"); - gate.received(ms(2500)); + gate.queued(ms(2500)); expect(Bun.peek.status(wait)).toBe("fulfilled"); }); @@ -293,7 +293,7 @@ describe("Backpressure", () => { const gate = new Backpressure(1000, true, Milli(2000)); gate.observe(ms(0), false); const wait = gate.wait(ms(1000), new AbortController().signal); - gate.received(ms(60_000)); + gate.queued(ms(60_000)); expect(Bun.peek.status(wait)).toBe("pending"); gate.observe(ms(500), false); expect(Bun.peek.status(wait)).toBe("fulfilled"); @@ -302,9 +302,9 @@ describe("Backpressure", () => { it("starts a stalled ring only once the queued media covers the hold", () => { const gate = new Backpressure(1000, true, Milli(2000)); const wait = gate.wait(ms(0), new AbortController().signal); - gate.received(ms(1499)); + gate.queued(ms(1499)); expect(Bun.peek.status(wait)).toBe("pending"); - gate.received(ms(1500)); + gate.queued(ms(1500)); expect(Bun.peek.status(wait)).toBe("fulfilled"); }); @@ -312,34 +312,32 @@ describe("Backpressure", () => { // outright would overflow the fixed ring before the worklet reports it playing. it("refills a covered buffered ring only to its floor", () => { const gate = new Backpressure(1000, true, Milli(2000)); - gate.received(ms(60_000)); + gate.queued(ms(60_000)); const signal = new AbortController().signal; expect(Bun.peek.status(gate.wait(ms(0), signal))).toBe("fulfilled"); expect(Bun.peek.status(gate.wait(ms(500), signal))).toBe("fulfilled"); expect(Bun.peek.status(gate.wait(ms(520), signal))).toBe("pending"); }); - // A park holds the ring until its next write, so it starts a fresh refill: the frames a refill - // released before the ring reported playing must not hold back the write that resumes it. - it("releases the frame that resumes a buffered ring parked after a refill", () => { + // A refill that released its floor before the ring reported playing holds the next frame back, so + // no write could end the park. The queued media covering the hold is what ends it, once. + it("resumes a park once the queued media covers the hold", () => { const gate = new Backpressure(1000, true, Milli(1000)); - gate.received(ms(1000)); + expect(gate.queued(ms(1000))).toBe(false); const signal = new AbortController().signal; for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); gate.observe(ms(0), true); expect(gate.setLatency(Milli(2000))).toBe(true); - const wait = gate.wait(ms(520), signal); - gate.received(ms(2019)); - expect(Bun.peek.status(wait)).toBe("pending"); - gate.received(ms(2020)); - expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(gate.queued(ms(1999))).toBe(false); + expect(gate.queued(ms(2000))).toBe(true); + expect(gate.queued(ms(3000))).toBe(false); }); // An utterance that arrived whole before the reset changes nothing queued, so nothing would // report it again: forgetting it would hold its first frame until the next utterance. it("keeps what is queued across a reset", () => { const gate = new Backpressure(1000, true, Milli(2000)); - gate.received(ms(5000)); + gate.queued(ms(5000)); gate.observe(ms(0), false); gate.reset(); expect(Bun.peek.status(gate.wait(ms(0), new AbortController().signal))).toBe("fulfilled"); @@ -394,9 +392,9 @@ describe("AudioBuffer past its decoded floor", () => { // The ring holds 500 ms decoded; the rest of the hour waits upstream. const hold = HOUR - 500; const wait = buffer.wait(ms(0), new AbortController().signal); - buffer.received(ms(hold - 1)); + buffer.queued(ms(hold - 1)); expect(Bun.peek.status(wait)).toBe("pending"); - buffer.received(ms(hold)); + buffer.queued(ms(hold)); expect(Bun.peek.status(wait)).toBe("fulfilled"); } finally { buffer.close(); @@ -415,7 +413,7 @@ describe("AudioBuffer past its decoded floor", () => { const { buffer, sent, state } = fake(Milli(1000)); try { state(ms(100), stalled, 0); - buffer.received(ms(queued)); + buffer.queued(ms(queued)); buffer.setLatency(Milli(1500)); expect(sent.slice(1)).toEqual([ { type: "latency", latency: Milli(500), ...(parked ? { park: 1 } : {}) }, @@ -428,20 +426,21 @@ describe("AudioBuffer past its decoded floor", () => { } // The ring resumes on any floor update without a park, so a fall that leaves the remaining hold - // uncovered parks again, and one the queued media covers lets the ring resume. + // uncovered parks again, and the queued media covering it resumes the ring. it("keeps a park across a fall until the queued media covers the hold", () => { const { buffer, sent, state } = fake(Milli(1000)); try { state(ms(0), false, 0); - buffer.received(ms(1000)); + buffer.queued(ms(1000)); buffer.setLatency(Milli(2000)); buffer.setLatency(Milli(1800)); - buffer.received(ms(1800)); + buffer.queued(ms(1800)); buffer.setLatency(Milli(1600)); expect(sent.slice(1)).toEqual([ { type: "latency", latency: Milli(500), park: 1 }, { type: "latency", latency: Milli(500), park: 2 }, { type: "latency", latency: Milli(500) }, + { type: "latency", latency: Milli(500) }, ]); } finally { buffer.close(); @@ -453,7 +452,7 @@ describe("AudioBuffer past its decoded floor", () => { const { buffer, state } = fake(Milli(1000)); try { state(ms(200), false, 0); - buffer.received(ms(1599)); + buffer.queued(ms(1599)); buffer.setLatency(Milli(1500)); const wait = buffer.wait(ms(700), new AbortController().signal); state(ms(200), false, 0); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index bc128b26d2..d86164f587 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -35,14 +35,15 @@ export class Backpressure { #headroom = 0 as Micro; #hold = 0 as Micro; - // The newest media timestamp received, decoded or not, unset until one is. - #received?: Micro; + // The newest media timestamp queued ahead of the decoder, unset while none is. + #queued?: Micro; // The ring as last observed. The playhead is undefined until one is known on the current timeline. #playhead?: Micro; #stalled = true; - // Whether the ring is parked for a hold the queued media does not cover yet. Only a write or a - // floor update releases a park, so a floor update has to carry it while the hold stays uncovered. + // Whether the ring is parked for a hold the queued media does not cover yet. The worklet keeps a + // park until the next floor update, so only this decides when it ends: a write of output decoded + // before the park cannot. #parked = false; // The first and latest frames released since the ring stalled. A stalled ring refills to the @@ -68,7 +69,8 @@ export class Backpressure { * Split `delay` between the floor and the hold. Returns true when the ring must park with the new * floor: a rise in the hold that the queued media does not cover yet holds playback until it does, * the way a deeper floor parks the ring until it refills, since video holds the extra delay on its - * own. A ring already parked stays parked while the remaining hold is uncovered. + * own. A floor update ends a park, so a ring already parked stays parked while the remaining hold + * is uncovered. */ setLatency(delay: Milli): boolean { this.#floor = ringSamples(this.#rate, delay); @@ -81,13 +83,7 @@ export class Backpressure { // parking a ring that is still stalled changes nothing. const playhead = this.#playhead; this.#parked = (rise || this.#parked) && playhead !== undefined && !this.#covered(playhead + this.#headroom); - if (this.#parked) { - // Only a write resumes a parked ring, so the park starts a fresh refill: the frames an - // earlier refill released must not hold back the one that resumes it. - this.#stalled = true; - this.#first = undefined; - this.#last = undefined; - } + if (this.#parked) this.#stalled = true; this.#release(); return this.#parked; } @@ -97,17 +93,23 @@ export class Backpressure { this.#playhead = playhead; this.#stalled = stalled; if (!stalled) { - this.#parked = false; this.#first = undefined; this.#last = undefined; } this.#release(); } - /** Note the newest media timestamp received, decoded or not, or undefined until one is. */ - received(newest: Micro | undefined): void { - this.#received = newest; + /** + * Note the newest media timestamp queued ahead of the decoder, or undefined once none is. Returns + * true when that covers the hold of a parked ring, which a floor update then resumes. + */ + queued(newest: Micro | undefined): boolean { + this.#queued = newest; + const playhead = this.#playhead; + const resume = this.#parked && playhead !== undefined && this.#covered(playhead + this.#headroom); + if (resume) this.#parked = false; this.#release(); + return resume; } wait(timestamp: Micro, signal: AbortSignal): Promise { @@ -160,7 +162,7 @@ export class Backpressure { // Whether the queued media reaches the hold past `timestamp`. #covered(timestamp: number): boolean { if (this.#hold === 0) return true; - return this.#received !== undefined && this.#received >= timestamp + this.#hold; + return this.#queued !== undefined && this.#queued >= timestamp + this.#hold; } #admit(timestamp: Micro): boolean { @@ -285,11 +287,13 @@ export class AudioBuffer { } /** - * Note the newest media timestamp received, decoded or not, or undefined until one is. The - * media is what covers the part of the target held encoded. + * Note the newest media timestamp queued ahead of the decoder, or undefined once none is. The + * queue is what covers the part of the target held encoded, so it is what resumes a parked ring. */ - received(newest: Micro | undefined): void { - this.#backpressure.received(newest); + queued(newest: Micro | undefined): void { + if (!this.#backpressure.queued(newest)) return; + const msg: Latency = { type: "latency", latency: this.#latency() }; + this.#worklet.port.postMessage(msg); } /** diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 8cf094e0b8..e7b27f83df 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -477,7 +477,10 @@ export class Decoder { decoder.configure(decoderConfig); const order = this.#order(); - const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); + const play = (reads: FrameRead[]) => { + read.set(order.newest(reads) as Micro | undefined); + return this.#play(effect, decoder, decoderConfig, order, reads); + }; for (;;) { const next = await nextMedia(consumer); if (next?.kind !== "frame") { @@ -493,7 +496,6 @@ export class Decoder { continue; } const { frame } = next; - read.set(frame.timestamp as Micro); // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Milli.fromMicro(frame.timestamp as Micro); @@ -562,7 +564,10 @@ export class Decoder { decoder.configure(decoderConfig); const order = this.#order(); - const play = (reads: FrameRead[]) => this.#play(effect, decoder, decoderConfig, order, reads); + const play = (reads: FrameRead[]) => { + read.set(order.newest(reads) as Micro | undefined); + return this.#play(effect, decoder, decoderConfig, order, reads); + }; for (;;) { const next = await nextMedia(consumer); if (next?.kind !== "frame") { @@ -578,7 +583,6 @@ export class Decoder { continue; } const { frame } = next; - read.set(frame.timestamp); const timestamp = Milli.fromMicro(frame.timestamp); const late = this.sync.received(timestamp, frame.arrival); @@ -685,10 +689,10 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); - // What has arrived is what covers the part of the target held encoded, so the decode loop's - // held frame is released once enough is queued behind it. That includes frames the loop has - // read but not decoded yet: a buffered head waits in the reorder buffer until it spans the - // whole target, leaving the consumer empty. + // What has arrived but not been decoded is what covers the part of the target held encoded, so + // the decode loop's held frame is released once enough is queued behind it. That includes the + // frames the loop has read but not decoded yet: a buffered head waits in the reorder buffer + // until it spans the whole target, leaving the consumer empty. const read = new Signal(undefined); effect.run((inner) => { const ring = inner.get(this.#ring); @@ -698,7 +702,7 @@ export class Decoder { const micro = Micro.fromMilli(end); if (newest === undefined || micro > newest) newest = micro; } - ring.received(newest); + ring.queued(newest); }); // Feed the arrival estimate into the playout target. It outlives this consumer, so the target diff --git a/js/watch/src/audio/order.test.ts b/js/watch/src/audio/order.test.ts index d8269b7e03..b0ed6d84a6 100644 --- a/js/watch/src/audio/order.test.ts +++ b/js/watch/src/audio/order.test.ts @@ -37,6 +37,16 @@ describe("Order", () => { expect(timestamps(order.push(read(0, 0)))).toEqual([0, 20, 40, 60]); }); + // The newest frame still to decode covers the hold, wherever it waits: a head arrives newest-first, + // so the frame just read is often the oldest. + it("reports the newest frame held or released", () => { + const order = new Order(() => ({ floor: 100, maxDelay: 60 })); + expect(order.newest([])).toBeUndefined(); + for (const t of [60, 40, 20]) expect(order.newest(order.push(read(t / 20, t)))).toBe(60); + expect(order.newest(order.push(read(0, 0)))).toBe(60); + expect(order.newest([])).toBeUndefined(); + }); + it("releases the head as it stands on a flush", () => { const order = new Order(() => ({ floor: 100, maxDelay: 1000 })); order.push(read(1, 20)); diff --git a/js/watch/src/audio/order.ts b/js/watch/src/audio/order.ts index 098ba25e40..95cad335bb 100644 --- a/js/watch/src/audio/order.ts +++ b/js/watch/src/audio/order.ts @@ -48,6 +48,15 @@ export class Order { return []; } + /** The newest timestamp among the held head and `released`, the frames still to decode. */ + newest(released: T[]): number | undefined { + let newest = this.#held.length > 0 ? this.#span?.newest : undefined; + for (const { frame } of released) { + if (newest === undefined || frame.timestamp > newest) newest = frame.timestamp; + } + return newest; + } + /** Release the head as it stands, ending the hold: the track ended, or a gap or end cut it short. */ flush(): T[] { this.#hold = undefined; diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index ac69d0e4f8..48eef2ecb3 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -82,21 +82,23 @@ describe("render dispatch", () => { ]); }); - // A covered ring parks only on the main thread's word, and the next data resumes it. The echoed - // epoch is what tells the main thread that state reports no longer predate the park. - it("parks until the next data and echoes the park's epoch", () => { + // A covered ring parks and resumes only on the main thread's word: data decoded before the park + // may still land. The echoed epoch is what tells the main thread that state reports no longer + // predate the park. + it("parks until the next floor update and echoes the park's epoch", () => { const { send, read, sent } = renderer(); send({ type: "data", timestamp: Micro(0), data: [new Float32Array(200).fill(1)] }); expect([...read(20)]).toEqual(Array(20).fill(1)); send({ type: "latency", latency: Milli(100), park: 3 }); expect([...read(20)]).toEqual(Array(20).fill(0)); send({ type: "data", timestamp: Micro(200_000), data: [new Float32Array(20).fill(2)] }); + expect([...read(20)]).toEqual(Array(20).fill(0)); + send({ type: "latency", latency: Milli(100) }); // It fades back in on the parked audio, not the new data. expect(read(20)[19]).toBe(1); read(20); - read(20); expect(sent).toEqual([ - { type: "state", epoch: 3, timestamp: Micro(80_000), stalled: false, anchored: true, underruns: 0 }, + { type: "state", epoch: 3, timestamp: Micro(60_000), stalled: false, anchored: true, underruns: 0 }, ]); }); diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 24eb8e9942..77ecb70fa2 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -38,8 +38,8 @@ export interface Data { export interface Latency { type: "latency"; latency: Milli; - // When set, also hold playback until the next data message, keeping what is buffered. Echoed like - // a reset's epoch, so state from before the park cannot report the ring playing. + // When set, also hold playback, keeping what is buffered, until the next latency message without + // it. Echoed like a reset's epoch, so state from before the park cannot report the ring playing. park?: number; } diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index ebe9344d89..e80241680d 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -155,14 +155,15 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato // The decode gate, and the worklet's ring on the floor it sets, the way `AudioBuffer` drives both. const gate = new Backpressure(rate, false, delay); const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, gate.floor) }); - const observe = () => gate.observe(ring.timestamp, ring.stalled); + const observe = () => gate.observe(ring.anchored ? ring.timestamp : undefined, ring.stalled); - // Arrived media, read or still in the consumer, is what covers the part of the target held encoded. + // Arrived but not yet decoded media, read or still in the consumer, is what covers the part of the + // target held encoded, and what resumes a parked ring. const read = new Signal(undefined); signals.run((effect) => { let newest = effect.get(read); for (const { end } of effect.get(consumer.buffered)) newest = Math.max(newest ?? 0, micros(end)) as Micro; - gate.received(newest); + if (gate.queued(newest)) ring.setLatency(millis(rate, gate.floor)); }); // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index 02ac28fabe..1d1c4f3d77 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -415,7 +415,7 @@ describe("setLatency", () => { }); describe("park", () => { - it("holds a covered ring until the next write, keeping what it buffered", () => { + it("holds a covered ring until the next floor update, keeping what it buffered", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 50 as Milli }); write(buffer, 0 as Milli, 100, { channels: 1, value: 1.0 }); expect(read(buffer, 10, 1)[0].length).toBe(10); @@ -424,9 +424,12 @@ describe("park", () => { expect(buffer.stalled).toBe(true); expect(read(buffer, 10, 1)[0].length).toBe(0); - // The main thread releases the next frame only once the media it holds encoded covers the - // target, so that write is what resumes playback. + // Output decoded before the park can still land, so a write does not resume it. write(buffer, 100 as Milli, 10, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + + // The main thread knows when the media it holds encoded covers the target. + buffer.setLatency(50 as Milli); expect(buffer.stalled).toBe(false); expect(read(buffer, 10, 1)[0]).toEqual(new Float32Array(10).fill(1.0)); }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 22850883ff..8706af6968 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -9,6 +9,8 @@ export class AudioRingBuffer { readonly rate: number; readonly channels: number; #stalled = true; + // Held stalled for the part of the target above the floor, until the next floor update. + #parked = false; #underruns = 0; // Samples in the most recent write, i.e. one decoded chunk. The overflow band tolerates this @@ -80,7 +82,7 @@ export class AudioRingBuffer { return this.#buffer[0]?.length; } - /** Move the floor, which never exceeds the fixed capacity. */ + /** Move the floor, which never exceeds the fixed capacity, and end a park. */ setLatency(latency: Milli): void { const latencySamples = Math.ceil(this.rate * Second.fromMilli(latency)); if (latencySamples <= 0) throw new Error("empty buffer"); @@ -92,6 +94,7 @@ export class AudioRingBuffer { // already covers the new floor. if (latencySamples > this.#latencySamples) this.#stalled = true; this.#latencySamples = latencySamples; + this.#parked = false; // Resume a refill that the buffer already covers, which `write` would only notice on the // next frame, never for a source that has stopped. @@ -99,14 +102,15 @@ export class AudioRingBuffer { } /** - * Hold playback until the next write, keeping what is buffered. + * Hold playback until the next floor update, keeping what is buffered. * * For a rise in the part of the target held encoded upstream: the ring already covers its floor, - * so only the main thread knows when the received media covers the rest, and it releases the next - * frame exactly then. + * so only the main thread knows when the received media covers the rest. A write does not resume + * it, since output decoded before the park may still land. */ park(): void { this.#stalled = true; + this.#parked = true; } write(timestamp: Micro, data: Float32Array[]): void { @@ -188,9 +192,9 @@ export class AudioRingBuffer { } // Start playback once we've buffered the latency target. This is the only way out of a - // stall, so an underrun mid-playback refills to the target before resuming rather than - // playing the next chunk on an empty cushion. - if (this.length >= this.#latencySamples) { + // stall but a park, so an underrun mid-playback refills to the target before resuming rather + // than playing the next chunk on an empty cushion. + if (!this.#parked && this.length >= this.#latencySamples) { this.#stalled = false; } } @@ -213,6 +217,7 @@ export class AudioRingBuffer { this.#readIndex = 0; this.#writeIndex = 0; this.#stalled = true; + this.#parked = false; this.#anchored = false; } From 527a52bc2e07d6e668a2b9d12449798686a8ce25 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 18:59:14 -0700 Subject: [PATCH 06/10] fix(watch): park a ring that started before reporting its playhead The ring starts playing a report interval before the gate learns its playhead, and a rise in that window skipped the park, leaving the new hold uncovered. The gate now measures from the first frame it released on the timeline, where the ring anchors, until a report carries one. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 11 +++++++++++ js/watch/src/audio/buffer.ts | 14 ++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 66c2aff2bb..d00df38ff9 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -333,6 +333,17 @@ describe("Backpressure", () => { expect(gate.queued(ms(3000))).toBe(false); }); + // The ring starts playing a report interval before the gate learns its playhead, so a rise in that + // window parks from the frame the ring anchors on. + it("parks a ring that started before reporting its playhead", () => { + const gate = new Backpressure(1000, false, Milli(1000)); + gate.queued(ms(1000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + expect(gate.setLatency(Milli(2000))).toBe(true); + expect(gate.queued(ms(2000))).toBe(true); + }); + // An utterance that arrived whole before the reset changes nothing queued, so nothing would // report it again: forgetting it would hold its first frame until the next utterance. it("keeps what is queued across a reset", () => { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index d86164f587..0b542405ee 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -41,6 +41,9 @@ export class Backpressure { // The ring as last observed. The playhead is undefined until one is known on the current timeline. #playhead?: Micro; #stalled = true; + // The first frame released on the current timeline, where the ring anchors. It stands in for the + // playhead until a report carries one, since the ring may already be playing from it. + #anchor?: Micro; // Whether the ring is parked for a hold the queued media does not cover yet. The worklet keeps a // park until the next floor update, so only this decides when it ends: a write of output decoded // before the park cannot. @@ -81,7 +84,7 @@ export class Backpressure { // Parks whatever the ring was last reported doing: a report of a stall may be stale, and // parking a ring that is still stalled changes nothing. - const playhead = this.#playhead; + const playhead = this.#playhead ?? this.#anchor; this.#parked = (rise || this.#parked) && playhead !== undefined && !this.#covered(playhead + this.#headroom); if (this.#parked) this.#stalled = true; this.#release(); @@ -105,7 +108,7 @@ export class Backpressure { */ queued(newest: Micro | undefined): boolean { this.#queued = newest; - const playhead = this.#playhead; + const playhead = this.#playhead ?? this.#anchor; const resume = this.#parked && playhead !== undefined && this.#covered(playhead + this.#headroom); if (resume) this.#parked = false; this.#release(); @@ -135,6 +138,7 @@ export class Backpressure { /** The ring was flushed for a new timeline: its playhead is unknown until the next frame anchors it. */ reset(): void { this.#playhead = undefined; + this.#anchor = undefined; this.#stalled = true; this.#parked = false; this.#first = undefined; @@ -166,6 +170,12 @@ export class Backpressure { } #admit(timestamp: Micro): boolean { + const due = this.#due(timestamp); + if (due) this.#anchor ??= timestamp; + return due; + } + + #due(timestamp: Micro): boolean { const playhead = this.#playhead; if (!this.#stalled && playhead !== undefined) { if (timestamp - this.#headroom <= playhead) return true; From e4996871e1514f587a27fec3069dfabeead71c6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 19:39:18 -0700 Subject: [PATCH 07/10] fix(watch): refill past warmup the decoder discards before the ring anchors A buffered refill stops at the first frame that reaches the floor from the first frame released. A warmup longer than the floor discards all of that output, so the ring never anchored and never reported the playhead the gate waited on. The decoder now tells the gate where discarded output ends, and an unanchored refill measures from there. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 15 +++++++++++++++ js/watch/src/audio/buffer.ts | 17 +++++++++++++++++ js/watch/src/audio/decoder.ts | 3 +++ 3 files changed, 35 insertions(+) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index d00df38ff9..d925ffbb1e 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -344,6 +344,21 @@ describe("Backpressure", () => { expect(gate.queued(ms(2000))).toBe(true); }); + // A warmup longer than the floor discards everything the refill releases toward it, so the ring + // would never anchor and report the playhead the gate waits on. + it("refills past output discarded before the ring anchors", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + const wait = gate.wait(ms(520), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.discarded(ms(520)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(1020), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(1040), signal))).toBe("pending"); + }); + // An utterance that arrived whole before the reset changes nothing queued, so nothing would // report it again: forgetting it would hold its first frame until the next utterance. it("keeps what is queued across a reset", () => { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 0b542405ee..45f76c685b 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -115,6 +115,18 @@ export class Backpressure { return resume; } + /** + * Note that decoded output before `timestamp` was discarded (codec warmup), so a ring that has not + * reported a playhead anchors no earlier. A refill measured from the first frame released would + * otherwise stop before anything reached the ring. + */ + discarded(timestamp: Micro): void { + if (this.#playhead !== undefined) return; + if (this.#first !== undefined && timestamp > this.#first) this.#first = timestamp; + if (this.#anchor !== undefined && timestamp > this.#anchor) this.#anchor = timestamp; + this.#release(); + } + wait(timestamp: Micro, signal: AbortSignal): Promise { if (this.#closed || signal.aborted || this.#admit(timestamp)) return Promise.resolve(); return new Promise((resolve) => { @@ -274,6 +286,11 @@ export class AudioBuffer { return Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); } + /** Note that decoded output before `timestamp` was discarded, so the ring anchors no earlier. */ + discarded(timestamp: Micro): void { + this.#backpressure.discarded(timestamp); + } + /** Insert audio samples at the given timestamp, transferring their buffers. Handles out-of-order writes. */ insert(timestamp: Micro, data: Float32Array[]): void { const msg: Data = { type: "data", data, timestamp }; diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index e7b27f83df..751fbb2c2c 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -732,6 +732,9 @@ export class Decoder { const { timestamp, frameOffset, frames } = this.#terminal.span(sample); const timestampMilli = Milli.fromMicro(timestamp); if (frames === 0) { + // Warmup can discard more than the gate releases toward the floor, so it learns that the + // ring anchors no earlier than here. + this.#ring.peek()?.discarded(timestamp); sample.close(); return; } From a3544ca366de33b908e2d4bec2fb8a301cb68559 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 19:48:06 -0700 Subject: [PATCH 08/10] fix(watch): count a stalled refill from discarded warmup behind the playhead A replacement rendition keeps the ring and its reported playhead, and its warmup can outlast the floor, so a refill counted from the playhead still ran out inside the discarded output. The gate now counts the floor from the end of output discarded since the ring stalled, playhead or not. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 15 +++++++++++++++ js/watch/src/audio/buffer.ts | 21 ++++++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index d925ffbb1e..ba79e3d0c0 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -359,6 +359,21 @@ describe("Backpressure", () => { expect(Bun.peek.status(gate.wait(ms(1040), signal))).toBe("pending"); }); + // A replacement rendition keeps the ring and its playhead, and its warmup can outlast the floor. + it("refills a stalled ring past output discarded behind its playhead", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + gate.observe(ms(1000), true); + const signal = new AbortController().signal; + for (let t = 1000; t <= 1500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + const wait = gate.wait(ms(1520), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.discarded(ms(2000)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2500), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2520), signal))).toBe("pending"); + }); + // An utterance that arrived whole before the reset changes nothing queued, so nothing would // report it again: forgetting it would hold its first frame until the next utterance. it("keeps what is queued across a reset", () => { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 45f76c685b..d115ccbde1 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -54,6 +54,9 @@ export class Backpressure { // before the decoder's output reaches the ring and un-stalls it. #first?: Micro; #last?: Micro; + // Where output the decoder discarded whole (codec warmup) ends, since the ring stalled. The ring + // holds nothing before it, so a refill counts the floor from there. + #discarded?: Micro; #waiters: Array<{ timestamp: Micro; resolve: () => void }> = []; @@ -98,6 +101,7 @@ export class Backpressure { if (!stalled) { this.#first = undefined; this.#last = undefined; + this.#discarded = undefined; } this.#release(); } @@ -116,14 +120,15 @@ export class Backpressure { } /** - * Note that decoded output before `timestamp` was discarded (codec warmup), so a ring that has not - * reported a playhead anchors no earlier. A refill measured from the first frame released would - * otherwise stop before anything reached the ring. + * Note that decoded output before `timestamp` was discarded (codec warmup). A refill counts the + * floor from there, or a warmup longer than the floor would use it up before anything reached the + * ring, and a ring that has not reported a playhead anchors no earlier. */ discarded(timestamp: Micro): void { - if (this.#playhead !== undefined) return; - if (this.#first !== undefined && timestamp > this.#first) this.#first = timestamp; - if (this.#anchor !== undefined && timestamp > this.#anchor) this.#anchor = timestamp; + if (this.#discarded === undefined || timestamp > this.#discarded) this.#discarded = timestamp; + if (this.#playhead === undefined && this.#anchor !== undefined && timestamp > this.#anchor) { + this.#anchor = timestamp; + } this.#release(); } @@ -155,6 +160,7 @@ export class Backpressure { this.#parked = false; this.#first = undefined; this.#last = undefined; + this.#discarded = undefined; this.#release(); } @@ -207,7 +213,8 @@ export class Backpressure { const first = this.#first; const last = this.#last; if (first !== undefined && last !== undefined) { - const from = playhead ?? first; + let from: number = playhead ?? first; + if (this.#discarded !== undefined && this.#discarded > from) from = this.#discarded; if (timestamp - this.#headroom > from && last - this.#headroom >= from) return false; } this.#first ??= timestamp; From 34f9ea8295a9a0fd9889582c1b2d9f1fdf745df0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 20:49:04 -0700 Subject: [PATCH 09/10] fix(watch): count a buffered refill by the output that reaches the ring A stalled buffered refill stopped once the frames it released spanned the floor. Output the decoder discards (warmup, Opus pre-skip, a terminal trim) never lands, so a pre-skip longer than the floor left the ring stalled with nothing left to decode. Count the PCM inserted past the playhead plus the frames still decoding instead, answered by each decoder output, and drop the in-flight frames of a decoder that closes. This replaces `discarded()`. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 49 ++++++++---- js/watch/src/audio/buffer.ts | 123 +++++++++++++++++------------ js/watch/src/audio/decoder.test.ts | 29 ++++++- js/watch/src/audio/decoder.ts | 22 +++++- js/watch/src/audio/replay.ts | 7 +- 5 files changed, 158 insertions(+), 72 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index ba79e3d0c0..658ff9c1de 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -185,18 +185,21 @@ describe("AudioBuffer", () => { } }); - // Decoded audio can land short of the timestamp its frame was released at (Opus pre-skip), so a - // refill that stops at the floor exactly leaves the ring a sample short of un-stalling, with the - // gate waiting on the ring. - it("refills until a frame reaches the floor", async () => { - const { buffer, state, close } = open(); + // Decoded audio can land short of the timestamp its frame was released at (Opus pre-skip), so the + // refill counts the PCM that reaches the ring: stopping at the released frames would leave the + // ring short of un-stalling, with the gate waiting on it. + it("refills until the ring holds the floor", () => { + const { buffer, close } = open(); const signal = new AbortController().signal; try { - // The ring anchored on a first frame at 6 ms, still short of the floor. - await state(6_000, true, 0); - expect(Bun.peek.status(buffer.wait(100_000 as Micro, signal))).toBe("fulfilled"); - expect(Bun.peek.status(buffer.wait(120_000 as Micro, signal))).toBe("fulfilled"); - expect(Bun.peek.status(buffer.wait(140_000 as Micro, signal))).toBe("pending"); + for (let t = 0; t <= 100_000; t += 20_000) { + expect(Bun.peek.status(buffer.wait(t as Micro, signal))).toBe("fulfilled"); + // The first output lost 6 ms to pre-skip. + const start = Math.max(t, 6_000); + buffer.insert(start as Micro, [new Float32Array(((t + 20_000 - start) * 48) / 1000)]); + buffer.decoded((t + 20_000) as Micro); + } + expect(Bun.peek.status(buffer.wait(120_000 as Micro, signal))).toBe("pending"); } finally { close(); } @@ -210,7 +213,9 @@ describe("AudioBuffer", () => { try { await state(100_000, true, 0); expect(Bun.peek.status(buffer.wait(10_000_000 as Micro, signal))).toBe("fulfilled"); - const next = buffer.wait(10_050_000 as Micro, signal); + buffer.insert(10_000_000 as Micro, [new Float32Array(960)]); + buffer.decoded(10_020_000 as Micro); + const next = buffer.wait(10_020_000 as Micro, signal); expect(Bun.peek.status(next)).toBe("pending"); // The frame past the gap un-stalls the ring, which plays the gap out up to it. @@ -353,7 +358,7 @@ describe("Backpressure", () => { for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); const wait = gate.wait(ms(520), signal); expect(Bun.peek.status(wait)).toBe("pending"); - gate.discarded(ms(520)); + for (let t = 0; t <= 500; t += 20) gate.decoded(ms(t + 20)); expect(Bun.peek.status(wait)).toBe("fulfilled"); expect(Bun.peek.status(gate.wait(ms(1020), signal))).toBe("fulfilled"); expect(Bun.peek.status(gate.wait(ms(1040), signal))).toBe("pending"); @@ -368,10 +373,24 @@ describe("Backpressure", () => { for (let t = 1000; t <= 1500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); const wait = gate.wait(ms(1520), signal); expect(Bun.peek.status(wait)).toBe("pending"); - gate.discarded(ms(2000)); + // The decoder may answer several frames with one output. + gate.decoded(ms(1500)); + gate.decoded(ms(1520)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2020), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2040), signal))).toBe("pending"); + }); + + // A replacement rendition keeps the ring, and the decoder it replaced never answers its frames. + it("refills past frames a closed decoder dropped", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + const signal = new AbortController().signal; + for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + const wait = gate.wait(ms(520), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.dropped(); expect(Bun.peek.status(wait)).toBe("fulfilled"); - expect(Bun.peek.status(gate.wait(ms(2500), signal))).toBe("fulfilled"); - expect(Bun.peek.status(gate.wait(ms(2520), signal))).toBe("pending"); }); // An utterance that arrived whole before the reset changes nothing queued, so nothing would diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index d115ccbde1..24901d1ae6 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -13,8 +13,8 @@ import type { Data, Init, Latency, Reset, State, Truncate } from "./render"; * * - while the ring is stalled, once the media queued behind it covers the hold, so playback starts * or refills with the whole target ahead of the playhead, decoded or not. A buffered refill also - * stops at the first frame that reaches the floor, since the ring plays through and cannot shed - * what overflows it; + * stops once the PCM in the ring plus the frames still decoding reach the floor, since the ring + * plays through and cannot shed what overflows it; * - while playing, once the playhead is within the floor of it, keeping the ring on its floor; * - while playing live, also once the media queued behind it covers the hold, so media past the * target reaches the ring, whose skip band catches up to the live edge. Buffered mode plays @@ -41,22 +41,23 @@ export class Backpressure { // The ring as last observed. The playhead is undefined until one is known on the current timeline. #playhead?: Micro; #stalled = true; - // The first frame released on the current timeline, where the ring anchors. It stands in for the - // playhead until a report carries one, since the ring may already be playing from it. + // The first frame released on the current timeline. Until PCM lands and a report carries a + // playhead, it stands in for where the ring anchors, since the ring may already be playing. #anchor?: Micro; // Whether the ring is parked for a hold the queued media does not cover yet. The worklet keeps a // park until the next floor update, so only this decides when it ends: a write of output decoded // before the park cannot. #parked = false; - // The first and latest frames released since the ring stalled. A stalled ring refills to the - // floor from its parked playhead, but releasing it outright would let the whole lookahead through - // before the decoder's output reaches the ring and un-stalls it. - #first?: Micro; - #last?: Micro; - // Where output the decoder discarded whole (codec warmup) ends, since the ring stalled. The ring - // holds nothing before it, so a refill counts the floor from there. - #discarded?: Micro; + // A buffered refill counts what reaches the ring, not what it released: output the decoder + // discards (warmup, Opus pre-skip, a terminal trim) never lands, so counting released frames would + // stop the refill short with the ring stalled and nothing left to decode. Releasing outright would + // instead let the whole lookahead through before the output reaches the ring and un-stalls it. + // + // The frames released and not yet decoded, oldest first, and where the first PCM inserted on this + // timeline starts (the ring anchors there) and the newest ends. + #decoding: Micro[] = []; + #inserted?: { start: Micro; end: Micro }; #waiters: Array<{ timestamp: Micro; resolve: () => void }> = []; @@ -87,7 +88,7 @@ export class Backpressure { // Parks whatever the ring was last reported doing: a report of a stall may be stale, and // parking a ring that is still stalled changes nothing. - const playhead = this.#playhead ?? this.#anchor; + const playhead = this.#start(); this.#parked = (rise || this.#parked) && playhead !== undefined && !this.#covered(playhead + this.#headroom); if (this.#parked) this.#stalled = true; this.#release(); @@ -98,11 +99,6 @@ export class Backpressure { observe(playhead: Micro | undefined, stalled: boolean): void { this.#playhead = playhead; this.#stalled = stalled; - if (!stalled) { - this.#first = undefined; - this.#last = undefined; - this.#discarded = undefined; - } this.#release(); } @@ -112,7 +108,7 @@ export class Backpressure { */ queued(newest: Micro | undefined): boolean { this.#queued = newest; - const playhead = this.#playhead ?? this.#anchor; + const playhead = this.#start(); const resume = this.#parked && playhead !== undefined && this.#covered(playhead + this.#headroom); if (resume) this.#parked = false; this.#release(); @@ -120,18 +116,37 @@ export class Backpressure { } /** - * Note that decoded output before `timestamp` was discarded (codec warmup). A refill counts the - * floor from there, or a warmup longer than the floor would use it up before anything reached the - * ring, and a ring that has not reported a playhead anchors no earlier. + * Note that the decoder produced output ending at `end`, kept or discarded. It answers the oldest + * frame still decoding and every one it reaches past: a decoder may emit nothing for a frame. */ - discarded(timestamp: Micro): void { - if (this.#discarded === undefined || timestamp > this.#discarded) this.#discarded = timestamp; - if (this.#playhead === undefined && this.#anchor !== undefined && timestamp > this.#anchor) { - this.#anchor = timestamp; - } + decoded(end: Micro): void { + let answered = 0; + while (answered < this.#decoding.length && (answered === 0 || this.#decoding[answered] < end)) answered++; + this.#decoding.splice(0, answered); + this.#release(); + } + + /** Note that the decoder closed, dropping every frame still decoding. */ + dropped(): void { + this.#decoding = []; + this.#release(); + } + + /** Note that PCM from `start` to `end` reached the ring. */ + inserted(start: Micro, end: Micro): void { + const inserted = this.#inserted; + this.#inserted = { start: inserted?.start ?? start, end: inserted && inserted.end > end ? inserted.end : end }; this.#release(); } + /** Note that the ring dropped what it held at or after `timestamp`, keeping what is already due. */ + truncated(timestamp: Micro): void { + const inserted = this.#inserted; + const start = this.#playhead ?? inserted?.start; + if (!inserted || start === undefined || timestamp >= inserted.end) return; + this.#inserted = { start: inserted.start, end: (timestamp > start ? timestamp : start) as Micro }; + } + wait(timestamp: Micro, signal: AbortSignal): Promise { if (this.#closed || signal.aborted || this.#admit(timestamp)) return Promise.resolve(); return new Promise((resolve) => { @@ -158,9 +173,8 @@ export class Backpressure { this.#anchor = undefined; this.#stalled = true; this.#parked = false; - this.#first = undefined; - this.#last = undefined; - this.#discarded = undefined; + this.#decoding = []; + this.#inserted = undefined; this.#release(); } @@ -187,9 +201,17 @@ export class Backpressure { return this.#queued !== undefined && this.#queued >= timestamp + this.#hold; } + // Where the ring plays from: its reported playhead, else where it anchored or will anchor. + #start(): Micro | undefined { + return this.#playhead ?? this.#inserted?.start ?? this.#anchor; + } + #admit(timestamp: Micro): boolean { const due = this.#due(timestamp); - if (due) this.#anchor ??= timestamp; + if (due) { + this.#anchor ??= timestamp; + if (this.#buffered) this.#decoding.push(timestamp); + } return due; } @@ -204,22 +226,15 @@ export class Backpressure { // A live ring sheds what overflows its floor, so it refills from everything that is due. if (!this.#buffered) return true; - // Refill from the parked playhead, or from the first frame while the ring has not anchored. The - // first frame always goes, so a ring that needs it to anchor, or to see past a gap, never waits - // on a playhead that will not move. Frames keep going until one reaches the floor: the ring - // only un-stalls once it holds that much, and decoded audio can land a little short of the - // timestamp it was released at (Opus pre-skip), so stopping at the floor exactly can leave the - // ring stalled with the gate waiting on it. - const first = this.#first; - const last = this.#last; - if (first !== undefined && last !== undefined) { - let from: number = playhead ?? first; - if (this.#discarded !== undefined && this.#discarded > from) from = this.#discarded; - if (timestamp - this.#headroom > from && last - this.#headroom >= from) return false; - } - this.#first ??= timestamp; - this.#last = last === undefined ? timestamp : (Math.max(last, timestamp) as Micro); - return true; + // Refill while the PCM in the ring past its playhead, plus the span of frames still decoding, is + // short of the floor. The ring un-stalls once it holds the floor, and output the decoder + // discards never counts, so the refill cannot stop with the ring short of it. + const inserted = this.#inserted; + const start = playhead ?? inserted?.start; + const buffered = inserted && start !== undefined ? Math.max(0, inserted.end - start) : 0; + const decoding = this.#decoding; + const pending = decoding.length > 0 ? (decoding[decoding.length - 1] as number) - (decoding[0] as number) : 0; + return buffered + pending < this.#headroom; } } @@ -293,13 +308,20 @@ export class AudioBuffer { return Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); } - /** Note that decoded output before `timestamp` was discarded, so the ring anchors no earlier. */ - discarded(timestamp: Micro): void { - this.#backpressure.discarded(timestamp); + /** Note that the decoder produced output ending at `end` on its own timeline, kept or discarded. */ + decoded(end: Micro): void { + this.#backpressure.decoded(end); + } + + /** Note that the decoder closed, dropping every frame still decoding. */ + dropped(): void { + this.#backpressure.dropped(); } /** Insert audio samples at the given timestamp, transferring their buffers. Handles out-of-order writes. */ insert(timestamp: Micro, data: Float32Array[]): void { + const end = (timestamp + Micro.fromSecond(((data[0]?.length ?? 0) / this.rate) as Second)) as Micro; + this.#backpressure.inserted(timestamp, end); const msg: Data = { type: "data", data, timestamp }; // Transfer the ArrayBuffers to avoid a copy: the caller loses access to them. this.#worklet.port.postMessage( @@ -338,6 +360,7 @@ export class AudioBuffer { * Unlike `reset()` this keeps playing, so there's no gap at the handover. */ truncate(timestamp: Micro): void { + this.#backpressure.truncated(timestamp); const msg: Truncate = { type: "truncate", timestamp }; this.#worklet.port.postMessage(msg); } diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index c1e977fb65..a5c6120d63 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -401,11 +401,12 @@ for (const cmaf of [false, true]) { } expect(playback.inserted()).toEqual([]); - // The live edge moves on, so the head is complete and plays from its oldest frame. + // The live edge moves on, so the head is complete and plays from its oldest frame, refilling + // the 100 ms floor. frameTimestamp = 140_000; playback.push({ ...frame(), group: 8 }); await microtasks(); - expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000, 100_000]); + expect(playback.inserted()).toEqual([0, 20_000, 40_000, 60_000, 80_000]); } finally { playback.close(); await microtasks(); @@ -430,6 +431,30 @@ for (const cmaf of [false, true]) { } }); + // Pre-skip longer than the decoded floor discards every output the refill released toward it, so + // the refill has to count what reaches the ring, not what it released. + if (!cmaf) { + it("starts a buffered Opus head whose pre-skip outlasts the floor", async () => { + const playback = await play(Milli(1000), true); + try { + // OpusHead, version 1, stereo, a 520 ms pre-skip of 24,960 samples: past the 500 ms floor, yet + // within the head the harness covers, since it queues nothing behind a released batch. + playback.restore({ description: "4f707573486561640102" + "8061" + "80bb0000" + "0000" + "00" }); + await microtasks(); + const target = playback.target(); + while (frameTimestamp <= (target + 2000) * 1000) { + playback.push(frame()); + await microtasks(); + } + // The first output to outlast the pre-skip lands at the start of the timeline. + expect(playback.inserted()[0]).toBe(20_000); + } finally { + playback.close(); + await microtasks(); + } + }); + } + // A fresh encoder after a declared end may restart its timeline below the old one, and the decoder // resets for it before anything is judged out of order. it(`decodes a ${cmaf ? "CMAF" : "legacy"} fresh encoder below the timeline it replaced`, async () => { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 751fbb2c2c..3fa3ec810d 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -1,4 +1,4 @@ -import { Micro, Milli } from "@moq/clock"; +import { Micro, Milli, type Second } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Util from "@moq/hang/util"; @@ -459,6 +459,8 @@ export class Decoder { }); effect.cleanup(() => { if (decoder.state !== "closed") decoder.close(); + // Closing drops what is still decoding, which a refill would otherwise wait on. + this.#ring.peek()?.dropped(); }); // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header. @@ -552,6 +554,8 @@ export class Decoder { }); effect.cleanup(() => { if (decoder.state !== "closed") decoder.close(); + // Closing drops what is still decoding, which a refill would otherwise wait on. + this.#ring.peek()?.dropped(); }); // Configure decoder with description from catalog @@ -729,12 +733,22 @@ export class Decoder { } #emit(sample: AudioData) { + // Whatever becomes of it, the output answers the frames it was decoded from, so the gate counts + // only what still decodes. Reported after any insert, so the gate always counts the frame as + // either decoding or in the ring. + const decoded = (sample.timestamp + + Micro.fromSecond((sample.numberOfFrames / sample.sampleRate) as Second)) as Micro; + try { + this.#place(sample); + } finally { + this.#ring.peek()?.decoded(decoded); + } + } + + #place(sample: AudioData) { const { timestamp, frameOffset, frames } = this.#terminal.span(sample); const timestampMilli = Milli.fromMicro(timestamp); if (frames === 0) { - // Warmup can discard more than the gate releases toward the floor, so it learns that the - // ring anchors no earlier than here. - this.#ring.peek()?.discarded(timestamp); sample.close(); return; } diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index e80241680d..b8cfebfac2 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -195,7 +195,12 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato frame.set(Milli(entry.duration)); await gate.wait(timestamp, abort.signal); if (abort.signal.aborted) return; - ring.write(timestamp, [new Float32Array(Math.round((entry.duration * rate) / 1000)).fill(0.5)]); + const pcm = new Float32Array(Math.round((entry.duration * rate) / 1000)).fill(0.5); + ring.write(timestamp, [pcm]); + // The decoder reports each output to the gate the way `AudioBuffer` and `Decoder.#emit` do. + const end = (timestamp + micros((pcm.length / rate) * 1000)) as Micro; + gate.inserted(timestamp, end); + gate.decoded(end); } })().catch((error: unknown) => { failure = { error }; From 15cc17f7d735612612ce79c87e367ae249cb9f57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 21:01:58 -0700 Subject: [PATCH 10/10] fix(watch): compare a playing frame to where its output lands Opus pre-skip stamps decoded PCM behind the input it came from, so the playing-state bound compared a frame's input timestamp to the PCM playhead and kept the ring short by the pre-skip: past the floor, it drained the ring on every refill. The decoder now reports where each kept output lands, and the gate maps the playhead back by that offset, for release and for park coverage alike. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 14 +++++++++++++ js/watch/src/audio/buffer.ts | 34 ++++++++++++++++++++++--------- js/watch/src/audio/decoder.ts | 9 +++++--- js/watch/src/audio/replay.ts | 2 +- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 658ff9c1de..aacabd46e2 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -381,6 +381,20 @@ describe("Backpressure", () => { expect(Bun.peek.status(gate.wait(ms(2040), signal))).toBe("pending"); }); + // Opus pre-skip stamps the PCM behind the input it was decoded from, so a playing ring compares a + // frame to its playhead where its output lands, or a pre-skip past the floor would drain it. + it("releases a playing frame by where its output lands", () => { + const gate = new Backpressure(1000, true, Milli(2000)); + gate.queued(ms(60_000)); + gate.observe(ms(0), false); + const signal = new AbortController().signal; + const wait = gate.wait(ms(1500), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.decoded(ms(1020), ms(20)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(1520), signal))).toBe("pending"); + }); + // A replacement rendition keeps the ring, and the decoder it replaced never answers its frames. it("refills past frames a closed decoder dropped", () => { const gate = new Backpressure(1000, true, Milli(2000)); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 24901d1ae6..7ab8c89ecc 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -15,7 +15,8 @@ import type { Data, Init, Latency, Reset, State, Truncate } from "./render"; * or refills with the whole target ahead of the playhead, decoded or not. A buffered refill also * stops once the PCM in the ring plus the frames still decoding reach the floor, since the ring * plays through and cannot shed what overflows it; - * - while playing, once the playhead is within the floor of it, keeping the ring on its floor; + * - while playing, once the playhead is within the floor of where its output lands, keeping the + * ring on its floor; * - while playing live, also once the media queued behind it covers the hold, so media past the * target reaches the ring, whose skip band catches up to the live edge. Buffered mode plays * through instead. @@ -58,6 +59,10 @@ export class Backpressure { // timeline starts (the ring anchors there) and the newest ends. #decoding: Micro[] = []; #inserted?: { start: Micro; end: Micro }; + // How far output lands in the ring behind the frame it was decoded from (Opus pre-skip). A playing + // ring compares a frame to its playhead where the output lands, or an offset past the floor would + // drain it. + #offset = 0; #waiters: Array<{ timestamp: Micro; resolve: () => void }> = []; @@ -116,10 +121,12 @@ export class Backpressure { } /** - * Note that the decoder produced output ending at `end`, kept or discarded. It answers the oldest - * frame still decoding and every one it reaches past: a decoder may emit nothing for a frame. + * Note that the decoder produced output ending at `end`, which lands in the ring up to `landed` + * unless it was discarded. It answers the oldest frame still decoding and every one it reaches + * past: a decoder may emit nothing for a frame. */ - decoded(end: Micro): void { + decoded(end: Micro, landed?: Micro): void { + if (landed !== undefined) this.#offset = end - landed; let answered = 0; while (answered < this.#decoding.length && (answered === 0 || this.#decoding[answered] < end)) answered++; this.#decoding.splice(0, answered); @@ -175,6 +182,7 @@ export class Backpressure { this.#parked = false; this.#decoding = []; this.#inserted = undefined; + this.#offset = 0; this.#release(); } @@ -201,9 +209,12 @@ export class Backpressure { return this.#queued !== undefined && this.#queued >= timestamp + this.#hold; } - // Where the ring plays from: its reported playhead, else where it anchored or will anchor. + // The frame the ring plays from: its reported playhead, else where it anchored or will anchor, + // mapped back to the frame its output came from, which is what the queued media covers. #start(): Micro | undefined { - return this.#playhead ?? this.#inserted?.start ?? this.#anchor; + const start = this.#playhead ?? this.#inserted?.start; + if (start !== undefined) return (start + this.#offset) as Micro; + return this.#anchor; } #admit(timestamp: Micro): boolean { @@ -218,7 +229,7 @@ export class Backpressure { #due(timestamp: Micro): boolean { const playhead = this.#playhead; if (!this.#stalled && playhead !== undefined) { - if (timestamp - this.#headroom <= playhead) return true; + if (timestamp - this.#offset - this.#headroom <= playhead) return true; return !this.#buffered && this.#covered(timestamp); } @@ -308,9 +319,12 @@ export class AudioBuffer { return Milli.fromSecond((this.#backpressure.floor / this.rate) as Second); } - /** Note that the decoder produced output ending at `end` on its own timeline, kept or discarded. */ - decoded(end: Micro): void { - this.#backpressure.decoded(end); + /** + * Note that the decoder produced output ending at `end` on its own timeline, which lands in the ring + * up to `landed` unless it was discarded. + */ + decoded(end: Micro, landed?: Micro): void { + this.#backpressure.decoded(end, landed); } /** Note that the decoder closed, dropping every frame still decoding. */ diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 3fa3ec810d..dc0cd40bd0 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -738,14 +738,16 @@ export class Decoder { // either decoding or in the ring. const decoded = (sample.timestamp + Micro.fromSecond((sample.numberOfFrames / sample.sampleRate) as Second)) as Micro; + let landed: Micro | undefined; try { - this.#place(sample); + landed = this.#place(sample); } finally { - this.#ring.peek()?.decoded(decoded); + this.#ring.peek()?.decoded(decoded, landed); } } - #place(sample: AudioData) { + // Trim the output and write it to the ring, returning where the PCM written ends. + #place(sample: AudioData): Micro | undefined { const { timestamp, frameOffset, frames } = this.#terminal.span(sample); const timestampMilli = Milli.fromMicro(timestamp); if (frames === 0) { @@ -808,6 +810,7 @@ export class Decoder { ring.insert(timestamp, channelData); sample.close(); + return (timestamp + durationMicro) as Micro; } #addDecodeBuffered(start: Milli, end: Milli): void { diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 0d3f4e9650..82a9b91d63 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -200,7 +200,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato // The decoder reports each output to the gate the way `AudioBuffer` and `Decoder.#emit` do. const end = (timestamp + micros((pcm.length / rate) * 1000)) as Micro; gate.inserted(timestamp, end); - gate.decoded(end); + gate.decoded(end, end); } })().catch((error: unknown) => { failure = { error };