Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion doc/concept/audio-jitter.md
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,9 @@ The advertised floor is the one term that can exceed that bound, and it is
deliberately not clamped to it. The ceiling is a property of the histogram's
size; the floor is a number the publisher declared about itself, so clamping
the floor to the ceiling would under-buffer precisely the publisher that was
honest about a long flush.
honest about a long flush. Honoring it must not cost memory in proportion to
the declared number, so the browser decodes at most 500 ms of any target and
holds the rest as the encoded frames it actually received.

Policy limits sit above the algorithm. A receiver's own configured maximum and
the track's retention window both cap what it may hold, and neither is the
Expand Down
4 changes: 2 additions & 2 deletions js/watch/bench/ring-page.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/** The page `ring.ts` loads: N players feeding the real render worklet through `AudioBuffer`. */
import { Micro } from "@moq/clock";
import { Micro, Milli } from "@moq/clock";
import { AudioBuffer } from "../src/audio/buffer";

const RATE = 48_000;
Expand All @@ -12,7 +12,7 @@ const players = Array.from({ length: count }, () => {
const worklet = new AudioWorkletNode(context, "render", { outputChannelCount: [2] });
worklet.connect(context.destination);
// The 200 ms floor, unbuffered: what a live viewer plays at.
const buffer = new AudioBuffer(worklet, { channels: 2, rate: RATE, latency: RATE / 5, buffered: false });
const buffer = new AudioBuffer(worklet, { channels: 2, rate: RATE, delay: Milli(200), buffered: false });
return { worklet, buffer };
});
await context.resume();
Expand Down
322 changes: 302 additions & 20 deletions js/watch/src/audio/buffer.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import { describe, expect, it } from "bun:test";
import { type Micro, Milli } from "@moq/clock";
import { AudioBuffer } from "./buffer";
import type { Message, State } from "./render";
import { Micro, Milli } from "@moq/clock";
import { AudioBuffer, Backpressure } from "./buffer";
import type { Init, Message, State } from "./render";
import { AudioRingBuffer } from "./ring-buffer";

const HOUR = Milli(60 * 60 * 1000);
const ms = (value: number) => Micro.fromMilli(Milli(value));

// A buffered buffer over a real port, with the worklet's end of it left to the test.
function open(): {
Expand All @@ -13,8 +17,8 @@ function open(): {
const channel = new MessageChannel();
const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, {
channels: 1,
rate: 1000,
latency: 100,
rate: 48_000,
delay: Milli(100),
buffered: true,
});
return {
Expand Down Expand Up @@ -49,8 +53,8 @@ describe("AudioBuffer", () => {
const channel = new MessageChannel();
const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, {
channels: 2,
rate: 1000,
latency: 100,
rate: 48_000,
delay: Milli(100),
buffered: true,
});
const messages: Message[] = [];
Expand All @@ -66,12 +70,12 @@ describe("AudioBuffer", () => {
buffer.insert(500_000 as Micro, [left, right]);
expect(left.byteLength).toBe(0);
expect(right.byteLength).toBe(0);
buffer.setLatency(200);
buffer.setLatency(Milli(200));
buffer.truncate(502_000 as Micro);
buffer.reset();
await received;
expect(messages).toEqual([
{ type: "init", channels: 2, rate: 1000, latency: Milli(100), buffered: true },
{ type: "init", channels: 2, rate: 48_000, latency: Milli(100), buffered: true },
{
type: "data",
timestamp: 500_000 as Micro,
Expand Down Expand Up @@ -181,18 +185,21 @@ describe("AudioBuffer", () => {
}
});

// Decoded audio can land short of the timestamp its frame was released at (Opus pre-skip), so a
// refill that stops at the floor exactly leaves the ring a sample short of un-stalling, with the
// gate waiting on the ring.
it("refills until a frame reaches the floor", async () => {
const { buffer, state, close } = open();
// Decoded audio can land short of the timestamp its frame was released at (Opus pre-skip), so the
// refill counts the PCM that reaches the ring: stopping at the released frames would leave the
// ring short of un-stalling, with the gate waiting on it.
it("refills until the ring holds the floor", () => {
const { buffer, close } = open();
const signal = new AbortController().signal;
try {
// The ring anchored on a first frame at 6 ms, still short of the floor.
await state(6_000, true, 0);
expect(Bun.peek.status(buffer.wait(100_000 as Micro, signal))).toBe("fulfilled");
expect(Bun.peek.status(buffer.wait(120_000 as Micro, signal))).toBe("fulfilled");
expect(Bun.peek.status(buffer.wait(140_000 as Micro, signal))).toBe("pending");
for (let t = 0; t <= 100_000; t += 20_000) {
expect(Bun.peek.status(buffer.wait(t as Micro, signal))).toBe("fulfilled");
// The first output lost 6 ms to pre-skip.
const start = Math.max(t, 6_000);
buffer.insert(start as Micro, [new Float32Array(((t + 20_000 - start) * 48) / 1000)]);
buffer.decoded((t + 20_000) as Micro);
}
expect(Bun.peek.status(buffer.wait(120_000 as Micro, signal))).toBe("pending");
} finally {
close();
}
Expand All @@ -206,7 +213,9 @@ describe("AudioBuffer", () => {
try {
await state(100_000, true, 0);
expect(Bun.peek.status(buffer.wait(10_000_000 as Micro, signal))).toBe("fulfilled");
const next = buffer.wait(10_050_000 as Micro, signal);
buffer.insert(10_000_000 as Micro, [new Float32Array(960)]);
buffer.decoded(10_020_000 as Micro);
const next = buffer.wait(10_020_000 as Micro, signal);
expect(Bun.peek.status(next)).toBe("pending");

// The frame past the gap un-stalls the ring, which plays the gap out up to it.
Expand Down Expand Up @@ -256,3 +265,276 @@ describe("AudioBuffer", () => {
}
});
});

describe("Backpressure", () => {
it("releases every live frame when the whole target fits the floor", () => {
const gate = new Backpressure(1000, false, Milli(100));
gate.observe(ms(0), false);
expect(Bun.peek.status(gate.wait(ms(10_000), new AbortController().signal))).toBe("fulfilled");
});

it("holds buffered playback to the floor ahead of the playhead", () => {
const gate = new Backpressure(1000, true, Milli(200));
gate.observe(ms(0), false);
const wait = gate.wait(ms(1000), new AbortController().signal);
gate.observe(ms(799), false);
expect(Bun.peek.status(wait)).toBe("pending");
gate.observe(ms(800), false);
expect(Bun.peek.status(wait)).toBe("fulfilled");
});

// Two seconds is a 500 ms floor and 1.5 s held encoded.
it("releases a live frame once the queued media covers the hold behind it", () => {
const gate = new Backpressure(1000, false, Milli(2000));
gate.observe(ms(0), false);
const wait = gate.wait(ms(1000), new AbortController().signal);
gate.queued(ms(2499));
expect(Bun.peek.status(wait)).toBe("pending");
gate.queued(ms(2500));
expect(Bun.peek.status(wait)).toBe("fulfilled");
});

it("plays buffered media through instead of releasing it ahead of the floor", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.observe(ms(0), false);
const wait = gate.wait(ms(1000), new AbortController().signal);
gate.queued(ms(60_000));
expect(Bun.peek.status(wait)).toBe("pending");
gate.observe(ms(500), false);
expect(Bun.peek.status(wait)).toBe("fulfilled");
});

it("starts a stalled ring only once the queued media covers the hold", () => {
const gate = new Backpressure(1000, true, Milli(2000));
const wait = gate.wait(ms(0), new AbortController().signal);
gate.queued(ms(1499));
expect(Bun.peek.status(wait)).toBe("pending");
gate.queued(ms(1500));
expect(Bun.peek.status(wait)).toBe("fulfilled");
});

// The ring un-stalls only once the decoder's output reaches it, so covered media released
// outright would overflow the fixed ring before the worklet reports it playing.
it("refills a covered buffered ring only to its floor", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.queued(ms(60_000));
const signal = new AbortController().signal;
expect(Bun.peek.status(gate.wait(ms(0), signal))).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(500), signal))).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(520), signal))).toBe("pending");
});

// A refill that released its floor before the ring reported playing holds the next frame back, so
// no write could end the park. The queued media covering the hold is what ends it, once.
it("resumes a park once the queued media covers the hold", () => {
const gate = new Backpressure(1000, true, Milli(1000));
expect(gate.queued(ms(1000))).toBe(false);
const signal = new AbortController().signal;
for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
gate.observe(ms(0), true);
expect(gate.setLatency(Milli(2000))).toBe(true);
expect(gate.queued(ms(1999))).toBe(false);
expect(gate.queued(ms(2000))).toBe(true);
expect(gate.queued(ms(3000))).toBe(false);
});

// The ring starts playing a report interval before the gate learns its playhead, so a rise in that
// window parks from the frame the ring anchors on.
it("parks a ring that started before reporting its playhead", () => {
const gate = new Backpressure(1000, false, Milli(1000));
gate.queued(ms(1000));
const signal = new AbortController().signal;
for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
expect(gate.setLatency(Milli(2000))).toBe(true);
expect(gate.queued(ms(2000))).toBe(true);
});

// A warmup longer than the floor discards everything the refill releases toward it, so the ring
// would never anchor and report the playhead the gate waits on.
it("refills past output discarded before the ring anchors", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.queued(ms(60_000));
const signal = new AbortController().signal;
for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
const wait = gate.wait(ms(520), signal);
expect(Bun.peek.status(wait)).toBe("pending");
for (let t = 0; t <= 500; t += 20) gate.decoded(ms(t + 20));
expect(Bun.peek.status(wait)).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(1020), signal))).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(1040), signal))).toBe("pending");
});

// A replacement rendition keeps the ring and its playhead, and its warmup can outlast the floor.
it("refills a stalled ring past output discarded behind its playhead", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.queued(ms(60_000));
gate.observe(ms(1000), true);
const signal = new AbortController().signal;
for (let t = 1000; t <= 1500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
const wait = gate.wait(ms(1520), signal);
expect(Bun.peek.status(wait)).toBe("pending");
// The decoder may answer several frames with one output.
gate.decoded(ms(1500));
gate.decoded(ms(1520));
expect(Bun.peek.status(wait)).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(2020), signal))).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(2040), signal))).toBe("pending");
});

// Opus pre-skip stamps the PCM behind the input it was decoded from, so a playing ring compares a
// frame to its playhead where its output lands, or a pre-skip past the floor would drain it.
it("releases a playing frame by where its output lands", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.queued(ms(60_000));
gate.observe(ms(0), false);
const signal = new AbortController().signal;
const wait = gate.wait(ms(1500), signal);
expect(Bun.peek.status(wait)).toBe("pending");
gate.decoded(ms(1020), ms(20));
expect(Bun.peek.status(wait)).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(1520), signal))).toBe("pending");
});

// A replacement rendition keeps the ring, and the decoder it replaced never answers its frames.
it("refills past frames a closed decoder dropped", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.queued(ms(60_000));
const signal = new AbortController().signal;
for (let t = 0; t <= 500; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
const wait = gate.wait(ms(520), signal);
expect(Bun.peek.status(wait)).toBe("pending");
gate.dropped();
expect(Bun.peek.status(wait)).toBe("fulfilled");
});

// An utterance that arrived whole before the reset changes nothing queued, so nothing would
// report it again: forgetting it would hold its first frame until the next utterance.
it("keeps what is queued across a reset", () => {
const gate = new Backpressure(1000, true, Milli(2000));
gate.queued(ms(5000));
gate.observe(ms(0), false);
gate.reset();
expect(Bun.peek.status(gate.wait(ms(0), new AbortController().signal))).toBe("fulfilled");
});
});

// A worklet whose port records what the buffer posts and delivers state reports synchronously.
function fake(delay: Milli, buffered = false) {
const sent: Message[] = [];
const port = Object.assign(new EventTarget(), {
postMessage: (msg: Message) => sent.push(msg),
start() {},
});
const buffer = new AudioBuffer({ port } as unknown as AudioWorkletNode, {
channels: 1,
rate: 1000,
delay,
buffered,
});
const state = (timestamp: Micro, stalled: boolean, epoch: number) => {
const data: State = { type: "state", timestamp, stalled, anchored: true, underruns: 0, epoch };
port.dispatchEvent(new MessageEvent("message", { data }));
};
return { buffer, sent, state };
}

describe("AudioBuffer past its decoded floor", () => {
// A catalog advertising an hour of jitter once sized the ring to two hours of stereo PCM, 4 GiB.
it("allocates the same ring for an hour of delay as for a tenth of a second", () => {
const bytes = [Milli(100), HOUR].map((delay) => {
const sent: Message[] = [];
const port = Object.assign(new EventTarget(), {
postMessage: (msg: Message) => sent.push(msg),
start() {},
});
new AudioBuffer({ port } as unknown as AudioWorkletNode, {
channels: 2,
rate: 48_000,
delay,
buffered: false,
}).close();
const ring = new AudioRingBuffer(sent[0] as Init);
return ring.capacity * ring.channels * Float32Array.BYTES_PER_ELEMENT;
});
expect(bytes[1]).toBe(bytes[0]);
expect(bytes[1]).toBeLessThanOrEqual(2 ** 20);
});

it("holds a frame encoded until an hour of queued media is behind it", () => {
const { buffer } = fake(HOUR);
try {
// The ring holds 500 ms decoded; the rest of the hour waits upstream.
const hold = HOUR - 500;
const wait = buffer.wait(ms(0), new AbortController().signal);
buffer.queued(ms(hold - 1));
expect(Bun.peek.status(wait)).toBe("pending");
buffer.queued(ms(hold));
expect(Bun.peek.status(wait)).toBe("fulfilled");
} finally {
buffer.close();
}
});

// One second is a 500 ms floor and 500 ms held encoded. Playing 100 ms in, a rise to 1.5 s needs
// queued media through 1600 ms to keep playing. A report of a stall may be stale, so it parks
// regardless: parking a ring that is still stalled changes nothing.
for (const stalled of [false, true]) {
for (const [queued, parked] of [
[1599, true],
[1600, false],
] as const) {
it(`${parked ? "parks" : "keeps playing"} on a rise with ${queued} ms queued, reported ${stalled ? "stalled" : "playing"}`, () => {
const { buffer, sent, state } = fake(Milli(1000));
try {
state(ms(100), stalled, 0);
buffer.queued(ms(queued));
buffer.setLatency(Milli(1500));
expect(sent.slice(1)).toEqual([
{ type: "latency", latency: Milli(500), ...(parked ? { park: 1 } : {}) },
]);
} finally {
buffer.close();
}
});
}
}

// The ring resumes on any floor update without a park, so a fall that leaves the remaining hold
// uncovered parks again, and the queued media covering it resumes the ring.
it("keeps a park across a fall until the queued media covers the hold", () => {
const { buffer, sent, state } = fake(Milli(1000));
try {
state(ms(0), false, 0);
buffer.queued(ms(1000));
buffer.setLatency(Milli(2000));
buffer.setLatency(Milli(1800));
buffer.queued(ms(1800));
buffer.setLatency(Milli(1600));
expect(sent.slice(1)).toEqual([
{ type: "latency", latency: Milli(500), park: 1 },
{ type: "latency", latency: Milli(500), park: 2 },
{ type: "latency", latency: Milli(500) },
{ type: "latency", latency: Milli(500) },
]);
} finally {
buffer.close();
}
});

// A report the worklet posted before it applied the park still says the ring is playing.
it("ignores state from before a park", () => {
const { buffer, state } = fake(Milli(1000));
try {
state(ms(200), false, 0);
buffer.queued(ms(1599));
buffer.setLatency(Milli(1500));
const wait = buffer.wait(ms(700), new AbortController().signal);
state(ms(200), false, 0);
expect(Bun.peek.status(wait)).toBe("pending");
state(ms(200), false, 1);
expect(Bun.peek.status(wait)).toBe("fulfilled");
} finally {
buffer.close();
}
});
});
Loading
Loading