Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion doc/concept/audio-jitter.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ group is observed when the stall clears. A missing group, one that has delivered
no frame, is waited for only the playout delay plus any lookahead, since past that
it would play late: a reordered backlog's holes waited out to the whole budget
would delay the frames behind them, which the estimator would measure as
jitter. A caller that estimates keeps the budget past the target, which is why
jitter. The browser player waits less still, only until its clock reaches the
group past the hole: a viewer resuming behind a replayed backlog, which arrives
newest first, would otherwise hold the live audio for the backlog's oldest group. A caller that estimates keeps the budget past the target, which is why
`moq play` waits on a stalled audio group for the delay plus the estimate's 2 s
ceiling. The observation is also only as honest as
the caller is prompt: `read()` is timed when it resolves, so a player drains it
Expand Down
6 changes: 6 additions & 0 deletions js/hang/api/container.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export class Consumer {
export interface ConsumerProps {
delay?: Signals.GetterInit<Net.Track.Delay>;
format: Format;
playhead?: Playhead;
predecessor?: Consumer;
wait?: Signals.GetterInit<Net.Clock.Duration | undefined>;
warmup?: Net.Clock.Duration;
Expand Down Expand Up @@ -79,6 +80,11 @@ export * as Loc from "@moq/loc";

export const MAX_AGE: Net.Clock.Duration;

export interface Playhead {
readonly anchor: Signals.Getter<unknown>;
now(): Net.Clock.Duration | undefined;
}

export type Read = {
frame: ReceivedFrame;
group: bigint;
Expand Down
114 changes: 113 additions & 1 deletion js/hang/src/container/consumer.holes.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { expect, test } from "bun:test";
import { Duration } from "@moq/clock";
import { Duration, now } from "@moq/clock";
import { Manual } from "@moq/clock/test";
import { Time, Track } from "@moq/net";
import { Signal } from "@moq/signals";
import { Consumer, type Read } from "./consumer.ts";
import { encodeFrame, Format } from "./legacy.ts";

Expand Down Expand Up @@ -475,3 +476,114 @@ test("a hole is waited for the wait while an arrived group plays up to the max d
}
}
});

test.each([
["the playout clock reaching the group past it", 900, 100],
["the wait, while the clock trails", 0, 500],
["the wait, without a clock", undefined, 500],
])("a hole ends at %s", async (_, start, after) => {
const clock = new Manual();
const installed = clock.install();
const origin = now();
// A playout clock presenting `start` ms of media now, advancing with the manual clock.
const playhead =
start === undefined
? undefined
: { now: () => Duration.millis(start).add(now().since(origin)), anchor: new Signal(0) };
const track = new Track.Producer("audio");
const consumer = new Consumer(track.subscribe({ delay: { max: Duration.millis(30_000) } }), {
format: new Format("audio"),
delay: { max: Duration.millis(2_000) },
wait: Duration.millis(500),
playhead,
});
const results: Read[] = [];
const reading = (async () => {
for (;;) {
const read = await consumer.next();
if (!read) return;
results.push(read);
}
})();
const write = (sequence: bigint, millis: number) => {
const group = track.createGroup(sequence);
group.writeFrame({
payload: encodeFrame(new Uint8Array([1]), Duration.millis(millis)),
timestamp: Time.Timestamp.fromMicros(millis * 1000),
});
group.finish();
};
try {
write(0n, 0);
await flush();
expect(results).toHaveLength(1);
// A resume: the groups past the hole start a second of media later, inside the max delay.
write(50n, 1_000);
await flush();
await clock.advance(Duration.millis(after - 1));
expect(results).toHaveLength(1);
await clock.advance(Duration.millis(1));
expect(results.slice(1).map((read) => read.kind)).toEqual(["gap", "frame"]);
} finally {
consumer.close();
track.finish();
track.close();
try {
await reading;
} finally {
installed.close();
}
}
});

test("a hole ends once the playout clock jumps past the group past it", async () => {
const clock = new Manual();
const installed = clock.install();
// A clock trailing the group past the hole until it re-anchors a second later.
let position = Duration.ZERO;
const anchor = new Signal(0);
const track = new Track.Producer("audio");
const consumer = new Consumer(track.subscribe({ delay: { max: Duration.millis(30_000) } }), {
format: new Format("audio"),
delay: { max: Duration.millis(2_000) },
wait: Duration.millis(500),
playhead: { now: () => position, anchor },
});
const results: Read[] = [];
const reading = (async () => {
for (;;) {
const read = await consumer.next();
if (!read) return;
results.push(read);
}
})();
const write = (sequence: bigint, millis: number) => {
const group = track.createGroup(sequence);
group.writeFrame({
payload: encodeFrame(new Uint8Array([1]), Duration.millis(millis)),
timestamp: Time.Timestamp.fromMicros(millis * 1000),
});
group.finish();
};
try {
write(0n, 0);
await flush();
write(50n, 1_000);
await flush();
await clock.advance(Duration.millis(10));
expect(results).toHaveLength(1);
position = Duration.millis(1_000);
anchor.set(1);
await flush();
expect(results.slice(1).map((read) => read.kind)).toEqual(["gap", "frame"]);
} finally {
consumer.close();
track.finish();
track.close();
try {
await reading;
} finally {
installed.close();
}
}
});
60 changes: 55 additions & 5 deletions js/hang/src/container/consumer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,21 @@ export interface ConsumerProps {
* switch neither lowers nor raises {@link Consumer.spread} on its own.
*/
predecessor?: Consumer;
/**
* The playout clock this reader feeds, such as the player's `Sync` (default: none). A missing
* group whose successor starts at or before the media time it presents now could only arrive
* too late to play, so it is skipped then rather than waited for: a reader resuming behind the
* live edge moves on once the clock passes the hole, not a whole {@link ConsumerProps.wait} later.
*/
playhead?: Playhead;
}

/** A playout clock, which a {@link Consumer} reads to tell when a missing group could no longer play. */
export interface Playhead {
/** The media time presenting now, or undefined while nothing presents. */
now(): Duration | undefined;
/** Changes whenever {@link Playhead.now} jumps rather than advancing with the wall clock: a re-anchor, a new delay, or a pause. */
readonly anchor: Getter<unknown>;
}

/** One item of a {@link Consumer.next} read, in delivery order. Mirrors Rust's `container::Read`. */
Expand Down Expand Up @@ -189,6 +204,7 @@ export class Consumer {
#format: Format;
#delay: Getter<Moq.Track.Delay>;
#wait: Getter<Duration | undefined>;
#playhead?: Playhead;
#maxBuffer: number;
// How much further than the min delay the start reaches, and the age skip tolerates while warming.
#warmup: Duration;
Expand Down Expand Up @@ -279,6 +295,7 @@ export class Consumer {
this.#format = props.format;
this.#delay = getter(props.delay ?? {});
this.#wait = getter(props.wait);
this.#playhead = props.playhead;
this.#maxBuffer = props.maxBuffer ?? DEFAULT_MAX_BUFFER;
this.#warmup = (props.warmup ?? Duration.ZERO).min(DELAY_MAX);
this.#jitter = props.predecessor ? props.predecessor.#jitter.fork() : new Jitter();
Expand Down Expand Up @@ -908,7 +925,8 @@ export class Consumer {
// A missing sequence also expires on elapsed monotonic time, so timestamp-free markers
// behind a hole can end a paused track without more arrivals. It is late since a group
// past it arrived, not since the cursor got here: once the edge stops moving, a backlog
// with every other group missing would otherwise drain one budget per hole.
// with every other group missing would otherwise drain one budget per hole. And once the
// playout clock reaches where the first group past it begins, nothing missing can still play.
if (starting === undefined && this.#active !== undefined && this.#groups.length > 0) {
const head = this.#groups[0];
if (head.consumer.sequence > this.#active) {
Expand All @@ -918,19 +936,30 @@ export class Consumer {
const contiguous = ptsContiguous(this.#presentedEnd, head.frames.at(0)?.timestamp.asMicros());
const empty = head.empty && head.consumer.done;
const wait = this.#holeWait(this.#delay.peek(), this.#wait.peek());
const bound = this.#groups.find((group) => group.start !== undefined)?.start;
const bound = this.#bound();
const edge = this.#liveEdge();
// Measured against the max delay, not the wait: a reader starved of CPU takes in a burst of
// groups at once, and the missing one may land in the same burst.
const aged =
bound !== undefined && edge !== undefined && edge - bound >= maxMicros(this.#delay.peek());
const elapsed = now().since(this.#hole).asMicros() >= wait;
const passed = this.#passed(bound) === 0;
// The ranges ask for nothing in between, so nothing there is owed.
const next = this.#track.nextInRange(this.#active);
const excluded = next === undefined || next >= head.consumer.sequence;
// The groups past the hole already fill the max buffer, so it can hold none of what would arrive into it.
const full = this.#full();
if (startup || empty || contiguous || aged || elapsed || excluded || full || ended !== undefined) {
if (
startup ||
empty ||
contiguous ||
aged ||
elapsed ||
passed ||
excluded ||
full ||
ended !== undefined
) {
if (!contiguous && !empty) {
this.#lost = true;
if (!startup) {
Expand Down Expand Up @@ -1040,11 +1069,19 @@ export class Consumer {
timer = new Effect();
timer.run((effect) => {
const delay = effect.get(this.#delay);
// A clock that jumps moves when it reaches the group past a hole.
if (this.#playhead) effect.get(this.#playhead.anchor);
// The start waits out its reach, and a hole the wait, which an unlimited one never does.
const budget =
starting === undefined ? this.#holeWait(delay, effect.get(this.#wait)) : this.#reach(delay);
if (!Number.isFinite(budget)) return;
effect.timer(() => this.#notify?.(), Duration.micros(budget).saturatingSub(now().since(since)));
let left = Number.isFinite(budget)
? Duration.micros(budget).saturatingSub(now().since(since))
: undefined;
// A hole also ends once the playout clock reaches the first group past it.
const passing = starting === undefined ? this.#passed(this.#bound()) : undefined;
if (passing !== undefined) left = Duration.micros(Math.min(passing, left?.asMicros() ?? passing));
if (left === undefined) return;
effect.timer(() => this.#notify?.(), left);
});
}

Expand All @@ -1063,6 +1100,19 @@ export class Consumer {
}
}

// Where the first stamped buffered group begins, which bounds everything a missing group before it presents.
#bound(): number | undefined {
return this.#groups.find((group) => group.start !== undefined)?.start;
}

// How long until the playout clock reaches `bound`, in microseconds: zero once it has, undefined
// without a clock that is presenting or without a bound.
#passed(bound: number | undefined): number | undefined {
const playhead = this.#playhead?.now()?.asMicros();
if (bound === undefined || playhead === undefined) return undefined;
return Math.max(0, bound - playhead);
}

// How long a missing group is waited for, in microseconds: the wait, at most the max delay.
#holeWait(delay: Moq.Track.Delay, wait: Duration | undefined): number {
const max = maxMicros(delay);
Expand Down
2 changes: 1 addition & 1 deletion js/hang/src/container/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

/** Re-export of {@link https://www.npmjs.com/package/@moq/loc | @moq/loc}, the LOC container. */
export * as Loc from "@moq/loc";
export { Consumer, type ConsumerProps, DELAY_MAX, type Read } from "./consumer";
export { Consumer, type ConsumerProps, DELAY_MAX, type Playhead, type Read } from "./consumer";
export type { Format } from "./format";
/** The legacy varint container. */
export * as Legacy from "./legacy";
Expand Down
1 change: 1 addition & 0 deletions js/watch/api/index.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ export * as Signals from "@moq/signals";

export class Sync {
constructor(props?: Signals.Inputs<SyncInput>);
readonly anchor: Signals.Getter<unknown>;
close(): void;
readonly in: Signals.Readonlys<SyncInput>;
now(): Clock.Duration | undefined;
Expand Down
1 change: 1 addition & 0 deletions js/watch/src/audio/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -921,6 +921,7 @@ export class Decoder {
wait: this.sync.out.maxDelay,
warmup: Duration.millis(config.warmup ?? 0).min(DELAY_MAX),
predecessor: continues ? outgoing.consumer : undefined,
playhead: this.sync,
});
effect.cleanup(() => consumer.close());
this.#phase.set({ ...phase, consumer: { path, codec: config.codec, consumer } });
Expand Down
6 changes: 6 additions & 0 deletions js/watch/src/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,8 @@ export class Sync {
readonly #clock: Clock;
/** Whether buffered startup has released the media consumers. @internal */
readonly playing: Getter<boolean>;
/** Changes whenever {@link Sync.now} jumps rather than advancing with the wall clock: a re-anchor, a new delay, or a pause. */
readonly anchor: Getter<unknown>;

#signals = new Effect();

Expand All @@ -475,6 +477,10 @@ export class Sync {
this.in = this.#clock.in;
this.out = readonlys(this.#clock.out);
this.playing = this.#clock.playing;
this.anchor = new Derived(
[this.#clock.out.reference, this.#clock.out.delay, this.#clock.playing],
(reference, delay, playing) => ({ reference, delay, playing }),
);

this.#signals.run(this.#runInstant.bind(this));
this.#signals.run(this.#runDelay.bind(this));
Expand Down
3 changes: 3 additions & 0 deletions js/watch/src/video/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,9 @@ class DecoderTrack {
format,
delay: this.#delay,
predecessor: this.consumer ?? this.#predecessor,
// The max-delay decoder reads older groups on purpose, to rebuild references for the clock's
// lane, so only the lane driving the clock skips what the clock passed.
playhead: this.#drives ? this.sync : undefined,
});
effect.cleanup(() => consumer.close());
this.consumer = consumer;
Expand Down
2 changes: 1 addition & 1 deletion quest/m0/nightly/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,5 @@ and the chromium `av_lag` rows
After #8635 landed, run
[43601796176](https://github.com/moq-dev/moq/actions/runs/43601796176) still
failed the firefox resume, now on audio arriving late rather than trailing:
[Firefox resume audio](/quest/m0/nightly/firefox-resume-audio.md) owns it.
[moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799) fixed that.

36 changes: 0 additions & 36 deletions quest/m0/nightly/firefox-resume-audio.md

This file was deleted.

Loading