Skip to content
34 changes: 13 additions & 21 deletions js/net/src/resume.model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -271,11 +271,6 @@ interface Reader {
floor?: number;
/** Its instance lost its last route while it read. */
doomed: boolean;
/**
* The newest group handed out when its routes last changed. A route change resumes from
* there, so a group below it that lands only afterwards may be lost.
*/
resumed?: number;
/** `null` once the track ended cleanly, the error once it failed. */
end?: Error | null;
/**
Expand Down Expand Up @@ -345,19 +340,6 @@ class Model {
}

async apply(op: Op): Promise<void> {
// Only adding or removing a route can move a reader to another one. Unlike Rust, which
// moves only when the cheapest route changes, a JS reader may keep reading from a route
// that is no longer the cheapest, so any change counts.
const rerouted =
(op.kind === "up" && !this.instances[op.epoch].routes[op.slot]) ||
(op.kind === "down" && this.instances[op.epoch].routes[op.slot] !== undefined);
if (rerouted) {
for (const reader of this.readers) {
if (reader?.instance !== op.epoch) continue;
const newest = max(reader.handed);
if (newest !== undefined) reader.resumed = Math.max(reader.resumed ?? newest, newest);
}
}
switch (op.kind) {
case "up":
this.up(op.epoch, op.slot);
Expand Down Expand Up @@ -786,7 +768,6 @@ class Model {
let owed: number[] = [];
for (let group = 0; group < GROUPS; group++) {
if (reader.floor !== undefined && group <= reader.floor) continue;
if (reader.resumed !== undefined && group <= reader.resumed) continue;
if (covers(reader, group) && carried(group)) owed.push(group);
}
if (reader.spec.ordered && before !== undefined) owed = owed.filter((group) => group > before);
Expand Down Expand Up @@ -873,7 +854,7 @@ test(
{ timeout: 3_600_000 },
);

/** An Up or Down that changes no route resumes nothing, so lower groups stay owed. */
/** An Up or Down that leaves the serving route still delivers a group only a standby route has. */
test("unchanged routes keep groups owed", async () => {
const clock = new Manual();
const installed = clock.install();
Expand All @@ -892,7 +873,6 @@ test("unchanged routes keep groups owed", async () => {
}
const reader = model.readers[0];
expect(reader?.handed).toContain(8);
expect(reader?.resumed).toBeUndefined();
} finally {
await model.close();
installed.close();
Expand Down Expand Up @@ -983,6 +963,18 @@ test("regressions", async () => {
],
finish: false,
},
// A widened cursor owes a group below where a route change resumed, landing late.
{
ops: [
{ kind: "join", reader: 0, spec: spec(false, "hour", [1, 2], false) },
{ kind: "up", epoch: 1, slot: 0 },
{ kind: "up", epoch: 0, slot: 1 },
{ kind: "bounds", reader: 0, range: undefined },
{ kind: "write", epoch: 0, slot: 0, group: 1, frames: 1 },
{ kind: "down", epoch: 0, slot: 0 },
],
finish: false,
},
];
const failures: string[] = [];
for (const [i, c] of regressions.entries()) {
Expand Down
83 changes: 81 additions & 2 deletions js/net/src/resume.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,11 @@ test("returning to a route reuses its subscription and held group", async () =>
}
});

test("a returning route resumes where a fresh copy would", async () => {
/**
* A returning route resumes where a fresh copy would, and what it buffered while replaced below
* what another route already handed out comes through the backfill: the late group once.
*/
test("a returning route backfills below the reader's newest", async () => {
const clock = new Manual();
const installed = clock.install();
const front = new Front();
Expand All @@ -169,7 +173,7 @@ test("a returning route resumes where a fresh copy would", async () => {
front.serve(handles[0]);
await clock.advance(Nano.zero);
expect(mirrored.mock.calls.length).toBe(1);
expect(drain(sub)).toEqual([]);
expect(drain(sub)).toEqual([15]);
ta.write(26);
expect(drain(sub)).toEqual([26]);
held.close();
Expand All @@ -184,6 +188,81 @@ test("a returning route resumes where a fresh copy would", async () => {
}
});

/**
* A group below the replacement's resume point that the reader never got is asked of the new
* route, as one cursor on the logical track would still get it late.
*/
/**
* A gap that stays open across route flaps keeps one backfill per distinct route: a later resume
* point widens the route's backfill rather than asking again.
*/
test("route flaps keep one backfill per distinct route", async () => {
const clock = new Manual();
const installed = clock.install();
const front = new Front();
const sources = [new BroadcastProducer(), new BroadcastProducer()];
const handles = sources.map((source) => source.consume());
const tracks = sources.map(copy);
front.serve(handles[0]);
const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) });
const subscribed = tracks.map((track) => spyOn(track.producer, "subscribe"));
try {
await clock.advance(Nano.zero);
for (let round = 1; round < 24; round++) {
front.serve(handles[round % 2]);
await clock.advance(Nano.zero);
// Every odd group never lands, so every resume point leaves a gap.
tracks[round % 2].write(round * 2);
await clock.advance(Nano.zero);
expect(drain(sub)).toEqual([round * 2]);
}
// Each route's own copy, returned to on every flap, and its one backfill.
for (const spy of subscribed) expect(spy.mock.calls.length).toBe(2);
tracks[1].write(1);
await clock.advance(Nano.zero);
expect(drain(sub)).toEqual([1]);
} finally {
for (const spy of subscribed) spy.mockRestore();
sub.close();
front.consumer.close();
for (const handle of handles) handle.close();
for (const source of sources) source.close();
installed.close();
}
});

test("a group below the replacement's resume point is backfilled", async () => {
const clock = new Manual();
const installed = clock.install();
const front = new Front();
const sources = [new BroadcastProducer(), new BroadcastProducer()];
const handles = sources.map((source) => source.consume());
const [ta, tb] = sources.map(copy);
front.serve(handles[0]);
const sub = front.consumer.track("seq").subscribe({ maxDelay: Milli(3_600_000) });
try {
await clock.advance(Nano.zero);
ta.write(5);
expect(drain(sub)).toEqual([5]);
front.serve(handles[1]);
await clock.advance(Nano.zero);
expect(drain(sub)).toEqual([]);
tb.write(2);
tb.write(6);
await clock.advance(Nano.zero);
expect(drain(sub)).toEqual([2, 6]);
tb.write(4);
await clock.advance(Nano.zero);
expect(drain(sub)).toEqual([4]);
} finally {
sub.close();
front.consumer.close();
for (const handle of handles) handle.close();
for (const source of sources) source.close();
installed.close();
}
});

test("a routed source start follows the serving copy and its subscriber floor", async () => {
const a = new BroadcastProducer();
const b = new BroadcastProducer();
Expand Down
52 changes: 45 additions & 7 deletions js/net/src/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
* A new copy is asked from where the reader got to, so it never has the groups below the reader's
* local floor that the copies it replaced cached. Lowering that floor asks the newest route for
* them through a backfill: another copy of that route, bounded to the groups the floor newly
* covers, and kept like a replaced copy.
* covers, and kept like a replaced copy. A switch asks the same way for the groups below where the
* reader got to that it still lacks, from the lowest one: one late, reordered, or in flight on the
* replaced route still arrives within the reader's budget, as it would on one cursor.
*
* A group carries on across the switch through its {@link Recover}: once its copy stalls while a
* newer route serves, it continues from the serving route's copy of the same group at the same
Expand Down Expand Up @@ -295,6 +297,11 @@ class Reader implements ResumeSource {
#start: number;
/** The newest group handed out: where a new copy is asked to start. */
#newest?: number;
/**
* Set once the route changed, so the serving copy's resume point may pass over groups the
* reader lacks. Kept until the gaps are backfilled; see {@link #backfillGaps}.
*/
#resumed = false;
/** The last datagram handed out, which never ages until another supersedes it. */
#datagram?: number;
/**
Expand Down Expand Up @@ -381,6 +388,9 @@ class Reader implements ResumeSource {
this.#drops = [];
const source = route.source;
if (!source) return;
// The new copy starts at the newest group handed out, so a lower one still missing is
// asked of its route once a read says which; see #backfillGaps.
this.#resumed = true;
// The replaced copies keep delivering until the new one delivers its first group: a
// group they get meanwhile may be one the new route never has. See #observe.
for (const copy of this.#copies) copy.replaced = true;
Expand Down Expand Up @@ -511,12 +521,22 @@ class Reader implements ResumeSource {
if (this.#delivered.covers(start, end)) return;
// A floor lowered again asks once.
const closed = newest.source.from.closed;
const asked = (copy: Copy) =>
copy.source.from.closed === closed &&
copy.floor !== undefined &&
copy.floor <= start &&
(copyEnd(copy) ?? Number.POSITIVE_INFINITY) >= end;
if (this.#copies.some(asked)) return;
const asks = (copy: Copy) =>
copy.source.from.closed === closed && copy.floor !== undefined && copy.floor <= start;
if (this.#copies.some((copy) => asks(copy) && (copyEnd(copy) ?? Number.POSITIVE_INFINITY) >= end)) return;
// A later resume point widens the route's backfill rather than asking again, so flaps over
// a gap keep one per route. Waited for as long as a new one would be.
const widened = this.#copies
.slice(0, -1)
.find((copy) => copy.done === undefined && copy.until !== undefined && asks(copy));
if (widened) {
widened.until = end - 1;
widened.since = now();
this.#update(widened);
this.#bound(widened);
this.wake();
return;
}
const backfill = { floor: start, until: end - 1 };
this.#add(
{
Expand All @@ -534,6 +554,23 @@ class Reader implements ResumeSource {
);
}

/**
* Ask the serving route for the groups below its copy's resume point that the reader still
* lacks: a late or reordered one, or one in flight on the route it replaced, as one cursor on
* the logical track would still get. From the lowest gap within the reader's cursor, which
* decides what it reads, so groups already handed out are not asked for again below it. A gap
* past the reader's budget was already forgotten as handed out. A lowered cursor asks through
* {@link #backfill} itself. Mirrors the Rust `Reader::backfill_gaps`.
*/
#backfillGaps(delivered: Delivered): void {
if (!this.#resumed) return;
this.#resumed = false;
const resumed = this.#copies.at(-1)?.floor;
if (resumed === undefined) return;
const [gap] = delivered.gaps(this.#shell.cursor.peek().start, resumed);
if (gap) this.#backfill(gap.start, resumed);
}

/** Wake parked reads, so they look again and let go of what nothing needs. */
wake(): void {
this.#changed.update((n) => n + 1);
Expand Down Expand Up @@ -594,6 +631,7 @@ class Reader implements ResumeSource {
this.#learnEnd();
if (this.#closed) return this.#refused ? { kind: "error", error: this.#refused } : { kind: "done" };
this.#groupsRead = true;
this.#backfillGaps(delivered);
if (dispositions) {
const dropped = this.#tryDrop(delivered);
if (dropped) return dropped;
Expand Down
1 change: 0 additions & 1 deletion quest/m2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,6 @@ lite-07; bitrate enforcement is not part of this release.
- [No MOQ_* knobs outside the binaries](/quest/m2/env-knobs.md) - test, bench, and library knobs leave the `MOQ_*` prefix
- [Clean ALSA playback onset](/quest/m2/audio-alsa-onset.md) - no onset restarts or clicks over PulseAudio
- [Interop audit](/quest/m2/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down or unsupported
- [Resume delivers late groups below its resume point](/quest/m2/resume-late-groups.md) - a bonus: groups that arrive late after a route change are delivered instead of written off
- [The HLS server follows each path once](/quest/m2/hls-follow-once.md) - `Following` reports synced absence, so moq-hls resolves with one follow
- [moq-bench pins the instances it measures](/quest/m2/bench-epoch-pin.md) - pooled requests pin their announced epoch, so a takeover is not counted as the same broadcast
- [We own our WebRTC stack](/quest/m2/webrtc/README.md) - moq-rtc becomes moq-webrtc with moq-ice, the data-channel transport, and a vendored, trimmed str0m/dimpl/is/sctp-proto
Expand Down
34 changes: 0 additions & 34 deletions quest/m2/resume-late-groups.md

This file was deleted.

Loading