diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index deb89674746..675ed6b8530 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -7,6 +7,30 @@ import { RUN_OVERHEAD as ARRIVAL_OVERHEAD } from "./ledger.ts"; import { Tail } from "./tail.ts"; import { Producer as TrackProducer } from "./track.ts"; +// A reader continued on a held re-offer keeps the frames the failed copy wrote that it has yet to +// read, so the pool still counts them once that copy leaves the cache. +test("a re-offer bills the failed copy's frames its readers still hold", async () => { + await withClock(async () => { + const pool = new cache.Pool({ capacity: 100 * billed(10_000) }); + const producer = new TrackProducer("test"); + hooks.poolTrack(producer, pool); + const track = producer.accept({ maxAge: null }); + hooks.receiveTrack(track); + const first = hooks.receiveGroup(track, { sequence: 0n, start: 0 }); + first.writeFrame({ payload: new Uint8Array(10_000) }); + const reader = first.consume(); + const arriving = hooks.arrivingGroup(track, 0n); + first.abort(new Error("reset")); + // The re-offer starts past the frame the reader still holds. + hooks.receiveGroup(track, { sequence: 0n, start: 1 }); + arriving?.release(); + expect(pool.used).toBeGreaterThan(10_000); + expect((await reader.readFrame())?.payload.byteLength).toBe(10_000); + reader.close(); + track.close(); + }); +}); + // Run `fn` on a manual clock, stepped in milliseconds. async function withClock(fn: (clock: { advance: (ms: number) => Promise }) => Promise | void) { const clock = new Manual(); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 4ec04c8e14c..bafcb9a61a7 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -130,6 +130,11 @@ class GroupState { // Tells the group this copy was mirrored from whether its reader will still read it, so a // copy buffered for a reader that passed it stops keeping the group wanted. reads?: (reads: boolean) => void; + // Stops the group teeing into this mirror from watching it close. + unwatch?: () => void; + // The first frame the group teeing into this mirror wrote here. A copy that took over a failed + // one tees from where that one stopped, so the frames below came from the failed copy. + from = 0; constructor(sequence: bigint) { this.sequence = sequence; @@ -261,6 +266,10 @@ export class Producer { // Called as the group aborts, before any signal subscriber runs (see `hooks.onGroupAbort`). #aborted?: ((error: Error) => void)[]; + // Duplicates waiting on this received copy (see `hooks.holdGroup`). Failing while held, it + // keeps its mirrors open for the duplicate that takes over to tee into. + #holds = 0; + constructor(sequence: bigint) { if (checkSequence(sequence) >= RESERVED) throw new RangeError(`group sequence out of range: ${sequence}`); this.#state = new GroupState(sequence); @@ -300,6 +309,55 @@ export class Producer { if (state.closed.peek() !== undefined) flate.finish(); }; hooks.groupOffset = (group) => group.#state.offset; + hooks.holdGroup = (group) => { + group.#holds++; + let held = true; + return () => { + if (!held) return; + held = false; + if (--group.#holds === 0) group.#failHeld(); + }; + }; + hooks.succeedGroup = (next, failed) => { + const error = failed.#state.closed.peek(); + if (!(error instanceof Error)) throw new Error("group has not failed"); + const state = next.#state; + // The failed copy's frames the continued mirrors hold, once each though mirrors share them. + const kept = new Set(); + for (const [mirror, reads] of failed.#mirrors ?? []) { + failed.#unmirror(mirror); + // The successor continues a mirror from the frame it reached, so it must hold that + // frame from its start. Compression binds a group to its prefix, so a compressed + // successor starts at 0. + if (mirror.closed.peek() !== undefined) continue; + const lost = state.start === mirror.total.peek() && headShed(state.frames.peek().first); + if (state.closed.peek() !== undefined || state.start > mirror.total.peek() || lost) { + mirror.closed.set(error); + continue; + } + // Its unread frames count against the tail it continues in, as they would on one copy, so a + // chain of takeovers cannot grow a stalled reader's buffer: past the tail, it is behind it. + let unread = 0; + for (const frame of mirror.frames.peek()) unread += frameBytes(frame); + if (unread > TAIL_BYTES) { + mirror.frames.set(new Queue()); + mirror.cacheBytes = 0; + mirror.flate = undefined; + mirror.closed.set(evicted()); + continue; + } + for (const frame of mirror.frames.peek()) kept.add(frame); + mirror.from = mirror.total.peek(); + mirror.charge = state.charge; + if (state.compressed) mirror.flate = state.flate; + let index = state.start; + for (const frame of state.frames.peek()) if (index++ >= mirror.from) appendFrame(mirror, frame); + next.#attach(mirror, reads); + } + let bytes = 0; + for (const frame of kept) bytes += frameBytes(frame); + return bytes; + }; hooks.startGroup = (group, start) => { if (!Number.isSafeInteger(start) || start < 0) throw new RangeError("invalid group frame start"); const state = group.#state; @@ -413,11 +471,17 @@ export class Producer { return makeConsumer(dst); } + this.#attach(dst, true); + return makeConsumer(dst); + } + + // Tee later writes into `dst`, counting its reader if it `reads`. + #attach(dst: GroupState, reads: boolean): void { this.#mirrors ??= new Map(); const mirrors = this.#mirrors; - mirrors.set(dst, true); - this.#readers++; - this.#wanted.set(true); + mirrors.set(dst, reads); + if (reads) this.#readers++; + this.#wanted.set(this.#readers > 0); dst.reads = (reads) => { const was = mirrors.get(dst); if (was === undefined || was === reads) return; @@ -428,13 +492,9 @@ export class Producer { // Track this mirror's close eagerly (not just lazily on the next write) so demand drops the // moment the last reader leaves, letting an abandoned fetch cancel even with no more frames. - const dispose = dst.closed.subscribe((c) => { - if (c === undefined) return; - this.#unmirror(dst); - dispose(); + dst.unwatch = dst.closed.subscribe((c) => { + if (c !== undefined) this.#unmirror(dst); }); - - return makeConsumer(dst); } /** @@ -453,6 +513,8 @@ export class Producer { this.#mirrors?.delete(mirror); // A consumer that outlives the group must not pin it, and its buffers, through this. mirror.reads = undefined; + mirror.unwatch?.(); + mirror.unwatch = undefined; if (reads) this.#readers--; this.#wanted.set(this.#readers > 0); } @@ -575,9 +637,11 @@ export class Producer { for (const commit of commits) commit(); } + const index = this.#state.total.peek() - 1; for (const mirror of this.#mirrors?.keys() ?? []) { if (mirror.closed.peek() !== undefined) this.#unmirror(mirror); - else appendFrame(mirror, frame); + // A mirror taken over from a failed copy already holds the frames that copy wrote. + else if (index >= mirror.total.peek()) appendFrame(mirror, frame); } this.#trim(); // Past the frame's own bookkeeping: paying the track's debt may evict other groups. @@ -653,7 +717,8 @@ export class Producer { for (const mirror of this.#mirrors?.keys() ?? []) { // A mirror that has not taken the front frame can no longer read it from its start. if (mirror.closed.peek() !== undefined) continue; - if (mirror.start > state.start || (mirror.start === state.start && !lost)) continue; + const front = Math.max(mirror.start, mirror.from); + if (front > state.start || (front === state.start && !lost)) continue; mirror.frames.set(new Queue()); mirror.cacheBytes = 0; mirror.flate = undefined; @@ -687,21 +752,44 @@ export class Producer { #end(outcome: Error | null) { if (this.#state.closed.peek() !== undefined) return; // A frame cut short leaves the group incomplete, so even a clean end fails it. + let cut: Chunked | undefined; if (this.#open) { this.#open = undefined; outcome ??= new Error("group finished while a frame was being written"); const last = this.#state.frames.peek().last; - if (last instanceof Chunked && last.done.peek() === undefined) last.done.set(outcome); + if (last instanceof Chunked && last.done.peek() === undefined) cut = last; } + const held = outcome !== null && this.#holds > 0; + // A held copy's duplicate continues each mirror from the frame it reached, so a mirror + // drops the frame cut short before its reader takes it. One whose reader took it lost it. + if (held && cut) { + const index = this.#state.total.peek() - 1; + for (const mirror of this.#mirrors?.keys() ?? []) { + // A mirror taken over past the cut frame never got it from this copy. + if (index < mirror.from) continue; + if (mirror.frames.peek().last === cut) { + mirror.frames.mutate((frames) => frames.pop()); + mirror.total.update((total) => total - 1); + } else if (mirror.closed.peek() === undefined) mirror.closed.set(outcome); + } + } + cut?.done.set(outcome); this.#state.flate?.finish(); this.#state.closed.set(outcome); + if (!held) this.#failHeld(); + if (outcome) for (const fn of this.#aborted ?? []) fn(outcome); + this.#aborted = undefined; + } + + // End the open mirrors, once this copy closed and no duplicate held for it can take over. + #failHeld(): void { + const outcome = this.#state.closed.peek(); + if (outcome === undefined) return; for (const mirror of this.#mirrors?.keys() ?? []) { if (mirror.closed.peek() === undefined) mirror.closed.set(outcome); this.#unmirror(mirror); } - if (outcome) for (const fn of this.#aborted ?? []) fn(outcome); - this.#aborted = undefined; } } @@ -1055,7 +1143,8 @@ export class Consumer { hooks.evictGroup = (group, lost) => group.#evict(lost); shedReader = (group, start) => { // A copy that took every shed frame keeps reading; one that had not lost them. - if (group.#state === group.#mirrored && group.#state.start < start) group.#evict(evicted()); + const state = group.#state; + if (state === group.#mirrored && Math.max(state.start, state.from) < start) group.#evict(evicted()); }; hooks.refreshGroup = (group) => group.#state.charge?.refresh(); hooks.onShedRead = (group, notify) => { diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 0de6cb0c948..20a5044b06a 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -419,6 +419,14 @@ export interface GroupRecover { replace(index: number, failed: Error | undefined, signal: AbortSignal): Promise; } +/** A copy of a group still arriving, held for a duplicate that waits on it. */ +export interface Arriving { + /** How the copy ends. */ + readonly settled: GetPromise; + /** Stop holding it, once the duplicate took the group or gave up. Idempotent. */ + release(): void; +} + /** A candidate copy of a group, committed only once the read can use it. */ export interface Replacement { readonly group: GroupConsumer; @@ -647,8 +655,12 @@ export const hooks: { * fill's subscription answered, which the feed never promised. Assigned by `track.ts`. */ receiveGroup: (producer: Producer, group: { sequence: bigint; start: number; fill?: boolean }) => GroupProducer; - /** How the copy of `sequence` still arriving ends, if one is, for a duplicate to wait on. Assigned by `track.ts`. */ - arrivingGroup: (producer: Producer, sequence: bigint) => GetPromise | undefined; + /** + * Hold the copy of `sequence` still arriving, if one is, for a duplicate to wait on: until + * released, that copy's failure leaves its readers open for the duplicate to continue. Assigned + * by `track.ts`. + */ + arrivingGroup: (producer: Producer, sequence: bigint) => Arriving | undefined; /** The producer's largest reported location and its subsequent changes. */ producerLargest: (producer: Producer) => Getter; @@ -935,6 +947,17 @@ export const hooks: { * out, so its `closed` never settles on a failure the recovery continues past. */ recoverGroup: (group: GroupConsumer, recover: GroupRecover) => void; + /** + * Hold a received copy still arriving for a duplicate: if it fails while held, its readers wait + * for the duplicate to take over (see {@link Arriving}). Assigned by `group.ts`. + */ + holdGroup: (group: GroupProducer) => () => void; + /** + * Hand `next` the readers `failed` kept open while held, teeing each from the frame it reached. + * Fails those `next` cannot serve. Returns the bytes of `failed`'s frames they still hold. + * Assigned by `group.ts`. + */ + succeedGroup: (next: GroupProducer, failed: GroupProducer) => number; /** Read a payload chunk, releasing its wait when the output stream closes. */ readFrameChunk: (frame: FrameConsumer, closed: Promise) => Promise; /** Read the next frame, as its stored DEFLATE slice when `compressed`, so a lite-07 publisher forwards it without inflating. */ @@ -1206,6 +1229,12 @@ export const hooks: { arrivingGroup: () => { throw new Error("track.ts not loaded"); }, + succeedGroup: () => { + throw new Error("group.ts not loaded"); + }, + holdGroup: () => { + throw new Error("group.ts not loaded"); + }, groupOffset: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index 8e3c92a4f72..5c940f7eb7b 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1646,7 +1646,8 @@ test("a lite-06 group reset with YIELDED ends", async () => { }); // A lite-07 re-offer can overtake the reset of the stream it replaces: it waits, unread, until that -// copy ends. It is refused as a duplicate once the copy finishes, and takes the group once it fails. +// copy ends. It is refused as a duplicate once the copy finishes, and takes the group once it fails, +// continuing the reads of that copy from their frame. test.each([false, true])("a duplicate of an arriving copy waits for it to settle (first fails: %p)", async (fails) => { const version = Version.DRAFT_07; const { quic, streams, opened } = fakeSession(); @@ -1681,7 +1682,30 @@ test.each([false, true])("a duplicate of an arriving copy waits for it to settle const group = await track.recvGroup(); expect((await group?.readFrame())?.payload).toEqual(new Uint8Array([7])); - const duplicate = await groupStream(0, version); + const resent: Uint8Array[] = []; + const resend = new Writer( + new WritableStream({ write: (chunk) => void resent.push(new Uint8Array(chunk)) }), + version, + ); + for (const [timestamp, payload] of [ + [0n, 7], + [2n, 8], + ] as const) { + // The size plus one first, as above. + await resend.u53(2); + await resend.u62(timestamp); + await resend.write(new Uint8Array([payload])); + } + const duplicate = new Reader( + new ReadableStream({ + start: (controller) => { + for (const chunk of resent) controller.enqueue(chunk); + controller.close(); + }, + }), + undefined, + version, + ); const stop = spyOn(duplicate, "stop"); let held = true; const receiving = subscriber.runGroup(new Group({ subscribe: 0n, sequence: 0n }), duplicate).finally(() => { @@ -1700,16 +1724,22 @@ test.each([false, true])("a duplicate of an arriving copy waits for it to settle return; } - // The reset lands: the held stream takes the group. + // The reset lands: the reader already handed the original waits on the held stream rather + // than failing, and continues on it from its frame once it takes the group. + const next = group?.readFrame(); first.error(new Error("peer reset")); await original; await receiving; - expect(await group?.closed).toBeInstanceOf(Error); + expect((await next)?.payload).toEqual(new Uint8Array([8])); + expect(await group?.readFrame()).toBeUndefined(); + expect(await group?.closed).toBeNull(); + // One that had not reached it reads the re-offer whole. const later = subscriber.consume(Path.from("room")).track("video").subscribe(); try { const again = await later.recvGroup(); expect(again?.sequence).toBe(0n); - expect((await again?.readFrame())?.payload).toEqual(new Uint8Array([0])); + expect((await again?.readFrame())?.payload).toEqual(new Uint8Array([7])); + expect((await again?.readFrame())?.payload).toEqual(new Uint8Array([8])); } finally { later.close(); } diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 13afdd14454..27be820bcec 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -28,6 +28,7 @@ import * as netGroup from "../group.ts"; import { Cost, type Hop, MAX_HOPS, routesEqual, UNKNOWN_HOP, WireRoute } from "../hop.ts"; import * as Instance from "../instance.ts"; import { + type Arriving, bestCandidate, type DropRun, hiddenBelow, @@ -3250,6 +3251,7 @@ export class Subscriber { // Asked before this stream accounts for the group: a drop that overtook it named it. let owed = !tail.covers(group.sequence, group.sequence + 1n); let producer: netGroup.Producer | undefined; + let arriving: Arriving | undefined; let ingest: Ingest | undefined; let yielding: Yielding | undefined; // Until the stream is read, or a pause hands it to the paused group. @@ -3269,23 +3271,31 @@ export class Subscriber { } // A duplicate of a copy still arriving, such as a lite-07 re-offer that overtook the reset // of the stream it replaces, waits unread, which flow control bounds. Created once that copy - // ends, it is refused as a duplicate if the copy finished and takes the group if it failed. - // Its own reset ends the wait; a clean end leaves it on the copy alone. A lite-07 copy a - // YIELDED reset pauses is continued by this stream instead. + // ends, it is refused as a duplicate if the copy finished and takes the group if it failed, + // continuing the reads of that copy, which wait while it is held. Its own reset ends the + // wait; a clean end leaves it on the copy alone. A lite-07 copy a YIELDED reset pauses is + // continued by this stream instead. for (;;) { const paused = yields ? this.#resume(entry, group) : undefined; if (paused) { + // The same copy continues, so its readers need no hold. + arriving?.release(); + arriving = undefined; producer = paused.group; ingest = { index: group.frameStart, partial: paused.partial }; // The publisher owes a paused group its continuation, which may yield again. owed = true; break; } - const arriving = hooks.arrivingGroup(track, group.sequence); - if (arriving === undefined) break; + // Kept held past a failure until this copy takes the group, below. + const next = hooks.arrivingGroup(track, group.sequence); + if (next === undefined) break; + arriving?.release(); + arriving = next; // Served again once that copy fails or pauses, so owed again. owed = true; - const waits: unknown[] = [arriving, stream.closed.then(() => arriving)]; + const settled = arriving.settled; + const waits: unknown[] = [settled, stream.closed.then(() => settled)]; if (yields) waits.push(Signal.race(pausesOf(entry).changed)); await race(waits); } @@ -3294,6 +3304,8 @@ export class Subscriber { start: group.frameStart, fill: entry.filling, }); + // Released once this copy took the group, so the failed copy's readers find it. + arriving?.release(); if (yields) { ingest ??= { index: group.frameStart }; yielding = { owed }; @@ -3338,6 +3350,7 @@ export class Subscriber { producer?.abort(e); stream.stop(e); } finally { + arriving?.release(); if (yielding) this.#endYielding(entry, group.sequence, yielding); read?.(); } diff --git a/js/net/src/queue.ts b/js/net/src/queue.ts index 5e1843bd9e4..b0524c961d2 100644 --- a/js/net/src/queue.ts +++ b/js/net/src/queue.ts @@ -28,6 +28,11 @@ export class Queue implements Iterable { this.#items.push(item); } + /** Remove the newest entry. */ + pop(): T | undefined { + return this.length === 0 ? undefined : this.#items.pop(); + } + shift(): T | undefined { if (this.length === 0) return undefined; const item = this.#items[this.#head]; diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 7738ac5977e..51d1cb4c65d 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -3402,6 +3402,371 @@ test("an upstream drop ends a group still arriving with its error", async () => producer.close(); }); +// A reader on a received copy that fails waits out a duplicate still held for its group, then +// continues on it from its frame with nothing repeated or missed. A released hold leaves the +// failure, even to a reader only watching `closed`, as does a duplicate starting past the reader's +// frame. One that sheds the reader's frame after taking over evicts it, as any copy does a reader +// behind it. A re-offer that fails too hands over to the next. A reader closed while it waits ends +// cleanly and can be closed again, and one closed with frames still buffered reads them first. +test.each([ + "reoffer", + "readable", + "released", + "observed", + "shedObserved", + "unserved", + "twice", + "leaving", + "closed", + "past", + "shed", +])("a failed copy continues on its held re-offer (%p)", async (kind) => { + const producer = new TrackProducer("video").accept({ maxAge: null }); + hooks.receiveTrack(producer); + const first = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + first.writeFrame({ payload: enc.encode("a") }); + const reader = first.consume(); + expect(await reader.readString()).toBe("a"); + + const arriving = hooks.arrivingGroup(producer, 3n); + expect(arriving).toBeDefined(); + const failed = new Error("reset"); + if (kind === "closed") { + // A frame still buffered when the copy fails stays readable after a close. + first.writeFrame({ payload: enc.encode("c") }); + first.abort(failed); + reader.close(); + expect(await reader.readString()).toBe("c"); + expect(await reader.readString()).toBeUndefined(); + arriving?.release(); + producer.close(); + return; + } + first.abort(failed); + if (kind === "observed") { + let closed: Error | null | undefined; + reader.closed.then((value) => { + closed = value; + }); + await settle(); + expect(closed).toBeUndefined(); + arriving?.release(); + await settle(); + expect(closed).toBe(failed); + // The failure it reported stands: a re-offer arriving later does not take over. + const late = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + late.writeFrame({ payload: enc.encode("a") }); + late.writeFrame({ payload: enc.encode("b") }); + await expect(reader.readString()).rejects.toBe(failed); + producer.close(); + return; + } + if (kind === "twice") { + // The reader reaches neither re-offer before the third copy takes over. + const second = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + arriving?.release(); + const held = hooks.arrivingGroup(producer, 3n); + second.abort(failed); + await settle(); + expect(reader.closed.peek()).toBeUndefined(); + const third = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + held?.release(); + third.writeFrame({ payload: enc.encode("a") }); + third.writeFrame({ payload: enc.encode("b") }); + third.finish(); + expect(await reader.readString()).toBe("b"); + expect(await reader.readString()).toBeUndefined(); + expect(await reader.closed).toBeNull(); + producer.close(); + return; + } + if (kind === "shedObserved") { + // A re-offer that took over and then shed the reader's frame settles `closed` then. + const again = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + arriving?.release(); + for (const payload of ["a", "b", "c"]) again.writeFrame({ payload: enc.encode(payload) }); + let closed: Error | null | undefined; + reader.closed.then((value) => { + closed = value; + }); + await settle(); + expect(closed).toBeUndefined(); + hooks.shedGroup(again, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, []); + await settle(); + expect(closed).toMatchObject({ code: StreamCode.Evicted }); + producer.close(); + return; + } + if (kind === "unserved") { + // A re-offer starting past the reader's frame cannot take over, so `closed` settles once + // nothing else is held. + hooks.receiveGroup(producer, { sequence: 3n, start: 2 }); + arriving?.release(); + await settle(); + expect(reader.closed.peek()).toBe(failed); + producer.close(); + return; + } + let read: string | Uint8Array | undefined | Error; + // A readiness poll waits the same way, and adopts the re-offer rather than spinning on the + // failed copy. + const pending = + kind === "readable" ? reader.readable().then(() => reader.tryReadFrame()?.payload) : reader.readString(); + const reading = pending.then( + (frame) => { + read = frame; + }, + (err: Error) => { + read = err; + }, + ); + await settle(); + expect(read).toBeUndefined(); + expect(reader.closed.peek()).toBeUndefined(); + + if (kind === "leaving") { + reader.close(); + await reading; + expect(read).toBeUndefined(); + // Closing is idempotent, even after the wait was cancelled. + reader.close(); + arriving?.release(); + producer.close(); + return; + } + if (kind === "released") { + arriving?.release(); + await reading; + expect(read).toBe(failed); + expect(await reader.closed).toBe(failed); + producer.close(); + return; + } + const again = hooks.receiveGroup(producer, { sequence: 3n, start: kind === "past" ? 2 : 0 }); + arriving?.release(); + if (kind === "shed") { + // The re-offer drops frames 0 and 1 to cache pressure before the reader reaches it. + again.writeFrame({ payload: enc.encode("a") }); + again.writeFrame({ payload: enc.encode("b") }); + again.writeFrame({ payload: enc.encode("c") }); + hooks.shedGroup(again, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, []); + await reading; + expect(read).toMatchObject({ code: StreamCode.Evicted }); + producer.close(); + return; + } + if (kind === "past") { + await reading; + expect(read).toBe(failed); + producer.close(); + return; + } + await settle(); + expect(read).toBeUndefined(); + again.writeFrame({ payload: enc.encode("a") }); + again.writeFrame({ payload: enc.encode("b") }); + again.finish(); + await reading; + expect(read).toEqual(kind === "readable" ? enc.encode("b") : "b"); + expect(await reader.readString()).toBeUndefined(); + expect(await reader.closed).toBeNull(); + producer.close(); +}); + +// A stalled reader continued across a chain of takeovers, each copy writing one frame, keeps no more +// than the tail: its unread frames count against each successor's, so it ends evicted past it. +test("a chain of takeovers keeps a stalled reader within the tail", async () => { + const producer = new TrackProducer("video").accept({ maxAge: null }); + hooks.receiveTrack(producer); + const failed = new Error("reset"); + const payload = new Uint8Array(9 * 1024 * 1024); + let copy = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + const reader = copy.consume(); + for (let index = 0; index < 8; index++) { + copy.writeFrame({ payload }); + const held = hooks.arrivingGroup(producer, 3n); + copy.abort(failed); + copy = hooks.receiveGroup(producer, { sequence: 3n, start: index + 1 }); + held?.release(); + } + copy.finish(); + expect(await reader.closed).toMatchObject({ code: StreamCode.Evicted }); + producer.close(); +}); + +// While a duplicate is held, a failed copy keeps each mirror open and the copy that takes over tees +// into it from the frame it reached: a sink still buffering the group hands it out once, a frame cut +// short is read whole from the successor unless a reader already took part of it, readers at +// different frames each continue from their own, and with no takeover every mirror fails once. +test.each(["buffered", "handed", "partial", "replay", "positions", "once"])( + "a held copy's mirrors continue on its successor (%p)", + async (kind) => { + const producer = new TrackProducer("video").accept({ maxAge: null }); + hooks.receiveTrack(producer); + const track = producer.subscribe(); + const first = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + first.writeFrame({ payload: enc.encode("a") }); + const arriving = hooks.arrivingGroup(producer, 3n); + const failed = new Error("reset"); + const takeover = () => { + const again = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + arriving?.release(); + for (const payload of ["a", "b", "c"]) again.writeFrame({ payload: enc.encode(payload) }); + again.finish(); + }; + + if (kind === "buffered" || kind === "handed") { + // The sink hands the group out before or after its copy fails. + const handed = kind === "handed" ? await track.recvGroup() : undefined; + if (handed) expect(await handed.readString()).toBe("a"); + first.abort(failed); + takeover(); + const group = handed ?? (await track.recvGroup()); + expect(group?.sequence).toBe(3n); + for (const payload of handed ? ["b", "c"] : ["a", "b", "c"]) + expect(await group?.readString()).toBe(payload); + expect(await group?.readString()).toBeUndefined(); + // The sink held one copy of the group, so it hands out nothing more. + expect(await settled(track.recvGroup())).toBe(false); + } else if (kind === "partial") { + const ahead = first.consume(); + const behind = first.consume(); + expect(await ahead.readString()).toBe("a"); + const cut = first.createFrame({ size: 2 }); + cut.write(enc.encode("b")); + // `ahead` joins the frame as it arrives; `behind` has yet to reach it. + const joining = ahead.readString(); + await settle(); + first.abort(failed); + await expect(joining).rejects.toBe(failed); + takeover(); + for (const payload of ["a", "b", "c"]) expect(await behind.readString()).toBe(payload); + expect(await behind.readString()).toBeUndefined(); + } else if (kind === "replay") { + // The successor fails partway through a frame the reader already has: no loss to it, so a + // third copy held for the group still continues it. + const reader = first.consume(); + expect(await reader.readString()).toBe("a"); + first.abort(failed); + const second = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + arriving?.release(); + const held = hooks.arrivingGroup(producer, 3n); + second.createFrame({ size: 2 }).write(enc.encode("a")); + second.abort(failed); + await settle(); + expect(reader.closed.peek()).toBeUndefined(); + const third = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + held?.release(); + for (const payload of ["a", "b"]) third.writeFrame({ payload: enc.encode(payload) }); + third.finish(); + expect(await reader.readString()).toBe("b"); + expect(await reader.readString()).toBeUndefined(); + } else if (kind === "positions") { + first.writeFrame({ payload: enc.encode("b") }); + const readers = [first.consume(), first.consume(), first.consume()]; + expect(await readers[1].readString()).toBe("a"); + expect(await readers[2].readString()).toBe("a"); + expect(await readers[2].readString()).toBe("b"); + first.abort(failed); + takeover(); + const rest = [["a", "b", "c"], ["b", "c"], ["c"]]; + for (const [i, reader] of readers.entries()) { + for (const payload of rest[i]) expect(await reader.readString()).toBe(payload); + expect(await reader.readString()).toBeUndefined(); + expect(await reader.closed).toBeNull(); + } + } else { + const group = await track.recvGroup(); + const reader = first.consume(); + first.writeFrame({ payload: enc.encode("b") }); + first.abort(failed); + await settle(); + expect(group?.closed.peek()).toBeUndefined(); + expect(reader.closed.peek()).toBeUndefined(); + arriving?.release(); + // A released hold ends each mirror with the copy's failure, once, after what it buffered. + arriving?.release(); + for (const mirror of [group, reader]) { + expect(await mirror?.readString()).toBe("a"); + expect(await mirror?.readString()).toBe("b"); + await expect(mirror?.readString()).rejects.toBe(failed); + expect(await mirror?.closed).toBe(failed); + } + // A re-offer arriving later serves new readers only. + const late = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + late.writeFrame({ payload: enc.encode("a") }); + await expect(reader.readString()).rejects.toBe(failed); + expect(await late.consume().readString()).toBe("a"); + } + track.close(); + producer.close(); + }, +); + +// A subscriber's reader of a failed copy keeps its place when a re-offer replaces that copy in the +// track, continuing on it, and its budget still gives up on it while it waits on a held re-offer. +test.each(["buffered", "expires"])("a subscriber's failed copy waits on its re-offer (%p)", async (kind) => { + const clock = manualTime(0); + try { + await subscriberReoffer(kind); + } finally { + clock.installed.close(); + } +}); + +async function subscriberReoffer(kind: string): Promise { + const producer = new TrackProducer("video").accept({ timescale: Timescale.MILLI, maxAge: null }); + hooks.receiveTrack(producer); + const track = producer.subscribe({ delay: { max: Duration.millis(500) } }); + const first = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + first.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.fromMillis(0) }); + first.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.fromMillis(1) }); + const reader = await track.recvGroup(); + if (!reader) throw new Error("missing group"); + expect(await reader.readString()).toBe("a"); + + const arriving = hooks.arrivingGroup(producer, 3n); + const failed = new Error("reset"); + first.abort(failed); + if (kind === "expires") { + expect(await reader.readString()).toBe("b"); + let read: unknown; + reader.readString().then( + (frame) => { + read = frame; + }, + (err: unknown) => { + read = err; + }, + ); + // Group 4 bounds group 3's reach at 1s, and group 5 puts the edge a second past it. + for (const [sequence, millis] of [ + [4n, 1000], + [5n, 2000], + ] as const) { + hooks + .receiveGroup(producer, { sequence, start: 0 }) + .writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(millis) }); + } + await settle(); + expect(read).toMatchObject({ code: StreamCode.Old }); + arriving?.release(); + producer.close(); + return; + } + // The re-offer replaces the failed copy in the track before the reader takes `b`. + const again = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); + arriving?.release(); + again.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.fromMillis(0) }); + again.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.fromMillis(1) }); + again.writeFrame({ payload: enc.encode("c"), timestamp: Timestamp.fromMillis(2) }); + again.finish(); + expect(await reader.readString()).toBe("b"); + expect(await reader.readString()).toBe("c"); + expect(await reader.readString()).toBeUndefined(); + producer.close(); +} + test("a drop below the window is reported once the window lowers to it", () => { const producer = new TrackProducer("video").accept({}); const reader = producer diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 66d3341ecfa..789912cb07e 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -655,6 +655,17 @@ function lose(sink: TrackState, sequence: bigint): void { if (lost.length > LOST_KEPT) lost.splice(0, lost.length, { start: lost[0].start, end: (lost.at(-1) as Run).end }); } +// Take a group's mirror out of `sink`, so its subscriber can no longer receive it. +function unlinkMirror(sink: TrackState, mirror: GroupConsumer): void { + sink.groups.mutate((groups) => { + const i = groups.indexOf(mirror); + if (i >= 0) groups.splice(i, 1); + }); + const index = timelineIndex(sink.timeline, mirror.sequence); + if (sink.timeline[index] === mirror) sink.timeline.splice(index, 1); + lose(sink, mirror.sequence); +} + // The index of the first timeline group at or after `sequence`. function timelineIndex(timeline: readonly { sequence: bigint }[], sequence: bigint): number { let lo = 0; @@ -1345,8 +1356,9 @@ export class Producer { }; hooks.receiveGroup = (producer, { sequence, start, fill }) => producer.#createGroup(sequence, start, fill); hooks.arrivingGroup = (producer, sequence) => { - const closed = producer.#cached.get(sequence)?.group.closed; - return closed?.peek() === undefined ? closed : undefined; + const group = producer.#cached.get(sequence)?.group; + if (!group || group.closed.peek() !== undefined) return undefined; + return { settled: group.closed, release: hooks.holdGroup(group) }; }; sourceStart = (producer) => producer.#wire?.served && producer.#feedStart(); fillFloor = (producer, named) => producer.#fillFloor(named); @@ -1793,15 +1805,7 @@ export class Producer { // Take a cached group's mirrors out of every sink, so a subscriber can no longer // receive them. A reader already holding one keeps it as is. #unlink(entry: CachedGroup): void { - for (const [sink, mirror] of entry.mirrors) { - sink.groups.mutate((groups) => { - const i = groups.indexOf(mirror); - if (i >= 0) groups.splice(i, 1); - }); - const index = timelineIndex(sink.timeline, mirror.sequence); - if (sink.timeline[index] === mirror) sink.timeline.splice(index, 1); - lose(sink, mirror.sequence); - } + for (const [sink, mirror] of entry.mirrors) unlinkMirror(sink, mirror); entry.charge.sub(entry.mirrors.size * cache.MIRROR_OVERHEAD); entry.mirrors.clear(); } @@ -1931,7 +1935,13 @@ export class Producer { // against this track when it lands. // `fetched` marks a group a fill answered, which the live feed never delivered: it accounts // for nothing the feed owes, as the Rust `insert` records arrivals only for visible groups. - #publish(group: GroupProducer, fetched = false): void { + // `succeeds` is a failed copy of the same sequence this one takes over: its mirrors held open + // for a duplicate continue on this one, and each sink in `kept` keeps its mirror. + #publish( + group: GroupProducer, + fetched = false, + succeeds?: { failed: GroupProducer; kept?: Map }, + ): void { // Before the group is cached, so the write creating it can never evict it. this.#chargeDebt(); this.#settle(); @@ -1943,6 +1953,9 @@ export class Producer { fetched: fetched || undefined, }; hooks.chargeGroup(group, entry.charge); + // The failed copy's frames its continued readers still hold, billed until this entry leaves + // the cache, which evicts the sink mirrors holding them. + if (succeeds) entry.charge.bill(hooks.succeedGroup(group, succeeds.failed)); this.#undrop(group.sequence); // No reader is handed a failed fill answer (see #withhold), withheld as it aborts so a // read that follows at once cannot take it: its sequence is unknown again, owed while the @@ -1988,7 +2001,17 @@ export class Producer { this.#state.latest = max(this.#state.latest ?? 0n, group.sequence); if (edge && edge !== entry && !this.#protects(edge)) this.#account.demote(edge); this.#account.insert(entry); - for (const sink of this.#sinks) this.#mirror(entry, sink); + for (const sink of this.#sinks) { + const mirror = succeeds?.kept?.get(sink); + // Continued, it stays the sink's copy of the group, so the sink never holds two. + if (mirror && mirror.closed.peek() === undefined) { + entry.mirrors.set(sink, mirror); + entry.charge.bill(cache.MIRROR_OVERHEAD); + continue; + } + if (mirror) unlinkMirror(sink, mirror); + this.#mirror(entry, sink); + } // Follow the group's frames as they land, past the head `#advance` already named. const stop = hooks.groupProgress(group, (total) => { if (total > hooks.groupOffset(group)) this.#advanceObject({ group: group.sequence, frame: total - 1 }); @@ -2188,10 +2211,19 @@ export class Producer { // publisher may re-create a group it cut or aborted. this.#refuseSkipped(sequence); const existing = this.#cached.get(sequence); + let kept: Map | undefined; if (existing) { if (!(existing.group.closed.peek() instanceof Error)) { throw new Error(`duplicate group: sequence=${sequence}`); } + // A mirror the failed copy kept open, held for a duplicate, is the new copy's to continue. + for (const [sink, mirror] of existing.mirrors) { + if (mirror.closed.peek() !== undefined || !hooks.readsMirrored(mirror)) continue; + kept ??= new Map(); + kept.set(sink, mirror); + existing.mirrors.delete(sink); + existing.charge.sub(cache.MIRROR_OVERHEAD); + } // Subscribers may have seen the aborted incarnation's start, so it stays a watermark. this.#evict(existing); this.#uncache(existing, true); @@ -2202,7 +2234,7 @@ export class Producer { hooks.startGroup(group, start); this.#bind(group); this.#cap(group); - this.#publish(group, fetched); + this.#publish(group, fetched, existing && { failed: existing.group, kept }); return group; } diff --git a/quest/m1/copy-recover.md b/quest/m1/copy-recover.md deleted file mode 100644 index aeabf7e4f7c..00000000000 --- a/quest/m1/copy-recover.md +++ /dev/null @@ -1,26 +0,0 @@ -# [M] A reader on a failed copy continues on the held re-offer - -## Goal - -A live reader already handed a copy of a group that then fails keeps the -group: its `group::Consumer` continues on the successor copy the held -re-offer carries in the same track, at the same frame, instead of surfacing -the failure. Rust and `js/net` behave the same. - -## Plan - -Planned 2026-11-05 (maintainer, /quest-plan of the follow-ups from the -2026-11-05 /quest-complete run), from #7670. Milestone: ✅ m1. - -`with_recover` already continues a reader across a front's routes; reuse it -or its mechanism for the same-track successor rather than adding a parallel -path. A successor that diverges from what the reader already consumed fails -the reader, as a route switch does. Test on mock time: a reader mid-group on a -copy that fails continues on the re-offer with no duplicate or missing frame. - -Decided 2026-11-05 (maintainer, /quest-audit): ✅ the wait on -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670), Drop set's own -PR, becomes a `Required` link to Drop set ([moq-dev/moq#8476](https://github.com/moq-dev/moq/pull/8476)), since -#7670 adds the held re-offer (its decision 24) this builds on. - -Public API: none expected. Wire: none. diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 48f72cd6c1e..902f0c1542e 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -293,7 +293,8 @@ struct Opened { /// overtook the reset of the stream it replaces. It waits unread, which its stream credit /// bounds, and is discarded if that copy finishes or takes the group if it fails. struct Duplicate { - arriving: group::Handle, + /// The copy still arriving, whose readers wait while this holds it. + arriving: track::Hold, /// Lite-07: where a YIELDED reset of the arriving copy pauses it, which this continues. paused: Option>, track: track::Producer, @@ -1730,7 +1731,7 @@ impl GroupRecv { } } } - match duplicate.arriving.poll_closed(waiter) { + match duplicate.arriving.arriving().poll_closed(waiter) { Poll::Pending => { *open = Some(Open::Duplicate(duplicate)); break Poll::Pending; @@ -1742,7 +1743,12 @@ impl GroupRecv { let Some(entry) = subs.get(&hdr.subscribe) else { break Poll::Ready(Err(Error::Cancel)); }; - match entry.reopen(hdr, self.subscriber.version, duplicate.reading) { + let Duplicate { arriving, reading, .. } = duplicate; + let reopened = entry.reopen(hdr, self.subscriber.version, reading); + // Released once the group is opened again, so the failed copy's readers + // waiting on it find this copy. + drop(arriving); + match reopened { Ok(reopened) => *open = Some(reopened), Err(err) => break Poll::Ready(Err(err)), } @@ -5461,7 +5467,7 @@ mod tests { /// A lite-07 re-offer can overtake the reset of the stream it replaces: its stream waits, /// unread, until that copy settles. It is discarded once the copy finishes, and takes the - /// group once the copy fails. + /// group once the copy fails, continuing the reads of that copy. #[kio::test] async fn a_duplicate_of_an_arriving_copy_waits_for_it_to_settle() { use crate::transport::poll::Session as _; @@ -5478,6 +5484,8 @@ mod tests { .encode(&mut encoder, version) .unwrap(); encoder.varint(crate::coding::varint::zigzag(300)).unwrap(); + encoder.bytes(b"first").unwrap(); + encoder.varint(crate::coding::varint::zigzag(1)).unwrap(); encoder.bytes(b"again").unwrap(); let session = crate::lite::test_transport::ScriptedSession::eof(script); let clock = crate::time::Deadlines::new(crate::time::Instant::now()); @@ -5537,26 +5545,29 @@ mod tests { assert_eq!(tail.read().streams(), 1); let mut original = reader.assert_group(); + let frame = original.read_frame().await.unwrap().unwrap(); + assert_eq!(&frame.payload[..], b"first"); if !first_fails { first.finish().unwrap(); let res = kio::wait(|waiter| duplicate.poll_serve(waiter)).await; assert!(matches!(res, Err(Error::Duplicate)), "{res:?}"); - let frame = original.read_frame().await.unwrap().unwrap(); - assert_eq!(&frame.payload[..], b"first"); assert!(original.read_frame().await.unwrap().is_none()); continue; } - // The reset lands: the held stream takes the group. + // The reset lands: the reader already handed the original waits on the held stream + // rather than failing, and continues on it from its frame once it takes the group. first.abort(Error::Cancel).unwrap(); + assert!(original.poll_read_frame(&waiter).is_pending()); let res = kio::wait(|waiter| duplicate.poll_serve(waiter)).await; assert!(res.is_ok(), "{res:?}"); - // A reader already handed the original sees it fail, since no sequence is handed - // out twice; one that had not reached it reads the re-offer. - assert!(original.read_frame().await.is_err()); + assert_eq!(&original.read_frame().await.unwrap().unwrap().payload[..], b"again"); + assert!(original.read_frame().await.unwrap().is_none()); + // One that had not reached it reads the re-offer whole. let mut later = track.subscribe(subscription); let mut group = later.assert_group(); assert_eq!(group.sequence, 3); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"first"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"again"); assert!(group.read_frame().await.unwrap().is_none()); } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index e28ed1925f7..17a3d008b47 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1222,7 +1222,13 @@ impl GroupState { /// (idle past the cache pool's expiry window, or evicted under its byte target). A reader /// that already consumed every frame is missing nothing, so it gets the clean end of /// group; one that fell short sees the abort rather than a silently truncated stream. + /// A copy cut short at the back holds nothing past its cut, which a reader reaches only + /// with its cap past it (one capped there ends before asking), so the frames it wants + /// are missing rather than the group ended. fn poll_terminal(&self, index: usize) -> Poll> { + if self.past_cut(index) { + return Poll::Ready(Err(Error::Evicted)); + } match &self.outcome { Outcome::Released(total, err) if index < *total => Poll::Ready(Err(err.clone())), Outcome::Finished(_) | Outcome::Released(..) => Poll::Ready(Ok(())), @@ -1238,6 +1244,17 @@ impl GroupState { } } + /// Where this copy settled at the back cut its transport bound made. One that finished + /// short of its bound holds the whole group, which ends there. + fn cut(&self) -> Option { + self.outcome.frames().filter(|&frames| self.range_end == Some(frames)) + } + + /// Whether this copy settled at its back cut, with `index` at or past it. + fn past_cut(&self, index: usize) -> bool { + self.cut().is_some_and(|cut| index >= cut) + } + /// The relay state once the ingest has settled it, which it does before the first frame, /// the end object, or the close: a group is visible before its header is read. #[cfg(feature = "async")] @@ -1265,8 +1282,13 @@ impl GroupState { /// Resolve whether a reader at `index`, which already prefetched the frames below /// `unread`, can still make progress, answering the same question as a read without /// consuming anything. - fn poll_end(&self, index: usize, unread: usize, raw: bool) -> Poll> { + /// A cut short of the reader's cap `end` leaves the frames past it missing, whatever + /// the reader reads before it. + fn poll_end(&self, index: usize, unread: usize, raw: bool, end: Option) -> Poll> { self.check_read(unread, raw)?; + if self.cut().is_some_and(|cut| end.is_none_or(|end| end > cut)) { + return Poll::Ready(Err(Error::Evicted)); + } self.poll_terminal(index) } @@ -2503,6 +2525,11 @@ impl Handle { Arc::ptr_eq(&self.alive, &reader.cursor.alive) } + /// Whether `other` is a handle of this same copy. + pub(super) fn is_copy(&self, other: &Handle) -> bool { + Arc::ptr_eq(&self.alive, &other.alive) + } + /// Abort the group without taking ownership of its writer. // Sessions cancel a group while its writer is in a frame. pub(crate) fn abort(&self, err: Error) -> Result<()> { @@ -3127,6 +3154,7 @@ impl Handle { end: None, prefetch: Prefetch::default(), raw: false, + partial: false, compressed: self.track.compressed(), cache: self.cache.clone(), alive: self.alive.clone(), @@ -3603,6 +3631,13 @@ pub struct Consumer { #[cfg(target_pointer_width = "64")] const _: () = assert!(size_of::() <= 184); +/// Drop an end check's held copy once the reader moved to another copy itself. +fn drop_finishing(carry: &mut Option>) { + if let Some(carry) = carry { + carry.finishing = None; + } +} + /// What carries a read past the copy it reads; see [`Consumer::with_recover`] and /// [`track::board`]. #[derive(Default)] @@ -3611,6 +3646,9 @@ struct Carry { recover: Option, // Boarding or riding a flight, once the copy read shed the frame read next. ride: Option, + // The copy an end check waits on past the frames this reader prefetched, with the frame + // it reads that copy from: held between polls so the copy keeps this reader's demand. + finishing: Option<(usize, Consumer)>, } impl Clone for Carry { @@ -3622,6 +3660,7 @@ impl Clone for Carry { boarding: None, ..*ride }), + finishing: None, } } } @@ -3700,6 +3739,11 @@ struct Cursor { // Read a compressed group's stored slices rather than its inflated copy. raw: bool, + // Whether the last frame handed out to stream from this copy was still being written, + // until a skip past it. Left uncommitted when the copy fails, it is torn. A whole read + // after it follows its commit, so it can leave this set. + partial: bool, + // Whether the group is compressed, so a plain read inflates under the state's lock. compressed: bool, @@ -3721,6 +3765,7 @@ impl Clone for Cursor { end: self.end, prefetch: Prefetch::default(), raw: self.raw, + partial: self.partial, compressed: self.compressed, cache: self.cache.clone(), alive: self.alive.clone(), @@ -3955,6 +4000,7 @@ impl Consumer { self.cursor.end = end; self.cursor.raw = raw; self.adopted = true; + drop_finishing(&mut self.carry); } } @@ -4340,7 +4386,13 @@ impl Consumer { ride.skip -= 1; continue; } - if !matches!(res, Poll::Ready(Err(_))) || !self.shed_next() { + if !matches!(res, Poll::Ready(Err(_))) { + return res; + } + if ready!(self.poll_successor(waiter))? { + continue; + } + if !self.shed_next() { return res; } // The copy just boarded shed the frame before the reader took it: pool debt @@ -4368,6 +4420,66 @@ impl Consumer { } } + /// Continue on another copy of this group in the same track once the copy read failed or + /// ended at a cut short of the reader's cap, such as a re-offer held while that copy was + /// still arriving (see [`track::successor`]). + /// Returns whether the reader moved to one, or [`Error::Old`] once its budget gives up + /// on the wait. + fn poll_successor(&mut self, waiter: &kio::Waiter) -> Poll> { + let Some(copy) = ready!(self.poll_successor_at(waiter, self.cursor.index))? else { + return Poll::Ready(Ok(false)); + }; + self.adopt_copy(copy); + Poll::Ready(Ok(true)) + } + + /// The copy that continues this reader from frame `index`, at or past its cursor, once + /// the copy read failed or ended at a cut short of its cap; see [`Self::poll_successor`]. + fn poll_successor_at(&mut self, waiter: &kio::Waiter, index: usize) -> Poll>> { + { + let state = self.cursor.state.read(); + // Only a delivery failure: content that failed to inflate is the group's under its + // name, and a frame this reader took from the copy partly is lost with it. Ordinals + // stand for frames only on a dense copy. A copy cut short at the back that the + // reader's raised cap passed lacks the rest, like a failed one. + if self.expired + || self.carry.as_ref().is_some_and(|carry| carry.ride.is_some()) + || (state.outcome.error().is_none() && !state.past_cut(index)) + || (!self.cursor.raw && state.malformed()) + || (self.cursor.partial && state.committed() < self.cursor.index) + || state.sparse() + { + return Poll::Ready(Ok(None)); + } + } + let Some(track) = self.cursor.cache.track() else { + return Poll::Ready(Ok(None)); + }; + match track::successor(&track, self, index, waiter) { + Poll::Ready(copy) => Poll::Ready(Ok(copy)), + // A stall the budget bounds, though the copy read can receive nothing more. + Poll::Pending => match self.poll_expiry(waiter, true) { + true => Poll::Ready(Err(Error::Old)), + false => Poll::Pending, + }, + } + } + + /// Whether this copy can continue `reader` from frame `index`. + pub(super) fn continues(&self, reader: &Consumer, index: usize) -> bool { + let raw = reader.cursor.raw; + let state = self.cursor.state.read(); + // A plain read inflates from frame 0, and a copy that shed the frame can't serve it. + !state.sparse() + && (raw || state.flate.is_none() || state.offset == 0) + && state.check_read(index, raw).is_ok() + // Inflating a frame past the inflated ones needs every slice since, so none shed. + && state.plain(raw).is_none_or(|plain| { + let len = plain.len.load(Ordering::Acquire); + len > index || len >= state.held() + }) + } + fn recover(&self) -> Option<&super::resume::Recover> { self.carry.as_ref()?.recover.as_ref() } @@ -4387,6 +4499,7 @@ impl Consumer { self.carry = Some(Box::new(Carry { recover: None, ride: Some(ride), + finishing: None, })) } (None, None) => {} @@ -4415,6 +4528,15 @@ impl Consumer { self.cursor.index = index; self.cursor.end = end; self.cursor.raw = raw; + drop_finishing(&mut self.carry); + } + + /// The frames this reader has yet to read from `index` on: up to its cap. + pub(super) fn unread(&self, index: usize) -> Bounds { + Bounds { + start: index as u64, + end: self.cursor.end.map(|end| end as u64), + } } /// One past the last frame the copy read committed. @@ -4530,7 +4652,89 @@ impl Consumer { res } + /// Like a frame read, a failed copy's end waits out a re-offer held for the group and + /// reports the end of the copy that took it over (see [`Self::poll_successor`]). fn poll_finished_once(&mut self, waiter: &kio::Waiter) -> Poll> { + loop { + let res = self.poll_finished_copy(waiter); + if !matches!(res, Poll::Ready(Err(_))) { + return res; + } + // Frames this reader already prefetched, and those a copy cut short of its cap + // still holds up to the cut, are read before a successor is needed, so the end is + // judged past them, without leaving them behind. + let mut drained = self.cursor.index + self.cursor.prefetch.buffered().0 as usize; + { + let state = self.cursor.state.read(); + if let Some(cut) = state.cut() + && self.cursor.end.is_none_or(|end| end > cut) + && state.check_read(drained, self.cursor.raw).is_ok() + { + drained = drained.max(cut); + } + } + if drained > self.cursor.index { + return self.poll_finished_past(waiter, drained, res); + } + if !ready!(self.poll_successor(waiter))? { + return res; + } + } + } + + /// This reader's end once it drained its prefetch up to `drained`: its cap's, if + /// that comes first, else the end of the copy that continues it from there, judged on a + /// copy of the reader so its cursor stays on the frames still to drain. `failed` is the + /// copy's own answer, which stands without one. + fn poll_finished_past( + &mut self, + waiter: &kio::Waiter, + drained: usize, + failed: Poll>, + ) -> Poll> { + let index = self.cursor.index as u64; + if self.cursor.end.is_some_and(|end| drained >= end) { + return Poll::Ready(Ok(index)); + } + let kept = self.carry.as_mut().and_then(|carry| carry.finishing.take()); + let mut next = match kept { + // It hands over to a later copy itself, should it fail too. A cap or mode changed + // since asks again: the copy kept may not cover the new range. + Some((at, next)) + if at == drained && next.cursor.end == self.cursor.end && next.cursor.raw == self.cursor.raw => + { + next + } + _ => { + let Some(copy) = ready!(self.poll_successor_at(waiter, drained))? else { + return failed; + }; + let mut next = self.clone(); + next.adopt_copy(copy); + next.cursor.index = drained; + next + } + }; + match next.poll_finished_once(waiter) { + Poll::Ready(res) => { + // The failed copy holds no end, so the copy that answered releases the demand. + if res.is_ok() { + next.release_if_finished(); + } + Poll::Ready(res.map(|_| index)) + } + Poll::Pending => { + // Dropped, it would leave the copy unwanted, and a fetch could abandon it. + self.carry.get_or_insert_default().finishing = Some((drained, next)); + match self.poll_expiry(waiter, true) { + true => Poll::Ready(Err(Error::Old)), + false => Poll::Pending, + } + } + } + } + + fn poll_finished_copy(&mut self, waiter: &kio::Waiter) -> Poll> { if self.expired { return Poll::Ready(Err(Error::Old)); } @@ -4539,13 +4743,13 @@ impl Consumer { if self.cursor.capped() { return Poll::Ready(Ok(index as u64)); } - let raw = self.cursor.raw; + let (raw, end) = (self.cursor.raw, self.cursor.end); // Frames already prefetched are this cursor's whatever the group sheds since, so // only the first one past them can be evicted. let unread = index + self.cursor.prefetch.buffered().0 as usize; let res = self .cursor - .poll(waiter, |state| state.poll_end(index, unread, raw)) + .poll(waiter, |state| state.poll_end(index, unread, raw, end)) .map(|res| res.map(|()| index as u64)); self.poll_expired_if_blocked(waiter, res) } @@ -4606,6 +4810,7 @@ impl Cursor { self.index = index; // The batch was drained from below the new cursor, so it can't be reused. self.prefetch = Prefetch::default(); + self.partial = false; } fn skip_to(&mut self, index: u64) { @@ -4615,6 +4820,8 @@ impl Cursor { } self.index = index; self.prefetch = Prefetch::default(); + // The frame handed out partly is given up below the skip, so a failure tears nothing. + self.partial = false; } fn poll_next_frame( @@ -4634,6 +4841,7 @@ impl Cursor { if let Some(frame) = self.prefetch.pop_cached() { self.refresh_if_stale(); self.index += 1; + self.partial = false; let tail = self.index.saturating_add(self.prefetch.buffered().0 as usize)..end; let info = frame::Info { size: frame.payload.len() as u64, @@ -4696,6 +4904,7 @@ impl Cursor { }; self.index += 1; + self.partial = matches!(source, frame::Source::Partial(_)); // A direct read (not prefetched): count the frame here; the frame::Consumer // counts its bytes per chunk as they're read out. stats.frames(1); @@ -6715,6 +6924,31 @@ mod test { assert!(matches!(raw.read_frame().now_or_never().unwrap(), Err(Error::Evicted))); } + /// A compressed copy that shed its front slices before any plain read inflated them + /// continues no plain reader, even past them: inflating its frame needs them. A raw + /// reader takes the stored slice. + #[test] + #[cfg_attr(miri, ignore = "fills the 32 MiB group tail, past Miri's 15-minute cap")] + #[cfg(feature = "async")] + fn a_copy_shed_before_inflating_continues_no_plain_reader() { + let pool = cache::Pool::unbounded(); + let mut group = compressed_group(&pool); + for frame in &mebibytes(40) { + group.write_frame(None, frame.clone()).unwrap(); + } + let held = group.handle.state.read().held(); + assert!(held > 1, "the slices were shed"); + let copy = group.consume(); + let mut plain = group.consume(); + plain.skip_to(held as u64); + assert!(!copy.continues(&plain, held)); + assert!( + matches!(read_plain(&mut plain), Err(Error::Evicted)), + "inflating needs them" + ); + assert!(copy.continues(&group.consume().compressed(), held)); + } + /// The inflated copy charges each frame's slot, so empty frames count toward its tail. #[test] #[cfg(feature = "async")] @@ -7349,6 +7583,27 @@ mod test { assert!(matches!(whole.poll_finished(&waiter), Poll::Ready(Err(Error::Cancel)))); } + /// A copy its transport bound at frame 4 that finishes after frame 2 ends there: the group + /// holds two frames, so a reader past them reads a clean end rather than a cut. + #[test] + fn a_bounded_copy_finished_short_of_its_bound_ends_cleanly() { + let mut producer = Info { sequence: 0 }.produce(); + producer.bound(Bounds { start: 0, end: Some(4) }).unwrap(); + let mut reader = producer.consume(); + for payload in [b"a", b"b"] { + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(payload)) + .unwrap(); + } + producer.finish().unwrap(); + reader.skip_to(4); + let waiter = kio::Waiter::noop(); + let res = reader.poll_read_frame(&waiter); + assert!(matches!(res, Poll::Ready(Ok(None))), "{res:?}"); + let res = reader.poll_finished(&waiter); + assert!(matches!(res, Poll::Ready(Ok(4))), "{res:?}"); + } + /// A publisher that read every frame of a finished group still learns when the budget /// gives up on it, so it can abandon the stream it is still writing. #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 8f8645fa559..5744e2664cf 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -42,7 +42,7 @@ pub(crate) use hole::Hole; pub(crate) use fill::{GroupFill, Readers as FillReaders}; use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet, VecDeque, btree_map}, ops::{self, Bound, RangeBounds}, sync::Arc, sync::OnceLock, @@ -583,6 +583,9 @@ pub(crate) struct TrackState { // the readers riding them; see [`Self::board`]. Held weakly, so the last rider leaving // frees one, and pruned as they are looked up. flights: BTreeMap>, + // The copies of each sequence still arriving that another received copy waits on, once per + // [`Hold`], which a reader of that copy waits out when it fails; see [`successor`]. + holds: BTreeMap>, // Publisher-produced groups in arrival order as (sequence, stamp), walked by // subscriptions; an entry only resolves while its stamp matches the slot's. @@ -3803,6 +3806,48 @@ impl TrackState { } } +/// The copy of its group `reader` continues on from frame `index` once the copy it reads +/// failed: one holding the group from there, once no re-offer of it is still held (see [`Hold`]). A +/// group's name fixes its content, so any such copy continues it. +pub(super) fn successor( + track: &kio::Producer, + reader: &group::Consumer, + index: usize, + waiter: &kio::Waiter, +) -> Poll> { + let sequence = reader.sequence; + // A copy cut short at the back still continues a reader capped no later than the cut. + let frames = reader.unread(index); + let found = track.poll_ref(waiter, |state| { + // A hold on the copy read, or on any copy of the group that failed, promises a copy + // that takes the group over, which continues this reader too. One on a copy still + // arriving waits on that copy, which the reader takes instead. + if state + .holds + .get(&sequence) + .is_some_and(|held| held.iter().any(|copy| copy.is_read_by(reader) || copy.is_aborted())) + { + return Poll::Pending; + } + // Every copy, since one that fails or cannot serve the frame may sit before one that can. + Poll::Ready( + state + .lookup + .get(&sequence) + .into_iter() + .flat_map(Slot::copies) + .filter(|copy| copy.covers(frames) && !copy.is_read_by(reader)) + .filter_map(group::Handle::try_consume) + .find(|copy| copy.continues(reader, index)), + ) + }); + Poll::Ready(match found { + Poll::Ready(Ok(copy)) => copy, + Poll::Ready(Err(_closed)) => None, + Poll::Pending => return Poll::Pending, + }) +} + /// Where a reader at a frame its copy shed goes: see [`board`]. pub(super) enum Board { /// Ride this copy, a flight or a cached copy, which holds the frame. @@ -4019,7 +4064,43 @@ pub(crate) enum Received { /// Another copy of the sequence that is still arriving: whether this one is needed /// depends on how that one settles. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - Held(group::Handle), + Held(Hold), +} + +/// A received copy waiting on another copy of its sequence still arriving. While it waits, +/// a reader of that copy that fails waits for this one to take the group instead of failing. +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +pub(crate) struct Hold { + arriving: group::Handle, + track: kio::Weak, + sequence: u64, +} + +#[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] +impl Hold { + /// The copy still arriving. + pub(crate) fn arriving(&self) -> &group::Handle { + &self.arriving + } +} + +impl Drop for Hold { + fn drop(&mut self) { + let Some(track) = self.track.upgrade() else { + return; + }; + let Ok(mut state) = track.write() else { + return; + }; + if let btree_map::Entry::Occupied(mut holds) = state.holds.entry(self.sequence) { + if let Some(at) = holds.get().iter().position(|held| held.is_copy(&self.arriving)) { + holds.get_mut().swap_remove(at); + } + if holds.get().is_empty() { + holds.remove(); + } + } + } } /// A producer for a track, used to create new groups. @@ -4175,10 +4256,16 @@ impl Producer { let start = received.unwrap_or(0); let content = if received.is_some() { if let Some(holding) = state.holding_group(info.sequence, start) { - return match holding.is_settled() { - true => Err(Error::Duplicate), - false => Ok(Received::Held(holding.clone())), - }; + if holding.is_settled() { + return Err(Error::Duplicate); + } + let arriving = holding.clone(); + state.holds.entry(info.sequence).or_default().push(arriving.clone()); + return Ok(Received::Held(Hold { + arriving, + track: self.state.downgrade(), + sequence: info.sequence, + })); } state.content_for_copy(info.sequence)? } else { @@ -17352,6 +17439,580 @@ mod test { producer } + /// A reader on a received copy that fails waits out a re-offer still held for its group, + /// then continues on it from its frame with nothing repeated or missed, even while another + /// duplicate waits on that re-offer. A released hold leaves the failure, as does a re-offer + /// starting past the reader's frame. + #[kio::test] + #[cfg(feature = "async")] + async fn a_failed_copy_continues_on_its_held_reoffer() { + for case in ["reoffer", "released", "past"] { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + let mut reader = first.consume(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a", "{case}"); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + let Ok(Received::Held(second)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("a second duplicate waits too"); + }; + first.abort(Error::Cancel).unwrap(); + let waiter = kio::Waiter::noop(); + assert!( + reader.poll_read_frame(&waiter).is_pending(), + "{case}: waits on the held re-offer" + ); + + if case == "released" { + drop((hold, second)); + assert!(reader.read_frame().await.is_err(), "{case}"); + continue; + } + let start = if case == "past" { 2 } else { 0 }; + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, start) else { + panic!("the re-offer takes the group"); + }; + drop(hold); + assert!( + reader.poll_read_frame(&waiter).is_pending(), + "{case}: the second still holds" + ); + // The second duplicate now waits on the re-offer, which holds nobody reading the + // failed copy. + let Ok(Received::Held(third)) = producer.create_group_at(group::Info { sequence: 3 }, start) else { + panic!("the second duplicate waits on the re-offer still arriving"); + }; + drop(second); + if case == "past" { + assert!(reader.read_frame().await.is_err(), "{case}: frame 1 is in no copy"); + continue; + } + // Parked on frame 1, which the re-offer has yet to write. + assert!(reader.poll_read_frame(&waiter).is_pending()); + again.write_frame(at(300), &b"a"[..]).unwrap(); + again.write_frame(at(301), &b"b"[..]).unwrap(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + again.finish().unwrap(); + drop(third); + assert!(reader.read_frame().await.unwrap().is_none()); + } + } + + /// A failed copy's end follows its frame reads: it waits out a held re-offer rather than + /// report the failure, so a publisher watching it keeps the stream the re-offer continues. + #[kio::test] + #[cfg(feature = "async")] + async fn a_failed_copy_finishes_on_its_held_reoffer() { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + let mut reader = first.consume(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let waiter = kio::Waiter::noop(); + assert!(reader.poll_finished(&waiter).is_pending(), "waits on the held re-offer"); + + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer takes the group"); + }; + drop(hold); + again.write_frame(at(300), &b"a"[..]).unwrap(); + again.write_frame(at(301), &b"b"[..]).unwrap(); + assert!(reader.poll_finished(&waiter).is_pending(), "the re-offer is still open"); + again.finish().unwrap(); + assert_eq!(reader.finished().await.unwrap(), 1); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + } + + /// A re-offer that fails while still sending frames the reader took from an earlier copy + /// hands over to the next held re-offer: only a frame the reader took from the failing copy + /// itself is torn. One it was streaming from that copy is lost with it. + #[kio::test] + #[cfg(feature = "async")] + async fn a_reoffer_failing_in_its_replay_hands_over_again() { + for torn in [false, true] { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + let mut reader = first.consume(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a", "{torn}"); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer takes the group"); + }; + let Ok(Received::Held(next)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("another re-offer waits on the first"); + }; + drop(hold); + let waiter = kio::Waiter::noop(); + let handle = again.handle(); + let streaming = if torn { + // The reader streams part of frame 1 from the re-offer before it fails. + again.write_frame(at(300), &b"a"[..]).unwrap(); + let mut frame = again + .create_frame(frame::Info { + size: 2, + timestamp: Some(at(301)), + }) + .unwrap(); + frame.write(&b"b"[..]).unwrap(); + let mut part = reader.next_frame().await.unwrap().expect("frame 1 streams"); + assert_eq!(&part.read_chunk().await.unwrap().unwrap()[..], b"b"); + frame + } else { + // The re-offer fails partway through frame 0, which the reader already holds. + let mut frame = again + .create_frame(frame::Info { + size: 2, + timestamp: Some(at(300)), + }) + .unwrap(); + frame.write(&b"a"[..]).unwrap(); + assert!(reader.poll_read_frame(&waiter).is_pending()); + frame + }; + handle.abort(Error::Cancel).unwrap(); + drop(streaming); + if torn { + drop(next); + assert!(reader.read_frame().await.is_err(), "frame 1 was handed out partly"); + continue; + } + assert!( + reader.poll_read_frame(&waiter).is_pending(), + "waits on the next re-offer" + ); + let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the next re-offer takes the group"); + }; + drop(next); + last.write_frame(at(300), &b"a"[..]).unwrap(); + last.write_frame(at(301), &b"b"[..]).unwrap(); + last.finish().unwrap(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + assert!(reader.read_frame().await.unwrap().is_none()); + } + } + + /// A reader whose end was judged on a re-offer past its prefetched frames keeps waiting + /// when that re-offer fails while another is held for it, then reads its prefetched frame + /// and continues on whichever copy holds the rest. + #[kio::test] + #[cfg(feature = "async")] + async fn a_failed_copy_finishes_on_a_reoffer_of_a_failed_reoffer() { + for again_fails in [false, true] { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + first.write_frame(at(301), &b"b"[..]).unwrap(); + let mut reader = first.consume(); + // Takes frame 1 into its prefetch with frame 0. + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 2) else { + panic!("the re-offer takes the group"); + }; + let Ok(Received::Held(next)) = producer.create_group_at(group::Info { sequence: 3 }, 2) else { + panic!("another re-offer waits on the first"); + }; + drop(hold); + let waiter = kio::Waiter::noop(); + assert!( + reader.poll_finished(&waiter).is_pending(), + "{again_fails}: the re-offer is open" + ); + if !again_fails { + drop(next); + again.write_frame(at(302), &b"c"[..]).unwrap(); + again.finish().unwrap(); + assert_eq!(reader.finished().await.unwrap(), 1); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); + assert!(reader.read_frame().await.unwrap().is_none()); + continue; + } + again.abort(Error::Cancel).unwrap(); + let res = reader.poll_finished(&waiter); + assert!(res.is_pending(), "waits on the next re-offer: {res:?}"); + let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the next re-offer takes the group"); + }; + drop(next); + for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { + last.write_frame(at(millis), &payload[..]).unwrap(); + } + last.finish().unwrap(); + assert_eq!(reader.finished().await.unwrap(), 1); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); + assert!(reader.read_frame().await.unwrap().is_none()); + } + } + + /// A failed copy's end agrees with its frame reads when the reader still has frames it + /// prefetched from it: a re-offer starting past the reader's frame but not past those + /// continues it, and one starting past both does not. + #[kio::test] + #[cfg(feature = "async")] + async fn a_failed_copy_finishes_past_its_prefetched_frames() { + for start in [2, 3] { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + first.write_frame(at(301), &b"b"[..]).unwrap(); + let mut reader = first.consume(); + // Takes frame 1 into its prefetch with frame 0. + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let waiter = kio::Waiter::noop(); + assert!(reader.poll_finished(&waiter).is_pending(), "{start}: waits on the hold"); + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, start) else { + panic!("the re-offer takes the group"); + }; + drop(hold); + if start == 3 { + assert!(reader.finished().await.is_err(), "frame 2 is in no copy"); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + assert!(reader.read_frame().await.is_err(), "nor can a read find it"); + continue; + } + again.write_frame(at(302), &b"c"[..]).unwrap(); + let res = reader.poll_finished(&waiter); + assert!(res.is_pending(), "the re-offer is still open: {res:?}"); + // The end check keeps reading the re-offer, so a fetch would not abandon it. + assert!( + again.handle().poll_unwanted(&waiter).is_pending(), + "the re-offer stays wanted" + ); + again.finish().unwrap(); + // From where the reader stands, with frame 1 still to read. + assert_eq!(reader.finished().await.unwrap(), 1); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); + assert!(reader.read_frame().await.unwrap().is_none()); + } + } + + /// A reader whose failed copy's end a re-offer answers past its prefetched frames reached + /// the group's end, so it no longer keeps the track wanted. + #[kio::test] + #[cfg(feature = "async")] + async fn a_failed_copy_finished_on_a_reoffer_releases_its_demand() { + let producer = received_producer(Info::timed()); + let mut subscriber = producer.subscribe(None); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + first.write_frame(at(301), &b"b"[..]).unwrap(); + let mut reader = subscriber.assert_group(); + drop(subscriber); + // Takes frame 1 into its prefetch with frame 0. + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 2) else { + panic!("the re-offer takes the group"); + }; + drop(hold); + again.write_frame(at(302), &b"c"[..]).unwrap(); + again.finish().unwrap(); + assert_eq!(producer.demand().wanted(), Some(0), "not read to its end yet"); + assert_eq!(reader.finished().await.unwrap(), 1); + assert_eq!(producer.demand().wanted(), None, "the re-offer answered the end"); + } + + /// A capped reader continues on a copy cut short at the back no earlier than its cap, which + /// holds every frame it still reads. + #[kio::test] + #[cfg(feature = "async")] + async fn a_capped_reader_continues_on_a_copy_cut_at_its_cap() { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + // Received from frame 1, so a fetch of the group's head can sit beside it. + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 1) else { + panic!("the first copy opens"); + }; + first.write_frame(at(301), &b"b"[..]).unwrap(); + let mut reader = first.consume().with_range(..3); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + + let mut head = producer + .state + .write() + .ok() + .unwrap() + .insert_group_request(3, group::Bounds { start: 0, end: Some(3) }, None) + .unwrap(); + for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { + head.write_frame(at(millis), &payload[..]).unwrap(); + } + head.finish().unwrap(); + first.abort(Error::Cancel).unwrap(); + // An uncapped reader needs frames past the head, which no copy holds. + assert!(reader.clone().with_end(None).read_frame().await.is_err()); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); + assert!(reader.read_frame().await.unwrap().is_none()); + } + + /// A reader that raises its cap past a copy's cut before reaching it learns from its end + /// what its frame reads will: the frames past the cut are missing, whether the ones before + /// it are still in the copy or already prefetched. + #[kio::test] + #[cfg(feature = "async")] + async fn a_raised_cap_before_a_cut_ends_in_its_error() { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let mut head = producer + .state + .write() + .ok() + .unwrap() + .insert_group_request(3, group::Bounds { start: 0, end: Some(3) }, None) + .unwrap(); + let mut reader = head.consume().with_range(..3); + for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { + head.write_frame(at(millis), &payload[..]).unwrap(); + } + head.finish().unwrap(); + // Takes frames 1 and 2 into its prefetch with frame 0. + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + let mut reader = reader.with_end(None); + // A clone shares the cursor but not the prefetch, so frames 1 and 2 are the copy's. + let mut unfetched = reader.clone(); + let res = unfetched.finished().await; + assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); + let res = reader.finished().await; + assert!(res.is_err(), "nor past the prefetch: {res:?}"); + for payload in [b"b", b"c"] { + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], payload); + } + assert!(reader.read_frame().await.is_err(), "the reads agree"); + } + + /// An end check waiting on a copy past the reader's prefetch follows the reader's cap: once + /// the cap is lifted, a successor cut at the old cap no longer ends the group cleanly. + #[kio::test] + #[cfg(feature = "async")] + async fn a_retained_end_check_follows_a_lifted_cap() { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(300), &b"a"[..]).unwrap(); + first.write_frame(at(301), &b"b"[..]).unwrap(); + let mut reader = first.consume().with_range(..3); + // Takes frame 1 into its prefetch with frame 0. + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let mut cut = producer + .state + .write() + .ok() + .unwrap() + .insert_group_request(3, group::Bounds { start: 2, end: Some(3) }, None) + .unwrap(); + drop(hold); + cut.write_frame(at(302), &b"c"[..]).unwrap(); + let waiter = kio::Waiter::noop(); + assert!(reader.poll_finished(&waiter).is_pending(), "the cut copy is still open"); + let mut reader = reader.with_range(..); + cut.finish().unwrap(); + let res = reader.finished().await; + assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); + } + + /// A reader that skipped ahead of the frames its copy committed tore none of them, so it + /// continues on a held re-offer from where it skipped to. + #[kio::test] + #[cfg(feature = "async")] + async fn a_skipped_reader_continues_on_a_held_reoffer() { + for chunked in [false, true] { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + let mut reader = first.consume(); + if chunked { + // Frame 0 streams to the reader while still being written, then commits. + let mut frame = first + .create_frame(frame::Info { + size: 1, + timestamp: Some(at(300)), + }) + .unwrap(); + let mut part = reader.next_frame().await.unwrap().expect("frame 0 streams"); + frame.write(&b"a"[..]).unwrap(); + assert_eq!(&part.read_chunk().await.unwrap().unwrap()[..], b"a"); + first = frame.finish().unwrap(); + } else { + first.write_frame(at(300), &b"a"[..]).unwrap(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + } + reader.skip_to(3); + + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let waiter = kio::Waiter::noop(); + assert!( + reader.poll_read_frame(&waiter).is_pending(), + "waits on the held re-offer" + ); + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer takes the group"); + }; + drop(hold); + for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c"), (303, b"d")] { + again.write_frame(at(millis), &payload[..]).unwrap(); + } + again.finish().unwrap(); + assert_eq!( + &reader.read_frame().await.unwrap().unwrap().payload[..], + b"d", + "chunked: {chunked}" + ); + assert!(reader.read_frame().await.unwrap().is_none()); + } + } + + /// A capped reader that continued on a copy cut at its cap and then raises the cap reads + /// no clean end at the cut: the frames past it are missing from that copy, so it continues + /// on a copy that holds them, or fails without one. + #[kio::test] + #[cfg(feature = "async")] + async fn a_raised_cap_past_an_adopted_cut_finds_the_rest() { + for reoffer in [false, true] { + let producer = received_producer(Info::timed()); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 1) else { + panic!("the first copy opens"); + }; + first.write_frame(at(301), &b"b"[..]).unwrap(); + let mut reader = first.consume().with_range(..3); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); + + let mut head = producer + .state + .write() + .ok() + .unwrap() + .insert_group_request(3, group::Bounds { start: 0, end: Some(3) }, None) + .unwrap(); + for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { + head.write_frame(at(millis), &payload[..]).unwrap(); + } + head.finish().unwrap(); + first.abort(Error::Cancel).unwrap(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); + + let mut reader = reader.with_end(None); + let mut finished = reader.clone(); + if !reoffer { + let res = reader.read_frame().await; + assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); + let res = finished.finished().await; + assert!(res.is_err(), "nor is the group's end: {res:?}"); + continue; + } + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer takes the group"); + }; + for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c"), (303, b"d")] { + again.write_frame(at(millis), &payload[..]).unwrap(); + } + again.finish().unwrap(); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"d"); + assert!(reader.read_frame().await.unwrap().is_none()); + // Its end, from where that reader stood, on the copy holding the rest. + assert_eq!(finished.finished().await.unwrap(), 3); + } + } + + /// A reader waiting out a held re-offer is stalled, so its budget still gives up on it. + #[kio::test] + #[cfg(feature = "async")] + async fn waiting_on_a_held_reoffer_expires() { + for finished in [false, true] { + let producer = received_producer(Info::timed()); + let mut subscriber = producer + .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); + let at = |millis| Timestamp::from_millis(millis).unwrap(); + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the first copy opens"); + }; + first.write_frame(at(0), &b"a"[..]).unwrap(); + let mut reader = subscriber.recv_group().await.unwrap().expect("group 3"); + assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); + + let Ok(Received::Held(_hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + panic!("the re-offer waits on the copy still arriving"); + }; + first.abort(Error::Cancel).unwrap(); + let waiter = kio::Waiter::noop(); + assert!(reader.poll_read_frame(&waiter).is_pending(), "{finished}"); + + // Group 4 bounds group 3's reach at 1s, and group 5 puts the edge a second past it. + let mut later = Vec::new(); + for (sequence, millis) in [(4, 1000), (5, 2000)] { + let Ok(Received::Group(mut next)) = producer.create_group_at(group::Info { sequence }, 0) else { + panic!("group {sequence} opens"); + }; + next.write_frame(at(millis), &b"x"[..]).unwrap(); + later.push(next); + } + let res = match finished { + true => reader.finished().await.map(|_| ()), + false => reader.read_frame().await.map(|_| ()), + }; + assert!(matches!(res, Err(Error::Old)), "{finished}: {res:?}"); + } + } + /// Create and finish group `sequence`, with one frame at `millis` on a timed track. #[cfg(feature = "async")] fn put(producer: &Producer, sequence: u64, millis: Option) {