From c9ac9a4385fb8ea643abb6ae625e94bb96cd2325 Mon Sep 17 00:00:00 2001 From: Matt Sheppard Date: Fri, 23 Oct 2026 05:26:05 -0500 Subject: [PATCH] fix(net)!: stop one silent dial from stalling the reconnect loop Co-authored-by: Claude Opus 5.5 --- js/net/MIGRATE.md | 5 + js/net/src/connection/connect.test.ts | 596 +++++++++++++++++++++++++- js/net/src/connection/connect.ts | 183 ++++++-- js/net/src/connection/pool.test.ts | 35 ++ js/net/src/connection/pool.ts | 16 +- js/net/src/connection/reload.test.ts | 114 ++++- js/net/src/connection/reload.ts | 16 +- js/net/src/error.ts | 19 + js/net/src/errors.ts | 1 + 9 files changed, 942 insertions(+), 43 deletions(-) diff --git a/js/net/MIGRATE.md b/js/net/MIGRATE.md index bc64e99cadd..ac922c5c311 100644 --- a/js/net/MIGRATE.md +++ b/js/net/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### A connect gives up after 30 seconds + +`connect()`, and each attempt of a `Connection`, now rejects with `Error.ConnectTimeout` once the dial and the MoQ handshake together take longer than 30 s, closing what it had opened. Before, an attempt nothing answered waited until the browser or the OS gave up, which for a WebSocket can be minutes, and a supplied `transport` whose handshake was never answered waited forever. +Pass `timeout` to change the bound, or `timeout: Milli(0)` for no deadline. A deadline set through `signal`, for example `AbortSignal.timeout(60_000)`, now only takes effect when it is shorter than `timeout`. + ### `Track.Producer.largest()` is a getter `Track.Producer.largest()` returns the track's largest location synchronously, `undefined` before any, instead of a promise for the next newer one. diff --git a/js/net/src/connection/connect.test.ts b/js/net/src/connection/connect.test.ts index 382099eb3a0..ac6d485348c 100644 --- a/js/net/src/connection/connect.test.ts +++ b/js/net/src/connection/connect.test.ts @@ -1,5 +1,6 @@ -import { expect, jest, test } from "bun:test"; -import { SessionCode, SessionError } from "../error.ts"; +import { expect, jest, spyOn, test } from "bun:test"; +import { Manual, Milli, Nano, sleep } from "@moq/clock"; +import { ConnectTimeout, SessionCode, SessionError } from "../error.ts"; import * as Ietf from "../ietf/index.ts"; import { ALPN, ALPN_05, ALPN_06, ALPN_07_WIP } from "../lite/version.ts"; import { createMockTransportPair } from "../mock.ts"; @@ -448,3 +449,594 @@ test("the WebSocket fallback offers the same versions", async () => { expect(asked[0]?.endsWith(ALPN_07_WIP)).toBe(true); expect(asked.some((protocol) => protocol.endsWith(ALPN_05))).toBe(false); }); + +// A WebTransport whose handshake nothing ever answers, as a dial to a dead target is. +function silentWebTransport(): { closes: () => number; restore: () => void } { + const original = globalThis.WebTransport; + let closes = 0; + class SilentWebTransport { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() { + closes++; + } + } + globalThis.WebTransport = SilentWebTransport as unknown as typeof WebTransport; + return { closes: () => closes, restore: () => (globalThis.WebTransport = original) }; +} + +// Record how a connect settles, without awaiting it. +function track(attempt: Promise): { outcome: () => unknown } { + let outcome: unknown; + attempt.then( + () => (outcome = "connected"), + (err: unknown) => (outcome = err), + ); + return { outcome: () => outcome }; +} + +test("a dial nothing answers fails with ConnectTimeout after the default timeout and closes the transport", async () => { + const clock = new Manual(); + const restore = clock.install(); + const silent = silentWebTransport(); + try { + const attempt = track(connect(url, { websocket: { enabled: false } })); + await clock.advance(Nano.fromMilli(Milli(29_999))); + expect(attempt.outcome()).toBeUndefined(); + + await clock.advance(Nano.fromMilli(Milli(1))); + await settle(); + expect(attempt.outcome()).toBeInstanceOf(ConnectTimeout); + expect((attempt.outcome() as ConnectTimeout).timeout).toBe(Milli(30_000)); + // The dial and its hedge. + expect(silent.closes()).toBe(2); + } finally { + silent.restore(); + restore(); + } +}); + +test("a timeout of 0 waits for the caller", async () => { + const clock = new Manual(); + const restore = clock.install(); + const silent = silentWebTransport(); + try { + const controller = new AbortController(); + const attempt = track( + connect(url, { websocket: { enabled: false }, timeout: Milli(0), signal: controller.signal }), + ); + await clock.advance(Nano.fromMilli(Milli(600_000))); + expect(attempt.outcome()).toBeUndefined(); + + const reason = new Error("teardown"); + controller.abort(reason); + await settle(); + expect(attempt.outcome()).toBe(reason); + } finally { + silent.restore(); + restore(); + } +}); + +test("the caller's abort wins over the timeout and disarms it", async () => { + const clock = new Manual(); + const restore = clock.install(); + const silent = silentWebTransport(); + try { + const controller = new AbortController(); + const attempt = track(connect(url, { websocket: { enabled: false }, signal: controller.signal })); + await clock.advance(Nano.fromMilli(Milli(10_000))); + const reason = new Error("teardown"); + controller.abort(reason); + await settle(); + expect(attempt.outcome()).toBe(reason); + + await clock.advance(Nano.fromMilli(Milli(60_000))); + expect(attempt.outcome()).toBe(reason); + // The dial and its hedge. + expect(silent.closes()).toBe(2); + } finally { + silent.restore(); + restore(); + } +}); + +test("a handshake the peer never answers times out on a supplied transport too", async () => { + const clock = new Manual(); + const restore = clock.install(); + try { + // The `moql` ALPN negotiates in SETUP, which nothing on the other side answers. + const pair = createMockTransportPair(ALPN); + let transportClosed = false; + void pair.client.closed.then( + () => (transportClosed = true), + () => (transportClosed = true), + ); + + const attempt = track(connect(url, { transport: pair.client, timeout: Milli(5_000) })); + await clock.advance(Nano.fromMilli(Milli(4_999))); + expect(attempt.outcome()).toBeUndefined(); + + await clock.advance(Nano.fromMilli(Milli(1))); + await settle(); + expect(attempt.outcome()).toBeInstanceOf(ConnectTimeout); + expect(transportClosed).toBe(true); + } finally { + restore(); + } +}); + +type Dial = "hang" | "ok" | { failAfter: Milli } | { openWhen: Promise }; + +// Stub WebTransport so each dial behaves as `plan` says, recording when it was dialed (on +// the Manual clock) and how often it was closed. +function webTransportPlan(clock: Manual, plan: (dial: number) => Dial) { + const original = globalThis.WebTransport; + const dials: { at: number; closes: number }[] = []; + globalThis.WebTransport = function StubWebTransport() { + const index = dials.length; + const record = { at: Nano.toMilli(Nano(clock.now())), closes: 0 }; + dials.push(record); + const dial = plan(index); + if (dial === "hang" || (typeof dial === "object" && "failAfter" in dial)) { + const ready = + dial === "hang" + ? new Promise(() => {}) + : sleep(Nano.fromMilli(dial.failAfter)).then(() => { + throw new Error(`refused ${index}`); + }); + ready.catch(() => {}); + return { + ready, + closed: new Promise(() => {}), + close() { + record.closes++; + }, + }; + } + const pair = createMockTransportPair(ALPN_06); + const close = pair.client.close.bind(pair.client); + pair.client.close = (info?: WebTransportCloseInfo) => { + record.closes++; + close(info); + }; + if (dial !== "ok") Object.defineProperty(pair.client, "ready", { value: dial.openWhen.then(() => undefined) }); + return pair.client; + } as unknown as typeof WebTransport; + return { dials, restore: () => (globalThis.WebTransport = original) }; +} + +test("a dial still unopened after 500 ms is hedged, and the hedge wins when the first hangs", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webTransportPlan(clock, (dial) => (dial === 0 ? "hang" : "ok")); + try { + const attempt = connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(499))); + expect(plan.dials.length).toBe(1); + + await clock.advance(Nano.fromMilli(Milli(1))); + const connection = await attempt; + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 500]); + await settle(); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 0]); + connection.abort(); + } finally { + plan.restore(); + restore(); + } +}); + +test("a dial that opens within 500 ms is not hedged", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webTransportPlan(clock, () => "ok"); + try { + const connection = await connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(5_000))); + expect(plan.dials.length).toBe(1); + connection.abort(); + } finally { + plan.restore(); + restore(); + } +}); + +test("a dial that fails outright is not hedged", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webTransportPlan(clock, () => ({ failAfter: Milli(100) })); + try { + const attempt = track(connect(url, { websocket: { enabled: false } })); + await clock.advance(Nano.fromMilli(Milli(5_000))); + await settle(); + expect(plan.dials.length).toBe(1); + expect((attempt.outcome() as Error).message).toBe("no transport connected: refused 0"); + } finally { + plan.restore(); + restore(); + } +}); + +test("when a dial and its hedge both fail, the first dial's reason is reported", async () => { + const clock = new Manual(); + const restore = clock.install(); + // The hedge dials at 500 ms and fails at 600 ms; the first fails at 1.5 s. + const plan = webTransportPlan(clock, (dial) => ({ failAfter: Milli(dial === 0 ? 1_500 : 100) })); + try { + const attempt = track(connect(url, { websocket: { enabled: false } })); + await clock.advance(Nano.fromMilli(Milli(2_000))); + await settle(); + expect(plan.dials.length).toBe(2); + expect((attempt.outcome() as Error).message).toBe("no transport connected: refused 0"); + } finally { + plan.restore(); + restore(); + } +}); + +test("when a dial and its hedge open in the same turn, one wins and the other is closed", async () => { + const clock = new Manual(); + const restore = clock.install(); + let open = () => {}; + const opened = new Promise((resolve) => (open = resolve)); + // The first opens the moment the hedge dials, and the hedge opens at once. + const plan = webTransportPlan(clock, (dial) => { + if (dial === 0) return { openWhen: opened }; + open(); + return "ok"; + }); + try { + const attempt = connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(500))); + const connection = await attempt; + await settle(); + expect(plan.dials.length).toBe(2); + expect(plan.dials.map((dial) => dial.closes).sort()).toEqual([0, 1]); + connection.abort(); + } finally { + plan.restore(); + restore(); + } +}); + +test("a hedge that opens after the first dial failed still wins", async () => { + const clock = new Manual(); + const restore = clock.install(); + // The hedge dials at 500 ms and opens at 2 s; the first fails at 1.5 s in between. + const plan = webTransportPlan(clock, (dial) => + dial === 0 ? { failAfter: Milli(1_500) } : { openWhen: sleep(Nano.fromMilli(Milli(1_500))) }, + ); + try { + const attempt = connect(url, { websocket: { enabled: false } }); + await clock.advance(Nano.fromMilli(Milli(2_000))); + const connection = await attempt; + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 500]); + connection.abort(); + } finally { + plan.restore(); + restore(); + } +}); + +test("when the WebSocket wins, every WebTransport dial is closed", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webTransportPlan(clock, () => "hang"); + // The WebSocket dials after its 500 ms head start, with the hedge, and opens 100 ms later. + const native = globalThis as { WebSocketStream?: unknown }; + const originalStream = native.WebSocketStream; + native.WebSocketStream = class { + url: string; + opened = sleep(Nano.fromMilli(Milli(100))).then(() => ({ + readable: new ReadableStream(), + writable: new WritableStream(), + protocol: "", + extensions: "", + })); + closed = new Promise(() => {}); + constructor(url: string) { + this.url = url; + } + close() {} + }; + const controller = new AbortController(); + try { + const pending = connect(new URL("https://example.com/websocket-wins"), { signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(700))); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 500]); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 1]); + + // The WebSocket's SETUP is never answered. + controller.abort(); + await pending.catch(() => undefined); + } finally { + native.WebSocketStream = originalStream; + plan.restore(); + restore(); + } +}); + +test("when WebTransport and WebSocket open in the same turn, the one not chosen is closed", async () => { + const clock = new Manual(); + const restore = clock.install(); + let open = () => {}; + const opened = new Promise((resolve) => (open = resolve)); + const plan = webTransportPlan(clock, () => ({ openWhen: opened })); + // A native-shaped WebSocketStream that opens on the same signal. + let websocketCloses = 0; + const native = globalThis as { WebSocketStream?: unknown }; + const originalStream = native.WebSocketStream; + native.WebSocketStream = class { + url: string; + opened = opened.then(() => ({ + readable: new ReadableStream(), + writable: new WritableStream(), + protocol: "", + extensions: "", + })); + closed = new Promise(() => {}); + constructor(url: string) { + this.url = url; + } + close() { + websocketCloses++; + } + }; + const controller = new AbortController(); + try { + // Its own URL: a WebSocket win drops WebTransport's head start for that URL afterwards. + const pending = connect(new URL("https://example.com/same-turn"), { + websocket: { delay: Milli(0) }, + signal: controller.signal, + }); + await settle(); + open(); + await settle(); + await settle(); + // Exactly one transport lost the race, and it was closed. + expect(plan.dials[0].closes + websocketCloses).toBe(1); + + // A WebSocket winner is still in SETUP, which nothing answers; a WebTransport one connected. + controller.abort(); + const connection = await pending.catch(() => undefined); + connection?.abort(); + } finally { + native.WebSocketStream = originalStream; + plan.restore(); + restore(); + } +}); + +test("the caller's abort wins over a timeout that fires in the same turn", async () => { + const clock = new Manual(); + const restore = clock.install(); + const silent = silentWebTransport(); + try { + const controller = new AbortController(); + const attempt = track(connect(url, { websocket: { enabled: false }, signal: controller.signal })); + await settle(); + const reason = new Error("teardown"); + // Fire the timeout without running any task, then abort before one runs. + clock.jump(Nano.fromMilli(Milli(30_000))); + controller.abort(reason); + await settle(); + expect(attempt.outcome()).toBe(reason); + } finally { + silent.restore(); + restore(); + } +}); + +test("a connected session outlives its connect timeout", async () => { + const clock = new Manual(); + const restore = clock.install(); + try { + const pair = createMockTransportPair(ALPN_06); + let closed = false; + void pair.client.closed.then(() => (closed = true)); + const connection = await connect(url, { transport: pair.client, timeout: Milli(1_000) }); + await clock.advance(Nano.fromMilli(Milli(2_000))); + await settle(); + expect(closed).toBe(false); + connection.abort(); + } finally { + restore(); + } +}); + +test("timeout race never returns a closed connection", async () => { + // Sweep the microtask window between winning the transport race and returning its + // connection, as "abort race never returns a closed connection" does for the signal. + for (let hops = 0; hops < 12; hops++) { + const clock = new Manual(); + const restore = clock.install(); + try { + const pair = createMockTransportPair(ALPN_05); + let transportClosed = false; + void pair.client.closed.then(() => { + transportClosed = true; + }); + const outcome = connect(url, { transport: pair.client, timeout: Milli(1) }).then( + (connection) => ({ connection }), + (err: unknown) => ({ err }), + ); + void (async () => { + for (let i = 0; i < hops; i++) await Promise.resolve(); + clock.jump(Nano.fromMilli(Milli(1))); + })(); + const result = await outcome; + await settle(); + if ("connection" in result) { + expect(transportClosed, `hops ${hops}`).toBe(false); + result.connection.abort(); + } else { + expect(result.err, `hops ${hops}`).toBeInstanceOf(ConnectTimeout); + expect(transportClosed, `hops ${hops}`).toBe(true); + } + } finally { + restore(); + } + } +}); + +test("an invalid timeout throws before touching the caller's signal", async () => { + const controller = new AbortController(); + const added = spyOn(controller.signal, "addEventListener"); + for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + await expect( + connect(url, { websocket: { enabled: false }, timeout: timeout as Milli, signal: controller.signal }), + ).rejects.toThrow(RangeError); + } + expect(added).not.toHaveBeenCalled(); + // In the caller's unit, not the nanoseconds it is converted to. + await expect(connect(url, { timeout: -1 as Milli })).rejects.toThrow("invalid time: -1"); +}); + +type WebSocketDial = "hang" | "open" | { failAfter: Milli } | { openAfter: Milli }; + +// Their own URL: a WebSocket that wins drops WebTransport's head start for its URL afterwards. +const websocketUrl = new URL("https://example.com/websocket-only"); + +// Stub a native WebSocketStream so each dial behaves as `plan` says, recording when it was +// dialed (on the Manual clock) and how often it was closed. WebTransport is removed, as in +// Safari, so the WebSocket is the only transport and dials at once. +function webSocketPlan(clock: Manual, plan: (dial: number) => WebSocketDial) { + const native = globalThis as { WebSocketStream?: unknown; WebTransport?: unknown }; + const originalStream = native.WebSocketStream; + const originalTransport = native.WebTransport; + native.WebTransport = undefined; + const dials: { at: number; closes: number }[] = []; + const streams = () => ({ + readable: new ReadableStream(), + writable: new WritableStream(), + protocol: "", + extensions: "", + }); + native.WebSocketStream = class { + url: string; + opened: Promise; + closed = new Promise(() => {}); + #record: { at: number; closes: number }; + constructor(url: string) { + this.url = url; + const index = dials.length; + this.#record = { at: Nano.toMilli(Nano(clock.now())), closes: 0 }; + dials.push(this.#record); + const dial = plan(index); + if (dial === "hang") this.opened = new Promise(() => {}); + else if (dial === "open") this.opened = Promise.resolve(streams()); + else if ("failAfter" in dial) + this.opened = sleep(Nano.fromMilli(dial.failAfter)).then(() => { + throw new Error(`refused ${index}`); + }); + else this.opened = sleep(Nano.fromMilli(dial.openAfter)).then(streams); + this.opened.catch(() => {}); + } + close() { + this.#record.closes++; + } + }; + return { + dials, + restore: () => { + native.WebSocketStream = originalStream; + native.WebTransport = originalTransport; + }, + }; +} + +test("a WebSocket dial still unopened after 10 s is closed and dialed again", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webSocketPlan(clock, (dial) => (dial === 0 ? "hang" : "open")); + const controller = new AbortController(); + try { + const pending = connect(websocketUrl, { signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(9_999))); + expect(plan.dials.length).toBe(1); + + await clock.advance(Nano.fromMilli(Milli(1))); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 10_000]); + // The session flushes its close for 100 ms before closing the socket. + await clock.advance(Nano.fromMilli(Milli(100))); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 0]); + + // The second dial opened; its SETUP is never answered. + controller.abort(); + await pending.catch(() => undefined); + } finally { + plan.restore(); + restore(); + } +}); + +test("each WebSocket redial within an attempt waits twice as long", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webSocketPlan(clock, () => "hang"); + const controller = new AbortController(); + try { + const pending = connect(websocketUrl, { timeout: Milli(0), signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(75_000))); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 10_000, 30_000, 70_000]); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 1, 1, 0]); + + controller.abort(); + await pending.catch(() => undefined); + } finally { + plan.restore(); + restore(); + } +}); + +test("a WebSocket dial that fails outright is not redialed", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webSocketPlan(clock, () => ({ failAfter: Milli(100) })); + try { + const attempt = track(connect(websocketUrl, {})); + await clock.advance(Nano.fromMilli(Milli(20_000))); + await settle(); + expect(plan.dials.length).toBe(1); + expect((attempt.outcome() as Error).message).toBe("no transport connected: refused 0"); + } finally { + plan.restore(); + restore(); + } +}); + +test("a WebSocket dial that opens within 10 s is not redialed", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webSocketPlan(clock, () => ({ openAfter: Milli(5_000) })); + const controller = new AbortController(); + try { + const pending = connect(websocketUrl, { signal: controller.signal }); + await clock.advance(Nano.fromMilli(Milli(15_000))); + expect(plan.dials.length).toBe(1); + + controller.abort(); + await pending.catch(() => undefined); + } finally { + plan.restore(); + restore(); + } +}); + +test("with the default timeout, a silent WebSocket is redialed once and the attempt then times out", async () => { + const clock = new Manual(); + const restore = clock.install(); + const plan = webSocketPlan(clock, () => "hang"); + try { + const attempt = track(connect(websocketUrl, {})); + await clock.advance(Nano.fromMilli(Milli(30_100))); + await settle(); + expect(attempt.outcome()).toBeInstanceOf(ConnectTimeout); + expect(plan.dials.map((dial) => dial.at)).toEqual([0, 10_000]); + expect(plan.dials.map((dial) => dial.closes)).toEqual([1, 1]); + } finally { + plan.restore(); + restore(); + } +}); diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index 2130654ce00..10d0cf95963 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -1,6 +1,7 @@ -import { Milli, Nano, sleep } from "@moq/clock"; +import { Elapsed, Milli, Nano, sleep, timeout } from "@moq/clock"; import Session, { type Version as QmuxVersion } from "@moq/qmux"; -import { error, fromClose, ProtocolViolation, sessionCause } from "../error.ts"; +import { race } from "@moq/signals"; +import { ConnectTimeout, error, fromClose, ProtocolViolation, sessionCause } from "../error.ts"; import * as Ietf from "../ietf/index.ts"; import * as Message from "../ietf/message.ts"; import * as Lite from "../lite/index.ts"; @@ -8,6 +9,7 @@ import type { Consumer as OriginConsumer, Producer as OriginProducer } from "../ import { Stream } from "../stream.ts"; import * as Hex from "../util/hex.ts"; import { redact } from "../util/log.ts"; +import { after } from "../util/timer.ts"; import { isWebTransportSupported } from "./browser.ts"; import type { Established } from "./established.ts"; import * as Extensions from "./extensions.ts"; @@ -17,6 +19,21 @@ import { exchangeSetup } from "./handshake.ts"; // Default head start for WebTransport before attempting the WebSocket fallback. const DEFAULT_WEBSOCKET_DELAY_MS = Milli(500); +// Default bound on one connect, the dial and the MoQ handshake; matches the Rust client. +const DEFAULT_TIMEOUT = Milli(30_000); + +// How long the first WebTransport dial may stay unopened before a second one dials alongside +// it. The same moment as the WebSocket's default head start, so a fresh WebTransport flow +// races the fallback rather than losing to it, which would leave the session on WebSocket for +// good. A healthy open is about two round trips, so this spares round trips up to about +// 250 ms, and it is inside the 2 s ceiling RFC 8305 recommends for staggered attempts. +const WEBTRANSPORT_HEDGE = DEFAULT_WEBSOCKET_DELAY_MS; + +// How long a WebSocket dial may stay unopened before it is closed and dialed again. Generous, +// since script cannot tell a silent connect from a slow one; Firefox and socket.io give a +// WebSocket 20 s to open. +const WEBSOCKET_REDIAL = Milli(10_000); + // RESET_STREAM and CONNECTION_CLOSE are separate transport notifications. Give the latter // time to surface its authoritative code without letting a reset-only peer stall SETUP forever. const SETUP_CLOSE_GRACE = Nano.fromMilli(Milli(100)); @@ -171,8 +188,9 @@ export interface ConnectProps { * Use a pre-existing WebTransport session instead of connecting; skips the * WebTransport/WebSocket race. The publisher acquires the session's datagram * writer lock (`datagrams.writable.getWriter()`) for the session's lifetime, - * so the caller must not hold it. Aborting via {@link ConnectProps.signal} - * closes this session, so don't share one you intend to reuse afterwards. + * so the caller must not hold it. Aborting via {@link ConnectProps.signal}, or + * {@link ConnectProps.timeout} passing, closes this session, so don't share one + * you intend to reuse afterwards. */ transport?: WebTransport; @@ -203,10 +221,17 @@ export interface ConnectProps { /** The moq-transport extensions to offer, each on unless set to `false`. */ extensions?: Extensions.Extensions; + /** + * Maximum time for one connect, covering the dial and the MoQ handshake, which has no + * deadline of its own (default: 30000ms). `0` waits forever. Past it, the connect rejects + * with {@link ConnectTimeout} and closes whatever it had opened. + */ + timeout?: Milli; + /** * Aborts the connection attempt with the signal's reason. An already-aborted * signal rejects before anything opens, and aborting after the connection is - * established has no effect. Use `AbortSignal.timeout(ms)` for a deadline. + * established has no effect. Set {@link ConnectProps.timeout} for a deadline. */ signal?: AbortSignal; } @@ -239,26 +264,40 @@ export async function connect({ url, ...props }: ConnectProps): Promise(); const onAbort = () => resolve(); signal.addEventListener("abort", onAbort, { once: true }); + let expired: ConnectTimeout | undefined; + const disarm = + limit > 0 + ? after(deadline, () => { + expired = new ConnectTimeout(limit); + resolve(); + }) + : () => {}; + const pending = connectInner(url, props, abort); try { - // A `pending` rejection propagates unless the abort beat it to the finish line. + // A `pending` rejection propagates unless the abort or timeout beat it to the finish line. const connection = await Promise.race([pending, abort.then(() => undefined)]); - if (connection && !signal.aborted) { + if (connection && !signal.aborted && !expired) { if (props.consume) forwardAnnounced(connection, props.consume); return connection; } - // Close a connection that settles after the abort. + // Close a connection that settles after the abort or timeout. pending.then((conn) => conn.abort()).catch(() => {}); - throw signal.reason; + // The caller's abort is teardown and wins over the timeout. + throw signal.aborted ? signal.reason : expired; } finally { signal.removeEventListener("abort", onAbort); + disarm(); } } @@ -294,7 +333,7 @@ async function connectInner(raw: URL, props: Omit, abort: P const cancel = Promise.race([raced, abort]); const webtransport = isWebTransportSupported() - ? connectWebTransport(url, cancel, offered.alpns, props.webtransport) + ? hedgeWebTransport(url, cancel, offered.alpns, props.webtransport) : undefined; // Give QUIC a head start to connect before trying WebSocket, unless WebSocket has won in the past. @@ -313,19 +352,19 @@ async function connectInner(raw: URL, props: Omit, abort: P // Race the available transports, using `.any` to ignore if one participant has an error. // `webtransport`/`websocket` are `Promise | undefined`, so test existence explicitly: a // promise is always truthy, so bare truthiness here would be a misused-promise. - const session = await Promise.any( - webtransport !== undefined - ? websocket !== undefined - ? [websocket, webtransport] - : [webtransport] - : [websocket], - ).catch((err: unknown) => { + const contenders: Promise[] = []; + if (websocket !== undefined) contenders.push(websocket); + if (webtransport !== undefined) contenders.push(webtransport); + const session = await Promise.any(contenders).catch((err: unknown) => { // Promise.any rejects with an empty message; name each transport's failure instead. if (!(err instanceof AggregateError)) throw err; const reasons = err.errors.map((e: unknown) => (e instanceof Error ? e.message : String(e))); throw new Error(`no transport connected: ${reasons.join("; ")}`, { cause: err }); }); done(); + // A contender that opened in the same turn as the winner has already returned its + // transport, so cancelling cannot close it; the rest close themselves on cancel. + closeLosers(contenders, session); if (!session) throw new Error("no transport available"); @@ -578,6 +617,58 @@ async function resolveCertificateHashes(options?: WebTransportProps): Promise 0 ? hashes : undefined; } +// Close every transport the race opened other than `winner`. +function closeLosers(contenders: Promise[], winner: unknown): void { + for (const contender of contenders) { + void contender.then( + (transport) => { + if (transport && transport !== winner) transport.close(); + }, + () => {}, + ); + } +} + +/** + * Dial WebTransport, and dial it again alongside if the first has not opened within + * {@link WEBTRANSPORT_HEDGE}. The first to open wins and the other is closed. + * + * A load balancer that still places some new flows on a target that just died sends every + * packet of such a flow there, and the browser waits seconds before failing the dial. A + * second dial is a new flow that the balancer places afresh. Neither dial is cut on a timer of + * its own, only by the attempt's {@link ConnectProps.timeout}: the browser's handshake timer + * tells a silent dial from a slow one, which script cannot, and every dial the browser fails + * or abandons counts against its WebTransport throttle. A first dial that fails outright is + * not hedged; that is left to the caller's retry. + */ +async function hedgeWebTransport( + url: URL, + cancel: Promise, + alpns: string[], + options?: WebTransportProps, +): Promise { + const first = connectWebTransport(url, cancel, alpns, options); + + // The first dial opening, failing, or being cancelled settles this before the delay does. + const hedge = await timeout(Nano.fromMilli(WEBTRANSPORT_HEDGE), first).then( + () => false, + (err: unknown) => err instanceof Elapsed, + ); + if (!hedge) return first; + console.debug(redact(url), `WebTransport not open after ${WEBTRANSPORT_HEDGE}ms; dialing a second`); + + const dials = [first, connectWebTransport(url, cancel, alpns, options)]; + try { + const winner = await Promise.any(dials); + closeLosers(dials, winner); + return winner; + } catch (err) { + // Both failed. Report the first dial's reason, as when there is no hedge, so `no transport + // connected` keeps one reason per transport; a browser words every handshake failure alike. + throw (err as AggregateError).errors[0]; + } +} + async function connectWebTransport( url: URL, cancel: Promise, @@ -648,6 +739,17 @@ async function connectWebTransport( } // TODO accept arguments to control the port/path used. +/** + * Dial the WebSocket fallback after `delay`, and if a dial has not opened within + * {@link WEBSOCKET_REDIAL}, close it and dial again, doubling the wait each time. + * + * A browser lets one WebSocket to an address be connecting at a time (RFC 6455 section 4.1), + * so a second dial alongside a silent one only queues behind it, and a TCP connect whose SYNs + * go unanswered waits for the OS's connect timeout. Closing the silent dial lets the next one + * start, on a new connection that a load balancer places afresh. The doubling lets a + * slow but working path through, and {@link ConnectProps.timeout} bounds the whole attempt. A + * dial that fails outright is not redialed. + */ async function connectWebSocket( url: URL, delay: number, @@ -659,23 +761,34 @@ async function connectWebSocket( const active = await Promise.race([cancel, timer.then(() => true)]); if (!active) return undefined; - // The default (`requireProtocol: false`) also advertises bare `qmux-01`, - // `qmux-00`, and `webtransport` so we still interop with relays that only - // know a wire-format version (today's moq-relay only accepts bare - // `webtransport`). - const quic = new Session(url, { - protocols: alpns, - versions: Object.fromEntries(alpns.map((alpn) => [alpn, QMUX[alpn] ?? null])), - }); + for (let wait = WEBSOCKET_REDIAL; ; wait = Milli(wait * 2)) { + // The default (`requireProtocol: false`) also advertises bare `qmux-01`, + // `qmux-00`, and `webtransport` so we still interop with relays that only + // know a wire-format version (today's moq-relay only accepts bare + // `webtransport`). + const quic = new Session(url, { + protocols: alpns, + versions: Object.fromEntries(alpns.map((alpn) => [alpn, QMUX[alpn] ?? null])), + }); - // Wait for the WebSocket to connect, or for the cancel promise to resolve. - // `ready` rejects on a refused/failed connection, so a throw here is the caller's - // cue to retry; a lost cancel race instead resolves and we close the loser. - const loaded = await Promise.race([quic.ready.then(() => true), cancel]); - if (!loaded) { - quic.close(); - return undefined; + // Wait for the WebSocket to connect, for the cancel promise to resolve, or for the + // redial. `ready` rejects on a refused/failed connection, so a throw here is the + // caller's cue to retry; a lost cancel race instead resolves and we close the loser. + const disarm = new AbortController(); + try { + // `race` rather than `Promise.race`, and `cancel` itself rather than something derived + // from it: `cancel` outlives every dial, and `race` releases what it attached. + const outcome = await race([ + quic.ready.then(() => "open" as const), + cancel, + sleep(Nano.fromMilli(wait), { signal: disarm.signal }).then(() => "redial" as const), + ]); + if (outcome === "open") return quic; + quic.close(); + if (outcome !== "redial") return undefined; + console.debug(redact(url), `WebSocket not open after ${wait}ms; dialing again`); + } finally { + disarm.abort(); + } } - - return quic; } diff --git a/js/net/src/connection/pool.test.ts b/js/net/src/connection/pool.test.ts index a44f6521899..6574be44a1d 100644 --- a/js/net/src/connection/pool.test.ts +++ b/js/net/src/connection/pool.test.ts @@ -301,6 +301,7 @@ test("connection-specific options infer a private loop and origin", async () => { webtransport: { congestionControl: "throughput" as const } }, { websocket: { enabled: false } }, { delay: { timeout: Milli(0) } }, + { timeout: Milli(10_000) }, { publish: origin.consume() }, { consume: origin }, ]; @@ -335,6 +336,7 @@ test("explicit share: true refuses connection-specific options", () => { ); expect(() => new Connection({ share: true, websocket: { enabled: false } })).toThrow(/share: false/); expect(() => new Connection({ share: true, delay: { timeout: Milli(0) } })).toThrow(/share: false/); + expect(() => new Connection({ share: true, timeout: Milli(10_000) })).toThrow(/share: false/); } finally { origin.close(); } @@ -775,3 +777,36 @@ test("GOAWAY keeps version pools distinct at the redirected URL", async () => { defaults?.close(); } }); + +test("a private handle's timeout bounds each attempt", async () => { + let attempts = 0; + class Hung { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() {} + } + globalThis.WebTransport = function StubWebTransport() { + attempts++; + return new Hung(); + } as unknown as typeof WebTransport; + + const handle = new Connection({ + url, + share: false, + websocket: { enabled: false }, + timeout: Milli(50), + delay: { initial: Milli(10), multiplier: 1, max: Milli(10) }, + }); + try { + // Each hung dial is cut at 50 ms and redialed after the backoff; the default 30 s + // timeout would still be holding the first. + await advance(1_000); + expect(attempts).toBeGreaterThanOrEqual(3); + } finally { + handle.close(); + } +}); + +test("an invalid timeout is refused when a private handle is built", () => { + expect(() => new Connection({ url, share: false, timeout: Number.NaN as Milli })).toThrow(RangeError); +}); diff --git a/js/net/src/connection/pool.ts b/js/net/src/connection/pool.ts index 3fe423048f0..cf7e8feba36 100644 --- a/js/net/src/connection/pool.ts +++ b/js/net/src/connection/pool.ts @@ -79,6 +79,13 @@ export interface ConnectionProps { /** Backoff settings for the reconnect loop; an unset field uses its default. */ delay?: ReloadDelay; + /** + * Maximum time for each connection attempt, the dial and the MoQ handshake (default: + * 30000ms; `0` waits forever). An attempt that outlives it is retried like any other + * failure. The same bound as the `timeout` option of {@link Connection.connect}. + */ + timeout?: Milli; + /** * How to react to the relay's GOAWAY; an unset field uses its default. Every connection * migrates on GOAWAY, dialing the replacement while the old session finishes its groups. @@ -114,7 +121,7 @@ export interface ConnectionProps { * session finishes its groups, and the handle and its origin carry across. A redirect the * {@link ConnectionProps.goaway} policy refuses stops the loop like an auth rejection does. * - * Options the pool cannot honor (transport options, delay, goaway, a pinned + * Options the pool cannot honor (transport options, delay, timeout, goaway, a pinned * certificate, caller-owned origins) use a private loop. An explicit `share: true` * refuses them. A supplied transport cannot reconnect at all; pass it to {@link Connection.connect} * instead. @@ -217,6 +224,8 @@ export class Connection { // A plain list is checked and copied now, so a bad name throws here and mutating the array // later changes nothing. A live list is checked as it changes, failing on `error`. const fixed = normalizeVersions({ ...props, versions: Array.isArray(given) ? given : undefined }); + // Likewise a bad timeout, before anything below exists to leak. + if (props?.timeout !== undefined) Milli(props.timeout); refuse(props); this.#signals = new Effect(); @@ -296,6 +305,7 @@ export class Connection { webtransport: props.webtransport, websocket: props.websocket, delay: props.delay, + timeout: props.timeout, goaway: props.goaway, }); signals.cleanup(() => loop.close()); @@ -434,6 +444,9 @@ function refuse(props?: ConnectionProps): void { if (props.delay !== undefined) { throw new Error("delay cannot be shared; pass share: false"); } + if (props.timeout !== undefined) { + throw new Error("timeout cannot be shared; pass share: false"); + } if (props.goaway !== undefined) { throw new Error("goaway cannot be shared; pass share: false"); } @@ -445,6 +458,7 @@ function requiresPrivate(props: ConnectionProps): boolean { props.webtransport !== undefined || props.websocket !== undefined || props.delay !== undefined || + props.timeout !== undefined || props.goaway !== undefined || props.publish !== undefined || props.consume !== undefined diff --git a/js/net/src/connection/reload.test.ts b/js/net/src/connection/reload.test.ts index 06892bb4885..d1ba0fa4498 100644 --- a/js/net/src/connection/reload.test.ts +++ b/js/net/src/connection/reload.test.ts @@ -1,9 +1,9 @@ import { expect, spyOn, test } from "bun:test"; -import { Milli } from "@moq/clock"; +import { Instant, Manual, Milli, Nano } from "@moq/clock"; import { Effect, Signal } from "@moq/signals"; import * as Announce from "../announced.ts"; import type { Producer as BroadcastProducer } from "../broadcast.ts"; -import { SessionCode, SessionError, StreamCode, toTransport } from "../error.ts"; +import { ConnectTimeout, SessionCode, SessionError, StreamCode, toTransport } from "../error.ts"; import { Route } from "../hop.ts"; import { ALPN as IetfALPN } from "../ietf/version.ts"; import * as Lite from "../lite/index.ts"; @@ -709,3 +709,113 @@ test("redirect loop detection catches any cycle in constant memory", () => { const redirects = new Redirects(); for (let hop = 0; hop < 1000; hop++) expect(redirects.move(`a${hop}`, `a${hop + 1}`)).toBe(false); }); + +// Fire the installed Manual clock timer by timer until `pred` holds; returns the Manual time +// that passed, in milliseconds. +async function elapse(clock: Manual, pred: () => boolean): Promise { + const start = clock.now(); + for (;;) { + for (let i = 0; i < 3 && !pred(); i++) await settle(); + if (pred()) return Nano.toMilli(Instant.durationSince(clock.now(), start)); + await clock.fire(); + } +} + +// Stub WebTransport so each dial either hangs, as one placed on a dead target does, or +// connects to a relay that accepts it. +function dialPlan(plan: (dial: number) => "hang" | "ok", url: URL) { + const original = globalThis.WebTransport; + const origin = new OriginProducer(); + let dials = 0; + let hungCloses = 0; + class Hung { + ready = new Promise(() => {}); + closed = new Promise(() => {}); + close() { + hungCloses++; + } + } + globalThis.WebTransport = function StubWebTransport() { + const dial = dials++; + if (plan(dial) === "hang") return new Hung(); + const pair = createMockTransportPair(Lite.ALPN_06); + void accept(pair.server, url, { publish: origin.consume() }); + return pair.client; + } as unknown as typeof WebTransport; + return { + dials: () => dials, + hungCloses: () => hungCloses, + restore: () => { + globalThis.WebTransport = original; + origin.close(); + }, + }; +} + +test("an attempt that outlives the connect timeout is retried after the backoff", async () => { + const clock = new Manual(); + const restore = clock.install(); + const url = new URL("https://relay.example/room"); + // Scaled down: the attempt times out before a WebTransport hedge would dial. + const plan = dialPlan((dial) => (dial === 0 ? "hang" : "ok"), url); + const statuses: string[] = []; + const reload = new Reload({ + url, + websocket: { enabled: false }, + timeout: Milli(50), + delay: { initial: Milli(20) }, + }); + const unsubscribe = reload.status.subscribe((status) => statuses.push(status)); + try { + await waitUntil(() => plan.dials() === 1); + const waited = await elapse(clock, () => plan.dials() === 2); + // The 50 ms timeout, then the first backoff with its equal jitter (10 to 20 ms). + expect(waited).toBeGreaterThanOrEqual(60); + expect(waited).toBeLessThanOrEqual(70); + expect(plan.hungCloses()).toBe(1); + + await waitUntil(() => reload.status.peek() === "connected"); + expect(statuses).toContain("disconnected"); + expect(reload.error.peek()).toBeUndefined(); + } finally { + unsubscribe(); + reload.close(); + plan.restore(); + restore(); + } +}); + +test("attempts that keep timing out give up at the retry window with ConnectTimeout", async () => { + const clock = new Manual(); + const restore = clock.install(); + const url = new URL("https://relay.example/room"); + const plan = dialPlan(() => "hang", url); + const reload = new Reload({ + url, + websocket: { enabled: false }, + timeout: Milli(30), + delay: { initial: Milli(10), multiplier: 1, max: Milli(10), timeout: Milli(100) }, + }); + try { + await waitUntil(() => plan.dials() === 1); + const waited = await elapse(clock, () => reload.error.peek() !== undefined); + // The window starts at the first failure (30 ms) and is checked at each failure after + // it, so giving up comes up to one attempt after the window. + expect(waited).toBeGreaterThanOrEqual(130); + expect(waited).toBeLessThan(170); + expect(reload.error.peek()).toBeInstanceOf(ConnectTimeout); + expect(plan.hungCloses()).toBe(plan.dials()); + } finally { + reload.close(); + plan.restore(); + restore(); + } +}); + +test("an invalid timeout is refused when the loop is built, not by every attempt", () => { + for (const timeout of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => new Reload({ url: new URL("https://example.com/"), timeout: timeout as Milli })).toThrow( + RangeError, + ); + } +}); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 81c8c6e10ed..926d593c99a 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -23,8 +23,10 @@ import type { Probe, Stats } from "./stats.ts"; * Exponential backoff settings for the reconnect loop. * * The delays carry jitter, so a fleet of tabs knocked offline together doesn't reconnect in - * lockstep. Every failure is retried; {@link ReloadDelay.timeout} is what stops the current - * URL. A new URL or a disable/re-enable starts another sequence. + * lockstep. Every failure is retried, including an attempt that outlives + * {@link ConnectProps.timeout}, until {@link ReloadDelay.timeout} runs out; only an auth + * rejection or a refused GOAWAY redirect stops the current URL sooner. A new URL or a + * disable/re-enable starts another sequence. * * A GOAWAY off a session that outlived {@link ReloadDelay.initial} migrates at once, unless it * sends the connection back to an endpoint that already redirected it elsewhere and the @@ -44,7 +46,9 @@ export type ReloadDelay = { /** * Maximum total time to spend retrying the current URL before giving up * (default: 0, unlimited). Resets after each successful connection outside a redirect loop, - * a URL change, or a disable/re-enable. + * a URL change, or a disable/re-enable. It decides whether another attempt starts and never + * cuts one that is running, so giving up can take up to one attempt longer than the window; + * {@link ConnectProps.timeout} bounds each attempt. */ timeout?: Milli; }; @@ -156,6 +160,9 @@ export class Reload { /** WebSocket fallback options applied to each connection attempt (not reactive). */ websocket: WebSocketProps | undefined; + /** The bound on each connection attempt. See {@link ConnectProps.timeout}. */ + readonly timeout?: Milli; + /** * The origin whose broadcasts are served, spanning reconnects (not reactive). * @@ -254,6 +261,8 @@ export class Reload { this.versions = normalizeVersions(props); this.webtransport = props?.webtransport; this.websocket = props?.websocket; + // Validated here, like the versions, rather than by every attempt. + this.timeout = props?.timeout === undefined ? undefined : Milli(props.timeout); this.publish = props?.publish; this.consume = props?.consume; @@ -411,6 +420,7 @@ export class Reload { websocket: this.#redirect.peek() ? { ...this.websocket, url: undefined } : this.websocket, versions: this.versions, webtransport: this.webtransport, + timeout: this.timeout, publish: this.publish, consume: this.consume, signal, diff --git a/js/net/src/error.ts b/js/net/src/error.ts index 170b8725547..1b861a5fa7f 100644 --- a/js/net/src/error.ts +++ b/js/net/src/error.ts @@ -4,6 +4,7 @@ * @module */ +import type { Milli } from "@moq/clock"; import { sharedStreamCode } from "./ietf/error.ts"; import type { IetfVersion } from "./ietf/version.ts"; import { TimeoutError } from "./util/timeout.ts"; @@ -302,6 +303,24 @@ export class RefusedRedirect extends Error { } } +/** + * The dial and the MoQ handshake together outlived the connect's `timeout` option. + * + * A caller can retry it. Mirrors the Rust `Error::ConnectTimeout`. + * + * @public + */ +export class ConnectTimeout extends Error { + /** The timeout that elapsed. */ + readonly timeout: Milli; + + constructor(timeout: Milli) { + super(`connect timed out after ${timeout}ms`); + this.name = "ConnectTimeout"; + this.timeout = timeout; + } +} + /** * A peer broke the protocol in a way the spec says must end the session. * diff --git a/js/net/src/errors.ts b/js/net/src/errors.ts index ec33cc0ab88..b8421c2a0fb 100644 --- a/js/net/src/errors.ts +++ b/js/net/src/errors.ts @@ -4,6 +4,7 @@ * @module */ export { + ConnectTimeout, FrameTooLarge, GroupTooLarge, NoStreamCredit,