From c76e11c7b91dea8361afe6d6814671345aa4850b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 07:44:01 -0700 Subject: [PATCH 1/6] quest: claim uring-teardown-close Co-Authored-By: Claude Opus 5.5 From 7540ee91ac2534a08a5e3c07c803b935a49a8dfe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 08:57:50 -0700 Subject: [PATCH 2/6] fix(uring): resolve a dial only once the client Finished is staged noq raises Connected on the client with its Finished still queued, and paces it like any other packet. Under load the first RTT sample is long, so the pacer can hold the Finished just as `connect` resolves. A close in that gap replaced it: the server never completed its handshake, saw only a handshake-level close without the application code, and its `accept` never resolved. That is the 60 s hang in the teardown close test. `connect` now resolves once the client has staged one more CRYPTO frame than it had at Connected. A new teardown test reproduces the gap on an idle host through a relay that slows the server-to-client leg. Co-Authored-By: Claude Opus 5.5 --- quest/m1/test-flakes-2/README.md | 1 - .../m1/test-flakes-2/uring-teardown-close.md | 31 -------- rs/moq-uring/src/quic/noq/connection.rs | 29 +++++++- rs/moq-uring/src/quic/noq/endpoint.rs | 3 + rs/moq-uring/tests/teardown.rs | 72 +++++++++++++++++-- 5 files changed, 98 insertions(+), 38 deletions(-) delete mode 100644 quest/m1/test-flakes-2/uring-teardown-close.md diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index f95604acd21..310b1b408c5 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -40,7 +40,6 @@ Public API: none. Wire: none. - [WebTransport close release](/quest/m1/test-flakes-2/interop-close-release.md) - a compatible published adapter contains the acknowledged-close fix - [Interop close code](/quest/m1/test-flakes-2/interop-close-code.md) - adopt that release so the browser reliably sees `unauthorized` for a refused session - [Auth refresh mocks time](/quest/m1/test-flakes-2/auth-refresh-mock-time.md) - moq-relay's auth refresh and fixed-lease tests measure a grant's life on one mocked clock -- [uring teardown close](/quest/m1/test-flakes-2/uring-teardown-close.md) - moq-uring's published-close teardown test holds under load, fixed at its cause - [Auth serve signals](/quest/m1/test-flakes-2/auth-serve-signals.md) - moq-cli's auth serve signal tests hold under load - [js -> go rewind](/quest/m1/test-flakes-2/js-go-rewind.md) - the `js -> go` interop cell never reads a frame below the previous group's start, fixed on the side that produced it diff --git a/quest/m1/test-flakes-2/uring-teardown-close.md b/quest/m1/test-flakes-2/uring-teardown-close.md deleted file mode 100644 index 55d8c697656..00000000000 --- a/quest/m1/test-flakes-2/uring-teardown-close.md +++ /dev/null @@ -1,31 +0,0 @@ -# [S] The uring teardown close test holds under load - -## Goal - -`moq-uring`'s `a_published_close_has_already_left_the_client` -(`rs/moq-uring/tests/teardown.rs`) passes under a loaded `just check`, -fixed at its cause. It still proves that a close the client reported as -published reaches the server as the application code, not an idle timeout. - -## Plan - -Seen 2026-10-13 on #5483's `just check` at a load average near 48: the test -hung until nextest's 60 s timeout. It passed alone and 30 of 30 times under -`--stress-count`. The test uses real UDP sockets and two `Worker` threads, -and the server waits on `poll_closed` with no deadline of its own, so a -CONNECTION_CLOSE that never leaves the client, or a server wake that is -lost, both look like this hang. - -Decided 2026-10-13 (maintainer): a child of this questline, m1. Reproduce it -under load first (for example the test in a loop beside a CPU hog, or -`--stress-count` with `just check` running). Then tell the two causes apart: - -- The client stops its runtime before the close datagram is on the wire. - That is a product bug: "published" must mean sent. Fix the teardown. -- The server's `poll_closed` misses the wake for a close it received. Fix - the wake. - -Never widen a timeout or add a retry. Land a regression that fails without -the fix where one is easy. - -Public API: none expected. Wire: none. diff --git a/rs/moq-uring/src/quic/noq/connection.rs b/rs/moq-uring/src/quic/noq/connection.rs index 9def53c5ee1..4b081f7fd0a 100644 --- a/rs/moq-uring/src/quic/noq/connection.rs +++ b/rs/moq-uring/src/quic/noq/connection.rs @@ -125,6 +125,12 @@ impl State { self.readable.remove(&id); } + /// The handshake is done as far as the peer can tell: hand the handle over. + fn establish(&mut self) { + self.established = true; + self.establish_waiters.wake(); + } + /// Terminate with `err` (the first one wins) and wake absolutely everyone, /// the driver included: an externally failed driver has to notice. fn fail(&mut self, err: Error) { @@ -379,6 +385,7 @@ pub(crate) fn launch( deadline: owner.timer(), scratch: Vec::with_capacity(TRAIN_SEGMENTS * SEGMENT), close_staged: false, + finished_after: None, }; let future = async move { kio::wait(|waiter| driver.poll(waiter)).await }; (shared, future) @@ -707,6 +714,14 @@ struct Driver { /// flush that came back empty: noq paces the close like any other packet, /// so it can sit behind a pacing timer after the application asked for it. close_staged: bool, + /// A client that has completed its handshake but not yet staged its + /// Finished: the CRYPTO frames it had sent when noq said `Connected`. + /// + /// noq raises `Connected` with the Finished still queued, and paces it + /// like any other packet. A close in that gap replaces it, so the server + /// never completes its side and only sees a handshake-level close with the + /// application code stripped. The handle waits for the Finished instead. + finished_after: Option, } impl Driver { @@ -792,8 +807,12 @@ impl Driver { let mut state = self.shared.state.borrow_mut(); match event { moq_noq_proto::Event::Connected => { - state.established = true; - state.establish_waiters.wake(); + let mut conn = self.shared.conn.borrow_mut(); + if conn.side().is_client() { + self.finished_after = Some(conn.stats().frame_tx.crypto); + } else { + state.establish(); + } } moq_noq_proto::Event::ConnectionLost { reason } => state.fail(reason.into()), moq_noq_proto::Event::DatagramReceived => state.datagram_recv_waiters.wake(), @@ -884,6 +903,12 @@ impl Driver { return Poll::Ready(Err(Error::Io(err.to_string()))); } self.close_staged |= closing; + if let Some(sent) = self.finished_after + && self.shared.conn.borrow_mut().stats().frame_tx.crypto > sent + { + self.finished_after = None; + self.shared.state.borrow_mut().establish(); + } // A flush frees datagram-send queue space. self.shared.state.borrow_mut().datagram_send_waiters.wake(); Poll::Ready(Ok(())) diff --git a/rs/moq-uring/src/quic/noq/endpoint.rs b/rs/moq-uring/src/quic/noq/endpoint.rs index 2a7ff9d8e5d..1fd99a01d90 100644 --- a/rs/moq-uring/src/quic/noq/endpoint.rs +++ b/rs/moq-uring/src/quic/noq/endpoint.rs @@ -156,6 +156,9 @@ impl Endpoint { /// Dial [`Config::peer`](crate::quic::client::Config::peer) through this /// endpoint's socket, driving the handshake to completion. + /// + /// Resolves once the client's Finished is staged, so the server can + /// complete its side even if the connection is closed right away. pub async fn connect(&self, config: &crate::quic::client::Config) -> Result { if let Some(err) = self.inner.closed() { return Err(err); diff --git a/rs/moq-uring/tests/teardown.rs b/rs/moq-uring/tests/teardown.rs index ad1383a5e9c..e549b5ca35b 100644 --- a/rs/moq-uring/tests/teardown.rs +++ b/rs/moq-uring/tests/teardown.rs @@ -15,8 +15,8 @@ #[path = "support.rs"] mod support; -use std::net::UdpSocket; -use std::time::Duration; +use std::net::{SocketAddr, UdpSocket}; +use std::time::{Duration, Instant}; use moq_uring::{Config, Error, Worker, quic, udp}; use web_transport_trait::poll::Session as _; @@ -85,7 +85,7 @@ fn server_config(certs: &support::Certs) -> quic::server::Config { ) } -fn dial_config(peer: std::net::SocketAddr) -> quic::client::Config { +fn dial_config(peer: SocketAddr) -> quic::client::Config { let mut config = quic::client::Config::new(peer, "localhost"); config.alpn = vec![ALPN.to_string()]; config.verify = false; @@ -98,11 +98,32 @@ fn dial_config(peer: std::net::SocketAddr) -> quic::client::Config { /// like. #[test] fn a_published_close_has_already_left_the_client() { + close_right_after_dialing(None); +} + +/// The same, with the client's Finished held back by its pacer, which is how +/// a loaded host first hit it. The slow return leg makes the client's first +/// RTT sample long, so its pacer refills slowly: the padded ACK it sends for +/// the server's first datagram drains it, and the Finished owed once the +/// second arrives has to wait. A close in that gap must not replace the +/// Finished, or the server never completes its side, sees only a +/// handshake-level close stripped of the application code, and never hands +/// the connection to `accept`. +#[test] +fn a_close_right_after_dialing_follows_the_handshake() { + close_right_after_dialing(Some(Duration::from_millis(100))); +} + +fn close_right_after_dialing(slow_return: Option) { let Some(mut client_worker) = worker() else { return }; let certs = support::certs().expect("certificates"); let server_sock = UdpSocket::bind("127.0.0.1:0").expect("bind server"); let server_addr = server_sock.local_addr().expect("server addr"); + let dial_addr = match slow_return { + Some(delay) => slow_return_path(server_addr, delay), + None => server_addr, + }; // The server outlives the client's teardown, so its verdict is only about // what the client managed to send. It reports in once serving, so a setup @@ -138,7 +159,7 @@ fn a_published_close_has_already_left_the_client() { client_worker .block_on(async move { - let mut conn = endpoint.connect(&dial_config(server_addr)).await.expect("dial"); + let mut conn = endpoint.connect(&dial_config(dial_addr)).await.expect("dial"); conn.close(CLOSE_CODE, CLOSE_REASON); std::future::poll_fn(|cx| conn.poll_closed(cx)).await; }) @@ -151,3 +172,46 @@ fn a_published_close_has_already_left_the_client() { "the server saw {err} instead of the application close" ); } + +/// A UDP relay to `server` that passes client datagrams straight through and +/// holds each server datagram for `delay`. Returns the address to dial; its +/// threads live as long as the test process. +/// +/// Released server datagrams are spaced a little apart, so the client handles +/// each on a turn of its own rather than as one batch, but closer than its +/// pacer refills at an RTT of `delay`. +fn slow_return_path(server: SocketAddr, delay: Duration) -> SocketAddr { + const SPACING: Duration = Duration::from_millis(2); + + let relay = UdpSocket::bind("127.0.0.1:0").expect("bind relay"); + let addr = relay.local_addr().expect("relay addr"); + let out = relay.try_clone().expect("clone relay"); + let (hold, release) = std::sync::mpsc::channel::<(Instant, Vec, SocketAddr)>(); + + std::thread::spawn(move || { + let mut client = None; + let mut buf = vec![0u8; u16::MAX as usize]; + while let Ok((len, from)) = relay.recv_from(&mut buf) { + if from != server { + client = Some(from); + relay.send_to(&buf[..len], server).expect("relay to server"); + } else if let Some(client) = client + && hold + .send((Instant::now() + delay, buf[..len].to_vec(), client)) + .is_err() + { + return; + } + } + }); + std::thread::spawn(move || { + let mut next = Instant::now(); + for (at, datagram, client) in release { + next = next.max(at); + std::thread::sleep(next.saturating_duration_since(Instant::now())); + out.send_to(&datagram, client).expect("relay to client"); + next += SPACING; + } + }); + addr +} From 2edc2a34cafd084fc93c9e50cd94b755cfb75c19 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:56:28 -0700 Subject: [PATCH 3/6] fix(quic): send a client's queued Finished ahead of its close A client is Connected with its Finished only queued, and pacing can hold it there. A close in that gap replaced it: the server never completed its handshake and only read an Initial or Handshake close, which carries no application code, so its accept never resolved. That was the 60 s hang in moq-uring's teardown close test, and moq-tokio shares it. The close flight now carries the queued Finished in its Handshake packet instead of a handshake-level close, and the server reads the 1-RTT close behind it. This replaces the moq-uring driver gate, which only covered one backend. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 27 +++++++++++++++++-- rs/moq-quic/src/tests/mod.rs | 43 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 9cdf4ae177a..8b83df7b7b6 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -552,6 +552,16 @@ impl Connection { _ => false, }; + // A client is `Connected` with its Finished only queued, and pacing can hold it there, so + // the application can close first. Without the Finished the server cannot complete its + // handshake and only reads an Initial or Handshake close, which carries no application + // code (RFC 9000 §10.2.3). So the Handshake packet carries the Finished instead, and the + // server reads the 1-RTT close behind it. + let finish_first = close + && self.side.is_client() + && self.highest_space == SpaceId::Data + && !self.spaces[SpaceId::Handshake].pending.crypto.is_empty(); + // Check whether we need to send an ACK_FREQUENCY frame if let Some(config) = &self.config.ack_frequency_config { self.spaces[SpaceId::Data].pending.ack_frequency = self @@ -597,7 +607,9 @@ impl Connection { // Is there data or a close message to send in this space? let can_send = self.space_can_send(space_id, frame_space_1rtt); - if can_send.is_empty() && (!close || self.spaces[space_id].crypto.is_none()) { + if (can_send.is_empty() && (!close || self.spaces[space_id].crypto.is_none())) + || (finish_first && space_id == SpaceId::Initial) + { space_idx += 1; continue; } @@ -868,7 +880,18 @@ impl Connection { ); } - if buf.len() + frame::ConnectionClose::SIZE_BOUND <= builder.max_size { + if finish_first && space_id == SpaceId::Handshake { + let pending = &mut self.spaces[space_id].pending.crypto; + while let Some(frame) = pending.front() + && buf.len() + frame::Crypto::SIZE_BOUND + frame.data.len() + <= builder.max_size + { + trace!("CRYPTO: off {} len {}", frame.offset, frame.data.len()); + frame.encode(buf); + self.stats.frame_tx.crypto += 1; + pending.pop_front(); + } + } else if buf.len() + frame::ConnectionClose::SIZE_BOUND <= builder.max_size { let max_frame_size = builder.max_size - buf.len(); match &self.state { State::Closed(state::Closed { reason }) => { diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 28ebabca77e..46095c8d758 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -220,6 +220,49 @@ fn lifecycle() { assert_eq!(pair.server.known_cids(), 0); } +/// A client that closes as soon as it is `Connected`, before its Finished has left, still lets +/// the server complete the handshake and read the application close. +#[test] +fn close_before_the_client_finished_leaves() { + let _guard = subscribe(); + let mut pair = Pair::default(); + let client_ch = pair.begin_connect(client_config()); + pair.drive_client(); + pair.drive_server(); + + // Hand the client the server's flight without letting it transmit, as a pacer would. + let mut buf = Vec::new(); + while let Some((recv_time, ecn, packet)) = pair.client.inbound.pop_front() { + let server = pair.server.addr; + let Some(DatagramEvent::ConnectionEvent(_, event)) = pair + .client + .endpoint + .handle(recv_time, server, None, ecn, packet, &mut buf) + else { + panic!("expected a connection event"); + }; + pair.client_conn_mut(client_ch).handle_event(event); + } + + const REASON: &[u8] = b"whee"; + let now = pair.time; + let client = pair.client_conn_mut(client_ch); + assert_matches!(client.poll(), Some(Event::HandshakeDataReady)); + assert_matches!(client.poll(), Some(Event::Connected)); + client.close(now, VarInt(42), REASON.into()); + pair.drive(); + + let server_ch = pair.server.assert_accept(); + let server = pair.server_conn_mut(server_ch); + assert_matches!(server.poll(), Some(Event::HandshakeDataReady)); + assert_matches!(server.poll(), Some(Event::HandshakeConfirmed)); + assert_matches!(server.poll(), Some(Event::Connected)); + assert_matches!(server.poll(), + Some(Event::ConnectionLost { reason: ConnectionError::ApplicationClosed( + ApplicationClose { error_code: VarInt(42), reason } + )}) if reason == REASON); +} + #[test] fn stats_include_congestion_controller_bandwidth_estimate() { const WINDOW: u64 = 12_000; From 059bd6d08ac17123bbd65094f37c2b86c44dd82a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 18:34:29 -0700 Subject: [PATCH 4/6] fix(quic): resend the queued Finished with every close flight A close resent after the first flight was lost carried a Handshake close instead, since the Finished had been taken off the queue. Leave it queued so every close flight carries it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 13 +++++++------ rs/moq-quic/src/tests/mod.rs | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 8b83df7b7b6..cd9a7761a3a 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -881,15 +881,16 @@ impl Connection { } if finish_first && space_id == SpaceId::Handshake { - let pending = &mut self.spaces[space_id].pending.crypto; - while let Some(frame) = pending.front() - && buf.len() + frame::Crypto::SIZE_BOUND + frame.data.len() - <= builder.max_size - { + // Left queued, so the close resent for a lost flight carries it too. + for frame in &self.spaces[space_id].pending.crypto { + if buf.len() + frame::Crypto::SIZE_BOUND + frame.data.len() + > builder.max_size + { + break; + } trace!("CRYPTO: off {} len {}", frame.offset, frame.data.len()); frame.encode(buf); self.stats.frame_tx.crypto += 1; - pending.pop_front(); } } else if buf.len() + frame::ConnectionClose::SIZE_BOUND <= builder.max_size { let max_frame_size = builder.max_size - buf.len(); diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 46095c8d758..bdec34c32f6 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -224,6 +224,17 @@ fn lifecycle() { /// the server complete the handshake and read the application close. #[test] fn close_before_the_client_finished_leaves() { + close_before_the_client_finished(false); +} + +/// The same, when the first close flight is lost and the client resends it in answer to the +/// server's retransmitted handshake. +#[test] +fn close_before_the_client_finished_leaves_survives_a_lost_flight() { + close_before_the_client_finished(true); +} + +fn close_before_the_client_finished(lose_first: bool) { let _guard = subscribe(); let mut pair = Pair::default(); let client_ch = pair.begin_connect(client_config()); @@ -250,6 +261,12 @@ fn close_before_the_client_finished_leaves() { assert_matches!(client.poll(), Some(Event::HandshakeDataReady)); assert_matches!(client.poll(), Some(Event::Connected)); client.close(now, VarInt(42), REASON.into()); + if lose_first { + let server = pair.server.addr; + pair.client.drive(now, server); + assert!(!pair.client.outbound.is_empty()); + pair.client.outbound.clear(); + } pair.drive(); let server_ch = pair.server.assert_accept(); From c5136b6781c684f710646ff92431e44445b48ddc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 18:47:38 -0700 Subject: [PATCH 5/6] fix(quic): carry an in-flight Finished in the close too A Finished already sent but lost left the same gap: the close went out as a Handshake close the server cannot complete its handshake from. The close flight now carries every unacknowledged Handshake CRYPTO frame, queued or in flight, until the client discards its Handshake keys. A flight too big for one packet keeps the plain close, since part of it would not complete the handshake either. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 38 ++++++++++++++++++---------- rs/moq-quic/src/connection/spaces.rs | 9 +++++++ rs/moq-quic/src/tests/mod.rs | 35 +++++++++++++++++++------ 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index cd9a7761a3a..e83f2d5535c 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -552,15 +552,19 @@ impl Connection { _ => false, }; - // A client is `Connected` with its Finished only queued, and pacing can hold it there, so - // the application can close first. Without the Finished the server cannot complete its - // handshake and only reads an Initial or Handshake close, which carries no application - // code (RFC 9000 §10.2.3). So the Handshake packet carries the Finished instead, and the - // server reads the 1-RTT close behind it. + // A client is `Connected` before the server has its Finished: it may still be queued + // behind the pacer, or lost. A close that replaces it leaves the server unable to complete + // its handshake, so it only reads an Initial or Handshake close, which carries no + // application code (RFC 9000 §10.2.3). So the Handshake packet carries the unacknowledged + // Finished instead, and the server reads the 1-RTT close behind it. let finish_first = close && self.side.is_client() && self.highest_space == SpaceId::Data - && !self.spaces[SpaceId::Handshake].pending.crypto.is_empty(); + && self.spaces[SpaceId::Handshake].crypto.is_some() + && self.spaces[SpaceId::Handshake] + .unacked_crypto() + .next() + .is_some(); // Check whether we need to send an ACK_FREQUENCY frame if let Some(config) = &self.config.ack_frequency_config { @@ -880,14 +884,20 @@ impl Connection { ); } - if finish_first && space_id == SpaceId::Handshake { - // Left queued, so the close resent for a lost flight carries it too. - for frame in &self.spaces[space_id].pending.crypto { - if buf.len() + frame::Crypto::SIZE_BOUND + frame.data.len() - > builder.max_size - { - break; - } + // All or nothing: part of the flight would not complete the handshake. A flight + // too big for one packet (a large client certificate chain) keeps the plain close. + // Nothing is dequeued, so a close resent for a lost flight carries it again. + let finished = &self.spaces[space_id]; + let carry_finished = finish_first + && space_id == SpaceId::Handshake + && buf.len() + + finished + .unacked_crypto() + .map(|frame| frame::Crypto::SIZE_BOUND + frame.data.len()) + .sum::() + <= builder.max_size; + if carry_finished { + for frame in finished.unacked_crypto() { trace!("CRYPTO: off {} len {}", frame.offset, frame.data.len()); frame.encode(buf); self.stats.frame_tx.crypto += 1; diff --git a/rs/moq-quic/src/connection/spaces.rs b/rs/moq-quic/src/connection/spaces.rs index 480f6fe8705..b2447adb334 100644 --- a/rs/moq-quic/src/connection/spaces.rs +++ b/rs/moq-quic/src/connection/spaces.rs @@ -182,6 +182,15 @@ impl PacketSpace { SendableFrames { acks, other } } + /// Outgoing CRYPTO frames the peer has not acknowledged: those in flight, then those queued. + pub(super) fn unacked_crypto(&self) -> impl Iterator { + self.sent_packets + .range(..) + .filter_map(|(_, packet)| packet.retransmits.get()) + .flat_map(|retransmits| &retransmits.crypto) + .chain(&self.pending.crypto) + } + /// Verifies sanity of an ECN block and returns whether congestion was encountered. pub(super) fn detect_ecn( &mut self, diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index bdec34c32f6..7db1cb2916d 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -220,21 +220,34 @@ fn lifecycle() { assert_eq!(pair.server.known_cids(), 0); } +/// Which client datagram a `close_before_the_server_has_the_finished` run loses. +#[derive(Clone, Copy, PartialEq, Eq)] +enum LostFlight { + None, + /// The one carrying the Finished, sent before the close. + Finished, + /// The first close, which the client resends for the server's retransmitted handshake. + Close, +} + /// A client that closes as soon as it is `Connected`, before its Finished has left, still lets /// the server complete the handshake and read the application close. #[test] fn close_before_the_client_finished_leaves() { - close_before_the_client_finished(false); + close_before_the_server_has_the_finished(LostFlight::None); +} + +#[test] +fn close_after_the_client_finished_is_lost() { + close_before_the_server_has_the_finished(LostFlight::Finished); } -/// The same, when the first close flight is lost and the client resends it in answer to the -/// server's retransmitted handshake. #[test] fn close_before_the_client_finished_leaves_survives_a_lost_flight() { - close_before_the_client_finished(true); + close_before_the_server_has_the_finished(LostFlight::Close); } -fn close_before_the_client_finished(lose_first: bool) { +fn close_before_the_server_has_the_finished(lost: LostFlight) { let _guard = subscribe(); let mut pair = Pair::default(); let client_ch = pair.begin_connect(client_config()); @@ -257,12 +270,18 @@ fn close_before_the_client_finished(lose_first: bool) { const REASON: &[u8] = b"whee"; let now = pair.time; + let server = pair.server.addr; let client = pair.client_conn_mut(client_ch); assert_matches!(client.poll(), Some(Event::HandshakeDataReady)); assert_matches!(client.poll(), Some(Event::Connected)); - client.close(now, VarInt(42), REASON.into()); - if lose_first { - let server = pair.server.addr; + if lost == LostFlight::Finished { + pair.client.drive(now, server); + assert!(!pair.client.outbound.is_empty()); + pair.client.outbound.clear(); + } + pair.client_conn_mut(client_ch) + .close(now, VarInt(42), REASON.into()); + if lost == LostFlight::Close { pair.client.drive(now, server); assert!(!pair.client.outbound.is_empty()); pair.client.outbound.clear(); From 774045d221532275a6ea7492c646642e88d0f0e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 17 Oct 2026 20:50:07 -0700 Subject: [PATCH 6/6] fix(quic): leave room for the 1-RTT close when carrying the Finished A client flight that fit the close packet but not the coalesced 1-RTT close made a one-datagram poll resend the Handshake packet forever. Co-Authored-By: Claude Opus 5.5 --- rs/moq-quic/src/connection/mod.rs | 5 ++ rs/moq-quic/src/tests/mod.rs | 122 ++++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index e83f2d5535c..9dfd186e25a 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -886,6 +886,9 @@ impl Connection { // All or nothing: part of the flight would not complete the handshake. A flight // too big for one packet (a large client certificate chain) keeps the plain close. + // It must also leave room to coalesce the 1-RTT close, or a one-datagram budget + // resends this packet on every poll and never reaches the close. Both spaces share + // the negotiated suite, so the 1-RTT tag is as long as this one. // Nothing is dequeued, so a close resent for a lost flight carries it again. let finished = &self.spaces[space_id]; let carry_finished = finish_first @@ -895,6 +898,8 @@ impl Connection { .unacked_crypto() .map(|frame| frame::Crypto::SIZE_BOUND + frame.data.len()) .sum::() + + MIN_PACKET_SPACE + + builder.tag_len <= builder.max_size; if carry_finished { for frame in finished.unacked_crypto() { diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 7db1cb2916d..9c86af25699 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -247,14 +247,12 @@ fn close_before_the_client_finished_leaves_survives_a_lost_flight() { close_before_the_server_has_the_finished(LostFlight::Close); } -fn close_before_the_server_has_the_finished(lost: LostFlight) { - let _guard = subscribe(); - let mut pair = Pair::default(); - let client_ch = pair.begin_connect(client_config()); +/// Hands the client the server's flight without letting it transmit, as a pacer would. +fn connect_before_the_finished_leaves(pair: &mut Pair, config: ClientConfig) -> ConnectionHandle { + let client_ch = pair.begin_connect(config); pair.drive_client(); pair.drive_server(); - // Hand the client the server's flight without letting it transmit, as a pacer would. let mut buf = Vec::new(); while let Some((recv_time, ecn, packet)) = pair.client.inbound.pop_front() { let server = pair.server.addr; @@ -268,12 +266,120 @@ fn close_before_the_server_has_the_finished(lost: LostFlight) { pair.client_conn_mut(client_ch).handle_event(event); } - const REASON: &[u8] = b"whee"; - let now = pair.time; - let server = pair.server.addr; let client = pair.client_conn_mut(client_ch); assert_matches!(client.poll(), Some(Event::HandshakeDataReady)); assert_matches!(client.poll(), Some(Event::Connected)); + client_ch +} + +/// A client certificate flight that fits the close packet but crowds out the coalesced 1-RTT close +/// falls back to the plain close, instead of resending the flight on every poll of a one-datagram +/// budget and never reaching the close. +#[test] +fn close_before_a_client_certificate_leaves_ends() { + let _guard = subscribe(); + let provider = Arc::new(default_provider()); + let mut roots = RootCertStore::empty(); + roots.add(CERTIFIED_KEY.cert.der().clone()).unwrap(); + + let (mut carried, mut fell_back) = (0, 0); + // Grow the client certificate by names, sweeping its flight past the room in the close packet. + for len in (0..1400).step_by(16) { + let mut names = vec!["localhost".to_string()]; + names.extend((0..len / 50).map(|i| format!("{i:03}{}.test", "a".repeat(42)))); + names.push(format!("{}.test", "b".repeat(len % 50 + 1))); + let identity = rcgen::generate_simple_self_signed(names).unwrap(); + + let mut store = RootCertStore::empty(); + store.add(identity.cert.der().clone()).unwrap(); + let server = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_client_cert_verifier( + WebPkiClientVerifier::builder_with_provider(Arc::new(store), provider.clone()) + .build() + .unwrap(), + ) + .with_single_cert( + vec![CERTIFIED_KEY.cert.der().clone()], + PrivatePkcs8KeyDer::from(CERTIFIED_KEY.signing_key.serialize_der()).into(), + ) + .unwrap(); + let client = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_root_certificates(roots.clone()) + .with_client_auth_cert( + vec![identity.cert.der().clone()], + PrivatePkcs8KeyDer::from(identity.signing_key.serialize_der()).into(), + ) + .unwrap(); + let mut client = ClientConfig::new(Arc::new( + crypto::rustls::QuicClientConfig::try_from(client).unwrap(), + )); + client.transport = Arc::new(cubic_transport()); + + let mut pair = Pair::new( + Default::default(), + ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(server).unwrap())), + ); + let client_ch = connect_before_the_finished_leaves(&mut pair, client); + + let now = pair.time; + let client = pair.client_conn_mut(client_ch); + client.close(now, VarInt(42), Bytes::new()); + let mut transmits = Vec::new(); + let mut buf = Vec::new(); + while transmits.len() <= 2 + && let Some(transmit) = client.poll_transmit(now, 1, &mut buf) + { + let datagram = Bytes::copy_from_slice(&buf[..transmit.size]); + transmits.push((transmit, datagram)); + buf.clear(); + } + assert_eq!(transmits.len(), 1, "{len} bytes of names"); + pair.client.outbound.extend(transmits); + pair.drive(); + + let server_ch = pair.server.assert_accept(); + let server = pair.server_conn_mut(server_ch); + let mut connected = false; + while let Some(event) = server.poll() { + match event { + Event::Connected => connected = true, + Event::ConnectionLost { reason } if connected => { + assert_matches!( + reason, + ConnectionError::ApplicationClosed(ApplicationClose { + error_code: VarInt(42), + .. + }) + ); + } + _ => {} + } + } + if connected { + carried += 1; + } else { + fell_back += 1; + } + } + // The sweep covers a flight carried with the close and one too big for it. + assert!( + carried > 0 && fell_back > 0, + "{carried} carried, {fell_back} fell back" + ); +} + +fn close_before_the_server_has_the_finished(lost: LostFlight) { + let _guard = subscribe(); + let mut pair = Pair::default(); + let client_ch = connect_before_the_finished_leaves(&mut pair, client_config()); + + const REASON: &[u8] = b"whee"; + let now = pair.time; + let server = pair.server.addr; if lost == LostFlight::Finished { pair.client.drive(now, server); assert!(!pair.client.outbound.is_empty());