diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index 80209fa9c89..b4cfc6451ea 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -37,7 +37,6 @@ Public API: none. Wire: none. - [TS MP2 join hole](/quest/m1/test-flakes-2/ts-mp2-join-hole.md) - the TS export skips no MPEG-1 audio group right after its live join, fixed at the cause - [Auth refresh mocks time](/quest/m1/test-flakes-2/auth-refresh-mock-time.md) - moq-relay's auth refresh and fixed-lease tests measure a grant's life on one mocked clock -- [uring teardown close](/quest/m1/test-flakes-2/uring-teardown-close.md) - moq-uring's published-close teardown test holds under load, fixed at its cause - [Relay burst drill](/quest/m1/test-flakes-2/relay-burst-impaired.md) - the impaired transport-peer burst drill never aborts its subscription while the link is up - [BBR marking seed](/quest/m1/test-flakes-2/bbr-marking-seed.md) - the BBR marking test seeds its probe schedule, replays a failing seed, and its rare drop is fixed at the cause diff --git a/quest/m1/test-flakes-2/uring-teardown-close.md b/quest/m1/test-flakes-2/uring-teardown-close.md deleted file mode 100644 index 3a24d79e80c..00000000000 --- a/quest/m1/test-flakes-2/uring-teardown-close.md +++ /dev/null @@ -1,31 +0,0 @@ -# [S] The uring teardown close test holds under load - -## Goal - -`moq-uring`'s `a_published_close_has_already_left_the_client` -(`rs/moq-uring/tests/integration/teardown.rs`) passes under a loaded `just check`, -fixed at its cause. It still proves that a close the client reported as -published reaches the server as the application code, not an idle timeout. - -## Plan - -Seen 2026-10-13 on #5483's `just check` at a load average near 48: the test -hung until nextest's 60 s timeout. It passed alone and 30 of 30 times under -`--stress-count`. The test uses real UDP sockets and two `Worker` threads, -and the server waits on `poll_closed` with no deadline of its own, so a -CONNECTION_CLOSE that never leaves the client, or a server wake that is -lost, both look like this hang. - -Decided 2026-10-13 (maintainer): a child of this questline, m1. Reproduce it -under load first (for example the test in a loop beside a CPU hog, or -`--stress-count` with `just check` running). Then tell the two causes apart: - -- The client stops its runtime before the close datagram is on the wire. - That is a product bug: "published" must mean sent. Fix the teardown. -- The server's `poll_closed` misses the wake for a close it received. Fix - the wake. - -Never widen a timeout or add a retry. Land a regression that fails without -the fix where one is easy. - -Public API: none expected. Wire: none. diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 4e55a1179cb..05dc462ab63 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -561,6 +561,20 @@ impl Connection { _ => false, }; + // A client is `Connected` before the server has its Finished: it may still be queued + // behind the pacer, or lost. A close that replaces it leaves the server unable to complete + // its handshake, so it only reads an Initial or Handshake close, which carries no + // application code (RFC 9000 ยง10.2.3). So the Handshake packet carries the unacknowledged + // Finished instead, and the server reads the 1-RTT close behind it. + let finish_first = close + && self.side.is_client() + && self.highest_space == SpaceId::Data + && self.spaces[SpaceId::Handshake].crypto.is_some() + && self.spaces[SpaceId::Handshake] + .unacked_crypto() + .next() + .is_some(); + // Check whether we need to send an ACK_FREQUENCY frame if let Some(config) = &self.config.ack_frequency_config { self.spaces[SpaceId::Data].pending.ack_frequency = self @@ -606,7 +620,9 @@ impl Connection { // Is there data or a close message to send in this space? let can_send = self.space_can_send(space_id, frame_space_1rtt); - if can_send.is_empty() && (!close || self.spaces[space_id].crypto.is_none()) { + if (can_send.is_empty() && (!close || self.spaces[space_id].crypto.is_none())) + || (finish_first && space_id == SpaceId::Initial) + { space_idx += 1; continue; } @@ -877,7 +893,30 @@ impl Connection { ); } - if buf.len() + frame::ConnectionClose::SIZE_BOUND <= builder.max_size { + // All or nothing: part of the flight would not complete the handshake. A flight + // too big for one packet (a large client certificate chain) keeps the plain close. + // It must also leave room to coalesce the 1-RTT close, or a one-datagram budget + // resends this packet on every poll and never reaches the close. Both spaces share + // the negotiated suite, so the 1-RTT tag is as long as this one. + // Nothing is dequeued, so a close resent for a lost flight carries it again. + let finished = &self.spaces[space_id]; + let carry_finished = finish_first + && space_id == SpaceId::Handshake + && buf.len() + + finished + .unacked_crypto() + .map(|frame| frame::Crypto::SIZE_BOUND + frame.data.len()) + .sum::() + + MIN_PACKET_SPACE + + builder.tag_len + <= builder.max_size; + if carry_finished { + for frame in finished.unacked_crypto() { + trace!("CRYPTO: off {} len {}", frame.offset, frame.data.len()); + frame.encode(buf); + self.stats.frame_tx.crypto += 1; + } + } else if buf.len() + frame::ConnectionClose::SIZE_BOUND <= builder.max_size { let max_frame_size = builder.max_size - buf.len(); match &self.state { State::Closed(state::Closed { reason }) => { diff --git a/rs/moq-quic/src/connection/spaces.rs b/rs/moq-quic/src/connection/spaces.rs index 480f6fe8705..b2447adb334 100644 --- a/rs/moq-quic/src/connection/spaces.rs +++ b/rs/moq-quic/src/connection/spaces.rs @@ -182,6 +182,15 @@ impl PacketSpace { SendableFrames { acks, other } } + /// Outgoing CRYPTO frames the peer has not acknowledged: those in flight, then those queued. + pub(super) fn unacked_crypto(&self) -> impl Iterator { + self.sent_packets + .range(..) + .filter_map(|(_, packet)| packet.retransmits.get()) + .flat_map(|retransmits| &retransmits.crypto) + .chain(&self.pending.crypto) + } + /// Verifies sanity of an ECN block and returns whether congestion was encountered. pub(super) fn detect_ecn( &mut self, diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index b0ded60c966..8c9597b9bd9 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -222,6 +222,191 @@ fn lifecycle() { assert_eq!(pair.server.known_cids(), 0); } +/// Which client datagram a `close_before_the_server_has_the_finished` run loses. +#[derive(Clone, Copy, PartialEq, Eq)] +enum LostFlight { + None, + /// The one carrying the Finished, sent before the close. + Finished, + /// The first close, which the client resends for the server's retransmitted handshake. + Close, +} + +/// A client that closes as soon as it is `Connected`, before its Finished has left, still lets +/// the server complete the handshake and read the application close. +#[test] +fn close_before_the_client_finished_leaves() { + close_before_the_server_has_the_finished(LostFlight::None); +} + +#[test] +fn close_after_the_client_finished_is_lost() { + close_before_the_server_has_the_finished(LostFlight::Finished); +} + +#[test] +fn close_before_the_client_finished_leaves_survives_a_lost_flight() { + close_before_the_server_has_the_finished(LostFlight::Close); +} + +/// Hands the client the server's flight without letting it transmit, as a pacer would. +fn connect_before_the_finished_leaves(pair: &mut Pair, config: ClientConfig) -> ConnectionHandle { + let client_ch = pair.begin_connect(config); + pair.drive_client(); + pair.drive_server(); + + let mut buf = Vec::new(); + while let Some((recv_time, ecn, packet)) = pair.client.inbound.pop_front() { + let server = pair.server.addr; + let Some(DatagramEvent::ConnectionEvent(_, event)) = pair + .client + .endpoint + .handle(recv_time, server, None, ecn, packet, &mut buf) + else { + panic!("expected a connection event"); + }; + pair.client_conn_mut(client_ch).handle_event(event); + } + + let client = pair.client_conn_mut(client_ch); + assert_matches!(client.poll(), Some(Event::HandshakeDataReady)); + assert_matches!(client.poll(), Some(Event::Connected)); + client_ch +} + +/// A client certificate flight that fits the close packet but crowds out the coalesced 1-RTT close +/// falls back to the plain close, instead of resending the flight on every poll of a one-datagram +/// budget and never reaching the close. +#[test] +fn close_before_a_client_certificate_leaves_ends() { + let _guard = subscribe(); + let provider = Arc::new(default_provider()); + let mut roots = RootCertStore::empty(); + roots.add(CERTIFIED_KEY.cert.der().clone()).unwrap(); + + let (mut carried, mut fell_back) = (0, 0); + // Grow the client certificate by names, sweeping its flight past the room in the close packet. + for len in (0..1400).step_by(16) { + let mut names = vec!["localhost".to_string()]; + names.extend((0..len / 50).map(|i| format!("{i:03}{}.test", "a".repeat(42)))); + names.push(format!("{}.test", "b".repeat(len % 50 + 1))); + let identity = rcgen::generate_simple_self_signed(names).unwrap(); + + let mut store = RootCertStore::empty(); + store.add(identity.cert.der().clone()).unwrap(); + let server = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_client_cert_verifier( + WebPkiClientVerifier::builder_with_provider(Arc::new(store), provider.clone()) + .build() + .unwrap(), + ) + .with_single_cert( + vec![CERTIFIED_KEY.cert.der().clone()], + PrivatePkcs8KeyDer::from(CERTIFIED_KEY.signing_key.serialize_der()).into(), + ) + .unwrap(); + let client = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_root_certificates(roots.clone()) + .with_client_auth_cert( + vec![identity.cert.der().clone()], + PrivatePkcs8KeyDer::from(identity.signing_key.serialize_der()).into(), + ) + .unwrap(); + let mut client = ClientConfig::new(Arc::new( + crypto::rustls::QuicClientConfig::try_from(client).unwrap(), + )); + client.transport = Arc::new(cubic_transport()); + + let mut pair = Pair::new( + Default::default(), + ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(server).unwrap())), + ); + let client_ch = connect_before_the_finished_leaves(&mut pair, client); + + let now = pair.time; + let client = pair.client_conn_mut(client_ch); + client.close(now, VarInt(42), Bytes::new()); + let mut transmits = Vec::new(); + let mut buf = Vec::new(); + while transmits.len() <= 2 + && let Some(transmit) = client.poll_transmit(now, 1, &mut buf) + { + let datagram = Bytes::copy_from_slice(&buf[..transmit.size]); + transmits.push((transmit, datagram)); + buf.clear(); + } + assert_eq!(transmits.len(), 1, "{len} bytes of names"); + pair.client.outbound.extend(transmits); + pair.drive(); + + let server_ch = pair.server.assert_accept(); + let server = pair.server_conn_mut(server_ch); + let mut connected = false; + while let Some(event) = server.poll() { + match event { + Event::Connected => connected = true, + Event::ConnectionLost { reason } if connected => { + assert_matches!( + reason, + ConnectionError::ApplicationClosed(ApplicationClose { + error_code: VarInt(42), + .. + }) + ); + } + _ => {} + } + } + if connected { + carried += 1; + } else { + fell_back += 1; + } + } + // The sweep covers a flight carried with the close and one too big for it. + assert!( + carried > 0 && fell_back > 0, + "{carried} carried, {fell_back} fell back" + ); +} + +fn close_before_the_server_has_the_finished(lost: LostFlight) { + let _guard = subscribe(); + let mut pair = Pair::default(); + let client_ch = connect_before_the_finished_leaves(&mut pair, client_config()); + + const REASON: &[u8] = b"whee"; + let now = pair.time; + let server = pair.server.addr; + if lost == LostFlight::Finished { + pair.client.drive(now, server); + assert!(!pair.client.outbound.is_empty()); + pair.client.outbound.clear(); + } + pair.client_conn_mut(client_ch) + .close(now, VarInt(42), REASON.into()); + if lost == LostFlight::Close { + pair.client.drive(now, server); + assert!(!pair.client.outbound.is_empty()); + pair.client.outbound.clear(); + } + pair.drive(); + + let server_ch = pair.server.assert_accept(); + let server = pair.server_conn_mut(server_ch); + assert_matches!(server.poll(), Some(Event::HandshakeDataReady)); + assert_matches!(server.poll(), Some(Event::HandshakeConfirmed)); + assert_matches!(server.poll(), Some(Event::Connected)); + assert_matches!(server.poll(), + Some(Event::ConnectionLost { reason: ConnectionError::ApplicationClosed( + ApplicationClose { error_code: VarInt(42), reason } + )}) if reason == REASON); +} + #[test] fn stats_include_congestion_controller_bandwidth_estimate() { const WINDOW: u64 = 12_000;