diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index 05cc7f87ff5..08b24f9e6fa 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -298,4 +298,5 @@ out: static and API peers then share one socket on that port, and a redial reuses it. The `/nodes` [internal endpoint](/bin/relay/http#get-nodes) lists the peers -this relay dialed and holds a session with. +this relay dials, including peers still retrying, with their live transports and +how long ago each last connected. diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 034c3943d53..f4b0881c981 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -132,10 +132,18 @@ curl -X POST 'http://127.0.0.1:9101/sessions/revalidate?id=00ff' ### GET /nodes -The peers this relay dialed and holds a session with: each node's URL, without -its query, and the connections to it, with the same `conn` id the logs use. +Every outbound peer this relay is configured to dial, including those still +connecting or retrying, so an entry with no `connections` is a peer that is down. +`node` is the peer's identity without credentials: its URL, or the mDNS instance +of a LAN peer that advertised no node URL. +Each live connection carries the `conn` ID its log lines use and its transport +(`quic`, `webtransport`, `websocket`, ...). +`last_connected_seconds_ago` is the time since a session to the peer last +connected, kept across disconnects, failed attempts, and a redial for a new +cost or token. It is null until the first successful connect, and the peer and +its history go away once it is removed from the peer list. Peers that dialed this relay are not listed, since they declare no URL. ```json -{ "nodes": [{ "node": "https://us-east.example.com/", "connections": [{ "id": 3 }] }] } +{ "nodes": [{ "node": "https://us-east.example.com/", "connections": [{ "id": 3, "transport": "webtransport" }], "last_connected_seconds_ago": 12.5 }] } ``` diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 045e9acf92e..a9afd9b04c5 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,13 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### `/nodes` lists peers that are down ([#7776](https://github.com/moq-dev/moq/pull/7776)) + +The internal `/nodes` endpoint lists every configured outbound peer, not only those with a live session. +A peer that is connecting, retrying, or has never connected has an empty `connections` array, so tooling that counts entries as live peers must count those with a nonempty `connections` instead. +Each connection adds `transport`, and each peer adds `last_connected_seconds_ago`, null until its first successful connect. +A LAN peer is named by its mDNS identity instead of the first address dialed, whose path carried its credential. + ### Stats are configured in `cluster::Options` ([#7673](https://github.com/moq-dev/moq/pull/7673)) `Cluster::with_stats` is removed, and `stats::Config::build` is no longer public. Pass the stats config to `Cluster::new` instead, which builds the producer on the cluster's origin and runs its publisher in the returned `Driver`: diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 390ac649e4b..877265ff503 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1213,6 +1213,15 @@ impl Mesh { #[cfg(feature = "cluster-lan")] DialCommand::Lost(peer) => self.dialed.release(&peer, DialSource::Mdns, &mut dial), } + // A replaced dial keeps its peer's `/nodes` history; a peer that left the + // dial set takes it along. + self.cluster.nodes.retain(|peer| self.dialed.contains(peer)); + } +} + +impl Drop for Mesh { + fn drop(&mut self) { + self.cluster.nodes.retain(|_| false); } } @@ -1221,9 +1230,11 @@ fn dialer<'a>(cluster: &'a Cluster, token: &'a str) -> impl FnMut(DialTarget) -> move |target| { // Logged by key, never by the target's URL, so an inline query stays out of the logs. tracing::info!(peer = %target.key, lan = target.lan, "dialing cluster peer"); + // Registered now, so `/nodes` lists the peer before its first attempt. + let dial = cluster.nodes.dial(&target.key); Child::new( "cluster dial", - cluster.clone().supervise_remote(target, token.to_string()), + cluster.clone().supervise_remote(target, token.to_string(), dial), ) } } @@ -2036,12 +2047,12 @@ impl Cluster { let _ = commands.try_push(DialCommand::Api(desired)); } - async fn supervise_remote(self, target: DialTarget, token: String) { + async fn supervise_remote(self, target: DialTarget, token: String, dial: crate::nodes::Dial) { let log_peer = target.key.clone(); let dialers = self.dialers.clone(); dialers .supervise(async move { - if let Err(err) = self.run_remote(&target, token).await { + if let Err(err) = self.run_remote(&target, token, &dial).await { tracing::warn!(%err, peer = %log_peer, "cluster peer connection ended"); } }) @@ -2049,7 +2060,7 @@ impl Cluster { } #[tracing::instrument("remote", skip_all, err, fields(remote = %target.key))] - async fn run_remote(self, target: &DialTarget, token: String) -> anyhow::Result<()> { + async fn run_remote(self, target: &DialTarget, token: String, dial: &crate::nodes::Dial) -> anyhow::Result<()> { let mut urls = target.addrs(); // Apply the shared cluster token unless the URL already carries its own // non-empty `?jwt=` (a per-peer inline token or object `token` wins; the @@ -2075,7 +2086,8 @@ impl Cluster { loop { let started = kio::clock::now(); - let result = self.run_remote_once(&addrs, target).await; + let result = self.run_remote_once(&addrs, target, dial).await; + dial.disconnected(); let elapsed = started.elapsed(); match result { @@ -2090,18 +2102,29 @@ impl Cluster { } /// One session to `target` over `addrs`, which carry any credential. - async fn run_remote_once(&self, addrs: &moq_tokio::Addrs, target: &DialTarget) -> anyhow::Result<()> { + async fn run_remote_once( + &self, + addrs: &moq_tokio::Addrs, + target: &DialTarget, + dial: &crate::nodes::Dial, + ) -> anyhow::Result<()> { // Each attempt is its own session, so it gets its own id. Matches the span an // accepted connection runs under, so both directions log the same way. let id = self.next_connection_id(); // A panic fails this attempt like any other error, so the peer is redialed. - moq_tokio::panic::isolate(self.run_remote_session(id, addrs, target)) + moq_tokio::panic::isolate(self.run_remote_session(id, addrs, target, dial)) .instrument(tracing::info_span!("conn", id)) .await .context("cluster peer session panicked")? } - async fn run_remote_session(&self, id: u64, addrs: &moq_tokio::Addrs, target: &DialTarget) -> anyhow::Result<()> { + async fn run_remote_session( + &self, + id: u64, + addrs: &moq_tokio::Addrs, + target: &DialTarget, + dial: &crate::nodes::Dial, + ) -> anyhow::Result<()> { // The peer URL may carry the cluster JWT in its query, so neither the log // line nor the node label below may show the raw URL. let first = addrs.as_slice().first().expect("Addrs is non-empty").url(); @@ -2161,20 +2184,15 @@ impl Cluster { // and replay both announce tables there for the rest of its window. Its // routes are already re-priced to the maximum, so the session serves on // until the node closes it, and the redial after that finds the replacement. - let mut reconnect = client + let reconnect = client .with_reconnect(true) .with_restart(moq_tokio::Restart::Wait) .connect(addrs.clone()); - let mut connection = None; + let monitor = reconnect.monitor(); + let mut observation = monitor.state(); loop { - match reconnect.status().await? { - moq_tokio::Status::Connected if connection.is_none() => { - connection = Some(self.nodes.connect_outbound(id, redacted.to_string())); - } - moq_tokio::Status::Disconnected => connection = None, - moq_tokio::Status::Migrating => {} - _ => {} - } + dial.observe(id, &observation); + observation = monitor.state_changed(observation.revision).await?; } } @@ -3220,6 +3238,50 @@ mod tests { assert_eq!(err.to_string(), "source stopped"); } + /// `/nodes` lists a peer from the moment it is configured, keeps its connect + /// history when a new cost or credential replaces its dial, and forgets it only + /// once the peer leaves the dial set. + #[tokio::test] + async fn mesh_keeps_peer_history_until_the_peer_is_removed() { + let (mut mesh, _commands) = test_mesh(); + let api = |peer: &str| { + let target = DialTarget::parse(peer).unwrap(); + DialCommand::Api(HashMap::from([(target.key.clone(), target)])) + }; + let nodes = mesh.cluster.nodes.clone(); + + mesh.apply(api("https://peer.example/?cost=1")); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1, "listed before its first attempt"); + assert!(snapshot.nodes[0].last_connected_seconds_ago.is_none()); + + // Stand in for the dial task's own session, which these unpolled tasks never open. + let session = nodes.dial("https://peer.example/"); + session.observe( + 0, + &moq_tokio::connection::Observation { + revision: 1, + phase: moq_tokio::connection::Phase::Connected, + connects: 1, + failure: None, + version: None, + last_connected: Some(kio::clock::now()), + transport: Some(moq_tokio::Transport::Quic), + }, + ); + drop(session); + + mesh.apply(api("https://peer.example/?cost=2")); + assert_eq!(mesh.dialed.tasks.len(), 1, "the cost change replaced the dial"); + assert!( + nodes.snapshot().nodes[0].last_connected_seconds_ago.is_some(), + "a replaced dial keeps its history" + ); + + mesh.apply(DialCommand::Api(HashMap::new())); + assert!(nodes.snapshot().nodes.is_empty(), "a removed peer takes its history"); + } + /// A dial task's panic ends the mesh with it, rather than unwinding the task /// polling the mesh. #[tokio::test] @@ -4118,7 +4180,8 @@ mod tests { .with_connect(Default::default(), quic); let broadcast = dialer.origin.create_broadcast("from-dialer").expect("create"); broadcast.announce(Default::default()).expect("announce"); - tokio::spawn(dialer.clone().supervise_remote(target, String::new())); + let dial = dialer.nodes.dial(&target.key); + tokio::spawn(dialer.clone().supervise_remote(target, String::new(), dial)); (dialer, broadcast) }) .await diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index ec881d8173e..8280b061e7d 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -18,7 +18,7 @@ //! crossing it, so they have no place on the `moq-stats` wire. //! - `/health` - a liveness mirror of the public probe, for internal checks //! that don't want to hit the customer port. -//! - `/nodes` - the cluster peers this relay dialed and holds a session with. +//! - `/nodes` - configured outbound cluster peers, live transports, and connect history. //! - `/sessions` and `/sessions/revalidate` - list or nudge live sessions on //! this node. A push only causes a re-check, so a caller on this trusted //! plane gains nothing a scheduled cadence would not do. @@ -475,7 +475,7 @@ async fn serve_metrics(State(state): State) -> Response { .into_response() } -/// Cluster peers this relay dialed and currently holds a session with. Accepted +/// Configured outbound cluster peers, their live sessions and connection history. Accepted /// peer sessions are omitted, since a peer declares no URL to list it under. async fn serve_nodes(State(state): State) -> Json { Json(state.nodes.map(|nodes| nodes.snapshot()).unwrap_or_default()) @@ -1664,7 +1664,7 @@ mod tests { #[tokio::test] async fn nodes_endpoint_uses_the_attached_cluster_registry() { let nodes = crate::nodes::Nodes::default(); - let _connection = nodes.connect_outbound(0, "https://relay-b.example/"); + let _connection = nodes.dial("https://relay-b.example/"); let state = InternalState { stats: moq_net::stats::Registry::disabled(), nodes: Some(nodes), diff --git a/rs/moq-relay/src/nodes.rs b/rs/moq-relay/src/nodes.rs index b4bea42c126..c6efe6fc170 100644 --- a/rs/moq-relay/src/nodes.rs +++ b/rs/moq-relay/src/nodes.rs @@ -1,97 +1,169 @@ //! The local cluster view behind the internal `/nodes` endpoint. //! -//! Lists the peers this relay dialed and currently holds a session with, keyed -//! by the URL it dialed. Accepted peer sessions are not listed: a peer declares +//! Lists every peer this relay is configured to dial, whether or not a session +//! is up, with its live sessions and when one last connected. A peer's history +//! is keyed by its dial identity, so it survives failed attempts and a dial +//! restarted for a new cost or credential, and is retired only when the peer +//! leaves the dial set. Accepted peer sessions are not listed: a peer declares //! no URL, so there is nothing to name it by. use std::{ - collections::{BTreeMap, HashMap}, + collections::BTreeMap, sync::{Arc, Mutex, PoisonError}, }; use serde::Serialize; -/// Established outbound cluster connections, by `conn` id. +/// Outbound cluster peers by dial identity. #[derive(Clone, Default)] pub(crate) struct Nodes { - connections: Arc>>, + state: Arc>, +} + +#[derive(Default)] +struct State { + /// Keyed by the dial set's peer identity, which may still carry userinfo. + peers: BTreeMap, + /// Distinguishes the dials of one peer, so a replaced dial clears only its own session. + next_dial: u64, +} + +#[derive(Default)] +struct Peer { + /// Live sessions by `conn` id, each with the dial that owns it. + connections: BTreeMap, + /// The latest successful handshake of any dial of this peer. + last_connected: Option, } /// The JSON document returned by the internal `/nodes` endpoint. #[derive(Debug, Default, Serialize)] pub(crate) struct Snapshot { - /// Dialed cluster nodes with a live session. + /// Configured outbound peers, connected or not. pub nodes: Vec, } -/// One dialed cluster node. +/// One configured outbound peer. #[derive(Debug, Serialize)] pub(crate) struct Node { - /// Canonical URL this relay dialed, without its query. + /// The peer's canonical identity, with query, userinfo, and repeated slashes + /// removed: its dial URL, or for a LAN peer that advertised no node URL, its + /// per-run mDNS instance in URL form. pub node: String, - /// Established connections to this node. + /// Live sessions to this peer; empty while it connects or retries. pub connections: Vec, + /// Seconds since a session to this peer last connected, including sessions + /// already closed. Null until the first successful handshake. + pub last_connected_seconds_ago: Option, } -/// An established outbound cluster connection. +/// A live outbound cluster session. +#[serde_with::serde_as] #[derive(Debug, Serialize)] pub(crate) struct Connection { /// The session's `conn` id, matching the `conn{id}` span its log lines carry. /// Process-local, and only meaningful while the session lasts. pub id: u64, + /// The transport the session runs over. + #[serde_as(as = "serde_with::DisplayFromStr")] + pub transport: moq_tokio::Transport, } -/// Removes a live connection from the view when its session ends. -pub(crate) struct ConnectionGuard { +/// One dial task's handle on its peer's entry. Dropping it clears the session it +/// holds, but keeps the peer's history: [`Nodes::retain`] retires that. +pub(crate) struct Dial { nodes: Nodes, + peer: String, id: u64, } impl Nodes { - /// Record a dial this relay initiated, keyed by the URL it dialed. - /// - /// `id` is the session's `conn` id from - /// [`Cluster::next_connection_id`](crate::cluster::Cluster::next_connection_id). - pub(crate) fn connect_outbound(&self, id: u64, node: impl Into) -> ConnectionGuard { - self.connections - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(id, node.into()); - ConnectionGuard { + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Register a dial of `peer`, the dial set's identity for it, keeping any + /// history an earlier dial of the same peer left. + pub(crate) fn dial(&self, peer: &str) -> Dial { + let mut state = self.lock(); + state.peers.entry(peer.to_string()).or_default(); + let id = state.next_dial; + state.next_dial += 1; + Dial { nodes: self.clone(), + peer: peer.to_string(), id, } } + /// Retire the history of every peer `keep` rejects, once it leaves the dial set. + pub(crate) fn retain(&self, mut keep: impl FnMut(&str) -> bool) { + self.lock().peers.retain(|peer, _| keep(peer)); + } + pub(crate) fn snapshot(&self) -> Snapshot { - let mut nodes = BTreeMap::>::new(); - let connections = self.connections.lock().unwrap_or_else(PoisonError::into_inner); - for (&id, node) in connections.iter() { - nodes - .entry(crate::cluster::canonicalize_peer_key(node)) - .or_default() - .push(Connection { id }); + let state = self.lock(); + // Two identities can share a label once userinfo is removed, so merge them. + let mut nodes = BTreeMap::::new(); + for (peer, entry) in &state.peers { + let label = crate::cluster::canonicalize_peer_key(peer); + let node = nodes.entry(label.clone()).or_insert_with(|| Node { + node: label, + connections: Vec::new(), + last_connected_seconds_ago: None, + }); + node.connections.extend( + entry + .connections + .iter() + .map(|(&id, &(_, transport))| Connection { id, transport }), + ); + if let Some(age) = entry.last_connected.map(|at| at.elapsed().as_secs_f64()) { + node.last_connected_seconds_ago = + Some(node.last_connected_seconds_ago.map_or(age, |prev| prev.min(age))); + } + } + for node in nodes.values_mut() { + node.connections.sort_by_key(|connection| connection.id); } - Snapshot { - nodes: nodes - .into_iter() - .map(|(node, mut connections)| { - connections.sort_by_key(|connection| connection.id); - Node { node, connections } - }) - .collect(), + nodes: nodes.into_values().collect(), } } } -impl Drop for ConnectionGuard { +impl Dial { + /// Apply one lifecycle observation of session `id`, which this dial owns. + pub(crate) fn observe(&self, id: u64, observation: &moq_tokio::connection::Observation) { + let mut state = self.nodes.lock(); + // A retired peer's dial is being cancelled; it records nothing more. + let Some(peer) = state.peers.get_mut(&self.peer) else { + return; + }; + if let Some(at) = observation.last_connected { + peer.last_connected = Some(peer.last_connected.map_or(at, |prev| prev.max(at))); + } + match observation.transport { + Some(transport) => { + peer.connections.insert(id, (self.id, transport)); + } + None => { + peer.connections.remove(&id); + } + } + } + + /// Clear this dial's session once it ends, keeping the peer's history. + pub(crate) fn disconnected(&self) { + if let Some(peer) = self.nodes.lock().peers.get_mut(&self.peer) { + peer.connections.retain(|_, (dial, _)| *dial != self.id); + } + } +} + +impl Drop for Dial { fn drop(&mut self) { - self.nodes - .connections - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&self.id); + self.disconnected(); } } @@ -99,39 +171,102 @@ impl Drop for ConnectionGuard { mod tests { use super::*; + fn observation(transport: Option) -> moq_tokio::connection::Observation { + moq_tokio::connection::Observation { + revision: 1, + phase: match transport { + Some(_) => moq_tokio::connection::Phase::Connected, + None => moq_tokio::connection::Phase::Disconnected, + }, + connects: 1, + failure: None, + version: None, + last_connected: Some(kio::clock::now()), + transport, + } + } + #[test] - fn snapshot_groups_live_connections_by_node() { + fn configured_peers_list_before_they_connect_without_credentials() { let nodes = Nodes::default(); - let _second = nodes.connect_outbound(1, "https://relay-b.example/"); - let _first = nodes.connect_outbound(0, "https://relay-b.example"); - let _other = nodes.connect_outbound(2, "https://relay-a.example/"); + let _dial = nodes.dial("https://user:secret@peer.example//relay/"); assert_eq!( serde_json::to_value(nodes.snapshot()).unwrap(), serde_json::json!({ - "nodes": [ - { "node": "https://relay-a.example/", "connections": [{ "id": 2 }] }, - { "node": "https://relay-b.example/", "connections": [{ "id": 0 }, { "id": 1 }] } - ] + "nodes": [{ "node": "https://peer.example/relay", "connections": [], "last_connected_seconds_ago": null }] }), ); } #[test] - fn snapshot_stops_reporting_closed_outbound_connections() { + fn history_survives_disconnects_and_a_replaced_dial() { + let nodes = Nodes::default(); + let dial = nodes.dial("https://peer.example/"); + dial.observe(2, &observation(Some(moq_tokio::Transport::WebSocket))); + assert_eq!( + serde_json::to_value(&nodes.snapshot().nodes[0].connections).unwrap(), + serde_json::json!([{ "id": 2, "transport": "websocket" }]), + ); + + // A connect and disconnect may both happen before the observer polls. + dial.observe(2, &observation(None)); + dial.disconnected(); + let snapshot = nodes.snapshot(); + assert!(snapshot.nodes[0].connections.is_empty()); + assert!(snapshot.nodes[0].last_connected_seconds_ago.is_some()); + + // A new cost or credential replaces the dial: the new one starts before + // the old one is dropped or after, and the history stays either way. + let replacement = nodes.dial("https://peer.example/"); + replacement.observe(4, &observation(Some(moq_tokio::Transport::Quic))); + drop(dial); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1); + assert_eq!( + snapshot.nodes[0].connections.len(), + 1, + "dropping one dial keeps the other's session" + ); + assert_eq!(snapshot.nodes[0].connections[0].id, 4); + + drop(replacement); + let snapshot = nodes.snapshot(); + assert!(snapshot.nodes[0].connections.is_empty()); + assert!(snapshot.nodes[0].last_connected_seconds_ago.is_some()); + } + + #[test] + fn retired_peers_drop_their_history() { let nodes = Nodes::default(); - let connection = nodes.connect_outbound(0, "https://relay-b.example/"); - assert_eq!(nodes.snapshot().nodes.len(), 1); + let kept = nodes.dial("https://kept.example/"); + let retired = nodes.dial("https://retired.example/"); + retired.observe(1, &observation(Some(moq_tokio::Transport::Quic))); + + drop(retired); + nodes.retain(|peer| peer == "https://kept.example/"); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1); + assert_eq!(snapshot.nodes[0].node, "https://kept.example/"); - drop(connection); + // A dial cancelled after its peer retired records nothing. + kept.observe(2, &observation(Some(moq_tokio::Transport::Quic))); + nodes.retain(|_| false); + kept.observe(3, &observation(Some(moq_tokio::Transport::Quic))); assert!(nodes.snapshot().nodes.is_empty()); } #[test] - fn outbound_node_omits_credentials_from_url() { + fn identities_sharing_a_label_merge() { let nodes = Nodes::default(); - let _connection = nodes.connect_outbound(0, "https://relay-b.example/?jwt=secret"); + let first = nodes.dial("https://a@peer.example/"); + let second = nodes.dial("https://b@peer.example/"); + second.observe(7, &observation(Some(moq_tokio::Transport::Quic))); + first.observe(3, &observation(Some(moq_tokio::Transport::Quic))); - assert_eq!(nodes.snapshot().nodes[0].node, "https://relay-b.example/"); + let snapshot = nodes.snapshot(); + assert_eq!(snapshot.nodes.len(), 1); + let ids: Vec = snapshot.nodes[0].connections.iter().map(|c| c.id).collect(); + assert_eq!(ids, [3, 7]); } } diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 68dffcdfe91..fb576f82429 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -67,7 +67,29 @@ async fn wait_dialed(http: &reqwest::Client, internal: u16, dialed: usize) { .await .expect("read /nodes"); let nodes: serde_json::Value = serde_json::from_str(&body).expect("parse /nodes"); - if nodes["nodes"].as_array().expect("a node list").len() == dialed { + if nodes["nodes"] + .as_array() + .expect("a node list") + .iter() + .filter(|node| !node["connections"].as_array().expect("connections").is_empty()) + .count() == dialed + { + for node in nodes["nodes"].as_array().unwrap() { + let connections = node["connections"].as_array().unwrap(); + if connections.is_empty() { + continue; + } + assert!( + node["last_connected_seconds_ago"].is_number(), + "live peer has no successful connect: {node}" + ); + for connection in connections { + assert!( + connection["transport"].is_string(), + "live peer has no transport: {node}" + ); + } + } return; } assert!(kio::clock::now() < deadline, "the mesh never formed: {nodes}"); diff --git a/rs/moq-tokio/MIGRATE.md b/rs/moq-tokio/MIGRATE.md index 3797c0906d1..9465084baa1 100644 --- a/rs/moq-tokio/MIGRATE.md +++ b/rs/moq-tokio/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `connection::Observation` reports the transport and last connect ([#7776](https://github.com/moq-dev/moq/pull/7776)) + +`Observation` adds `transport`, the live session's transport or none while offline, and `last_connected`, when a session last connected, kept while offline. +Code that builds an `Observation` with a struct literal, such as a test double, sets both. + ### QUIC waits have caller-owned poll forms ([#7707](https://github.com/moq-dev/moq/pull/7707)) The QUIC driver connection's stream opens, accepts, datagram receive and close waits, diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index a2dbf6837b4..4cf3ea1d8ec 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -590,6 +590,7 @@ struct State { /// Advanced by `Shared::connected` with the same write that publishes `status`, /// so a reader can't see `Connected` without its count. connects: u64, + last_connected: Option, /// Cumulative connects and disconnects, bumped by the reconnect loop itself so a session that /// connects and drops before a consumer polls still counts. presence: moq_net::stats::Presence, @@ -650,6 +651,7 @@ impl Shared { state.revision += 1; state.status = Some(Status::Connected); state.connects += 1; + state.last_connected = Some(kio::clock::now()); state.presence.sessions_started += 1; state.failure = None; state.version = Some(session.version()); @@ -775,6 +777,10 @@ pub struct Observation { pub phase: Phase, /// How many sessions have connected, including replacements. pub connects: u64, + /// When a session last connected, retained while offline. + pub last_connected: Option, + /// The live session's transport, or none while offline. + pub transport: Option, /// The latest failed attempt or terminal failure, cleared by a successful connect. pub failure: Option, /// The live session's negotiated version, or none while offline. @@ -811,6 +817,8 @@ impl State { } }, connects: self.connects, + last_connected: self.last_connected, + transport: self.transport, failure: self.error.clone().or_else(|| self.failure.clone()), version: self.version, } diff --git a/test/drill/mutations/peer-link-never-redialed.patch b/test/drill/mutations/peer-link-never-redialed.patch index 874e580a4b2..006191d7fca 100644 --- a/test/drill/mutations/peer-link-never-redialed.patch +++ b/test/drill/mutations/peer-link-never-redialed.patch @@ -9,16 +9,15 @@ diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 6e3e22cbc..7bc6d15cf 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs -@@ -1728,7 +1728,11 @@ impl Cluster { - moq_tokio::Status::Connected if connection.is_none() => { - connection = Some(self.nodes.connect_outbound(id, redacted.to_string())); - } -- moq_tokio::Status::Disconnected => connection = None, -+ // MUTATION: a lost peer link is never redialed. -+ moq_tokio::Status::Disconnected => { -+ drop(reconnect); -+ return std::future::pending().await; -+ } - moq_tokio::Status::Migrating => {} - _ => {} - } +@@ -2170,6 +2170,11 @@ impl Cluster { + let mut observation = monitor.state(); + loop { + dial.observe(id, &observation); ++ // MUTATION: a lost peer link is never redialed. ++ if observation.phase == moq_tokio::connection::Phase::Disconnected { ++ drop(reconnect); ++ return std::future::pending().await; ++ } + observation = monitor.state_changed(observation.revision).await?; + } + }