diff --git a/.config/nextest.toml b/.config/nextest.toml index b6e2b141de7..14df5d4a8de 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -99,3 +99,10 @@ path = "junit.xml" slow-timeout = { period = "300s", terminate-after = 3 } retries = 0 failure-output = "immediate-final" + +# A sweep run past its gate count on purpose, such as `MOQ_ROUTE_SCENARIOS=20000 +# just rs test -P sweep ...`, takes minutes. Still bounded, so a lost wakeup +# ends in a TIMEOUT rather than a wedge. +[profile.sweep] +slow-timeout = { period = "60s", terminate-after = 60 } +retries = 0 diff --git a/quest/m2/cluster-routing/routes/handoff-revival.md b/quest/m2/cluster-routing/routes/handoff-revival.md index 04cbe0ff62a..244b711ccd1 100644 --- a/quest/m2/cluster-routing/routes/handoff-revival.md +++ b/quest/m2/cluster-routing/routes/handoff-revival.md @@ -16,10 +16,8 @@ the same whichever neighbours the router believes. It has not been traced. Decided the same day (maintainer, /quest-plan): ✅ plan it. - Reproduce it: the sweep draws every scenario from one random stream, so run - the first 16,317. Record the scenario's steps as it runs and remove them - one at a time while the revival remains, as - [the false END](/quest/m2/cluster-routing/routes/widen-false-end.md) was - shrunk. + the first 16,317 with `MOQ_ROUTE_SCENARIOS=16317`. Record the scenario's + steps as it runs and remove them one at a time while the revival remains. - Decide whether it is a revival the contract forbids or one it excuses: a replay after a collection or a restart is allowed only until the withdrawal behind it lands @@ -31,7 +29,3 @@ Decided the same day (maintainer, /quest-plan): ✅ plan it. pin it as a `residual_` test naming that quest instead of fixing it twice. Public API: none. Wire: none. - -## Related - -- [Widening an interest ends no live claim](/quest/m2/cluster-routing/routes/widen-false-end.md) - found the same way, and adds the way to run more scenarios diff --git a/quest/m2/cluster-routing/routes/interest-set.md b/quest/m2/cluster-routing/routes/interest-set.md index fd471af3227..6591368fb05 100644 --- a/quest/m2/cluster-routing/routes/interest-set.md +++ b/quest/m2/cluster-routing/routes/interest-set.md @@ -108,4 +108,3 @@ Public API: none (the router is crate-private). Wire: none. ## Related - [The router keeps a route per scope](/quest/m2/cluster-routing/routes/scope.md) - rewrites how the router keeps a floor per prefix; whichever lands second adapts -- [Widening an interest ends no live claim](/quest/m2/cluster-routing/routes/widen-false-end.md) - a false END on an interest change today, which the edit in place may remove diff --git a/quest/m2/cluster-routing/routes/widen-false-end.md b/quest/m2/cluster-routing/routes/widen-false-end.md deleted file mode 100644 index f02f902167a..00000000000 --- a/quest/m2/cluster-routing/routes/widen-false-end.md +++ /dev/null @@ -1,93 +0,0 @@ -# [M] Widening an interest ends no live claim - -## Goal - -A client that widens its interest never sees END for a claim its publisher -still has. `sweep_scoped_handoff` in `rs/moq-net/src/route/tests/handoff.rs` -reports no false END over 20,000 scenarios, and the case below is a plain -test. - -## Plan - -Found 2026-11-11 by running the sweep past its 150 scenarios: 25 of 20,000 -end a live claim, the first at scenario 750, on `main` at `688312b9eb`. It is -the same whichever neighbours the router believes, so -[the route per scope](/quest/m2/cluster-routing/routes/scope.md) does not fix -it. Decided the same day (maintainer, /quest-plan): ✅ plan it (rejected: -leaving it to the interest set). - -Scenario 750, shrunk to 13 steps. The client sits behind two relays, each -under a grant for one prefix, and sees the publisher's `live` at whichever -prefix its interest and a grant meet. It narrows to `live/b/z`, then widens -to everything while its sessions and the publisher's flap. Its reader sees -`live/b/z` start, go offline, start, and END, where the claim only moved up -to `live/b`: - -```rust -let (origin, r1, r2, client) = (0, 1, 2, 3); -let mut sim = Sim::new(4); -sim.leaf(origin); -sim.leaf(client); -sim.join(origin, r1, 1); -sim.join(origin, r2, 1); -sim.join(r1, r2, 1); -for (relay, scope) in [(r1, "live/a/"), (r2, "live/b/")] { - let grant = Grant { subject: "local", scope }; - sim.link(client, relay, SimLink::cost(1).grant(grant), SimLink::cost(1)); -} -sim.originate(origin, 7, 0); -assert!(sim.run()); - -sim.announce(origin, "live/", 0, None); -sim.settle(1); -sim.deliver(r1, origin); -sim.settle(1); -sim.disconnect(client, r1); -sim.settle(1); -sim.disconnect(origin, r1); -sim.settle(1); -sim.deliver(r2, r1); -sim.settle(1); -sim.disconnect(origin, r2); -sim.settle(1); -sim.connect(origin, r1); -sim.settle(1); -sim.watch(client, "live/b/z"); -sim.settle(1); -sim.disconnect(origin, r1); -sim.settle(1); -sim.announce(origin, "live/", 1, None); -sim.settle(1); -sim.connect(origin, r2); -sim.settle(1); -sim.watch(client, ""); -sim.settle(1); -sim.disconnect(client, r2); -sim.settle(1); - -sim.connect(origin, r1); -sim.connect(client, r1); -sim.connect(client, r2); -assert!(sim.run()); -assert_eq!(false_ends(&sim.events), 0); -``` - -`show` in `route/mod.rs` reports a claim that left as offline, not ended, -when a claim above it with the same item is still shown. Here that check -missed. Find why before changing it: the order the widened stream's first -batch and the session close arrive in is the likely cause. - -- Shrink further if it helps, then fix the cause and keep the case as a - test. -- Give the sweep a way to run more scenarios than the gate's 150 without - editing it, and say in the PR how many it ran clean. -- [The interest set](/quest/m2/cluster-routing/routes/interest-set.md) - replaces "an interest change opens a new stream" with an edit in place. If - it lands first, run the case and the 20,000 scenarios on it, and delete - this quest if both are clean. - -Public API: none. Wire: none. - -## Related - -- [The route model and router take an interest as an edited set of prefixes](/quest/m2/cluster-routing/routes/interest-set.md) - rewrites interest changes, and may remove the case diff --git a/rs/moq-net/src/route/mod.rs b/rs/moq-net/src/route/mod.rs index 576b86528c4..a30d2296c53 100644 --- a/rs/moq-net/src/route/mod.rs +++ b/rs/moq-net/src/route/mod.rs @@ -228,6 +228,9 @@ pub(crate) struct Shown { /// Whether the node has a usable route to the claim's origin through a /// neighbor certifying it. pub available: bool, + /// Whether the claim sits at the root of a region its view was presented at, + /// where it stands for whichever claim covers that region. + pub presented: bool, } /// A change a node's announcement reader sees. @@ -1971,11 +1974,12 @@ impl Router { .as_ref() .is_some_and(|regions| !regions.iter().any(|region| within(claim, region))) || !ctx.own && self.views.get(&id).is_none_or(|view| view.as_of(claim).is_none()); - // A claim presented at a narrower interest moved up to the claim it - // stood for, which is still shown. + // A claim presented at a narrower region moved up to the claim it + // stood for, which is still shown. That claim may have changed + // meanwhile, so a presented claim needs only something shown above it. let moved = ancestors(claim) .filter(|above| above != claim) - .any(|above| shown(&above).is_some_and(|now| now.item == was.item)); + .any(|above| shown(&above).is_some_and(|now| was.presented || now.item == was.item)); let kind = if !within(claim, &self.watch) { Kind::Unwatched } else if ctx.live && !uncertified && !moved { @@ -2056,6 +2060,7 @@ impl Router { Some(Shown { item: item.clone(), available, + presented: merged.is_some_and(|merged| merged.floors.contains_key(claim)), }) }; claims.map(|claim| (claim.clone(), at(claim))).collect() diff --git a/rs/moq-net/src/route/tests/handoff.rs b/rs/moq-net/src/route/tests/handoff.rs index 1db1d141593..7b6ca299a5a 100644 --- a/rs/moq-net/src/route/tests/handoff.rs +++ b/rs/moq-net/src/route/tests/handoff.rs @@ -566,6 +566,54 @@ fn a_claim_above_an_interest_is_presented_at_it() { sim.check().unwrap(); } +/// A client presented a claim at a narrow interest widens it after the claim +/// changed, while cut off from the change. Its stale presentation may come +/// back first, and moves up to the changed claim without ending. +#[test] +fn widening_past_a_changed_claim_moves_it_up() { + let (origin, relay, client) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, relay, 1); + sim.join(client, relay, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/", 0, None); + sim.watch(client, "live/b/z"); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/b/z"), Some(true)); + + sim.disconnect(client, relay); + sim.announce(origin, "live/", 1, None); + sim.watch(client, ""); + assert!(sim.run()); + sim.connect(client, relay); + assert!(sim.run()); + assert_eq!(sim.shows(client, 7, "live/"), Some(true)); + assert_eq!(sim.shown(client).len(), 1); + assert_eq!(false_ends(&sim.events), 0); +} + +/// A claim withdrawn below one its origin still has ends: the claim above +/// covers its path but is not the claim it was. +#[test] +fn a_claim_withdrawn_under_a_live_one_ends() { + let (origin, relay, client) = (0, 1, 2); + let mut sim = Sim::new(3); + sim.leaf(origin); + sim.leaf(client); + sim.join(origin, relay, 1); + sim.join(client, relay, 1); + sim.originate(origin, 7, 0); + sim.announce(origin, "live/", 0, None); + sim.announce(origin, "live/a/x", 0, None); + assert!(sim.run()); + + sim.unannounce(origin, "live/a/x"); + assert!(sim.run()); + assert_eq!(life(&sim, client, "live/a/x"), [Kind::Start, Kind::End]); +} + /// Two origins hold the same wildcard capability for one epoch, and a client /// watches a region below it. Withdrawing one origin's wildcard ends only that /// origin's claim, and a lagging relay's older copy of it is refused after the @@ -1015,7 +1063,7 @@ fn sweep_scoped_handoff() { let (origin, r1, r2, client) = (0, 1, 2, 3); let mut rng = XorShift(0x5c0b_ed00); let mut live_starts = 0; - for scenario in 0..150 { + for scenario in 0..scenarios() { let mut sim = Sim::new(4); sim.leaf(origin); sim.leaf(client); @@ -1132,7 +1180,7 @@ const CLAIMS: [&str; 4] = ["live/", "live/a/", "live/a/x", "live/b"]; #[test] fn sweep_handoff() { let mut rng = XorShift(0x4a4d_0ff5); - for scenario in 0..150 { + for scenario in 0..scenarios() { let mut sim = churn(&mut rng); let label = format!("scenario {scenario}"); assert_eq!(revivals(&sim.events), 0, "{label}"); diff --git a/rs/moq-net/src/route/tests/mod.rs b/rs/moq-net/src/route/tests/mod.rs index 0bee4fa960a..30d75ac9d56 100644 --- a/rs/moq-net/src/route/tests/mod.rs +++ b/rs/moq-net/src/route/tests/mod.rs @@ -764,6 +764,14 @@ impl Sim { } } +/// How many scenarios a sweep runs: `MOQ_ROUTE_SCENARIOS`, or the gate's 150. +/// Each sweep draws its scenarios from one seeded stream, so a larger count +/// replays the gate's first and continues past them. Run a large count under +/// nextest's `sweep` profile (`just rs test -P sweep`), which waits up to an hour. +pub(super) fn scenarios() -> usize { + std::env::var("MOQ_ROUTE_SCENARIOS").map_or(150, |count| count.parse().expect("MOQ_ROUTE_SCENARIOS")) +} + pub(super) struct XorShift(pub u64); impl XorShift {