From 356ccce8f8fbdde7f32a98f9b4cb384e207ce00e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 10 Sep 2026 12:09:19 +0200 Subject: [PATCH 1/3] WIP --- compiler/rustc_ast/src/tokenstream.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index df71aad0111cd..340c805803826 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -249,6 +249,15 @@ impl LazyAttrTokenStreamInner { break_last_token, node_replacements, } => { + let mut snapshot = cursor_snapshot.clone(); + let stack_size = snapshot.stack.len(); + for i in 0..num_calls.saturating_sub(1) { + snapshot.next_and_bump(); + if snapshot.stack.len() < stack_size { + assert_eq!(snapshot.stack.len(), stack_size - 1); + assert_eq!(i, num_calls - 2); + } + } // The token produced by the final call to `{,inlined_}next_and_bump` was not // actually consumed by the callback. The combination of chaining the // initial token and using `take` produces the desired result - we @@ -973,6 +982,11 @@ impl TokenCursor { TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] } } + pub fn clone_without_stack(&self) -> Self { + let stack = self.stack.last().cloned().into_iter().collect(); + Self { curr: self.curr.clone(), stack } + } + /// Gets the next token and advances the cursor by one. pub fn next_and_bump(&mut self) -> (Token, Spacing) { self.inlined_next_and_bump() From e2006eb5a6075e1ec10ae3d51a98f91820d0b780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 10 Sep 2026 14:23:11 +0200 Subject: [PATCH 2/3] Implement `clone_without_stack` --- compiler/rustc_ast/src/tokenstream.rs | 12 +++--------- compiler/rustc_parse/src/parser/attr_wrapper.rs | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 340c805803826..b2d191896f87a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -249,15 +249,6 @@ impl LazyAttrTokenStreamInner { break_last_token, node_replacements, } => { - let mut snapshot = cursor_snapshot.clone(); - let stack_size = snapshot.stack.len(); - for i in 0..num_calls.saturating_sub(1) { - snapshot.next_and_bump(); - if snapshot.stack.len() < stack_size { - assert_eq!(snapshot.stack.len(), stack_size - 1); - assert_eq!(i, num_calls - 2); - } - } // The token produced by the final call to `{,inlined_}next_and_bump` was not // actually consumed by the callback. The combination of chaining the // initial token and using `take` produces the desired result - we @@ -982,6 +973,9 @@ impl TokenCursor { TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] } } + /// Clone the cursor while keeping only its immediate parent (if present). + /// This can be used as a faster clone for situations where you only want to continue parsing + /// the currently delimited sequence or its children, but not return back to its parents. pub fn clone_without_stack(&self) -> Self { let stack = self.stack.last().cloned().into_iter().collect(); Self { curr: self.curr.clone(), stack } diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index fe34d9951dc5f..a94d4758997d9 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -95,7 +95,7 @@ impl<'a> Parser<'a> { pub(super) fn collect_pos(&self) -> CollectPos { CollectPos { start_token: (self.token, self.token_spacing), - cursor_snapshot: self.token_cursor.clone(), + cursor_snapshot: self.token_cursor.clone_without_stack(), start_pos: self.num_bump_calls, } } From 30c69a5842dcb3dd4180690cea451a2d6dad178b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 10 Sep 2026 14:31:31 +0200 Subject: [PATCH 3/3] Store a separate parent in `TokenCursor` --- compiler/rustc_ast/src/tokenstream.rs | 28 +++++++++++++++++--------- compiler/rustc_parse/src/parser/mod.rs | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index b2d191896f87a..649d1767bf5cc 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -961,6 +961,8 @@ pub struct TokenCursor { // stream which never has delimiters. curr: TokenTreeCursor, + parent: Option, + // Token streams surrounding the current one. The `next_idx` within each cursor // is always greater than zero and always points one past the current // `TokenTree::Delimited`. @@ -970,15 +972,14 @@ pub struct TokenCursor { impl TokenCursor { #[inline] pub fn new(stream: TokenStream) -> Self { - TokenCursor { curr: TokenTreeCursor::new(stream), stack: vec![] } + TokenCursor { curr: TokenTreeCursor::new(stream), parent: None, stack: vec![] } } /// Clone the cursor while keeping only its immediate parent (if present). /// This can be used as a faster clone for situations where you only want to continue parsing /// the currently delimited sequence or its children, but not return back to its parents. pub fn clone_without_stack(&self) -> Self { - let stack = self.stack.last().cloned().into_iter().collect(); - Self { curr: self.curr.clone(), stack } + Self { curr: self.curr.clone(), parent: self.parent.clone(), stack: vec![] } } /// Gets the next token and advances the cursor by one. @@ -997,14 +998,14 @@ impl TokenCursor { /// delimited sequence. Panics if we are not within a delimited sequence. #[inline] pub fn look_ahead_past_close_delim(&self) -> Option<&TokenTree> { - self.stack.last().unwrap().next() + self.parent.as_ref().unwrap().next() } /// Clones the `TokenTree::Delimited` that we are currently within. Panics if we are not within /// a delimited sequence. #[inline] pub fn clone_enclosing_delim(&self) -> TokenTree { - self.stack.last().unwrap().curr().unwrap().clone() + self.parent.as_ref().unwrap().curr().unwrap().clone() } /// For skipping to the end of the current sequence, in rare circumstances. @@ -1016,13 +1017,13 @@ impl TokenCursor { /// Note: the outermost stream has depth of 0. #[inline] pub fn depth(&self) -> usize { - self.stack.len() + self.parent.is_some() as usize + self.stack.len() } /// Returns details about the parent delimited sequence, if there is one. #[inline] pub fn parent_delim_and_span(&self) -> Option<(Delimiter, DelimSpan)> { - if let Some(last) = self.stack.last() + if let Some(last) = &self.parent && let Some(TokenTree::Delimited(span, _, delim, _)) = last.curr() { Some((*delim, *span)) @@ -1049,19 +1050,26 @@ impl TokenCursor { &TokenTree::Delimited(sp, spacing, delim, ref tts) => { let trees = TokenTreeCursor::new(tts.clone()); self.curr.bump(); // move past the `Delimited` - self.stack.push(mem::replace(&mut self.curr, trees)); + + let prev_curr = mem::replace(&mut self.curr, trees); + let prev_parent = mem::replace(&mut self.parent, Some(prev_curr)); + if let Some(parent) = prev_parent { + self.stack.push(parent); + } + if !delim.skip() { return (Token::new(delim.as_open_token_kind(), sp.open), spacing.open); } // No open delimiter to return; continue on to the next iteration. } }; - } else if let Some(parent) = self.stack.pop() { + } else if let Some(parent) = self.parent.take() { // We have exhausted this token stream. Move back to its parent token stream. let Some(&TokenTree::Delimited(span, spacing, delim, _)) = parent.curr() else { panic!("parent should be Delimited") }; self.curr = parent; + self.parent = self.stack.pop(); if !delim.skip() { return (Token::new(delim.as_close_token_kind(), span.close), spacing.close); } @@ -1123,7 +1131,7 @@ mod size_asserts { static_assert_size!(AttrTokenStream, 8); static_assert_size!(AttrTokenTree, 32); static_assert_size!(LazyAttrTokenStream, 8); - static_assert_size!(LazyAttrTokenStreamInner, 88); + static_assert_size!(LazyAttrTokenStreamInner, 104); static_assert_size!(Option, 8); // must be small, used in many AST nodes static_assert_size!(TokenStream, 8); static_assert_size!(TokenTree, 32); diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 80c1eeb4ef041..fb45b2e556be9 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -247,7 +247,7 @@ pub struct Parser<'a> { // nonterminals. Make sure it doesn't unintentionally get bigger. We only check a few arches // though, because `TokenTypeSet(u128)` alignment varies on others, changing the total size. #[cfg(all(target_pointer_width = "64", any(target_arch = "aarch64", target_arch = "x86_64")))] -rustc_data_structures::static_assert_size!(Parser<'_>, 288); +rustc_data_structures::static_assert_size!(Parser<'_>, 304); /// Stores span information about a closure. #[derive(Clone, Debug)]