Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 29ae3e8

Browse files
borsbors
committed
Auto merge of rust-lang#134161 - nnethercote:overhaul-token-cursors, r=<try>
Overhaul token cursors r? `@ghost`
2 parents 5a6036a + 7874297 commit 29ae3e8

File tree

19 files changed

+190
-204
lines changed

19 files changed

+190
-204
lines changed

‎compiler/rustc_ast/src/attr/mod.rs‎

Lines changed: 30 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
//! Functions dealing with attributes and meta items.
22
3-
use std::iter;
43
use std::sync::atomic::{AtomicU32, Ordering};
54

65
use rustc_index::bit_set::GrowableBitSet;
@@ -16,7 +15,9 @@ use crate::ast::{
1615
};
1716
use crate::ptr::P;
1817
use crate::token::{self, CommentKind, Delimiter, Token};
19-
use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenTree};
18+
use crate::tokenstream::{
19+
DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree,
20+
};
2021
use crate::util::comments;
2122
use crate::util::literal::escape_string_symbol;
2223

@@ -368,22 +369,19 @@ impl MetaItem {
368369
}
369370
}
370371

371-
fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
372-
where
373-
I: Iterator<Item = &'a TokenTree>,
374-
{
372+
fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItem> {
375373
// FIXME: Share code with `parse_path`.
376-
let tt = tokens.next().map(|tt| TokenTree::uninterpolate(tt));
374+
let tt = iter.next().map(|tt| TokenTree::uninterpolate(tt));
377375
let path = match tt.as_deref() {
378376
Some(&TokenTree::Token(
379377
Token { kind: ref kind @ (token::Ident(..) | token::PathSep), span },
380378
_,
381379
)) => 'arm: {
382380
let mut segments = if let &token::Ident(name, _) = kind {
383381
if let Some(TokenTree::Token(Token { kind: token::PathSep, .. }, _)) =
384-
tokens.peek()
382+
iter.peek()
385383
{
386-
tokens.next();
384+
iter.next();
387385
thin_vec![PathSegment::from_ident(Ident::new(name, span))]
388386
} else {
389387
break 'arm Path::from_ident(Ident::new(name, span));
@@ -393,16 +391,16 @@ impl MetaItem {
393391
};
394392
loop {
395393
if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
396-
tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
394+
iter.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
397395
{
398396
segments.push(PathSegment::from_ident(Ident::new(name, span)));
399397
} else {
400398
return None;
401399
}
402400
if let Some(TokenTree::Token(Token { kind: token::PathSep, .. }, _)) =
403-
tokens.peek()
401+
iter.peek()
404402
{
405-
tokens.next();
403+
iter.next();
406404
} else {
407405
break;
408406
}
@@ -423,8 +421,8 @@ impl MetaItem {
423421
}
424422
_ => return None,
425423
};
426-
let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());
427-
let kind = MetaItemKind::from_tokens(tokens)?;
424+
let list_closing_paren_pos = iter.peek().map(|tt| tt.span().hi());
425+
let kind = MetaItemKind::from_tokens(iter)?;
428426
let hi = match &kind {
429427
MetaItemKind::NameValue(lit) => lit.span.hi(),
430428
MetaItemKind::List(..) => list_closing_paren_pos.unwrap_or(path.span.hi()),
@@ -440,25 +438,23 @@ impl MetaItem {
440438

441439
impl MetaItemKind {
442440
fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<MetaItemInner>> {
443-
let mut tokens = tokens.trees().peekable();
441+
let mut iter = tokens.iter();
444442
let mut result = ThinVec::new();
445-
while tokens.peek().is_some() {
446-
let item = MetaItemInner::from_tokens(&mut tokens)?;
443+
while iter.peek().is_some() {
444+
let item = MetaItemInner::from_tokens(&mut iter)?;
447445
result.push(item);
448-
match tokens.next() {
446+
match iter.next() {
449447
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
450448
_ => return None,
451449
}
452450
}
453451
Some(result)
454452
}
455453

456-
fn name_value_from_tokens<'a>(
457-
tokens: &mut impl Iterator<Item = &'a TokenTree>,
458-
) -> Option<MetaItemKind> {
459-
match tokens.next() {
454+
fn name_value_from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
455+
match iter.next() {
460456
Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
461-
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
457+
MetaItemKind::name_value_from_tokens(&mut inner_tokens.iter())
462458
}
463459
Some(TokenTree::Token(token, _)) => {
464460
MetaItemLit::from_token(token).map(MetaItemKind::NameValue)
@@ -467,19 +463,17 @@ impl MetaItemKind {
467463
}
468464
}
469465

470-
fn from_tokens<'a>(
471-
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
472-
) -> Option<MetaItemKind> {
473-
match tokens.peek() {
466+
fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemKind> {
467+
match iter.peek() {
474468
Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
475469
let inner_tokens = inner_tokens.clone();
476-
tokens.next();
470+
iter.next();
477471
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
478472
}
479473
Some(TokenTree::Delimited(..)) => None,
480474
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
481-
tokens.next();
482-
MetaItemKind::name_value_from_tokens(tokens)
475+
iter.next();
476+
MetaItemKind::name_value_from_tokens(iter)
483477
}
484478
_ => Some(MetaItemKind::Word),
485479
}
@@ -598,22 +592,19 @@ impl MetaItemInner {
598592
self.meta_item().is_some()
599593
}
600594

601-
fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItemInner>
602-
where
603-
I: Iterator<Item = &'a TokenTree>,
604-
{
605-
match tokens.peek() {
595+
fn from_tokens(iter: &mut TokenStreamIter<'_>) -> Option<MetaItemInner> {
596+
match iter.peek() {
606597
Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
607-
tokens.next();
598+
iter.next();
608599
return Some(MetaItemInner::Lit(lit));
609600
}
610601
Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => {
611-
tokens.next();
612-
return MetaItemInner::from_tokens(&mut inner_tokens.trees().peekable());
602+
iter.next();
603+
return MetaItemInner::from_tokens(&mut inner_tokens.iter());
613604
}
614605
_ => {}
615606
}
616-
MetaItem::from_tokens(tokens).map(MetaItemInner::MetaItem)
607+
MetaItem::from_tokens(iter).map(MetaItemInner::MetaItem)
617608
}
618609
}
619610

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 22 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ where
9999
CTX: crate::HashStableContext,
100100
{
101101
fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
102-
for sub_tt in self.trees() {
102+
for sub_tt in self.iter() {
103103
sub_tt.hash_stable(hcx, hasher);
104104
}
105105
}
@@ -406,7 +406,7 @@ impl Eq for TokenStream {}
406406

407407
impl PartialEq<TokenStream> for TokenStream {
408408
fn eq(&self, other: &TokenStream) -> bool {
409-
self.trees().eq(other.trees())
409+
self.iter().eq(other.iter())
410410
}
411411
}
412412

@@ -423,24 +423,24 @@ impl TokenStream {
423423
self.0.len()
424424
}
425425

426-
pub fn trees(&self) -> RefTokenTreeCursor<'_> {
427-
RefTokenTreeCursor::new(self)
426+
pub fn get(&self, index: usize) -> Option<&TokenTree> {
427+
self.0.get(index)
428428
}
429429

430-
pub fn into_trees(self) -> TokenTreeCursor {
431-
TokenTreeCursor::new(self)
430+
pub fn iter(&self) -> TokenStreamIter<'_> {
431+
TokenStreamIter::new(self)
432432
}
433433

434434
/// Compares two `TokenStream`s, checking equality without regarding span information.
435435
pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
436-
let mut t1 = self.trees();
437-
let mut t2 = other.trees();
438-
for (t1, t2) in iter::zip(&mut t1, &mut t2) {
439-
if !t1.eq_unspanned(t2) {
436+
let mut iter1 = self.iter();
437+
let mut iter2 = other.iter();
438+
for (tt1, tt2) in iter::zip(&mut iter1, &mut iter2) {
439+
if !tt1.eq_unspanned(tt2) {
440440
return false;
441441
}
442442
}
443-
t1.next().is_none() && t2.next().is_none()
443+
iter1.next().is_none() && iter2.next().is_none()
444444
}
445445

446446
/// Create a token stream containing a single token with alone spacing. The
@@ -509,7 +509,7 @@ impl TokenStream {
509509
#[must_use]
510510
pub fn flattened(&self) -> TokenStream {
511511
fn can_skip(stream: &TokenStream) -> bool {
512-
stream.trees().all(|tree| match tree {
512+
stream.iter().all(|tree| match tree {
513513
TokenTree::Token(token, _) => !matches!(
514514
token.kind,
515515
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
@@ -522,7 +522,7 @@ impl TokenStream {
522522
return self.clone();
523523
}
524524

525-
self.trees().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
525+
self.iter().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
526526
}
527527

528528
// If `vec` is not empty, try to glue `tt` onto its last token. The return
@@ -665,25 +665,26 @@ impl TokenStream {
665665
}
666666
}
667667

668-
/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
669-
/// items.
670668
#[derive(Clone)]
671-
pub struct RefTokenTreeCursor<'t> {
669+
pub struct TokenStreamIter<'t> {
672670
stream: &'t TokenStream,
673671
index: usize,
674672
}
675673

676-
impl<'t> RefTokenTreeCursor<'t> {
674+
impl<'t> TokenStreamIter<'t> {
677675
fn new(stream: &'t TokenStream) -> Self {
678-
RefTokenTreeCursor { stream, index: 0 }
676+
TokenStreamIter { stream, index: 0 }
679677
}
680678

681-
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
682-
self.stream.0.get(self.index + n)
679+
// Peeking could be done via `Peekable`, but most iterators need peeking,
680+
// and this is simple and avoids the need to use `peekable` and `Peekable`
681+
// at all the use sites.
682+
pub fn peek(&self) -> Option<&'t TokenTree> {
683+
self.stream.0.get(self.index)
683684
}
684685
}
685686

686-
impl<'t> Iterator for RefTokenTreeCursor<'t> {
687+
impl<'t> Iterator for TokenStreamIter<'t> {
687688
type Item = &'t TokenTree;
688689

689690
fn next(&mut self) -> Option<&'t TokenTree> {
@@ -694,39 +695,6 @@ impl<'t> Iterator for RefTokenTreeCursor<'t> {
694695
}
695696
}
696697

697-
/// Owning by-value iterator over a [`TokenStream`], that produces `&TokenTree`
698-
/// items.
699-
///
700-
/// Doesn't impl `Iterator` because Rust doesn't permit an owning iterator to
701-
/// return `&T` from `next`; the need for an explicit lifetime in the `Item`
702-
/// associated type gets in the way. Instead, use `next_ref` (which doesn't
703-
/// involve associated types) for getting individual elements, or
704-
/// `RefTokenTreeCursor` if you really want an `Iterator`, e.g. in a `for`
705-
/// loop.
706-
#[derive(Clone, Debug)]
707-
pub struct TokenTreeCursor {
708-
pub stream: TokenStream,
709-
index: usize,
710-
}
711-
712-
impl TokenTreeCursor {
713-
fn new(stream: TokenStream) -> Self {
714-
TokenTreeCursor { stream, index: 0 }
715-
}
716-
717-
#[inline]
718-
pub fn next_ref(&mut self) -> Option<&TokenTree> {
719-
self.stream.0.get(self.index).map(|tree| {
720-
self.index += 1;
721-
tree
722-
})
723-
}
724-
725-
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
726-
self.stream.0.get(self.index + n)
727-
}
728-
}
729-
730698
#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
731699
pub struct DelimSpan {
732700
pub open: Span,

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -732,7 +732,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
732732
// E.g. we have seen cases where a proc macro can handle `a :: b` but not
733733
// `a::b`. See #117433 for some examples.
734734
fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) {
735-
let mut iter = tts.trees().peekable();
735+
let mut iter = tts.iter().peekable();
736736
while let Some(tt) = iter.next() {
737737
let spacing = self.print_tt(tt, convert_dollar_crate);
738738
if let Some(next) = iter.peek() {

‎compiler/rustc_builtin_macros/src/concat_idents.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ pub(crate) fn expand_concat_idents<'cx>(
1919
}
2020

2121
let mut res_str = String::new();
22-
for (i, e) in tts.trees().enumerate() {
22+
for (i, e) in tts.iter().enumerate() {
2323
if i & 1 == 1 {
2424
match e {
2525
TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}

‎compiler/rustc_builtin_macros/src/trace_macros.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,17 @@ pub(crate) fn expand_trace_macros(
1010
sp: Span,
1111
tt: TokenStream,
1212
) -> MacroExpanderResult<'static> {
13-
let mut cursor = tt.trees();
13+
let mut iter = tt.iter();
1414
let mut err = false;
15-
let value = match &cursor.next() {
15+
let value = match iter.next() {
1616
Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true,
1717
Some(TokenTree::Token(token, _)) if token.is_keyword(kw::False) => false,
1818
_ => {
1919
err = true;
2020
false
2121
}
2222
};
23-
err |= cursor.next().is_some();
23+
err |= iter.next().is_some();
2424
if err {
2525
cx.dcx().emit_err(errors::TraceMacros { span: sp });
2626
} else {

0 commit comments

Comments
 (0)