From beb3a4a98b5c73f51fa2c4216949b652393de460 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Mon, 29 Jun 2020 09:55:28 -0400 Subject: [PATCH] Handle `None`-delimited groups when parsing `macro_rules!` macro When a `macro_rules!` macro expands to another `macro_rules!` macro, we may see `None`-delimited groups in odd places when another crate deserializes the 'inner' macro. This commit 'unwraps' an outer `None`-delimited group to avoid breaking existing code. See https://github.com/rust-lang/rust/pull/73569#issuecomment-650860457 for more details. The proper fix is to handle `None`-delimited groups systematically throughout the parser, but that will require significant work. In the meantime, this hack lets us fix important hygiene bugs in macros --- src/librustc_expand/mbe/quoted.rs | 110 ++++++++++-------- .../ui/proc-macro/auxiliary/meta-delim.rs | 12 ++ src/test/ui/proc-macro/meta-delim.rs | 12 ++ 3 files changed, 87 insertions(+), 47 deletions(-) create mode 100644 src/test/ui/proc-macro/auxiliary/meta-delim.rs create mode 100644 src/test/ui/proc-macro/meta-delim.rs diff --git a/src/librustc_expand/mbe/quoted.rs b/src/librustc_expand/mbe/quoted.rs index de66c2ada40e6..09306f26ee0ad 100644 --- a/src/librustc_expand/mbe/quoted.rs +++ b/src/librustc_expand/mbe/quoted.rs @@ -90,7 +90,7 @@ pub(super) fn parse( /// # Parameters /// /// - `tree`: the tree we wish to convert. -/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish +/// - `outer_trees`: an iterator over trees. We may need to read more tokens from it in order to finish /// converting `tree` /// - `expect_matchers`: same as for `parse` (see above). /// - `sess`: the parsing session. Any errors will be emitted to this session. @@ -98,7 +98,7 @@ pub(super) fn parse( /// unstable features or not. fn parse_tree( tree: tokenstream::TokenTree, - trees: &mut impl Iterator, + outer_trees: &mut impl Iterator, expect_matchers: bool, sess: &ParseSess, node_id: NodeId, @@ -106,56 +106,72 @@ fn parse_tree( // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { - // `tree` is followed by a delimited set of token trees. This indicates the beginning - // of a repetition sequence in the macro (e.g. `$(pat)*`). - Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { - // Must have `(` not `{` or `[` - if delim != token::Paren { - let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); - let msg = format!("expected `(`, found `{}`", tok); - sess.span_diagnostic.span_err(span.entire(), &msg); - } - // Parse the contents of the sequence itself - let sequence = parse(tts, expect_matchers, sess, node_id); - // Get the Kleene operator and optional separator - let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); - // Count the number of captured "names" (i.e., named metavars) - let name_captures = macro_parser::count_names(&sequence); - TokenTree::Sequence( - span, - Lrc::new(SequenceRepetition { - tts: sequence, - separator, - kleene, - num_captures: name_captures, - }), - ) + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => { + // FIXME: Handle `None`-delimited groups in a more systematic way + // during parsing. + let mut next = outer_trees.next(); + let mut trees: Box>; + if let Some(tokenstream::TokenTree::Delimited(_, token::NoDelim, tts)) = next { + trees = Box::new(tts.into_trees()); + next = trees.next(); + } else { + trees = Box::new(outer_trees); } - // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special - // metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { - let (ident, is_raw) = token.ident().unwrap(); - let span = ident.span.with_lo(span.lo()); - if ident.name == kw::Crate && !is_raw { - TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) - } else { - TokenTree::MetaVar(span, ident) + match next { + // `tree` is followed by a delimited set of token trees. This indicates the beginning + // of a repetition sequence in the macro (e.g. `$(pat)*`). + Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { + // Must have `(` not `{` or `[` + if delim != token::Paren { + let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); + let msg = format!("expected `(`, found `{}`", tok); + sess.span_diagnostic.span_err(span.entire(), &msg); + } + // Parse the contents of the sequence itself + let sequence = parse(tts, expect_matchers, sess, node_id); + // Get the Kleene operator and optional separator + let (separator, kleene) = + parse_sep_and_kleene_op(&mut trees, span.entire(), sess); + // Count the number of captured "names" (i.e., named metavars) + let name_captures = macro_parser::count_names(&sequence); + TokenTree::Sequence( + span, + Lrc::new(SequenceRepetition { + tts: sequence, + separator, + kleene, + num_captures: name_captures, + }), + ) } - } - // `tree` is followed by a random token. This is an error. - Some(tokenstream::TokenTree::Token(token)) => { - let msg = - format!("expected identifier, found `{}`", pprust::token_to_string(&token),); - sess.span_diagnostic.span_err(token.span, &msg); - TokenTree::MetaVar(token.span, Ident::invalid()) - } + // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special + // metavariable that names the crate of the invocation. + Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { + let (ident, is_raw) = token.ident().unwrap(); + let span = ident.span.with_lo(span.lo()); + if ident.name == kw::Crate && !is_raw { + TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) + } else { + TokenTree::MetaVar(span, ident) + } + } - // There are no more tokens. Just return the `$` we already have. - None => TokenTree::token(token::Dollar, span), - }, + // `tree` is followed by a random token. This is an error. + Some(tokenstream::TokenTree::Token(token)) => { + let msg = format!( + "expected identifier, found `{}`", + pprust::token_to_string(&token), + ); + sess.span_diagnostic.span_err(token.span, &msg); + TokenTree::MetaVar(token.span, Ident::invalid()) + } + + // There are no more tokens. Just return the `$` we already have. + None => TokenTree::token(token::Dollar, span), + } + } // `tree` is an arbitrary token. Keep it. tokenstream::TokenTree::Token(token) => TokenTree::Token(token), diff --git a/src/test/ui/proc-macro/auxiliary/meta-delim.rs b/src/test/ui/proc-macro/auxiliary/meta-delim.rs new file mode 100644 index 0000000000000..54e3d7857267b --- /dev/null +++ b/src/test/ui/proc-macro/auxiliary/meta-delim.rs @@ -0,0 +1,12 @@ +macro_rules! produce_it { + ($dollar_one:tt $foo:ident $my_name:ident) => { + #[macro_export] + macro_rules! meta_delim { + ($dollar_one ($dollar_one $my_name:ident)*) => { + stringify!($dollar_one ($dollar_one $my_name)*) + } + } + } +} + +produce_it!($my_name name); diff --git a/src/test/ui/proc-macro/meta-delim.rs b/src/test/ui/proc-macro/meta-delim.rs new file mode 100644 index 0000000000000..964291bc6784c --- /dev/null +++ b/src/test/ui/proc-macro/meta-delim.rs @@ -0,0 +1,12 @@ +// aux-build:meta-delim.rs +// edition:2018 +// run-pass + +// Tests that we can properly deserialize a macro with strange delimiters +// See https://github.com/rust-lang/rust/pull/73569#issuecomment-650860457 + +extern crate meta_delim; + +fn main() { + assert_eq!("a bunch of idents", meta_delim::meta_delim!(a bunch of idents)); +}