Emit an error instead of reconstructing token stream.

Author: cjgillot

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -7,6 +7,7 @@ use rustc_ast::visit::Visitor;
 use rustc_ast::NodeId;
 use rustc_ast::{mut_visit, visit};
 use rustc_ast::{Attribute, HasAttrs, HasTokens};
+use rustc_errors::PResult;
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_expand::config::StripUnconfigured;
 use rustc_expand::configure;
@@ -144,33 +145,34 @@ impl CfgEval<'_, '_> {
         // the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization
         // process is lossless, so this process is invisible to proc-macros.
 
-        let parse_annotatable_with: fn(&mut Parser<'_>) -> _ = match annotatable {
-            Annotatable::Item(_) => {
-                |parser| Annotatable::Item(parser.parse_item(ForceCollect::Yes).unwrap().unwrap())
-            }
-            Annotatable::TraitItem(_) => |parser| {
-                Annotatable::TraitItem(
-                    parser.parse_trait_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
-                )
-            },
-            Annotatable::ImplItem(_) => |parser| {
-                Annotatable::ImplItem(
-                    parser.parse_impl_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
-                )
-            },
-            Annotatable::ForeignItem(_) => |parser| {
-                Annotatable::ForeignItem(
-                    parser.parse_foreign_item(ForceCollect::Yes).unwrap().unwrap().unwrap(),
-                )
-            },
-            Annotatable::Stmt(_) => |parser| {
-                Annotatable::Stmt(P(parser.parse_stmt(ForceCollect::Yes).unwrap().unwrap()))
-            },
-            Annotatable::Expr(_) => {
-                |parser| Annotatable::Expr(parser.parse_expr_force_collect().unwrap())
-            }
-            _ => unreachable!(),
-        };
+        let parse_annotatable_with: for<'a> fn(&mut Parser<'a>) -> PResult<'a, _> =
+            match annotatable {
+                Annotatable::Item(_) => {
+                    |parser| Ok(Annotatable::Item(parser.parse_item(ForceCollect::Yes)?.unwrap()))
+                }
+                Annotatable::TraitItem(_) => |parser| {
+                    Ok(Annotatable::TraitItem(
+                        parser.parse_trait_item(ForceCollect::Yes)?.unwrap().unwrap(),
+                    ))
+                },
+                Annotatable::ImplItem(_) => |parser| {
+                    Ok(Annotatable::ImplItem(
+                        parser.parse_impl_item(ForceCollect::Yes)?.unwrap().unwrap(),
+                    ))
+                },
+                Annotatable::ForeignItem(_) => |parser| {
+                    Ok(Annotatable::ForeignItem(
+                        parser.parse_foreign_item(ForceCollect::Yes)?.unwrap().unwrap(),
+                    ))
+                },
+                Annotatable::Stmt(_) => |parser| {
+                    Ok(Annotatable::Stmt(P(parser.parse_stmt(ForceCollect::Yes)?.unwrap())))
+                },
+                Annotatable::Expr(_) => {
+                    |parser| Ok(Annotatable::Expr(parser.parse_expr_force_collect()?))
+                }
+                _ => unreachable!(),
+            };
 
         // 'Flatten' all nonterminals (i.e. `TokenKind::Interpolated`)
         // to `None`-delimited groups containing the corresponding tokens. This
@@ -193,7 +195,13 @@ impl CfgEval<'_, '_> {
         let mut parser =
             rustc_parse::stream_to_parser(&self.cfg.sess.parse_sess, orig_tokens, None);
         parser.capture_cfg = true;
-        annotatable = parse_annotatable_with(&mut parser);
+        match parse_annotatable_with(&mut parser) {
+            Ok(a) => annotatable = a,
+            Err(mut err) => {
+                err.emit();
+                return Some(annotatable);
+            }
+        }
 
         // Now that we have our re-parsed `AttrTokenStream`, recursively configuring
         // our attribute target will correctly the tokens as well.

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -442,38 +442,22 @@ fn make_token_stream(
         }
         token_and_spacing = iter.next();
     }
-    while let Some(FrameData { open_delim_sp, mut inner }) = stack.pop() {
-        // A former macro expansion could give us malformed tokens.
-        // In that case, manually close all open delimitors so downstream users
-        // don't ICE on them.
-        if let Some((delim, open_sp)) = open_delim_sp {
-            let dspan = DelimSpan::from_pair(open_sp, rustc_span::DUMMY_SP);
-            let stream = AttrTokenStream::new(inner);
-            let delimited = AttrTokenTree::Delimited(dspan, delim, stream);
-            stack
-                .last_mut()
-                .unwrap_or_else(|| panic!("Bottom token frame is missing for recovered token"))
+    let mut final_buf = stack.pop().expect("Missing final buf!");
+    if break_last_token {
+        let last_token = final_buf.inner.pop().unwrap();
+        if let AttrTokenTree::Token(last_token, spacing) = last_token {
+            let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
+
+            // An 'unglued' token is always two ASCII characters
+            let mut first_span = last_token.span.shrink_to_lo();
+            first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
+
+            final_buf
                 .inner
-                .push(delimited);
+                .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing));
         } else {
-            if break_last_token {
-                let last_token = inner.pop().unwrap();
-                if let AttrTokenTree::Token(last_token, spacing) = last_token {
-                    let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
-
-                    // An 'unglued' token is always two ASCII characters
-                    let mut first_span = last_token.span.shrink_to_lo();
-                    first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
-
-                    inner
-                        .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing));
-                } else {
-                    panic!("Unexpected last token {:?}", last_token)
-                }
-            }
-            assert!(stack.is_empty(), "Stack should be empty: stack={:?}", stack);
-            return AttrTokenStream::new(inner);
+            panic!("Unexpected last token {:?}", last_token)
         }
     }
-    panic!("Missing final buf!")
+    AttrTokenStream::new(final_buf.inner)
 }

src/test/ui/macros/syntax-error-recovery.rs

@@ -13,5 +13,6 @@ macro_rules! values {
 //~| ERROR macro expansion ignores token `(String)` and any following
 
 values!(STRING(1) as (String) => cfg(test),);
+//~^ ERROR expected one of `!` or `::`, found `<eof>`
 
 fn main() {}

src/test/ui/macros/syntax-error-recovery.stderr

@@ -20,5 +20,11 @@ LL | values!(STRING(1) as (String) => cfg(test),);
    |
    = note: the usage of `values!` is likely invalid in item context
 
-error: aborting due to 2 previous errors
+error: expected one of `!` or `::`, found `<eof>`
+  --> $DIR/syntax-error-recovery.rs:15:9
+   |
+LL | values!(STRING(1) as (String) => cfg(test),);
+   |         ^^^^^^ expected one of `!` or `::`
+
+error: aborting due to 3 previous errors