Rollup merge of rust-lang#146340 - fmease:frontmatter-containment, r=fee1-dead,Urgau

Strip frontmatter in fewer places

* Stop stripping frontmatter in `proc_macro::Literal::from_str` (RUST-146132)
* Stop stripping frontmatter in expr-ctxt (but not item-ctxt!) `include`s (RUST-145945)
* Stop stripping shebang (!) in `proc_macro::Literal::from_str`
  * Not a breaking change because it did compare spans already to ensure there wasn't extra whitespace or comments (`Literal::from_str("#!\n0")` already yields `Err(_)` thankfully!)
* Stop stripping frontmatter+shebang inside some rustdoc code where it doesn't make any observable difference (see self review comments)
* (Stop stripping frontmatter+shebang inside internal test code)

Fixes rust-lang#145945.
Fixes rust-lang#146132.

r? fee1-dead

Author: matthiaskrgr

compiler/rustc_builtin_macros/src/source_util.rs

@@ -1,3 +1,5 @@
+//! The implementation of built-in macros which relate to the file system.
+
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::sync::Arc;
@@ -11,9 +13,11 @@ use rustc_expand::base::{
 };
 use rustc_expand::module::DirOwnership;
 use rustc_lint_defs::BuiltinLintDiag;
-use rustc_parse::parser::{ForceCollect, Parser};
+use rustc_parse::lexer::StripTokens;
+use rustc_parse::parser::ForceCollect;
 use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal, utf8_error};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
+use rustc_session::parse::ParseSess;
 use rustc_span::source_map::SourceMap;
 use rustc_span::{ByteSymbol, Pos, Span, Symbol};
 use smallvec::SmallVec;
@@ -23,11 +27,7 @@ use crate::util::{
     check_zero_tts, get_single_str_from_tts, get_single_str_spanned_from_tts, parse_expr,
 };
 
-// These macros all relate to the file system; they either return
-// the column/row/filename of the expression, or they include
-// a given file into the current one.
-
-/// line!(): expands to the current line number
+/// Expand `line!()` to the current line number.
 pub(crate) fn expand_line(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -42,7 +42,7 @@ pub(crate) fn expand_line(
     ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.line as u32)))
 }
 
-/* column!(): expands to the current column number */
+/// Expand `column!()` to the current column number.
 pub(crate) fn expand_column(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -57,9 +57,7 @@ pub(crate) fn expand_column(
     ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize() as u32 + 1)))
 }
 
-/// file!(): expands to the current filename */
-/// The source_file (`loc.file`) contains a bunch more information we could spit
-/// out if we wanted.
+/// Expand `file!()` to the current filename.
 pub(crate) fn expand_file(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -81,6 +79,7 @@ pub(crate) fn expand_file(
     )))
 }
 
+/// Expand `stringify!($input)`.
 pub(crate) fn expand_stringify(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -91,6 +90,7 @@ pub(crate) fn expand_stringify(
     ExpandResult::Ready(MacEager::expr(cx.expr_str(sp, Symbol::intern(&s))))
 }
 
+/// Expand `module_path!()` to (a textual representation of) the current module path.
 pub(crate) fn expand_mod(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -104,9 +104,9 @@ pub(crate) fn expand_mod(
     ExpandResult::Ready(MacEager::expr(cx.expr_str(sp, Symbol::intern(&string))))
 }
 
-/// include! : parse the given file as an expr
-/// This is generally a bad idea because it's going to behave
-/// unhygienically.
+/// Expand `include!($input)`.
+///
+/// This works in item and expression position. Notably, it doesn't work in pattern position.
 pub(crate) fn expand_include<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
@@ -116,64 +116,76 @@ pub(crate) fn expand_include<'cx>(
     let ExpandResult::Ready(mac) = get_single_str_from_tts(cx, sp, tts, "include!") else {
         return ExpandResult::Retry(());
     };
-    let file = match mac {
-        Ok(file) => file,
+    let path = match mac {
+        Ok(path) => path,
         Err(guar) => return ExpandResult::Ready(DummyResult::any(sp, guar)),
     };
     // The file will be added to the code map by the parser
-    let file = match resolve_path(&cx.sess, file.as_str(), sp) {
-        Ok(f) => f,
+    let path = match resolve_path(&cx.sess, path.as_str(), sp) {
+        Ok(path) => path,
         Err(err) => {
             let guar = err.emit();
             return ExpandResult::Ready(DummyResult::any(sp, guar));
         }
     };
-    let p = unwrap_or_emit_fatal(new_parser_from_file(cx.psess(), &file, Some(sp)));
 
     // If in the included file we have e.g., `mod bar;`,
-    // then the path of `bar.rs` should be relative to the directory of `file`.
+    // then the path of `bar.rs` should be relative to the directory of `path`.
     // See https://github.com/rust-lang/rust/pull/69838/files#r395217057 for a discussion.
     // `MacroExpander::fully_expand_fragment` later restores, so "stack discipline" is maintained.
-    let dir_path = file.parent().unwrap_or(&file).to_owned();
+    let dir_path = path.parent().unwrap_or(&path).to_owned();
     cx.current_expansion.module = Rc::new(cx.current_expansion.module.with_dir_path(dir_path));
     cx.current_expansion.dir_ownership = DirOwnership::Owned { relative: None };
 
     struct ExpandInclude<'a> {
-        p: Parser<'a>,
+        psess: &'a ParseSess,
+        path: PathBuf,
         node_id: ast::NodeId,
+        span: Span,
     }
     impl<'a> MacResult for ExpandInclude<'a> {
-        fn make_expr(mut self: Box<ExpandInclude<'a>>) -> Option<Box<ast::Expr>> {
-            let expr = parse_expr(&mut self.p).ok()?;
-            if self.p.token != token::Eof {
-                self.p.psess.buffer_lint(
+        fn make_expr(self: Box<ExpandInclude<'a>>) -> Option<Box<ast::Expr>> {
+            let mut p = unwrap_or_emit_fatal(new_parser_from_file(
+                self.psess,
+                &self.path,
+                // Don't strip frontmatter for backward compatibility, `---` may be the start of a
+                // manifold negation. FIXME: Ideally, we wouldn't strip shebangs here either.
+                StripTokens::Shebang,
+                Some(self.span),
+            ));
+            let expr = parse_expr(&mut p).ok()?;
+            if p.token != token::Eof {
+                p.psess.buffer_lint(
                     INCOMPLETE_INCLUDE,
-                    self.p.token.span,
+                    p.token.span,
                     self.node_id,
                     BuiltinLintDiag::IncompleteInclude,
                 );
             }
             Some(expr)
         }
 
-        fn make_items(mut self: Box<ExpandInclude<'a>>) -> Option<SmallVec<[Box<ast::Item>; 1]>> {
+        fn make_items(self: Box<ExpandInclude<'a>>) -> Option<SmallVec<[Box<ast::Item>; 1]>> {
+            let mut p = unwrap_or_emit_fatal(new_parser_from_file(
+                self.psess,
+                &self.path,
+                StripTokens::ShebangAndFrontmatter,
+                Some(self.span),
+            ));
             let mut ret = SmallVec::new();
             loop {
-                match self.p.parse_item(ForceCollect::No) {
+                match p.parse_item(ForceCollect::No) {
                     Err(err) => {
                         err.emit();
                         break;
                     }
                     Ok(Some(item)) => ret.push(item),
                     Ok(None) => {
-                        if self.p.token != token::Eof {
-                            self.p
-                                .dcx()
-                                .create_err(errors::ExpectedItem {
-                                    span: self.p.token.span,
-                                    token: &pprust::token_to_string(&self.p.token),
-                                })
-                                .emit();
+                        if p.token != token::Eof {
+                            p.dcx().emit_err(errors::ExpectedItem {
+                                span: p.token.span,
+                                token: &pprust::token_to_string(&p.token),
+                            });
                         }
 
                         break;
@@ -184,10 +196,17 @@ pub(crate) fn expand_include<'cx>(
         }
     }
 
-    ExpandResult::Ready(Box::new(ExpandInclude { p, node_id: cx.current_expansion.lint_node_id }))
+    ExpandResult::Ready(Box::new(ExpandInclude {
+        psess: cx.psess(),
+        path,
+        node_id: cx.current_expansion.lint_node_id,
+        span: sp,
+    }))
 }
 
-/// `include_str!`: read the given file, insert it as a literal string expr
+/// Expand `include_str!($input)` to the content of the UTF-8-encoded file given by path `$input` as a string literal.
+///
+/// This works in expression, pattern and statement position.
 pub(crate) fn expand_include_str(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -206,6 +225,7 @@ pub(crate) fn expand_include_str(
         Ok((bytes, bsp)) => match std::str::from_utf8(&bytes) {
             Ok(src) => {
                 let interned_src = Symbol::intern(src);
+                // MacEager converts the expr into a pat if need be.
                 MacEager::expr(cx.expr_str(cx.with_def_site_ctxt(bsp), interned_src))
             }
             Err(utf8err) => {
@@ -218,6 +238,9 @@ pub(crate) fn expand_include_str(
     })
 }
 
+/// Expand `include_bytes!($input)` to the content of the file given by path `$input`.
+///
+/// This works in expression, pattern and statement position.
 pub(crate) fn expand_include_bytes(
     cx: &mut ExtCtxt<'_>,
     sp: Span,
@@ -237,6 +260,7 @@ pub(crate) fn expand_include_bytes(
             // Don't care about getting the span for the raw bytes,
             // because the console can't really show them anyway.
             let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(ByteSymbol::intern(&bytes)));
+            // MacEager converts the expr into a pat if need be.
             MacEager::expr(expr)
         }
         Err(dummy) => dummy,

compiler/rustc_driver_impl/src/lib.rs

@@ -51,6 +51,7 @@ use rustc_lint::unerased_lint_store;
 use rustc_metadata::creader::MetadataLoader;
 use rustc_metadata::locator;
 use rustc_middle::ty::TyCtxt;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_session::config::{
     CG_OPTIONS, CrateType, ErrorOutputType, Input, OptionDesc, OutFileName, OutputType, Sysroot,
@@ -1288,10 +1289,15 @@ fn warn_on_confusing_output_filename_flag(
 
 fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> {
     let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
-        Input::File(file) => new_parser_from_file(&sess.psess, file, None),
-        Input::Str { name, input } => {
-            new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
+        Input::File(file) => {
+            new_parser_from_file(&sess.psess, file, StripTokens::ShebangAndFrontmatter, None)
         }
+        Input::Str { name, input } => new_parser_from_source_str(
+            &sess.psess,
+            name.clone(),
+            input.clone(),
+            StripTokens::ShebangAndFrontmatter,
+        ),
     });
     parser.parse_inner_attributes()
 }

compiler/rustc_expand/src/module.rs

@@ -4,6 +4,7 @@ use std::path::{self, Path, PathBuf};
 use rustc_ast::{AttrVec, Attribute, Inline, Item, ModSpans};
 use rustc_attr_parsing::validate_attr;
 use rustc_errors::{Diag, ErrorGuaranteed};
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{exp, new_parser_from_file, unwrap_or_emit_fatal};
 use rustc_session::Session;
 use rustc_session::parse::ParseSess;
@@ -67,8 +68,12 @@ pub(crate) fn parse_external_mod(
         }
 
         // Actually parse the external file as a module.
-        let mut parser =
-            unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span)));
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
+            &sess.psess,
+            &mp.file_path,
+            StripTokens::ShebangAndFrontmatter,
+            Some(span),
+        ));
         let (inner_attrs, items, inner_span) =
             parser.parse_mod(exp!(Eof)).map_err(|err| ModError::ParserError(err))?;
         attrs.extend(inner_attrs);

compiler/rustc_expand/src/proc_macro_server.rs

@@ -8,7 +8,7 @@ use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
-use rustc_parse::lexer::nfc_normalize;
+use rustc_parse::lexer::{StripTokens, nfc_normalize};
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
 use rustc_proc_macro::bridge::{
@@ -485,8 +485,13 @@ impl server::FreeFunctions for Rustc<'_, '_> {
 
     fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
         let name = FileName::proc_macro_source_code(s);
-        let mut parser =
-            unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned()));
+
+        let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
+            self.psess(),
+            name,
+            s.to_owned(),
+            StripTokens::Nothing,
+        ));
 
         let first_span = parser.token.span.data();
         let minus_present = parser.eat(exp!(Minus));

compiler/rustc_interface/src/interface.rs

@@ -13,7 +13,8 @@ use rustc_lint::LintStore;
 use rustc_middle::ty;
 use rustc_middle::ty::CurrentGcx;
 use rustc_middle::util::Providers;
-use rustc_parse::new_parser_from_simple_source_str;
+use rustc_parse::lexer::StripTokens;
+use rustc_parse::new_parser_from_source_str;
 use rustc_parse::parser::attr::AllowLeadingUnsafe;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
@@ -68,7 +69,8 @@ pub(crate) fn parse_cfg(dcx: DiagCtxtHandle<'_>, cfgs: Vec<String>) -> Cfg {
                 };
             }
 
-            match new_parser_from_simple_source_str(&psess, filename, s.to_string()) {
+            match new_parser_from_source_str(&psess, filename, s.to_string(), StripTokens::Nothing)
+            {
                 Ok(mut parser) => match parser.parse_meta_item(AllowLeadingUnsafe::No) {
                     Ok(meta_item) if parser.token == token::Eof => {
                         if meta_item.path.segments.len() != 1 {
@@ -166,13 +168,15 @@ pub(crate) fn parse_check_cfg(dcx: DiagCtxtHandle<'_>, specs: Vec<String>) -> Ch
             error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
         };
 
-        let mut parser = match new_parser_from_simple_source_str(&psess, filename, s.to_string()) {
-            Ok(parser) => parser,
-            Err(errs) => {
-                errs.into_iter().for_each(|err| err.cancel());
-                expected_error();
-            }
-        };
+        let mut parser =
+            match new_parser_from_source_str(&psess, filename, s.to_string(), StripTokens::Nothing)
+            {
+                Ok(parser) => parser,
+                Err(errs) => {
+                    errs.into_iter().for_each(|err| err.cancel());
+                    expected_error();
+                }
+            };
 
         let meta_item = match parser.parse_meta_item(AllowLeadingUnsafe::No) {
             Ok(meta_item) if parser.token == token::Eof => meta_item,

compiler/rustc_interface/src/passes.rs

@@ -28,6 +28,7 @@ use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepsType;
 use rustc_middle::ty::{self, CurrentGcx, GlobalCtxt, RegisteredTools, TyCtxt};
 use rustc_middle::util::Providers;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_passes::{abi_test, input_stats, layout_test};
 use rustc_resolve::{Resolver, ResolverOutputs};
@@ -52,10 +53,18 @@ pub fn parse<'a>(sess: &'a Session) -> ast::Crate {
     let mut krate = sess
         .time("parse_crate", || {
             let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
-                Input::File(file) => new_parser_from_file(&sess.psess, file, None),
-                Input::Str { input, name } => {
-                    new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
-                }
+                Input::File(file) => new_parser_from_file(
+                    &sess.psess,
+                    file,
+                    StripTokens::ShebangAndFrontmatter,
+                    None,
+                ),
+                Input::Str { input, name } => new_parser_from_source_str(
+                    &sess.psess,
+                    name.clone(),
+                    input.clone(),
+                    StripTokens::ShebangAndFrontmatter,
+                ),
             });
             parser.parse_crate_mod()
         })

compiler/rustc_parse/src/lexer/mod.rs

@@ -45,7 +45,7 @@ pub(crate) struct UnmatchedDelim {
 }
 
 /// Which tokens should be stripped before lexing the tokens.
-pub(crate) enum StripTokens {
+pub enum StripTokens {
     /// Strip both shebang and frontmatter.
     ShebangAndFrontmatter,
     /// Strip the shebang but not frontmatter.