Add unstable frontmatter support

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -511,6 +511,7 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
     gate_all!(contracts, "contracts are incomplete");
     gate_all!(contracts_internals, "contract internal machinery is for internal use only");
     gate_all!(where_clause_attrs, "attributes in `where` clause are unstable");
+    gate_all!(frontmatter, "frontmatter syntax is unstable");
 
     if !visitor.features.never_patterns() {
         if let Some(spans) = spans.get(&sym::never_patterns) {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -120,6 +120,15 @@ fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec<Comment>
         pos += shebang_len;
     }
 
+    if let Some(frontmatter_len) = rustc_lexer::strip_frontmatter(&text[pos..]) {
+        comments.push(Comment {
+            style: CommentStyle::Isolated,
+            lines: vec![text[pos..pos + frontmatter_len].to_string()],
+            pos: start_bpos + BytePos(pos as u32),
+        });
+        pos += frontmatter_len;
+    }
+
     for token in rustc_lexer::tokenize(&text[pos..]) {
         let token_text = &text[pos..pos + token.len as usize];
         match token.kind {

compiler/rustc_feature/src/unstable.rs

@@ -501,6 +501,8 @@ declare_features! (
     (incomplete, fn_delegation, "1.76.0", Some(118212)),
     /// Allows impls for the Freeze trait.
     (internal, freeze_impls, "1.78.0", Some(121675)),
+    /// Frontmatter blocks for build tool metadata.
+    (unstable, frontmatter, "CURRENT_RUSTC_VERSION", Some(136889)),
     /// Allows defining gen blocks and `gen fn`.
     (unstable, gen_blocks, "1.75.0", Some(117078)),
     /// Infer generic args for both consts and types.

compiler/rustc_lexer/src/lib.rs

@@ -276,6 +276,58 @@ pub fn strip_shebang(input: &str) -> Option<usize> {
     None
 }
 
+/// Frontmatter is a special attribute type reserved for use by external tools
+///
+/// This must be called after [`strip_shebang`]
+pub fn strip_frontmatter(input: &str) -> Option<usize> {
+    // Whitespace may precede a frontmatter but must end with a newline
+    let rest = input.trim_start_matches(is_whitespace);
+    if rest.len() != input.len() {
+        let trimmed_len = input.len() - rest.len();
+        let last_trimmed_index = trimmed_len - 1;
+        if input.as_bytes()[last_trimmed_index] != b'\n' {
+            // either not a frontmatter or invalid opening
+            return None;
+        }
+    }
+
+    // Opens with a line that starts with 3 or more `-` followed by an optional identifier
+    const FENCE_CHAR: char = '-';
+    let fence_length =
+        rest.char_indices().find_map(|(i, c)| (c != FENCE_CHAR).then_some(i)).unwrap_or(rest.len());
+    if fence_length < 3 {
+        // either not a frontmatter or invalid frontmatter opening
+        return None;
+    }
+    let (fence_pattern, rest) = rest.split_at(fence_length);
+    let Some(info_end_index) = rest.find('\n') else {
+        // frontmatter close is required
+        return None;
+    };
+    let (info, rest) = rest.split_at(info_end_index);
+    let info = info.trim_matches(is_whitespace);
+    if !info.is_empty() && !is_ident(info) {
+        // optional infostring is not an identifier
+        return None;
+    }
+
+    // Ends with a line that starts with a matching number of `-` only followed by whitespace
+    let nl_fence_pattern = format!("\n{fence_pattern}");
+    let Some(frontmatter_nl) = rest.find(&nl_fence_pattern) else {
+        // frontmatter close is required
+        return None;
+    };
+    let rest = &rest[frontmatter_nl + nl_fence_pattern.len()..];
+
+    let (after_closing_fence, rest) = rest.split_once("\n").unwrap_or((rest, ""));
+    let after_closing_fence = after_closing_fence.trim_matches(is_whitespace);
+    if !after_closing_fence.is_empty() {
+        // extra characters beyond the original fence pattern, even if they are extra `-`
+        return None;
+    }
+    Some(input.len() - rest.len())
+}
+
 /// Validates a raw string literal. Used for getting more information about a
 /// problem with a `RawStr`/`RawByteStr` with a `None` field.
 #[inline]

compiler/rustc_lexer/src/tests.rs

@@ -124,6 +124,119 @@ fn test_valid_shebang() {
     assert_eq!(strip_shebang(input), None);
 }
 
+#[test]
+fn test_frontmatter() {
+    let input = "---
+---
+";
+    assert_eq!(strip_frontmatter(input), Some(input.len()));
+
+    let input = "---
+package.edition = '2024'
+
+[dependencies]
+regex = '1'
+---
+";
+    assert_eq!(strip_frontmatter(input), Some(input.len()));
+
+    // allow ident infostring
+    let input = "---cargo
+
+---
+";
+    assert_eq!(strip_frontmatter(input), Some(input.len()));
+
+    // disallow non-ident infostring
+    let input = "---cargo hello
+
+---
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // ignore extra whitespace
+    let input = "
+
+
+---\u{0020}
+
+---\u{0020}
+";
+    assert_eq!(strip_frontmatter(input), Some(input.len()));
+
+    // disallow indented opening/close
+    let input = "  ---
+  ---
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // ignore inner dashes not at line start
+    let input = "---
+
+  ---
+  ---
+
+---
+";
+    assert_eq!(strip_frontmatter(input), Some(input.len()));
+
+    // ignore fewer dashes inside
+    let input = "-----
+
+---
+---
+
+-----
+";
+    assert_eq!(strip_frontmatter(input), Some(input.len()));
+
+    // disallow more dashes inside
+    let input = "---
+
+-----
+-----
+
+---
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // disallow mismatch close
+    let input = "----
+
+---
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // disallow unclosed
+    let input = "---
+
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // disallow short open/close
+    let input = "--
+
+--
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // disallow content before
+    let input = "#![feature(frontmatter)]
+
+---
+---
+";
+    assert_eq!(strip_frontmatter(input), None);
+
+    // disallow trailing text
+    let input = "#![feature(frontmatter)]
+
+---
+---cargo
+";
+    assert_eq!(strip_frontmatter(input), None);
+}
+
 fn check_lexing(src: &str, expect: Expect) {
     let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect();
     expect.assert_eq(&actual)

compiler/rustc_parse/src/lexer/mod.rs

@@ -14,7 +14,7 @@ use rustc_session::lint::builtin::{
     TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
 };
 use rustc_session::parse::ParseSess;
-use rustc_span::{BytePos, Pos, Span, Symbol};
+use rustc_span::{BytePos, Pos, Span, Symbol, sym};
 use tracing::debug;
 
 use crate::lexer::diagnostics::TokenTreeDiagInfo;
@@ -55,6 +55,16 @@ pub(crate) fn lex_token_trees<'psess, 'src>(
         start_pos = start_pos + BytePos::from_usize(shebang_len);
     }
 
+    // Skip frontmatter, if present.
+    if let Some(frontmatter_len) = rustc_lexer::strip_frontmatter(src) {
+        src = &src[frontmatter_len..];
+        let lo = start_pos;
+        start_pos = start_pos + BytePos::from_usize(frontmatter_len);
+        let hi = start_pos;
+        let span = Span::with_root_ctxt(lo, hi);
+        psess.gated_spans.gate(sym::frontmatter, span);
+    }
+
     let cursor = Cursor::new(src);
     let mut lexer = Lexer {
         psess,

compiler/rustc_span/src/symbol.rs

@@ -1044,6 +1044,7 @@ symbols! {
         from_u16,
         from_usize,
         from_yeet,
+        frontmatter,
         fs_create_dir,
         fsub_algebraic,
         fsub_fast,

src/doc/unstable-book/src/language-features/frontmatter.md

@@ -0,0 +1,42 @@
+# `frontmatter`
+
+The tracking issue for this feature is: [#136889]
+
+[#136889]: https://github.com/rust-lang/rust/issues/136889
+
+------------------------
+
+The `frontmatter` feature adds support for a specialized and simplified attribute syntax
+intended for external tools to consume.
+
+For example, when used with Cargo:
+```rust,ignore (frontmatter/shebang are not intended for doctests)
+#!/usr/bin/env -S cargo -Zscript
+
+---
+[dependencies]
+clap = "4"
+---
+
+#![feature(frontmatter)]
+
+use clap::Parser;
+
+#[derive(Parser)]
+struct Cli {
+}
+
+fn main () {
+    Cli::parse();
+}
+```
+
+A frontmatter may come after a shebang and must come before any other syntax except whitespace.
+The open delimiter is three or more dashes (`-`) at the start of a new line.
+The open delimiter may be followed by whitespace and / or an identifier to mark the interpretation of the frontmatter within an external tool.
+It is then concluded at a newline.
+The close delimiter is a series of dashes that matches the open delimiter, at the start of a line.
+The close delimiter may be followed by whitespace.
+Any other trailing content, including more dashes than the open delimiter, is an error.
+It is then concluded at a newline.
+All content between the open and close delimiter lines is ignored.

src/tools/rustfmt/tests/source/frontmatter.rs

@@ -0,0 +1,11 @@
+
+
+---   identifier
+[dependencies]
+regex = "1"
+
+---
+
+#![feature(frontmatter)]
+
+fn main() {}

src/tools/rustfmt/tests/target/frontmatter.rs

@@ -0,0 +1,9 @@
+---   identifier
+[dependencies]
+regex = "1"
+
+---
+
+#![feature(frontmatter)]
+
+fn main() {}

tests/ui/feature-gates/feature-gate-frontmatter.rs

@@ -0,0 +1,7 @@
+---
+//~^ frontmatter syntax is unstable [E0658]
+---
+
+
+pub fn main() {
+}

tests/ui/feature-gates/feature-gate-frontmatter.stderr

@@ -0,0 +1,16 @@
+error[E0658]: frontmatter syntax is unstable
+  --> $DIR/feature-gate-frontmatter.rs:1:1
+   |
+LL | / ---
+LL | |
+LL | | ---
+LL | |
+   | |_^
+   |
+   = note: see issue #136889 <https://github.com/rust-lang/rust/issues/136889> for more information
+   = help: add `#![feature(frontmatter)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.

tests/ui/frontmatter/frontmatter-escaped.rs

@@ -0,0 +1,13 @@
+-----
+
+---
+---
+
+-----
+
+//@ check-pass
+
+#![feature(frontmatter)]
+
+fn main() {
+}

tests/ui/frontmatter/frontmatter-ignored-space.rs

@@ -0,0 +1,13 @@
+
+
+---  
+
+---  
+
+//@ check-pass
+#![feature(frontmatter)]
+// ignore-tidy-end-whitespace
+// ignore-tidy-leading-newlines
+
+fn main() {
+}

tests/ui/frontmatter/frontmatter-invalid-after-attribute.rs

@@ -0,0 +1,8 @@
+#![feature(frontmatter)]
+
+---
+//~^ ERROR expected item, found `-`
+---
+
+fn main() {
+}

tests/ui/frontmatter/frontmatter-invalid-after-attribute.stderr

@@ -0,0 +1,10 @@
+error: expected item, found `-`
+  --> $DIR/frontmatter-invalid-after-attribute.rs:3:1
+   |
+LL | ---
+   | ^ expected item
+   |
+   = note: for a full list of items that can appear in modules, see <https://doc.rust-lang.org/reference/items.html>
+
+error: aborting due to 1 previous error
+

tests/ui/frontmatter/frontmatter-invalid-after-comment.rs

@@ -0,0 +1,10 @@
+// Comment
+
+---
+//~^ ERROR expected item, found `-`
+---
+
+#![feature(frontmatter)]
+
+fn main() {
+}