Skip to content

Commit a7e51b5

Browse files
CentrilCentril
authored
Rollup merge of rust-lang#66935 - petrochenkov:attrtok2, r=Centril
syntax: Unify macro and attribute arguments in AST The unified form (`ast::MacArgs`) represents parsed arguments instead of an unstructured token stream that was previously used for attributes. It also tracks some spans and delimiter kinds better for fn-like macros and macro definitions. I've been talking about implementing this with @nnethercote in rust-lang#65750 (comment). The parsed representation is closer to `MetaItem` and requires less token juggling during conversions, so it potentially may be faster. r? @Centril
2 parents c1190d4 + 498737c commit a7e51b5

40 files changed

+395
-327
lines changed

‎src/librustc/hir/lowering.rs

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1003,7 +1003,7 @@ impl<'a> LoweringContext<'a> {
10031003
AttrKind::Normal(ref item) => {
10041004
AttrKind::Normal(AttrItem {
10051005
path: item.path.clone(),
1006-
tokens: self.lower_token_stream(item.tokens.clone()),
1006+
args: self.lower_mac_args(&item.args),
10071007
})
10081008
}
10091009
AttrKind::DocComment(comment) => AttrKind::DocComment(comment)
@@ -1017,6 +1017,16 @@ impl<'a> LoweringContext<'a> {
10171017
}
10181018
}
10191019

1020+
fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {
1021+
match *args {
1022+
MacArgs::Empty => MacArgs::Empty,
1023+
MacArgs::Delimited(dspan, delim, ref tokens) =>
1024+
MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone())),
1025+
MacArgs::Eq(eq_span, ref tokens) =>
1026+
MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone())),
1027+
}
1028+
}
1029+
10201030
fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
10211031
tokens
10221032
.into_trees()

‎src/librustc/hir/lowering/item.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ impl LoweringContext<'_> {
233233

234234
if let ItemKind::MacroDef(ref def) = i.kind {
235235
if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) {
236-
let body = self.lower_token_stream(def.stream());
236+
let body = self.lower_token_stream(def.body.inner_tokens());
237237
let hir_id = self.lower_node_id(i.id);
238238
self.exported_macros.push(hir::MacroDef {
239239
name: ident.name,

‎src/librustc_lint/builtin.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1450,10 +1450,10 @@ impl KeywordIdents {
14501450

14511451
impl EarlyLintPass for KeywordIdents {
14521452
fn check_mac_def(&mut self, cx: &EarlyContext<'_>, mac_def: &ast::MacroDef, _id: ast::NodeId) {
1453-
self.check_tokens(cx, mac_def.stream());
1453+
self.check_tokens(cx, mac_def.body.inner_tokens());
14541454
}
14551455
fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::Mac) {
1456-
self.check_tokens(cx, mac.tts.clone().into());
1456+
self.check_tokens(cx, mac.args.inner_tokens());
14571457
}
14581458
fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: ast::Ident) {
14591459
self.check_ident_token(cx, UnderMacro(false), ident);

‎src/librustc_metadata/rmeta/decoder/cstore_impl.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ use syntax::source_map;
3232
use syntax::source_map::Spanned;
3333
use syntax::symbol::Symbol;
3434
use syntax::expand::allocator::AllocatorKind;
35+
use syntax::ptr::P;
36+
use syntax::tokenstream::DelimSpan;
3537
use syntax_pos::{Span, FileName};
3638

3739
macro_rules! provide {
@@ -427,6 +429,7 @@ impl CStore {
427429

428430
let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
429431
let local_span = Span::with_root_ctxt(source_file.start_pos, source_file.end_pos);
432+
let dspan = DelimSpan::from_single(local_span);
430433
let (body, mut errors) = source_file_to_stream(&sess.parse_sess, source_file, None);
431434
emit_unclosed_delims(&mut errors, &sess.parse_sess);
432435

@@ -448,7 +451,7 @@ impl CStore {
448451
span: local_span,
449452
attrs: attrs.iter().cloned().collect(),
450453
kind: ast::ItemKind::MacroDef(ast::MacroDef {
451-
tokens: body.into(),
454+
body: P(ast::MacArgs::Delimited(dspan, ast::MacDelimiter::Brace, body)),
452455
legacy: def.legacy,
453456
}),
454457
vis: source_map::respan(local_span.shrink_to_lo(), ast::VisibilityKind::Inherited),

‎src/librustc_parse/config.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ impl<'a> StripUnconfigured<'a> {
101101
if !attr.has_name(sym::cfg_attr) {
102102
return vec![attr];
103103
}
104-
if attr.get_normal_item().tokens.is_empty() {
104+
if let ast::MacArgs::Empty = attr.get_normal_item().args {
105105
self.sess.span_diagnostic
106106
.struct_span_err(
107107
attr.span,

‎src/librustc_parse/lib.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,9 @@ pub fn parse_in_attr<'a, T>(
277277
) -> PResult<'a, T> {
278278
let mut parser = Parser::new(
279279
sess,
280-
attr.get_normal_item().tokens.clone(),
280+
// FIXME(#66940, Centril | petrochenkov): refactor this function so it doesn't
281+
// require reconstructing and immediately re-parsing delimiters.
282+
attr.get_normal_item().args.outer_tokens(),
281283
None,
282284
false,
283285
false,
@@ -409,7 +411,7 @@ fn prepend_attrs(
409411
brackets.push(stream);
410412
}
411413

412-
brackets.push(item.tokens.clone());
414+
brackets.push(item.args.outer_tokens());
413415

414416
// The span we list here for `#` and for `[ ... ]` are both wrong in
415417
// that it encompasses more than each token, but it hopefully is "good

‎src/librustc_parse/parser/attr.rs

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@ use super::{SeqSep, Parser, TokenType, PathStyle};
22
use syntax::attr;
33
use syntax::ast;
44
use syntax::util::comments;
5-
use syntax::token::{self, Nonterminal, DelimToken};
6-
use syntax::tokenstream::{TokenStream, TokenTree};
5+
use syntax::token::{self, Nonterminal};
76
use syntax_pos::{Span, Symbol};
87
use errors::PResult;
98

@@ -181,31 +180,8 @@ impl<'a> Parser<'a> {
181180
item
182181
} else {
183182
let path = self.parse_path(PathStyle::Mod)?;
184-
let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
185-
self.check(&token::OpenDelim(DelimToken::Bracket)) ||
186-
self.check(&token::OpenDelim(DelimToken::Brace)) {
187-
self.parse_token_tree().into()
188-
} else if self.eat(&token::Eq) {
189-
let eq = TokenTree::token(token::Eq, self.prev_span);
190-
let mut is_interpolated_expr = false;
191-
if let token::Interpolated(nt) = &self.token.kind {
192-
if let token::NtExpr(..) = **nt {
193-
is_interpolated_expr = true;
194-
}
195-
}
196-
let token_tree = if is_interpolated_expr {
197-
// We need to accept arbitrary interpolated expressions to continue
198-
// supporting things like `doc = $expr` that work on stable.
199-
// Non-literal interpolated expressions are rejected after expansion.
200-
self.parse_token_tree()
201-
} else {
202-
self.parse_unsuffixed_lit()?.token_tree()
203-
};
204-
TokenStream::new(vec![eq.into(), token_tree.into()])
205-
} else {
206-
TokenStream::default()
207-
};
208-
ast::AttrItem { path, tokens }
183+
let args = self.parse_attr_args()?;
184+
ast::AttrItem { path, args }
209185
})
210186
}
211187

@@ -244,7 +220,7 @@ impl<'a> Parser<'a> {
244220
Ok(attrs)
245221
}
246222

247-
fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
223+
pub(super) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
248224
let lit = self.parse_lit()?;
249225
debug!("checking if {:?} is unusuffixed", lit);
250226

‎src/librustc_parse/parser/expr.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -922,13 +922,11 @@ impl<'a> Parser<'a> {
922922
// `!`, as an operator, is prefix, so we know this isn't that.
923923
if self.eat(&token::Not) {
924924
// MACRO INVOCATION expression
925-
let (delim, tts) = self.expect_delimited_token_tree()?;
925+
let args = self.parse_mac_args()?;
926926
hi = self.prev_span;
927927
ex = ExprKind::Mac(Mac {
928928
path,
929-
tts,
930-
delim,
931-
span: lo.to(hi),
929+
args,
932930
prior_type_ascription: self.last_type_ascription,
933931
});
934932
} else if self.check(&token::OpenDelim(token::Brace)) {

‎src/librustc_parse/parser/item.rs

Lines changed: 25 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ use syntax::ast::{ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, Us
88
use syntax::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness, Extern, StrLit};
99
use syntax::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind};
1010
use syntax::ast::{Ty, TyKind, Generics, TraitRef, EnumDef, Variant, VariantData, StructField};
11-
use syntax::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
11+
use syntax::ast::{Mac, MacArgs, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
1212
use syntax::print::pprust;
1313
use syntax::ptr::P;
1414
use syntax::ThinVec;
1515
use syntax::token;
16-
use syntax::tokenstream::{TokenTree, TokenStream};
16+
use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream};
1717
use syntax::source_map::{self, respan, Span};
1818
use syntax::struct_span_err;
1919
use syntax_pos::BytePos;
@@ -432,22 +432,18 @@ impl<'a> Parser<'a> {
432432
let prev_span = self.prev_span;
433433
self.complain_if_pub_macro(&visibility.node, prev_span);
434434

435-
let mac_lo = self.token.span;
436-
437435
// Item macro
438436
let path = self.parse_path(PathStyle::Mod)?;
439437
self.expect(&token::Not)?;
440-
let (delim, tts) = self.expect_delimited_token_tree()?;
441-
if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
438+
let args = self.parse_mac_args()?;
439+
if args.need_semicolon() && !self.eat(&token::Semi) {
442440
self.report_invalid_macro_expansion_item();
443441
}
444442

445443
let hi = self.prev_span;
446444
let mac = Mac {
447445
path,
448-
tts,
449-
delim,
450-
span: mac_lo.to(hi),
446+
args,
451447
prior_type_ascription: self.last_type_ascription,
452448
};
453449
let item =
@@ -500,7 +496,6 @@ impl<'a> Parser<'a> {
500496
if self.token.is_path_start() &&
501497
!(self.is_async_fn() && self.token.span.rust_2015()) {
502498
let prev_span = self.prev_span;
503-
let lo = self.token.span;
504499
let path = self.parse_path(PathStyle::Mod)?;
505500

506501
if path.segments.len() == 1 {
@@ -518,16 +513,14 @@ impl<'a> Parser<'a> {
518513
*at_end = true;
519514

520515
// eat a matched-delimiter token tree:
521-
let (delim, tts) = self.expect_delimited_token_tree()?;
522-
if delim != MacDelimiter::Brace {
516+
let args = self.parse_mac_args()?;
517+
if args.need_semicolon() {
523518
self.expect_semi()?;
524519
}
525520

526521
Ok(Some(Mac {
527522
path,
528-
tts,
529-
delim,
530-
span: lo.to(self.prev_span),
523+
args,
531524
prior_type_ascription: self.last_type_ascription,
532525
}))
533526
} else {
@@ -1624,33 +1617,31 @@ impl<'a> Parser<'a> {
16241617
vis: &Visibility,
16251618
lo: Span
16261619
) -> PResult<'a, Option<P<Item>>> {
1627-
let token_lo = self.token.span;
16281620
let (ident, def) = if self.eat_keyword(kw::Macro) {
16291621
let ident = self.parse_ident()?;
1630-
let tokens = if self.check(&token::OpenDelim(token::Brace)) {
1631-
match self.parse_token_tree() {
1632-
TokenTree::Delimited(_, _, tts) => tts,
1633-
_ => unreachable!(),
1634-
}
1622+
let body = if self.check(&token::OpenDelim(token::Brace)) {
1623+
self.parse_mac_args()?
16351624
} else if self.check(&token::OpenDelim(token::Paren)) {
1636-
let args = self.parse_token_tree();
1625+
let params = self.parse_token_tree();
1626+
let pspan = params.span();
16371627
let body = if self.check(&token::OpenDelim(token::Brace)) {
16381628
self.parse_token_tree()
16391629
} else {
1640-
self.unexpected()?;
1641-
unreachable!()
1630+
return self.unexpected();
16421631
};
1643-
TokenStream::new(vec![
1644-
args.into(),
1645-
TokenTree::token(token::FatArrow, token_lo.to(self.prev_span)).into(),
1632+
let bspan = body.span();
1633+
let tokens = TokenStream::new(vec![
1634+
params.into(),
1635+
TokenTree::token(token::FatArrow, pspan.between(bspan)).into(),
16461636
body.into(),
1647-
])
1637+
]);
1638+
let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
1639+
P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens))
16481640
} else {
1649-
self.unexpected()?;
1650-
unreachable!()
1641+
return self.unexpected();
16511642
};
16521643

1653-
(ident, ast::MacroDef { tokens: tokens.into(), legacy: false })
1644+
(ident, ast::MacroDef { body, legacy: false })
16541645
} else if self.check_keyword(sym::macro_rules) &&
16551646
self.look_ahead(1, |t| *t == token::Not) &&
16561647
self.look_ahead(2, |t| t.is_ident()) {
@@ -1660,12 +1651,12 @@ impl<'a> Parser<'a> {
16601651
self.bump();
16611652

16621653
let ident = self.parse_ident()?;
1663-
let (delim, tokens) = self.expect_delimited_token_tree()?;
1664-
if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
1654+
let body = self.parse_mac_args()?;
1655+
if body.need_semicolon() && !self.eat(&token::Semi) {
16651656
self.report_invalid_macro_expansion_item();
16661657
}
16671658

1668-
(ident, ast::MacroDef { tokens, legacy: true })
1659+
(ident, ast::MacroDef { body, legacy: true })
16691660
} else {
16701661
return Ok(None);
16711662
};

‎src/librustc_parse/parser/mod.rs

Lines changed: 43 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use crate::lexer::UnmatchedBrace;
1616

1717
use syntax::ast::{
1818
self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Extern, Ident, StrLit,
19-
IsAsync, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
19+
IsAsync, MacArgs, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
2020
};
2121

2222
use syntax::print::pprust;
@@ -1010,27 +1010,49 @@ impl<'a> Parser<'a> {
10101010
}
10111011
}
10121012

1013-
fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> {
1014-
let delim = match self.token.kind {
1015-
token::OpenDelim(delim) => delim,
1016-
_ => {
1017-
let msg = "expected open delimiter";
1018-
let mut err = self.fatal(msg);
1019-
err.span_label(self.token.span, msg);
1020-
return Err(err)
1013+
fn parse_mac_args(&mut self) -> PResult<'a, P<MacArgs>> {
1014+
self.parse_mac_args_common(true).map(P)
1015+
}
1016+
1017+
fn parse_attr_args(&mut self) -> PResult<'a, MacArgs> {
1018+
self.parse_mac_args_common(false)
1019+
}
1020+
1021+
fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, MacArgs> {
1022+
Ok(if self.check(&token::OpenDelim(DelimToken::Paren)) ||
1023+
self.check(&token::OpenDelim(DelimToken::Bracket)) ||
1024+
self.check(&token::OpenDelim(DelimToken::Brace)) {
1025+
match self.parse_token_tree() {
1026+
TokenTree::Delimited(dspan, delim, tokens) =>
1027+
// We've confirmed above that there is a delimiter so unwrapping is OK.
1028+
MacArgs::Delimited(dspan, MacDelimiter::from_token(delim).unwrap(), tokens),
1029+
_ => unreachable!(),
10211030
}
1022-
};
1023-
let tts = match self.parse_token_tree() {
1024-
TokenTree::Delimited(_, _, tts) => tts,
1025-
_ => unreachable!(),
1026-
};
1027-
let delim = match delim {
1028-
token::Paren => MacDelimiter::Parenthesis,
1029-
token::Bracket => MacDelimiter::Bracket,
1030-
token::Brace => MacDelimiter::Brace,
1031-
token::NoDelim => self.bug("unexpected no delimiter"),
1032-
};
1033-
Ok((delim, tts.into()))
1031+
} else if !delimited_only {
1032+
if self.eat(&token::Eq) {
1033+
let eq_span = self.prev_span;
1034+
let mut is_interpolated_expr = false;
1035+
if let token::Interpolated(nt) = &self.token.kind {
1036+
if let token::NtExpr(..) = **nt {
1037+
is_interpolated_expr = true;
1038+
}
1039+
}
1040+
let token_tree = if is_interpolated_expr {
1041+
// We need to accept arbitrary interpolated expressions to continue
1042+
// supporting things like `doc = $expr` that work on stable.
1043+
// Non-literal interpolated expressions are rejected after expansion.
1044+
self.parse_token_tree()
1045+
} else {
1046+
self.parse_unsuffixed_lit()?.token_tree()
1047+
};
1048+
1049+
MacArgs::Eq(eq_span, token_tree.into())
1050+
} else {
1051+
MacArgs::Empty
1052+
}
1053+
} else {
1054+
return self.unexpected();
1055+
})
10341056
}
10351057

10361058
fn parse_or_use_outer_attributes(

0 commit comments

Comments
 (0)