Skip to content

Commit 5a6a41e

Browse files
borsbors
committed
Auto merge of #78782 - petrochenkov:nodoctok, r=Aaron1011
Do not collect tokens for doc comments Doc comment is a single token and AST has all the information to re-create it precisely. Doc comments are also responsible for majority of calls to `collect_tokens` (with `num_calls == 1` and `num_calls == 0`, cc #78736). (I also moved token collection into `fn parse_attribute` to deduplicate code a bit.) r? `@Aaron1011`
2 parents 77180db + 12de1e8 commit 5a6a41e

File tree

17 files changed

+140
-161
lines changed

17 files changed

+140
-161
lines changed

‎compiler/rustc_ast/src/ast.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2439,13 +2439,12 @@ pub struct Attribute {
24392439
/// or the construct this attribute is contained within (inner).
24402440
pub style: AttrStyle,
24412441
pub span: Span,
2442-
pub tokens: Option<LazyTokenStream>,
24432442
}
24442443

24452444
#[derive(Clone, Encodable, Decodable, Debug)]
24462445
pub enum AttrKind {
24472446
/// A normal attribute.
2448-
Normal(AttrItem),
2447+
Normal(AttrItem, Option<LazyTokenStream>),
24492448

24502449
/// A doc comment (e.g. `/// ...`, `//! ...`, `/** ... */`, `/*! ... */`).
24512450
/// Doc attributes (e.g. `#[doc="..."]`) are represented with the `Normal`

‎compiler/rustc_ast/src/attr/mod.rs‎

Lines changed: 32 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
88
use crate::mut_visit::visit_clobber;
99
use crate::ptr::P;
1010
use crate::token::{self, CommentKind, Token};
11-
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
11+
use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree, TreeAndSpacing};
1212

1313
use rustc_index::bit_set::GrowableBitSet;
1414
use rustc_span::source_map::{BytePos, Spanned};
@@ -120,15 +120,15 @@ impl NestedMetaItem {
120120
impl Attribute {
121121
pub fn has_name(&self, name: Symbol) -> bool {
122122
match self.kind {
123-
AttrKind::Normal(ref item) => item.path == name,
123+
AttrKind::Normal(ref item, _) => item.path == name,
124124
AttrKind::DocComment(..) => false,
125125
}
126126
}
127127

128128
/// For a single-segment attribute, returns its name; otherwise, returns `None`.
129129
pub fn ident(&self) -> Option<Ident> {
130130
match self.kind {
131-
AttrKind::Normal(ref item) => {
131+
AttrKind::Normal(ref item, _) => {
132132
if item.path.segments.len() == 1 {
133133
Some(item.path.segments[0].ident)
134134
} else {
@@ -144,14 +144,14 @@ impl Attribute {
144144

145145
pub fn value_str(&self) -> Option<Symbol> {
146146
match self.kind {
147-
AttrKind::Normal(ref item) => item.meta(self.span).and_then(|meta| meta.value_str()),
147+
AttrKind::Normal(ref item, _) => item.meta(self.span).and_then(|meta| meta.value_str()),
148148
AttrKind::DocComment(..) => None,
149149
}
150150
}
151151

152152
pub fn meta_item_list(&self) -> Option<Vec<NestedMetaItem>> {
153153
match self.kind {
154-
AttrKind::Normal(ref item) => match item.meta(self.span) {
154+
AttrKind::Normal(ref item, _) => match item.meta(self.span) {
155155
Some(MetaItem { kind: MetaItemKind::List(list), .. }) => Some(list),
156156
_ => None,
157157
},
@@ -160,7 +160,7 @@ impl Attribute {
160160
}
161161

162162
pub fn is_word(&self) -> bool {
163-
if let AttrKind::Normal(item) = &self.kind {
163+
if let AttrKind::Normal(item, _) = &self.kind {
164164
matches!(item.args, MacArgs::Empty)
165165
} else {
166166
false
@@ -246,15 +246,15 @@ impl AttrItem {
246246
impl Attribute {
247247
pub fn is_doc_comment(&self) -> bool {
248248
match self.kind {
249-
AttrKind::Normal(_) => false,
249+
AttrKind::Normal(..) => false,
250250
AttrKind::DocComment(..) => true,
251251
}
252252
}
253253

254254
pub fn doc_str(&self) -> Option<Symbol> {
255255
match self.kind {
256256
AttrKind::DocComment(.., data) => Some(data),
257-
AttrKind::Normal(ref item) if item.path == sym::doc => {
257+
AttrKind::Normal(ref item, _) if item.path == sym::doc => {
258258
item.meta(self.span).and_then(|meta| meta.value_str())
259259
}
260260
_ => None,
@@ -263,25 +263,37 @@ impl Attribute {
263263

264264
pub fn get_normal_item(&self) -> &AttrItem {
265265
match self.kind {
266-
AttrKind::Normal(ref item) => item,
266+
AttrKind::Normal(ref item, _) => item,
267267
AttrKind::DocComment(..) => panic!("unexpected doc comment"),
268268
}
269269
}
270270

271271
pub fn unwrap_normal_item(self) -> AttrItem {
272272
match self.kind {
273-
AttrKind::Normal(item) => item,
273+
AttrKind::Normal(item, _) => item,
274274
AttrKind::DocComment(..) => panic!("unexpected doc comment"),
275275
}
276276
}
277277

278278
/// Extracts the MetaItem from inside this Attribute.
279279
pub fn meta(&self) -> Option<MetaItem> {
280280
match self.kind {
281-
AttrKind::Normal(ref item) => item.meta(self.span),
281+
AttrKind::Normal(ref item, _) => item.meta(self.span),
282282
AttrKind::DocComment(..) => None,
283283
}
284284
}
285+
286+
pub fn tokens(&self) -> TokenStream {
287+
match self.kind {
288+
AttrKind::Normal(_, ref tokens) => tokens
289+
.as_ref()
290+
.unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
291+
.create_token_stream(),
292+
AttrKind::DocComment(comment_kind, data) => TokenStream::from(TokenTree::Token(
293+
Token::new(token::DocComment(comment_kind, self.style, data), self.span),
294+
)),
295+
}
296+
}
285297
}
286298

287299
/* Constructors */
@@ -321,11 +333,16 @@ crate fn mk_attr_id() -> AttrId {
321333
}
322334

323335
pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute {
324-
mk_attr_from_item(style, AttrItem { path, args, tokens: None }, span)
336+
mk_attr_from_item(AttrItem { path, args, tokens: None }, None, style, span)
325337
}
326338

327-
pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {
328-
Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span, tokens: None }
339+
pub fn mk_attr_from_item(
340+
item: AttrItem,
341+
tokens: Option<LazyTokenStream>,
342+
style: AttrStyle,
343+
span: Span,
344+
) -> Attribute {
345+
Attribute { kind: AttrKind::Normal(item, tokens), id: mk_attr_id(), style, span }
329346
}
330347

331348
/// Returns an inner attribute with the given value and span.
@@ -344,13 +361,7 @@ pub fn mk_doc_comment(
344361
data: Symbol,
345362
span: Span,
346363
) -> Attribute {
347-
Attribute {
348-
kind: AttrKind::DocComment(comment_kind, data),
349-
id: mk_attr_id(),
350-
style,
351-
span,
352-
tokens: None,
353-
}
364+
Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span }
354365
}
355366

356367
pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {

‎compiler/rustc_ast/src/mut_visit.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -586,17 +586,17 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
586586
}
587587

588588
pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
589-
let Attribute { kind, id: _, style: _, span, tokens } = attr;
589+
let Attribute { kind, id: _, style: _, span } = attr;
590590
match kind {
591-
AttrKind::Normal(AttrItem { path, args, tokens }) => {
591+
AttrKind::Normal(AttrItem { path, args, tokens }, attr_tokens) => {
592592
vis.visit_path(path);
593593
visit_mac_args(args, vis);
594594
visit_lazy_tts(tokens, vis);
595+
visit_lazy_tts(attr_tokens, vis);
595596
}
596597
AttrKind::DocComment(..) => {}
597598
}
598599
vis.visit_span(span);
599-
visit_lazy_tts(tokens, vis);
600600
}
601601

602602
pub fn noop_visit_mac<T: MutVisitor>(mac: &mut MacCall, vis: &mut T) {

‎compiler/rustc_ast/src/visit.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -881,7 +881,7 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) {
881881

882882
pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) {
883883
match attr.kind {
884-
AttrKind::Normal(ref item) => walk_mac_args(visitor, &item.args),
884+
AttrKind::Normal(ref item, ref _tokens) => walk_mac_args(visitor, &item.args),
885885
AttrKind::DocComment(..) => {}
886886
}
887887
}

‎compiler/rustc_ast_lowering/src/lib.rs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -966,17 +966,20 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
966966
// Note that we explicitly do not walk the path. Since we don't really
967967
// lower attributes (we use the AST version) there is nowhere to keep
968968
// the `HirId`s. We don't actually need HIR version of attributes anyway.
969+
// Tokens are also not needed after macro expansion and parsing.
969970
let kind = match attr.kind {
970-
AttrKind::Normal(ref item) => AttrKind::Normal(AttrItem {
971-
path: item.path.clone(),
972-
args: self.lower_mac_args(&item.args),
973-
tokens: None,
974-
}),
971+
AttrKind::Normal(ref item, _) => AttrKind::Normal(
972+
AttrItem {
973+
path: item.path.clone(),
974+
args: self.lower_mac_args(&item.args),
975+
tokens: None,
976+
},
977+
None,
978+
),
975979
AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data),
976980
};
977981

978-
// Tokens aren't needed after macro expansion and parsing
979-
Attribute { kind, id: attr.id, style: attr.style, span: attr.span, tokens: None }
982+
Attribute { kind, id: attr.id, style: attr.style, span: attr.span }
980983
}
981984

982985
fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
426426
}
427427
self.maybe_print_comment(attr.span.lo());
428428
match attr.kind {
429-
ast::AttrKind::Normal(ref item) => {
429+
ast::AttrKind::Normal(ref item, _) => {
430430
match attr.style {
431431
ast::AttrStyle::Inner => self.word("#!["),
432432
ast::AttrStyle::Outer => self.word("#["),

‎compiler/rustc_expand/src/config.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -291,8 +291,7 @@ impl<'a> StripUnconfigured<'a> {
291291
expanded_attrs
292292
.into_iter()
293293
.flat_map(|(item, span)| {
294-
let orig_tokens =
295-
attr.tokens.as_ref().unwrap_or_else(|| panic!("Missing tokens for {:?}", attr));
294+
let orig_tokens = attr.tokens();
296295

297296
// We are taking an attribute of the form `#[cfg_attr(pred, attr)]`
298297
// and producing an attribute of the form `#[attr]`. We
@@ -302,7 +301,7 @@ impl<'a> StripUnconfigured<'a> {
302301

303302
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
304303
// for `attr` when we expand it to `#[attr]`
305-
let pound_token = orig_tokens.create_token_stream().trees().next().unwrap();
304+
let pound_token = orig_tokens.trees().next().unwrap();
306305
if !matches!(pound_token, TokenTree::Token(Token { kind: TokenKind::Pound, .. })) {
307306
panic!("Bad tokens for attribute {:?}", attr);
308307
}
@@ -316,13 +315,12 @@ impl<'a> StripUnconfigured<'a> {
316315
.unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
317316
.create_token_stream(),
318317
);
319-
320-
let mut attr = attr::mk_attr_from_item(attr.style, item, span);
321-
attr.tokens = Some(LazyTokenStream::new(TokenStream::new(vec![
318+
let tokens = Some(LazyTokenStream::new(TokenStream::new(vec![
322319
(pound_token, Spacing::Alone),
323320
(bracket_group, Spacing::Alone),
324321
])));
325-
self.process_cfg_attr(attr)
322+
323+
self.process_cfg_attr(attr::mk_attr_from_item(item, tokens, attr.style, span))
326324
})
327325
.collect()
328326
}

‎compiler/rustc_expand/src/expand.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1776,15 +1776,13 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
17761776

17771777
let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items);
17781778
*at = ast::Attribute {
1779-
kind: ast::AttrKind::Normal(AttrItem {
1780-
path: meta.path,
1781-
args: meta.kind.mac_args(meta.span),
1782-
tokens: None,
1783-
}),
1779+
kind: ast::AttrKind::Normal(
1780+
AttrItem { path: meta.path, args: meta.kind.mac_args(meta.span), tokens: None },
1781+
None,
1782+
),
17841783
span: at.span,
17851784
id: at.id,
17861785
style: at.style,
1787-
tokens: None,
17881786
};
17891787
} else {
17901788
noop_visit_attribute(at, self)

‎compiler/rustc_middle/src/ich/impls_syntax.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ impl<'ctx> rustc_ast::HashStableContext for StableHashingContext<'ctx> {
4040
debug_assert!(!attr.ident().map_or(false, |ident| self.is_ignored_attr(ident.name)));
4141
debug_assert!(!attr.is_doc_comment());
4242

43-
let ast::Attribute { kind, id: _, style, span, tokens } = attr;
44-
if let ast::AttrKind::Normal(item) = kind {
43+
let ast::Attribute { kind, id: _, style, span } = attr;
44+
if let ast::AttrKind::Normal(item, tokens) = kind {
4545
item.hash_stable(self, hasher);
4646
style.hash_stable(self, hasher);
4747
span.hash_stable(self, hasher);

‎compiler/rustc_parse/src/lib.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -616,12 +616,7 @@ fn prepend_attrs(
616616
if attr.style == ast::AttrStyle::Inner {
617617
return None;
618618
}
619-
builder.push(
620-
attr.tokens
621-
.as_ref()
622-
.unwrap_or_else(|| panic!("Attribute {:?} is missing tokens!", attr))
623-
.create_token_stream(),
624-
);
619+
builder.push(attr.tokens());
625620
}
626621
builder.push(tokens);
627622
Some(builder.build())

0 commit comments

Comments
 (0)