Skip to content

Commit

Permalink
Auto merge of #114571 - nnethercote:improve-print_tts, r=petrochenkov
Browse files Browse the repository at this point in the history 
Improve `print_tts`

By slightly changing the meaning of `tokenstream::Spacing` we can greatly improve the output of `print_tts`.

r? `@ghost`
  • Loading branch information
@bors
bors committed Dec 11, 2023
2 parents d86d65b + 940c885 commit a9cb8ee
Show file tree
Hide file tree
Showing 78 changed files with 816 additions and 525 deletions.
6 changes: 3 additions & 3 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ impl MetaItemKind {
tokens: &mut impl Iterator<Item = &'a TokenTree>,
) -> Option<MetaItemKind> {
match tokens.next() {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
}
Some(TokenTree::Token(token, _)) => {
Expand All @@ -401,7 +401,7 @@ impl MetaItemKind {
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
) -> Option<MetaItemKind> {
match tokens.peek() {
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
let inner_tokens = inner_tokens.clone();
tokens.next();
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
Expand Down Expand Up @@ -524,7 +524,7 @@ impl NestedMetaItem {
tokens.next();
return Some(NestedMetaItem::Lit(lit));
}
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
tokens.next();
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
}
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_ast/src/mut_visit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,7 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
AttrTokenTree::Token(token, _) => {
visit_token(token, vis);
}
AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
AttrTokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
vis.visit_span(open);
vis.visit_span(close);
visit_attr_tts(tts, vis);
Expand All @@ -709,7 +709,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
TokenTree::Token(token, _) => {
visit_token(token, vis);
}
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
TokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
vis.visit_span(open);
vis.visit_span(close);
visit_tts(tts, vis);
Expand Down
137 changes: 99 additions & 38 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pub enum TokenTree {
/// delimiters are implicitly represented by `Delimited`.
Token(Token, Spacing),
/// A delimited sequence of token trees.
Delimited(DelimSpan, Delimiter, TokenStream),
Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
}

// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
Expand All @@ -62,11 +62,11 @@ where
}

impl TokenTree {
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
/// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
match (self, other) {
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
(TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
delim == delim2 && tts.eq_unspanned(tts2)
}
_ => false,
Expand Down Expand Up @@ -99,6 +99,11 @@ impl TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
}

/// Create a `TokenTree::Token` with joint-hidden spacing.
pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
}

pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
match self {
TokenTree::Token(token, spacing) => match token.uninterpolate() {
Expand Down Expand Up @@ -183,7 +188,7 @@ pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>);
#[derive(Clone, Debug, Encodable, Decodable)]
pub enum AttrTokenTree {
Token(Token, Spacing),
Delimited(DelimSpan, Delimiter, AttrTokenStream),
Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
/// Stores the attributes for an attribute target,
/// along with the tokens for that attribute target.
/// See `AttributesData` for more information
Expand All @@ -208,9 +213,14 @@ impl AttrTokenStream {
AttrTokenTree::Token(inner, spacing) => {
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
}
AttrTokenTree::Delimited(span, delim, stream) => {
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
.into_iter()
AttrTokenTree::Delimited(span, spacing, delim, stream) => {
smallvec![TokenTree::Delimited(
*span,
*spacing,
*delim,
stream.to_tokenstream()
),]
.into_iter()
}
AttrTokenTree::Attributes(data) => {
let idx = data
Expand All @@ -230,7 +240,7 @@ impl AttrTokenStream {
let mut found = false;
// Check the last two trees (to account for a trailing semi)
for tree in target_tokens.iter_mut().rev().take(2) {
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
// Inner attributes are only supported on extern blocks, functions,
// impls, and modules. All of these have their inner attributes
// placed at the beginning of the rightmost outermost braced group:
Expand All @@ -250,7 +260,7 @@ impl AttrTokenStream {
stream.push_stream(inner_attr.tokens());
}
stream.push_stream(delim_tokens.clone());
*tree = TokenTree::Delimited(*span, *delim, stream);
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
found = true;
break;
}
Expand Down Expand Up @@ -303,21 +313,64 @@ pub struct AttributesData {
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);

/// Similar to `proc_macro::Spacing`, but for tokens.
///
/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when
/// we convert to `proc_macro::TokenTree` for proc macros only `Punct`
/// `TokenTree`s have a `proc_macro::Spacing`.
/// Indicates whether a token can join with the following token to form a
/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
/// guide pretty-printing, which is where the `JointHidden` value (which isn't
/// part of `proc_macro::Spacing`) comes in useful.
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub enum Spacing {
/// The token is not immediately followed by an operator token (as
/// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`,
/// `+/*foo*/=`, `+ident`, and `+()`.
/// The token cannot join with the following token to form a compound
/// token.
///
/// In token streams parsed from source code, the compiler will use `Alone`
/// for any token immediately followed by whitespace, a non-doc comment, or
/// EOF.
///
/// When constructing token streams within the compiler, use this for each
/// token that (a) should be pretty-printed with a space after it, or (b)
/// is the last token in the stream. (In the latter case the choice of
/// spacing doesn't matter because it is never used for the last token. We
/// arbitrarily use `Alone`.)
///
/// Converts to `proc_macro::Spacing::Alone`, and
/// `proc_macro::Spacing::Alone` converts back to this.
Alone,

/// The token is immediately followed by an operator token. E.g. a `+`
/// token is `Joint` in `+=` and `++`.
/// The token can join with the following token to form a compound token.
///
/// In token streams parsed from source code, the compiler will use `Joint`
/// for any token immediately followed by punctuation (as determined by
/// `Token::is_punct`).
///
/// When constructing token streams within the compiler, use this for each
/// token that (a) should be pretty-printed without a space after it, and
/// (b) is followed by a punctuation token.
///
/// Converts to `proc_macro::Spacing::Joint`, and
/// `proc_macro::Spacing::Joint` converts back to this.
Joint,

/// The token can join with the following token to form a compound token,
/// but this will not be visible at the proc macro level. (This is what the
/// `Hidden` means; see below.)
///
/// In token streams parsed from source code, the compiler will use
/// `JointHidden` for any token immediately followed by anything not
/// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
/// literal, delimiter, doc comment.
///
/// When constructing token streams, use this for each token that (a)
/// should be pretty-printed without a space after it, and (b) is followed
/// by a non-punctuation token.
///
/// Converts to `proc_macro::Spacing::Alone`, but
/// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
/// Because of that, pretty-printing of `TokenStream`s produced by proc
/// macros is unavoidably uglier (with more whitespace between tokens) than
/// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
/// source code, internally constructed token streams, and token streams
/// produced by declarative macros).
JointHidden,
}

impl TokenStream {
Expand Down Expand Up @@ -421,21 +474,14 @@ impl TokenStream {
self
}

/// Create a token stream containing a single token with alone spacing.
/// Create a token stream containing a single token with alone spacing. The
/// spacing used for the final token in a constructed stream doesn't matter
/// because it's never used. In practice we arbitrarily use
/// `Spacing::Alone`.
pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
}

/// Create a token stream containing a single token with joint spacing.
pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_joint(kind, span)])
}

/// Create a token stream containing a single `Delimited`.
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
}

pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
let Some(tokens) = node.tokens() else {
panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
Expand Down Expand Up @@ -482,6 +528,7 @@ impl TokenStream {
}
token::Interpolated(nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
DelimSpacing::new(Spacing::JointHidden, spacing),
Delimiter::Invisible,
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
),
Expand All @@ -492,8 +539,8 @@ impl TokenStream {
fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
match tree {
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
TokenTree::Delimited(span, delim, tts) => {
TokenTree::Delimited(*span, *delim, tts.flattened())
TokenTree::Delimited(span, spacing, delim, tts) => {
TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
}
}
}
Expand All @@ -503,7 +550,7 @@ impl TokenStream {
fn can_skip(stream: &TokenStream) -> bool {
stream.trees().all(|tree| match tree {
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
TokenTree::Delimited(_, _, inner) => can_skip(inner),
TokenTree::Delimited(.., inner) => can_skip(inner),
})
}

Expand All @@ -517,7 +564,7 @@ impl TokenStream {
// If `vec` is not empty, try to glue `tt` onto its last token. The return
// value indicates if gluing took place.
fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
&& let TokenTree::Token(tok, spacing) = tt
&& let Some(glued_tok) = last_tok.glue(tok)
{
Expand Down Expand Up @@ -592,9 +639,10 @@ impl TokenStream {

&TokenTree::Token(..) => i += 1,

&TokenTree::Delimited(sp, delim, ref delim_stream) => {
&TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
let new_tt =
TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
Lrc::make_mut(&mut stream.0)[i] = new_tt;
modified = true;
}
Expand Down Expand Up @@ -622,10 +670,11 @@ impl TokenStream {
num_of_hashes = cmp::max(num_of_hashes, count);
}

// `/// foo` becomes `doc = r"foo"`.
// `/// foo` becomes `[doc = r"foo"]`.
let delim_span = DelimSpan::from_single(span);
let body = TokenTree::Delimited(
delim_span,
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
Delimiter::Bracket,
[
TokenTree::token_alone(token::Ident(sym::doc, false), span),
Expand All @@ -641,7 +690,7 @@ impl TokenStream {

if attr_style == AttrStyle::Inner {
vec![
TokenTree::token_alone(token::Pound, span),
TokenTree::token_joint(token::Pound, span),
TokenTree::token_alone(token::Not, span),
body,
]
Expand Down Expand Up @@ -738,6 +787,18 @@ impl DelimSpan {
}
}

#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub struct DelimSpacing {
pub open: Spacing,
pub close: Spacing,
}

impl DelimSpacing {
pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
DelimSpacing { open, close }
}
}

// Some types are used a lot. Make sure they don't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
mod size_asserts {
Expand Down
Loading

0 comments on commit a9cb8ee

Please sign in to comment.