Skip to content

proc_macro/bridge: stop using a remote object handle for proc_macro Punct and Group #98188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 28, 2022
Merged

proc_macro/bridge: stop using a remote object handle for proc_macro Punct and Group #98188

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
72bfe61
proc_macro: stop using a remote object handle for Punct
mystor Jun 29, 2021
f28dfdf
proc_macro: stop using a remote object handle for Group
mystor Jul 1, 2021
64a7d57
review changes
mystor Jun 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
review changes 
longer names for RPC generics and reduced dependency on macros in the server.
  • Loading branch information
@mystor
mystor committed Jun 28, 2022
commit 64a7d57046bc4653dddb346b51cd66a8980f3533
199 changes: 95 additions & 104 deletions compiler/rustc_expand/src/proc_macro_server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,10 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)>
fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
use rustc_ast::token::*;

// Estimate the capacity as `stream.len()` rounded up to the next power
// of two to limit the number of required reallocations.
let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
let mut cursor = stream.into_trees();
let mut trees = Vec::new();

while let Some((tree, spacing)) = cursor.next_with_spacing() {
let joint = spacing == Joint;
Expand All @@ -77,105 +79,88 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)>
tokenstream::TokenTree::Token(token) => token,
};

macro_rules! tt {
($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
trees.push(TokenTree::$ty(self::$ty {
$($field $(: $value)*,)+
span,
}))
);
($ty:ident::$method:ident($($value:expr),*)) => (
trees.push(TokenTree::$ty(self::$ty::$method($($value,)* span)))
);
}
macro_rules! op {
($a:expr) => {{
tt!(Punct { ch: $a, joint });
}};
($a:expr, $b:expr) => {{
tt!(Punct { ch: $a, joint: true });
tt!(Punct { ch: $b, joint });
}};
($a:expr, $b:expr, $c:expr) => {{
tt!(Punct { ch: $a, joint: true });
tt!(Punct { ch: $b, joint: true });
tt!(Punct { ch: $c, joint });
}};
}
let mut op = |s: &str| {
assert!(s.is_ascii());
trees.extend(s.as_bytes().iter().enumerate().map(|(idx, &ch)| {
TokenTree::Punct(Punct { ch, joint: joint || idx != s.len() - 1, span })
}));
};

match kind {
Eq => op!('='),
Lt => op!('<'),
Le => op!('<', '='),
EqEq => op!('=', '='),
Ne => op!('!', '='),
Ge => op!('>', '='),
Gt => op!('>'),
AndAnd => op!('&', '&'),
OrOr => op!('|', '|'),
Not => op!('!'),
Tilde => op!('~'),
BinOp(Plus) => op!('+'),
BinOp(Minus) => op!('-'),
BinOp(Star) => op!('*'),
BinOp(Slash) => op!('/'),
BinOp(Percent) => op!('%'),
BinOp(Caret) => op!('^'),
BinOp(And) => op!('&'),
BinOp(Or) => op!('|'),
BinOp(Shl) => op!('<', '<'),
BinOp(Shr) => op!('>', '>'),
BinOpEq(Plus) => op!('+', '='),
BinOpEq(Minus) => op!('-', '='),
BinOpEq(Star) => op!('*', '='),
BinOpEq(Slash) => op!('/', '='),
BinOpEq(Percent) => op!('%', '='),
BinOpEq(Caret) => op!('^', '='),
BinOpEq(And) => op!('&', '='),
BinOpEq(Or) => op!('|', '='),
BinOpEq(Shl) => op!('<', '<', '='),
BinOpEq(Shr) => op!('>', '>', '='),
At => op!('@'),
Dot => op!('.'),
DotDot => op!('.', '.'),
DotDotDot => op!('.', '.', '.'),
DotDotEq => op!('.', '.', '='),
Comma => op!(','),
Semi => op!(';'),
Colon => op!(':'),
ModSep => op!(':', ':'),
RArrow => op!('-', '>'),
LArrow => op!('<', '-'),
FatArrow => op!('=', '>'),
Pound => op!('#'),
Dollar => op!('$'),
Question => op!('?'),
SingleQuote => op!('\''),

Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
Ident(name, is_raw) => tt!(Ident::new(rustc.sess(), name, is_raw)),
Eq => op("="),
Lt => op("<"),
Le => op("<="),
EqEq => op("=="),
Ne => op("!="),
Ge => op(">="),
Gt => op(">"),
AndAnd => op("&&"),
OrOr => op("||"),
Not => op("!"),
Tilde => op("~"),
BinOp(Plus) => op("+"),
BinOp(Minus) => op("-"),
BinOp(Star) => op("*"),
BinOp(Slash) => op("/"),
BinOp(Percent) => op("%"),
BinOp(Caret) => op("^"),
BinOp(And) => op("&"),
BinOp(Or) => op("|"),
BinOp(Shl) => op("<<"),
BinOp(Shr) => op(">>"),
BinOpEq(Plus) => op("+="),
BinOpEq(Minus) => op("-="),
BinOpEq(Star) => op("*="),
BinOpEq(Slash) => op("/="),
BinOpEq(Percent) => op("%="),
BinOpEq(Caret) => op("^="),
BinOpEq(And) => op("&="),
BinOpEq(Or) => op("|="),
BinOpEq(Shl) => op("<<="),
BinOpEq(Shr) => op(">>="),
At => op("@"),
Dot => op("."),
DotDot => op(".."),
DotDotDot => op("..."),
DotDotEq => op("..="),
Comma => op(","),
Semi => op(";"),
Colon => op(":"),
ModSep => op("::"),
RArrow => op("->"),
LArrow => op("<-"),
FatArrow => op("=>"),
Pound => op("#"),
Dollar => op("$"),
Question => op("?"),
SingleQuote => op("'"),

Ident(name, false) if name == kw::DollarCrate => trees.push(TokenTree::Ident(Ident::dollar_crate(span))),
Ident(name, is_raw) => trees.push(TokenTree::Ident(Ident::new(rustc.sess(), name, is_raw, span))),
Lifetime(name) => {
let ident = symbol::Ident::new(name, span).without_first_quote();
tt!(Punct { ch: '\'', joint: true });
tt!(Ident::new(rustc.sess(), ident.name, false));
trees.extend([
TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
TokenTree::Ident(Ident::new(rustc.sess(), ident.name, false, span)),
]);
}
Literal(lit) => tt!(Literal { lit }),
Literal(lit) => trees.push(TokenTree::Literal(self::Literal { lit, span })),
DocComment(_, attr_style, data) => {
let mut escaped = String::new();
for ch in data.as_str().chars() {
escaped.extend(ch.escape_debug());
}
let stream = vec![
let stream = [
Ident(sym::doc, false),
Eq,
TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
]
.into_iter()
.map(|kind| tokenstream::TokenTree::token(kind, span))
.collect();
tt!(Punct { ch: '#', joint: false });
trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
if attr_style == ast::AttrStyle::Inner {
tt!(Punct { ch: '!', joint: false });
trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
}
trees.push(TokenTree::Group(Group {
delimiter: pm::Delimiter::Bracket,
Expand All @@ -190,6 +175,12 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)>

Interpolated(nt) => {
let stream = TokenStream::from_nonterminal_ast(&nt);
// A hack used to pass AST fragments to attribute and derive
// macros as a single nonterminal token instead of a token
// stream. Such token needs to be "unwrapped" and not
// represented as a delimited group.
// FIXME: It needs to be removed, but there are some
// compatibility issues (see #73345).
if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) {
trees.extend(Self::from_internal((stream, rustc)));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this Self::from_internal didn't go through a weird method trait, I would definitely suggest passing &mut Vec here (would help with any future experiments like using SmallVec<[_; 32]> instead of Vec, as there would be no expensive moves).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW I'm not sure this codepath is worth optimizing given it's only for that pretty-printing compatibility hack, meaning it only impacts enums with the name ProceduralMasqueradeDummyType.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair, though in the SmallVec<[_; 32]> case, I'm worried the rarely-used codepath would still cause worse cache behavior for the whole function.

} else {
Expand Down Expand Up @@ -254,28 +245,28 @@ impl ToInternal<TokenStream> for TokenTree<TokenStream, Span, Ident, Literal> {
};

let kind = match ch {
'=' => Eq,
'<' => Lt,
'>' => Gt,
'!' => Not,
'~' => Tilde,
'+' => BinOp(Plus),
'-' => BinOp(Minus),
'*' => BinOp(Star),
'/' => BinOp(Slash),
'%' => BinOp(Percent),
'^' => BinOp(Caret),
'&' => BinOp(And),
'|' => BinOp(Or),
'@' => At,
'.' => Dot,
',' => Comma,
';' => Semi,
':' => Colon,
'#' => Pound,
'$' => Dollar,
'?' => Question,
'\'' => SingleQuote,
b'=' => Eq,
b'<' => Lt,
b'>' => Gt,
b'!' => Not,
b'~' => Tilde,
b'+' => BinOp(Plus),
b'-' => BinOp(Minus),
b'*' => BinOp(Star),
b'/' => BinOp(Slash),
b'%' => BinOp(Percent),
b'^' => BinOp(Caret),
b'&' => BinOp(And),
b'|' => BinOp(Or),
b'@' => At,
b'.' => Dot,
b',' => Comma,
b';' => Semi,
b':' => Colon,
b'#' => Pound,
b'$' => Dollar,
b'?' => Question,
b'\'' => SingleQuote,
_ => unreachable!(),
};

Expand Down
53 changes: 27 additions & 26 deletions library/proc_macro/src/bridge/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ mark_noop! {
&'_ [u8],
&'_ str,
String,
u8,
usize,
Delimiter,
Level,
Expand Down Expand Up @@ -431,48 +432,48 @@ compound_traits!(
);

#[derive(Copy, Clone)]
pub struct DelimSpan<S> {
pub open: S,
pub close: S,
pub entire: S,
pub struct DelimSpan<Span> {
pub open: Span,
pub close: Span,
pub entire: Span,
}

impl<S: Copy> DelimSpan<S> {
pub fn from_single(span: S) -> Self {
impl<Span: Copy> DelimSpan<Span> {
pub fn from_single(span: Span) -> Self {
DelimSpan { open: span, close: span, entire: span }
}
}

compound_traits!(struct DelimSpan<Sp> { open, close, entire });
compound_traits!(struct DelimSpan<Span> { open, close, entire });

#[derive(Clone)]
pub struct Group<T, S> {
pub struct Group<TokenStream, Span> {
pub delimiter: Delimiter,
pub stream: Option<T>,
pub span: DelimSpan<S>,
pub stream: Option<TokenStream>,
pub span: DelimSpan<Span>,
}

compound_traits!(struct Group<T, Sp> { delimiter, stream, span });
compound_traits!(struct Group<TokenStream, Span> { delimiter, stream, span });

#[derive(Clone)]
pub struct Punct<S> {
pub ch: char,
pub struct Punct<Span> {
pub ch: u8,
pub joint: bool,
pub span: S,
pub span: Span,
}

compound_traits!(struct Punct<Sp> { ch, joint, span });
compound_traits!(struct Punct<Span> { ch, joint, span });

#[derive(Clone)]
pub enum TokenTree<T, S, I, L> {
Group(Group<T, S>),
Punct(Punct<S>),
Ident(I),
Literal(L),
pub enum TokenTree<TokenStream, Span, Ident, Literal> {
Group(Group<TokenStream, Span>),
Punct(Punct<Span>),
Ident(Ident),
Literal(Literal),
}

compound_traits!(
enum TokenTree<T, Sp, I, L> {
enum TokenTree<TokenStream, Span, Ident, Literal> {
Group(tt),
Punct(tt),
Ident(tt),
Expand All @@ -483,12 +484,12 @@ compound_traits!(
/// Globals provided alongside the initial inputs for a macro expansion.
/// Provides values such as spans which are used frequently to avoid RPC.
#[derive(Clone)]
pub struct ExpnGlobals<S> {
pub def_site: S,
pub call_site: S,
pub mixed_site: S,
pub struct ExpnGlobals<Span> {
pub def_site: Span,
pub call_site: Span,
pub mixed_site: Span,
}

compound_traits!(
struct ExpnGlobals<Sp> { def_site, call_site, mixed_site }
struct ExpnGlobals<Span> { def_site, call_site, mixed_site }
);
8 changes: 6 additions & 2 deletions library/proc_macro/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -969,13 +969,17 @@ impl Punct {
if !LEGAL_CHARS.contains(&ch) {
panic!("unsupported character `{:?}`", ch);
}
Punct(bridge::Punct { ch, joint: spacing == Spacing::Joint, span: Span::call_site().0 })
Punct(bridge::Punct {
ch: ch as u8,
joint: spacing == Spacing::Joint,
span: Span::call_site().0,
})
}

/// Returns the value of this punctuation character as `char`.
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
pub fn as_char(&self) -> char {
self.0.ch
self.0.ch as char
}

/// Returns the spacing of this punctuation character, indicating whether it's immediately
Expand Down