Skip to content

Commit

Permalink
Rollup merge of rust-lang#76285 - matklad:censor-spacing, r=petrochenkov
Browse files Browse the repository at this point in the history
Move jointness censoring to proc_macro

Proc-macro API currently exposes jointness in `Punct` tokens. That is,
`+` in `+one` is **non** joint.

Our lexer produces jointness info for all tokens, so we need to censor
it *somewhere*

Previously we did this in a lexer, but it makes more sense to do this
in a proc-macro server.

r? @petrochenkov
  • Loading branch information
Dylan-DPC authored Sep 5, 2020
2 parents b4d3873 + 09d3db2 commit 85cee57
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 16 deletions.
4 changes: 2 additions & 2 deletions compiler/rustc_ast/src/tokenstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -403,8 +403,8 @@ impl Cursor {
self.index = index;
}

pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone())
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
}
}

Expand Down
22 changes: 17 additions & 5 deletions compiler/rustc_expand/src/proc_macro_server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,26 @@ impl ToInternal<token::DelimToken> for Delimiter {
}
}

impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
for TokenTree<Group, Punct, Ident, Literal>
impl
FromInternal<(
TreeAndJoint,
Option<&'_ tokenstream::TokenTree>,
&'_ ParseSess,
&'_ mut Vec<Self>,
)> for TokenTree<Group, Punct, Ident, Literal>
{
fn from_internal(
((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
((tree, is_joint), look_ahead, sess, stack): (
TreeAndJoint,
Option<&tokenstream::TokenTree>,
&ParseSess,
&mut Vec<Self>,
),
) -> Self {
use rustc_ast::token::*;

let joint = is_joint == Joint;
let joint = is_joint == Joint
&& matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op());
let Token { kind, span } = match tree {
tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = Delimiter::from_internal(delim);
Expand Down Expand Up @@ -445,7 +456,8 @@ impl server::TokenStreamIter for Rustc<'_> {
loop {
let tree = iter.stack.pop().or_else(|| {
let next = iter.cursor.next_with_joint()?;
Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
let lookahead = iter.cursor.look_ahead(0);
Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack)))
})?;
// A hack used to pass AST fragments to attribute and derive macros
// as a single nonterminal token instead of a token stream.
Expand Down
5 changes: 1 addition & 4 deletions compiler/rustc_parse/src/lexer/tokentrees.rs
Original file line number Diff line number Diff line change
Expand Up @@ -262,10 +262,7 @@ impl<'a> TokenTreesReader<'a> {
}
_ => {
let tt = TokenTree::Token(self.token.take());
let mut is_joint = self.bump();
if !self.token.is_op() {
is_joint = NonJoint;
}
let is_joint = self.bump();
Ok((tt, is_joint))
}
}
Expand Down
10 changes: 5 additions & 5 deletions compiler/rustc_parse/src/parser/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -822,15 +822,15 @@ impl<'a> Parser<'a> {
}

let frame = &self.token_cursor.frame;
looker(&match frame.tree_cursor.look_ahead(dist - 1) {
match frame.tree_cursor.look_ahead(dist - 1) {
Some(tree) => match tree {
TokenTree::Token(token) => token,
TokenTree::Token(token) => looker(token),
TokenTree::Delimited(dspan, delim, _) => {
Token::new(token::OpenDelim(delim), dspan.open)
looker(&Token::new(token::OpenDelim(delim.clone()), dspan.open))
}
},
None => Token::new(token::CloseDelim(frame.delim), frame.span.close),
})
None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
}
}

/// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
Expand Down

0 comments on commit 85cee57

Please sign in to comment.