Skip to content

Commit

Permalink
Auto merge of #60679 - petrochenkov:lit2, r=<try>
Browse files Browse the repository at this point in the history
Keep original literal tokens in AST

The original literal tokens (`token::Lit`) are kept in AST until lowering to HIR.

The tokens are kept together with their lowered "semantic" representation (`ast::LitKind`), so the size of `ast::Lit` is increased (this also increases the size of meta-item structs used for processing built-in attributes).
However, the size of `ast::Expr` stays the same.

The intent is to remove the "semantic" representation from AST eventually and keep literals as tokens until lowering to HIR (at least), and I'm going to work on that, but it would be good to land this sooner to unblock progress on the [lexer refactoring](#59706).

Fixes a part of #43081 (literal tokens that are passed to proc macros are always precise, including hexadecimal numbers, strings with their original escaping, etc)
Fixes a part of #60495 (everything except for proc macro API doesn't need escaping anymore)
This also allows to eliminate a certain hack from the lexer (https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/pretty-printing.20comments/near/165005357).

cc @matklad
  • Loading branch information
bors committed May 9, 2019
2 parents ef01f29 + 865451e commit 11c353b
Show file tree
Hide file tree
Showing 17 changed files with 221 additions and 272 deletions.
2 changes: 1 addition & 1 deletion src/librustc/hir/lowering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4124,7 +4124,7 @@ impl<'a> LoweringContext<'a> {
let ohs = P(self.lower_expr(ohs));
hir::ExprKind::Unary(op, ohs)
}
ExprKind::Lit(ref l) => hir::ExprKind::Lit((*l).clone()),
ExprKind::Lit(ref l) => hir::ExprKind::Lit(respan(l.span, l.node.clone())),
ExprKind::Cast(ref expr, ref ty) => {
let expr = P(self.lower_expr(expr));
hir::ExprKind::Cast(expr, self.lower_ty(ty, ImplTraitContext::disallowed()))
Expand Down
5 changes: 4 additions & 1 deletion src/librustc/hir/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use syntax_pos::{Span, DUMMY_SP, symbol::InternedString};
use syntax::source_map::Spanned;
use rustc_target::spec::abi::Abi;
use syntax::ast::{self, CrateSugar, Ident, Name, NodeId, AsmDialect};
use syntax::ast::{Attribute, Label, Lit, StrStyle, FloatTy, IntTy, UintTy};
use syntax::ast::{Attribute, Label, LitKind, StrStyle, FloatTy, IntTy, UintTy};
use syntax::attr::{InlineAttr, OptimizeAttr};
use syntax::ext::hygiene::SyntaxContext;
use syntax::ptr::P;
Expand Down Expand Up @@ -1331,6 +1331,9 @@ impl BodyOwnerKind {
}
}

/// A literal.
pub type Lit = Spanned<LitKind>;

/// A constant (expression) that's not an item or associated item,
/// but needs its own `DefId` for type-checking, const-eval, etc.
/// These are usually found nested inside types (e.g., array lengths)
Expand Down
98 changes: 63 additions & 35 deletions src/librustc/hir/print.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ use crate::hir;
use crate::hir::{PatKind, GenericBound, TraitBoundModifier, RangeEnd};
use crate::hir::{GenericParam, GenericParamKind, GenericArg};

use std::ascii;
use std::borrow::Cow;
use std::cell::Cell;
use std::io::{self, Write, Read};
use std::iter::Peekable;
use std::vec;

pub enum AnnNode<'a> {
Expand Down Expand Up @@ -76,7 +76,6 @@ pub struct State<'a> {
pub s: pp::Printer<'a>,
cm: Option<&'a SourceMap>,
comments: Option<Vec<comments::Comment>>,
literals: Peekable<vec::IntoIter<comments::Literal>>,
cur_cmnt: usize,
boxes: Vec<pp::Breaks>,
ann: &'a (dyn PpAnn + 'a),
Expand All @@ -98,14 +97,6 @@ impl<'a> PrintState<'a> for State<'a> {
fn cur_cmnt(&mut self) -> &mut usize {
&mut self.cur_cmnt
}

fn cur_lit(&mut self) -> Option<&comments::Literal> {
self.literals.peek()
}

fn bump_lit(&mut self) -> Option<comments::Literal> {
self.literals.next()
}
}

#[allow(non_upper_case_globals)]
Expand All @@ -116,18 +107,16 @@ pub const default_columns: usize = 78;


/// Requires you to pass an input filename and reader so that
/// it can scan the input text for comments and literals to
/// copy forward.
/// it can scan the input text for comments to copy forward.
pub fn print_crate<'a>(cm: &'a SourceMap,
sess: &ParseSess,
krate: &hir::Crate,
filename: FileName,
input: &mut dyn Read,
out: Box<dyn Write + 'a>,
ann: &'a dyn PpAnn,
is_expanded: bool)
ann: &'a dyn PpAnn)
-> io::Result<()> {
let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded);
let mut s = State::new_from_input(cm, sess, filename, input, out, ann);

// When printing the AST, we sometimes need to inject `#[no_std]` here.
// Since you can't compile the HIR, it's not necessary.
Expand All @@ -143,36 +132,21 @@ impl<'a> State<'a> {
filename: FileName,
input: &mut dyn Read,
out: Box<dyn Write + 'a>,
ann: &'a dyn PpAnn,
is_expanded: bool)
ann: &'a dyn PpAnn)
-> State<'a> {
let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input);

State::new(cm,
out,
ann,
Some(cmnts),
// If the code is post expansion, don't use the table of
// literals, since it doesn't correspond with the literals
// in the AST anymore.
if is_expanded {
None
} else {
Some(lits)
})
let comments = comments::gather_comments(sess, filename, input);
State::new(cm, out, ann, Some(comments))
}

pub fn new(cm: &'a SourceMap,
out: Box<dyn Write + 'a>,
ann: &'a dyn PpAnn,
comments: Option<Vec<comments::Comment>>,
literals: Option<Vec<comments::Literal>>)
comments: Option<Vec<comments::Comment>>)
-> State<'a> {
State {
s: pp::mk_printer(out, default_columns),
cm: Some(cm),
comments,
literals: literals.unwrap_or_default().into_iter().peekable(),
cur_cmnt: 0,
boxes: Vec::new(),
ann,
Expand All @@ -189,7 +163,6 @@ pub fn to_string<F>(ann: &dyn PpAnn, f: F) -> String
s: pp::mk_printer(Box::new(&mut wr), default_columns),
cm: None,
comments: None,
literals: vec![].into_iter().peekable(),
cur_cmnt: 0,
boxes: Vec::new(),
ann,
Expand Down Expand Up @@ -1335,6 +1308,61 @@ impl<'a> State<'a> {
self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
}

fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
self.maybe_print_comment(lit.span.lo())?;
match lit.node {
hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style),
hir::LitKind::Err(st) => {
let st = st.as_str().escape_debug().to_string();
let mut res = String::with_capacity(st.len() + 2);
res.push('\'');
res.push_str(&st);
res.push('\'');
self.writer().word(res)
}
hir::LitKind::Byte(byte) => {
let mut res = String::from("b'");
res.extend(ascii::escape_default(byte).map(|c| c as char));
res.push('\'');
self.writer().word(res)
}
hir::LitKind::Char(ch) => {
let mut res = String::from("'");
res.extend(ch.escape_default());
res.push('\'');
self.writer().word(res)
}
hir::LitKind::Int(i, t) => {
match t {
ast::LitIntType::Signed(st) => {
self.writer().word(st.val_to_string(i as i128))
}
ast::LitIntType::Unsigned(ut) => {
self.writer().word(ut.val_to_string(i))
}
ast::LitIntType::Unsuffixed => {
self.writer().word(i.to_string())
}
}
}
hir::LitKind::Float(ref f, t) => {
self.writer().word(format!("{}{}", &f, t.ty_to_string()))
}
hir::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
hir::LitKind::Bool(val) => {
if val { self.writer().word("true") } else { self.writer().word("false") }
}
hir::LitKind::ByteStr(ref v) => {
let mut escaped: String = String::new();
for &ch in v.iter() {
escaped.extend(ascii::escape_default(ch)
.map(|c| c as char));
}
self.writer().word(format!("b\"{}\"", escaped))
}
}
}

pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
self.maybe_print_comment(expr.span.lo())?;
self.print_outer_attributes(&expr.attrs)?;
Expand Down
41 changes: 24 additions & 17 deletions src/librustc/ich/impls_syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,13 @@ impl_stable_hash_for!(enum ::syntax::ast::LitIntType {
Unsuffixed
});

impl_stable_hash_for_spanned!(::syntax::ast::LitKind);
impl_stable_hash_for!(struct ::syntax::ast::Lit {
node,
token,
suffix,
span
});

impl_stable_hash_for!(enum ::syntax::ast::LitKind {
Str(value, style),
Err(value),
Expand All @@ -175,6 +181,8 @@ impl_stable_hash_for!(enum ::syntax::ast::LitKind {
Bool(value)
});

impl_stable_hash_for_spanned!(::syntax::ast::LitKind);

impl_stable_hash_for!(enum ::syntax::ast::IntTy { Isize, I8, I16, I32, I64, I128 });
impl_stable_hash_for!(enum ::syntax::ast::UintTy { Usize, U8, U16, U32, U64, U128 });
impl_stable_hash_for!(enum ::syntax::ast::FloatTy { F32, F64 });
Expand Down Expand Up @@ -280,6 +288,19 @@ for tokenstream::TokenStream {
}
}

impl_stable_hash_for!(enum token::Lit {
Bool(val),
Byte(val),
Char(val),
Err(val),
Integer(val),
Float(val),
Str_(val),
ByteStr(val),
StrRaw(val, n),
ByteStrRaw(val, n)
});

fn hash_token<'a, 'gcx, W: StableHasherResult>(
token: &token::Token,
hcx: &mut StableHashingContext<'a>,
Expand Down Expand Up @@ -327,22 +348,8 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
token::Token::CloseDelim(delim_token) => {
std_hash::Hash::hash(&delim_token, hasher);
}
token::Token::Literal(ref lit, ref opt_name) => {
mem::discriminant(lit).hash_stable(hcx, hasher);
match *lit {
token::Lit::Byte(val) |
token::Lit::Char(val) |
token::Lit::Err(val) |
token::Lit::Integer(val) |
token::Lit::Float(val) |
token::Lit::Str_(val) |
token::Lit::ByteStr(val) => val.hash_stable(hcx, hasher),
token::Lit::StrRaw(val, n) |
token::Lit::ByteStrRaw(val, n) => {
val.hash_stable(hcx, hasher);
n.hash_stable(hcx, hasher);
}
};
token::Token::Literal(lit, opt_name) => {
lit.hash_stable(hcx, hasher);
opt_name.hash_stable(hcx, hasher);
}

Expand Down
6 changes: 2 additions & 4 deletions src/librustc_driver/pretty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -806,8 +806,7 @@ pub fn print_after_hir_lowering<'tcx>(
src_name,
&mut rdr,
box out,
annotation.pp_ann(),
true)
annotation.pp_ann())
})
}

Expand All @@ -830,8 +829,7 @@ pub fn print_after_hir_lowering<'tcx>(
src_name,
&mut rdr,
box out,
annotation.pp_ann(),
true);
annotation.pp_ann());
for node_id in uii.all_matching_node_ids(hir_map) {
let node = hir_map.get(node_id);
pp_state.print_node(node)?;
Expand Down
10 changes: 5 additions & 5 deletions src/librustc_lint/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ impl TypeLimits {
/// Returns `true` iff the lint was overridden.
fn lint_overflowing_range_endpoint<'a, 'tcx>(
cx: &LateContext<'a, 'tcx>,
lit: &ast::Lit,
lit: &hir::Lit,
lit_val: u128,
max: u128,
expr: &'tcx hir::Expr,
Expand Down Expand Up @@ -132,7 +132,7 @@ fn uint_ty_range(uint_ty: ast::UintTy) -> (u128, u128) {
}
}

fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &ast::Lit) -> Option<String> {
fn get_bin_hex_repr(cx: &LateContext<'_, '_>, lit: &hir::Lit) -> Option<String> {
let src = cx.sess().source_map().span_to_snippet(lit.span).ok()?;
let firstch = src.chars().next()?;

Expand Down Expand Up @@ -249,7 +249,7 @@ fn lint_int_literal<'a, 'tcx>(
cx: &LateContext<'a, 'tcx>,
type_limits: &TypeLimits,
e: &'tcx hir::Expr,
lit: &ast::Lit,
lit: &hir::Lit,
t: ast::IntTy,
v: u128,
) {
Expand Down Expand Up @@ -301,7 +301,7 @@ fn lint_int_literal<'a, 'tcx>(
fn lint_uint_literal<'a, 'tcx>(
cx: &LateContext<'a, 'tcx>,
e: &'tcx hir::Expr,
lit: &ast::Lit,
lit: &hir::Lit,
t: ast::UintTy,
) {
let uint_type = if let ast::UintTy::Usize = t {
Expand Down Expand Up @@ -363,7 +363,7 @@ fn lint_literal<'a, 'tcx>(
cx: &LateContext<'a, 'tcx>,
type_limits: &TypeLimits,
e: &'tcx hir::Expr,
lit: &ast::Lit,
lit: &hir::Lit,
) {
match cx.tables.node_type(e.hir_id).sty {
ty::Int(t) => {
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_typeck/check/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3084,7 +3084,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {

// AST fragment checking
fn check_lit(&self,
lit: &ast::Lit,
lit: &hir::Lit,
expected: Expectation<'tcx>)
-> Ty<'tcx>
{
Expand Down
16 changes: 8 additions & 8 deletions src/librustdoc/clean/cfg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -414,10 +414,9 @@ impl<'a> fmt::Display for Html<'a> {
mod test {
use super::Cfg;

use syntax::symbol::Symbol;
use syntax::ast::*;
use syntax::source_map::dummy_spanned;
use syntax_pos::DUMMY_SP;
use syntax::ast::*;
use syntax::symbol::Symbol;
use syntax::with_globals;

fn word_cfg(s: &str) -> Cfg {
Expand Down Expand Up @@ -592,12 +591,11 @@ mod test {
let mi = dummy_meta_item_word("all");
assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));

let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked);
let (token, suffix) = node.lit_token();
let mi = MetaItem {
path: Path::from_ident(Ident::from_str("all")),
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Str(
Symbol::intern("done"),
StrStyle::Cooked,
))),
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
span: DUMMY_SP,
};
assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));
Expand Down Expand Up @@ -627,9 +625,11 @@ mod test {
#[test]
fn test_parse_err() {
with_globals(|| {
let node = LitKind::Bool(false);
let (token, suffix) = node.lit_token();
let mi = MetaItem {
path: Path::from_ident(Ident::from_str("foo")),
node: MetaItemKind::NameValue(dummy_spanned(LitKind::Bool(false))),
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
span: DUMMY_SP,
};
assert!(Cfg::parse(&mi).is_err());
Expand Down
2 changes: 2 additions & 0 deletions src/librustdoc/html/highlight.rs
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,8 @@ impl<'a> Classifier<'a> {

// Number literals.
token::Integer(..) | token::Float(..) => Class::Number,

token::Bool(..) => panic!("literal token contains `Lit::Bool`"),
}
}

Expand Down
Loading

0 comments on commit 11c353b

Please sign in to comment.