Skip to content

Commit 8dbf8f5

Browse files
petrochenkovpetrochenkov
committed
syntax: Don't rely on token::IdentStyle in the parser
1 parent 8d0dd78 commit 8dbf8f5

File tree

3 files changed

+55
-135
lines changed

3 files changed

+55
-135
lines changed

‎src/libsyntax/parse/parser.rs

Lines changed: 27 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,8 @@ impl TokenType {
297297
}
298298
}
299299

300-
fn is_plain_ident_or_underscore(t: &token::Token) -> bool {
301-
t.is_plain_ident() || *t == token::Underscore
300+
fn is_ident_or_underscore(t: &token::Token) -> bool {
301+
t.is_ident() || *t == token::Underscore
302302
}
303303

304304
/// Information about the path to a module.
@@ -585,14 +585,6 @@ impl<'a> Parser<'a> {
585585
}
586586
}
587587

588-
pub fn parse_ident_or_self_type(&mut self) -> PResult<'a, ast::Ident> {
589-
if self.is_self_type_ident() {
590-
self.expect_self_type_ident()
591-
} else {
592-
self.parse_ident()
593-
}
594-
}
595-
596588
/// Check if the next token is `tok`, and return `true` if so.
597589
///
598590
/// This method will automatically add `tok` to `expected_tokens` if `tok` is not
@@ -1476,9 +1468,7 @@ impl<'a> Parser<'a> {
14761468
self.parse_qualified_path(NoTypesAllowed)?;
14771469

14781470
TyKind::Path(Some(qself), path)
1479-
} else if self.check(&token::ModSep) ||
1480-
self.token.is_ident() ||
1481-
self.token.is_path() {
1471+
} else if self.is_path_start() {
14821472
let path = self.parse_path(LifetimeAndTypesWithoutColons)?;
14831473
if self.check(&token::Not) {
14841474
// MACRO INVOCATION
@@ -1541,10 +1531,10 @@ impl<'a> Parser<'a> {
15411531
debug!("parser is_named_argument offset:{}", offset);
15421532

15431533
if offset == 0 {
1544-
is_plain_ident_or_underscore(&self.token)
1534+
is_ident_or_underscore(&self.token)
15451535
&& self.look_ahead(1, |t| *t == token::Colon)
15461536
} else {
1547-
self.look_ahead(offset, |t| is_plain_ident_or_underscore(t))
1537+
self.look_ahead(offset, |t| is_ident_or_underscore(t))
15481538
&& self.look_ahead(offset + 1, |t| *t == token::Colon)
15491539
}
15501540
}
@@ -1707,6 +1697,16 @@ impl<'a> Parser<'a> {
17071697
}
17081698
}
17091699

1700+
pub fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> {
1701+
match self.token {
1702+
token::Ident(sid, _) if self.token.is_path_segment_keyword() => {
1703+
self.bump();
1704+
Ok(sid)
1705+
}
1706+
_ => self.parse_ident(),
1707+
}
1708+
}
1709+
17101710
/// Parses qualified path.
17111711
///
17121712
/// Assumes that the leading `<` has been parsed already.
@@ -1813,7 +1813,7 @@ impl<'a> Parser<'a> {
18131813
let mut segments = Vec::new();
18141814
loop {
18151815
// First, parse an identifier.
1816-
let identifier = self.parse_ident_or_self_type()?;
1816+
let identifier = self.parse_path_segment_ident()?;
18171817

18181818
// Parse types, optionally.
18191819
let parameters = if self.eat_lt() {
@@ -1866,7 +1866,7 @@ impl<'a> Parser<'a> {
18661866
let mut segments = Vec::new();
18671867
loop {
18681868
// First, parse an identifier.
1869-
let identifier = self.parse_ident_or_self_type()?;
1869+
let identifier = self.parse_path_segment_ident()?;
18701870

18711871
// If we do not see a `::`, stop.
18721872
if !self.eat(&token::ModSep) {
@@ -1913,7 +1913,7 @@ impl<'a> Parser<'a> {
19131913
let mut segments = Vec::new();
19141914
loop {
19151915
// First, parse an identifier.
1916-
let identifier = self.parse_ident_or_self_type()?;
1916+
let identifier = self.parse_path_segment_ident()?;
19171917

19181918
// Assemble and push the result.
19191919
segments.push(ast::PathSegment {
@@ -2212,15 +2212,6 @@ impl<'a> Parser<'a> {
22122212
let lo = self.span.lo;
22132213
return self.parse_lambda_expr(lo, CaptureBy::Ref, attrs);
22142214
},
2215-
token::Ident(id @ ast::Ident {
2216-
name: token::SELF_KEYWORD_NAME,
2217-
ctxt: _
2218-
}, token::Plain) => {
2219-
self.bump();
2220-
let path = ast::Path::from_ident(mk_sp(lo, hi), id);
2221-
ex = ExprKind::Path(None, path);
2222-
hi = self.last_span.hi;
2223-
}
22242215
token::OpenDelim(token::Bracket) => {
22252216
self.bump();
22262217

@@ -2350,12 +2341,8 @@ impl<'a> Parser<'a> {
23502341
let mut db = self.fatal("expected expression, found statement (`let`)");
23512342
db.note("variable declaration using `let` is a statement");
23522343
return Err(db);
2353-
} else if self.check(&token::ModSep) ||
2354-
self.token.is_ident() &&
2355-
!self.check_keyword(keywords::True) &&
2356-
!self.check_keyword(keywords::False) {
2357-
let pth =
2358-
self.parse_path(LifetimeAndTypesWithColons)?;
2344+
} else if self.is_path_start() {
2345+
let pth = self.parse_path(LifetimeAndTypesWithColons)?;
23592346

23602347
// `!`, as an operator, is prefix, so we know this isn't that
23612348
if self.check(&token::Not) {
@@ -2694,7 +2681,7 @@ impl<'a> Parser<'a> {
26942681
op: repeat,
26952682
num_captures: name_num
26962683
})));
2697-
} else if self.token.is_keyword_allow_following_colon(keywords::Crate) {
2684+
} else if self.token.is_keyword(keywords::Crate) {
26982685
self.bump();
26992686
return Ok(TokenTree::Token(sp, SpecialVarNt(SpecialMacroVar::CrateMacroVar)));
27002687
} else {
@@ -3663,10 +3650,9 @@ impl<'a> Parser<'a> {
36633650
pat = PatKind::Box(subpat);
36643651
} else if self.is_path_start() {
36653652
// Parse pattern starting with a path
3666-
if self.token.is_plain_ident() && self.look_ahead(1, |t| *t != token::DotDotDot &&
3653+
if self.token.is_ident() && self.look_ahead(1, |t| *t != token::DotDotDot &&
36673654
*t != token::OpenDelim(token::Brace) &&
36683655
*t != token::OpenDelim(token::Paren) &&
3669-
// Contrary to its definition, a plain ident can be followed by :: in macros
36703656
*t != token::ModSep) {
36713657
// Plain idents have some extra abilities here compared to general paths
36723658
if self.look_ahead(1, |t| *t == token::Not) {
@@ -4626,16 +4612,9 @@ impl<'a> Parser<'a> {
46264612
}))
46274613
}
46284614

4629-
fn is_self_ident(&mut self) -> bool {
4630-
match self.token {
4631-
token::Ident(id, token::Plain) => id.name == special_idents::self_.name,
4632-
_ => false
4633-
}
4634-
}
4635-
46364615
fn expect_self_ident(&mut self) -> PResult<'a, ast::Ident> {
46374616
match self.token {
4638-
token::Ident(id, token::Plain) if id.name == special_idents::self_.name => {
4617+
token::Ident(id, _) if id.name == special_idents::self_.name => {
46394618
self.bump();
46404619
Ok(id)
46414620
},
@@ -4647,27 +4626,6 @@ impl<'a> Parser<'a> {
46474626
}
46484627
}
46494628

4650-
fn is_self_type_ident(&mut self) -> bool {
4651-
match self.token {
4652-
token::Ident(id, token::Plain) => id.name == special_idents::type_self.name,
4653-
_ => false
4654-
}
4655-
}
4656-
4657-
fn expect_self_type_ident(&mut self) -> PResult<'a, ast::Ident> {
4658-
match self.token {
4659-
token::Ident(id, token::Plain) if id.name == special_idents::type_self.name => {
4660-
self.bump();
4661-
Ok(id)
4662-
},
4663-
_ => {
4664-
let token_str = self.this_token_to_string();
4665-
Err(self.fatal(&format!("expected `Self`, found `{}`",
4666-
token_str)))
4667-
}
4668-
}
4669-
}
4670-
46714629
/// Parse the argument list and result type of a function
46724630
/// that may have a self type.
46734631
fn parse_fn_decl_with_self<F>(&mut self,
@@ -4736,7 +4694,7 @@ impl<'a> Parser<'a> {
47364694
} else {
47374695
Mutability::Immutable
47384696
};
4739-
if self.is_self_ident() {
4697+
if self.token.is_keyword(keywords::SelfValue) {
47404698
let span = self.span;
47414699
self.span_err(span, "cannot pass self by raw pointer");
47424700
self.bump();
@@ -4745,7 +4703,7 @@ impl<'a> Parser<'a> {
47454703
SelfKind::Value(special_idents::self_)
47464704
}
47474705
token::Ident(..) => {
4748-
if self.is_self_ident() {
4706+
if self.token.is_keyword(keywords::SelfValue) {
47494707
let self_ident = self.expect_self_ident()?;
47504708

47514709
// Determine whether this is the fully explicit form, `self:
@@ -6044,7 +6002,7 @@ impl<'a> Parser<'a> {
60446002
) -> PResult<'a, Option<P<Item>>> {
60456003
if macros_allowed && !self.token.is_any_keyword()
60466004
&& self.look_ahead(1, |t| *t == token::Not)
6047-
&& (self.look_ahead(2, |t| t.is_plain_ident())
6005+
&& (self.look_ahead(2, |t| t.is_ident())
60486006
|| self.look_ahead(2, |t| *t == token::OpenDelim(token::Paren))
60496007
|| self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace))) {
60506008
// MACRO INVOCATION ITEM
@@ -6061,7 +6019,7 @@ impl<'a> Parser<'a> {
60616019
// a 'special' identifier (like what `macro_rules!` uses)
60626020
// is optional. We should eventually unify invoc syntax
60636021
// and remove this.
6064-
let id = if self.token.is_plain_ident() {
6022+
let id = if self.token.is_ident() {
60656023
self.parse_ident()?
60666024
} else {
60676025
token::special_idents::invalid // no special identifier

‎src/libsyntax/parse/token.rs

Lines changed: 26 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ use std::fmt;
2626
use std::ops::Deref;
2727
use std::rc::Rc;
2828

29-
#[allow(non_camel_case_types)]
3029
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
3130
pub enum BinOpToken {
3231
Plus,
@@ -99,7 +98,6 @@ impl Lit {
9998
}
10099
}
101100

102-
#[allow(non_camel_case_types)]
103101
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
104102
pub enum Token {
105103
/* Expression-operator symbols. */
@@ -185,7 +183,7 @@ impl Token {
185183
pub fn can_begin_expr(&self) -> bool {
186184
match *self {
187185
OpenDelim(_) => true,
188-
Ident(_, _) => true,
186+
Ident(..) => true,
189187
Underscore => true,
190188
Tilde => true,
191189
Literal(_, _) => true,
@@ -218,7 +216,7 @@ impl Token {
218216
/// Returns `true` if the token is an identifier.
219217
pub fn is_ident(&self) -> bool {
220218
match *self {
221-
Ident(_, _) => true,
219+
Ident(..) => true,
222220
_ => false,
223221
}
224222
}
@@ -239,16 +237,6 @@ impl Token {
239237
}
240238
}
241239

242-
/// Returns `true` if the token is a path that is not followed by a `::`
243-
/// token.
244-
#[allow(non_upper_case_globals)]
245-
pub fn is_plain_ident(&self) -> bool {
246-
match *self {
247-
Ident(_, Plain) => true,
248-
_ => false,
249-
}
250-
}
251-
252240
/// Returns `true` if the token is a lifetime.
253241
pub fn is_lifetime(&self) -> bool {
254242
match *self {
@@ -289,77 +277,53 @@ impl Token {
289277
}
290278

291279
/// Returns `true` if the token is a given keyword, `kw`.
292-
#[allow(non_upper_case_globals)]
293280
pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
294281
match *self {
295-
Ident(sid, Plain) => kw.to_name() == sid.name,
296-
_ => false,
282+
Ident(id, _) => id.name == kw.to_name(),
283+
_ => false,
297284
}
298285
}
299286

300-
pub fn is_keyword_allow_following_colon(&self, kw: keywords::Keyword) -> bool {
287+
pub fn is_path_segment_keyword(&self) -> bool {
301288
match *self {
302-
Ident(sid, _) => { kw.to_name() == sid.name }
303-
_ => { false }
289+
Ident(id, _) => id.name == SUPER_KEYWORD_NAME ||
290+
id.name == SELF_KEYWORD_NAME ||
291+
id.name == SELF_TYPE_KEYWORD_NAME,
292+
_ => false,
304293
}
305294
}
306295

307-
/// Returns `true` if the token is either a special identifier, or a strict
308-
/// or reserved keyword.
309-
#[allow(non_upper_case_globals)]
296+
/// Returns `true` if the token is either a strict or reserved keyword.
310297
pub fn is_any_keyword(&self) -> bool {
311298
match *self {
312-
Ident(sid, Plain) => {
313-
let n = sid.name;
314-
315-
n == SELF_KEYWORD_NAME
316-
|| n == STATIC_KEYWORD_NAME
317-
|| n == SUPER_KEYWORD_NAME
318-
|| n == SELF_TYPE_KEYWORD_NAME
319-
|| STRICT_KEYWORD_START <= n
320-
&& n <= RESERVED_KEYWORD_FINAL
321-
},
299+
Ident(id, _) => id.name == SELF_KEYWORD_NAME ||
300+
id.name == STATIC_KEYWORD_NAME ||
301+
id.name == SUPER_KEYWORD_NAME ||
302+
id.name == SELF_TYPE_KEYWORD_NAME ||
303+
id.name >= STRICT_KEYWORD_START &&
304+
id.name <= RESERVED_KEYWORD_FINAL,
322305
_ => false
323306
}
324307
}
325308

326-
/// Returns `true` if the token may not appear as an identifier.
327-
#[allow(non_upper_case_globals)]
309+
/// Returns `true` if the token is either a strict keyword.
328310
pub fn is_strict_keyword(&self) -> bool {
329311
match *self {
330-
Ident(sid, Plain) => {
331-
let n = sid.name;
332-
333-
n == SELF_KEYWORD_NAME
334-
|| n == STATIC_KEYWORD_NAME
335-
|| n == SUPER_KEYWORD_NAME
336-
|| n == SELF_TYPE_KEYWORD_NAME
337-
|| STRICT_KEYWORD_START <= n
338-
&& n <= STRICT_KEYWORD_FINAL
339-
},
340-
Ident(sid, ModName) => {
341-
let n = sid.name;
342-
343-
n != SELF_KEYWORD_NAME
344-
&& n != SUPER_KEYWORD_NAME
345-
&& STRICT_KEYWORD_START <= n
346-
&& n <= STRICT_KEYWORD_FINAL
347-
}
312+
Ident(id, _) => id.name == SELF_KEYWORD_NAME ||
313+
id.name == STATIC_KEYWORD_NAME ||
314+
id.name == SUPER_KEYWORD_NAME ||
315+
id.name == SELF_TYPE_KEYWORD_NAME ||
316+
id.name >= STRICT_KEYWORD_START &&
317+
id.name <= STRICT_KEYWORD_FINAL,
348318
_ => false,
349319
}
350320
}
351321

352-
/// Returns `true` if the token is a keyword that has been reserved for
353-
/// possible future use.
354-
#[allow(non_upper_case_globals)]
322+
/// Returns `true` if the token is either a keyword reserved for possible future use.
355323
pub fn is_reserved_keyword(&self) -> bool {
356324
match *self {
357-
Ident(sid, Plain) => {
358-
let n = sid.name;
359-
360-
RESERVED_KEYWORD_START <= n
361-
&& n <= RESERVED_KEYWORD_FINAL
362-
},
325+
Ident(id, _) => id.name >= RESERVED_KEYWORD_START &&
326+
id.name <= RESERVED_KEYWORD_FINAL,
363327
_ => false,
364328
}
365329
}

‎src/test/parse-fail/use-mod-4.rs renamed to ‎src/test/compile-fail/use-mod-4.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@
88
// option. This file may not be copied, modified, or distributed
99
// except according to those terms.
1010

11-
// compile-flags: -Z parse-only
12-
13-
use foo::self;
14-
//~^ ERROR expected identifier, found keyword `self`
11+
use foo::self; //~ ERROR unresolved import `foo::self`
12+
//~^ ERROR `self` imports are only allowed within a { } list
1513

1614
fn main() {}

0 commit comments

Comments
 (0)