Skip to content

Commit 93b284d

Browse files
bvanjoikdy1
bvanjoi
authored and
kdy1
committed
refactor(es/lexer): EOF Token (#10880)
1 parent 5fde348 commit 93b284d

File tree

83 files changed

+1323
-1316
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+1323
-1316
lines changed

‎Cargo.lock‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/swc/tests/tsc-references/parserArrowFunctionExpression11.1.normal.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
//! x Expected ':', got '<eof>'
44
//! ,-[1:1]
55
//! 1 | a ? b ? c : (d) : e => f // Legal JS
6-
//! : ^
6+
//! : ^^
77
//! 2 |
88
//! `----
99
//// [fileTs.ts]
1010
//! x Expected ':', got '<eof>'
1111
//! ,----
1212
//! 1 | a ? b ? c : (d) : e => f
13-
//! : ^
13+
//! : ^^
1414
//! `----

‎crates/swc/tests/tsc-references/parserArrowFunctionExpression11.2.minified.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
//! x Expected ':', got '<eof>'
44
//! ,-[1:1]
55
//! 1 | a ? b ? c : (d) : e => f // Legal JS
6-
//! : ^
6+
//! : ^^
77
//! 2 |
88
//! `----
99
//// [fileTs.ts]
1010
//! x Expected ':', got '<eof>'
1111
//! ,----
1212
//! 1 | a ? b ? c : (d) : e => f
13-
//! : ^
13+
//! : ^^
1414
//! `----

‎crates/swc_common/src/syntax_pos.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,7 @@ impl Span {
407407
}
408408

409409
#[inline]
410+
#[track_caller]
410411
pub fn new_with_checked(lo: BytePos, hi: BytePos) -> Self {
411412
debug_assert!(lo <= hi, "lo: {lo:#?}, hi: {hi:#?}");
412413
Span { lo, hi }

‎crates/swc_ecma_lexer/Cargo.toml‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,17 @@ typescript = []
2525
verify = ["swc_ecma_visit"]
2626

2727
[dependencies]
28-
arrayvec = { workspace = true }
29-
bitflags = { workspace = true }
30-
either = { workspace = true }
31-
new_debug_unreachable = { workspace = true }
32-
num-bigint = { workspace = true }
33-
phf = { workspace = true, features = ["macros"] }
34-
rustc-hash = { workspace = true }
35-
seq-macro = { workspace = true }
36-
serde = { workspace = true, features = ["derive"] }
37-
smallvec = { workspace = true }
38-
smartstring = { workspace = true }
39-
tracing = { workspace = true }
28+
arrayvec = { workspace = true }
29+
bitflags = { workspace = true }
30+
either = { workspace = true }
31+
num-bigint = { workspace = true }
32+
phf = { workspace = true, features = ["macros"] }
33+
rustc-hash = { workspace = true }
34+
seq-macro = { workspace = true }
35+
serde = { workspace = true, features = ["derive"] }
36+
smallvec = { workspace = true }
37+
smartstring = { workspace = true }
38+
tracing = { workspace = true }
4039

4140
swc_atoms = { version = "7.0.0", path = "../swc_atoms" }
4241
swc_common = { version = "14.0.1", path = "../swc_common" }

‎crates/swc_ecma_lexer/src/common/lexer/mod.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1730,7 +1730,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
17301730
}
17311731

17321732
/// `#`
1733-
fn read_token_number_sign(&mut self) -> LexResult<Option<Self::Token>> {
1733+
fn read_token_number_sign(&mut self) -> LexResult<Self::Token> {
17341734
debug_assert!(self.cur().is_some_and(|c| c == '#'));
17351735

17361736
self.bump(); // '#'
@@ -1741,7 +1741,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
17411741
!self.input().is_at_start() || self.cur() != Some('!'),
17421742
"#! should have already been handled by read_shebang()"
17431743
);
1744-
Ok(Some(Self::Token::HASH))
1744+
Ok(Self::Token::HASH)
17451745
}
17461746

17471747
/// Read a token given `.`.
@@ -1936,14 +1936,14 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
19361936
}
19371937

19381938
#[inline(never)]
1939-
fn read_slash(&mut self) -> LexResult<Option<Self::Token>> {
1939+
fn read_slash(&mut self) -> LexResult<Self::Token> {
19401940
debug_assert_eq!(self.cur(), Some('/'));
19411941
self.bump(); // '/'
1942-
Ok(Some(if self.eat(b'=') {
1942+
Ok(if self.eat(b'=') {
19431943
Self::Token::DIV_EQ
19441944
} else {
19451945
Self::Token::DIV
1946-
}))
1946+
})
19471947
}
19481948

19491949
/// This can be used if there's no keyword starting with the first
@@ -2084,7 +2084,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
20842084
fn read_keyword_with(
20852085
&mut self,
20862086
convert: &dyn Fn(&str) -> Option<Self::Token>,
2087-
) -> LexResult<Option<Self::Token>> {
2087+
) -> LexResult<Self::Token> {
20882088
debug_assert!(self.cur().is_some());
20892089

20902090
let start = self.cur_pos();
@@ -2100,11 +2100,11 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
21002100
SyntaxError::EscapeInReservedWord { word: Atom::new(s) },
21012101
)
21022102
} else {
2103-
Ok(Some(word))
2103+
Ok(word)
21042104
}
21052105
} else {
21062106
let atom = self.atom(s);
2107-
Ok(Some(Self::Token::unknown_ident(atom, self)))
2107+
Ok(Self::Token::unknown_ident(atom, self))
21082108
}
21092109
}
21102110

‎crates/swc_ecma_lexer/src/common/lexer/token.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,15 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
149149
const CASE: Self;
150150
const DEFAULT: Self;
151151
const DEBUGGER: Self;
152+
const EOF: Self;
152153

153154
fn jsx_name(name: &'a str, lexer: &mut Self::Lexer) -> Self;
154155
fn is_jsx_name(&self) -> bool;
155156
fn take_jsx_name(self, buffer: &mut Self::Buffer) -> Atom;
156157

157158
fn str(value: Atom, raw: Atom, lexer: &mut Self::Lexer) -> Self;
158159
fn is_str(&self) -> bool;
160+
fn is_str_raw_content(&self, content: &str, buffer: &Self::Buffer) -> bool;
159161
fn take_str(self, buffer: &mut Self::Buffer) -> (Atom, Atom);
160162

161163
fn template(cooked: LexResult<Atom>, raw: Atom, lexer: &mut Self::Lexer) -> Self;
@@ -586,6 +588,10 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
586588
fn is_exp(&self) -> bool {
587589
Self::EXP.eq(self)
588590
}
591+
#[inline(always)]
592+
fn is_eof(&self) -> bool {
593+
Self::EOF.eq(self)
594+
}
589595
fn is_no_substitution_template_literal(&self) -> bool;
590596
fn is_template_head(&self) -> bool;
591597
}

‎crates/swc_ecma_lexer/src/common/parser/buffer.rs‎

Lines changed: 32 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1-
use debug_unreachable::debug_unreachable;
1+
use swc_atoms::Atom;
22
use swc_common::{BytePos, Span};
33
use swc_ecma_ast::EsVersion;
44

55
use super::token_and_span::TokenAndSpan as TokenAndSpanTrait;
66
use crate::common::{
7-
context::Context, input::Tokens, lexer::token::TokenFactory, syntax::SyntaxFlags,
7+
context::Context,
8+
input::Tokens,
9+
lexer::{token::TokenFactory, LexResult},
10+
syntax::SyntaxFlags,
811
};
912

1013
pub trait NextTokenAndSpan {
@@ -30,9 +33,9 @@ pub trait Buffer<'a> {
3033
fn set_next(&mut self, token: Option<Self::Next>);
3134
fn next_mut(&mut self) -> &mut Option<Self::Next>;
3235

33-
fn cur(&mut self) -> Option<&Self::Token>;
34-
fn get_cur(&self) -> Option<&Self::TokenAndSpan>;
35-
fn get_cur_mut(&mut self) -> &mut Option<Self::TokenAndSpan>;
36+
fn cur(&self) -> &Self::Token;
37+
fn get_cur(&self) -> &Self::TokenAndSpan;
38+
fn get_cur_mut(&mut self) -> &mut Self::TokenAndSpan;
3639

3740
fn prev_span(&self) -> Span;
3841
fn set_prev_span(&mut self, span: Span);
@@ -43,47 +46,34 @@ pub trait Buffer<'a> {
4346

4447
fn store(&mut self, token: Self::Token) {
4548
debug_assert!(self.next().is_none());
46-
debug_assert!(self.get_cur().is_none());
49+
debug_assert!(!self.cur().is_eof());
4750
let span = self.prev_span();
4851
let token = Self::TokenAndSpan::new(token, span, false);
4952
self.set_cur(token);
5053
}
5154

52-
#[allow(dead_code)]
53-
fn cur_debug<'b>(&'b self) -> Option<&'b Self::Token>
54-
where
55-
Self::TokenAndSpan: 'b,
56-
{
57-
self.get_cur().map(|it| it.token())
58-
}
59-
60-
fn dump_cur(&mut self) -> String;
61-
62-
/// Returns current token.
63-
fn bump(&mut self) -> Self::Token {
64-
let prev = match self.get_cur_mut().take() {
65-
Some(t) => t,
66-
None => unsafe {
67-
debug_unreachable!(
68-
"Current token is `None`. Parser should not call bump() without knowing \
69-
current token"
70-
)
71-
},
72-
};
73-
self.set_prev_span(prev.span());
74-
prev.take_token()
75-
}
55+
fn dump_cur(&self) -> String;
56+
57+
/// find next token.
58+
fn bump(&mut self);
59+
fn expect_word_token_and_bump(&mut self) -> Atom;
60+
fn expect_number_token_and_bump(&mut self) -> (f64, Atom);
61+
fn expect_string_token_and_bump(&mut self) -> (Atom, Atom);
62+
fn expect_bigint_token_and_bump(&mut self) -> (Box<num_bigint::BigInt>, Atom);
63+
fn expect_regex_token_and_bump(&mut self) -> (Atom, Atom);
64+
fn expect_template_token_and_bump(&mut self) -> (LexResult<Atom>, Atom);
65+
fn expect_error_token_and_bump(&mut self) -> crate::error::Error;
66+
fn expect_jsx_name_token_and_bump(&mut self) -> Atom;
67+
fn expect_jsx_text_token_and_bump(&mut self) -> (Atom, Atom);
68+
fn expect_shebang_token_and_bump(&mut self) -> Atom;
7669

7770
#[inline]
7871
fn knows_cur(&self) -> bool {
79-
self.get_cur().is_some()
72+
!self.cur().is_eof()
8073
}
8174

82-
fn had_line_break_before_cur(&mut self) -> bool {
83-
self.cur();
84-
self.get_cur()
85-
.map(|it| it.had_line_break())
86-
.unwrap_or_else(|| true)
75+
fn had_line_break_before_cur(&self) -> bool {
76+
self.get_cur().had_line_break()
8777
}
8878

8979
/// This returns true on eof.
@@ -118,8 +108,8 @@ pub trait Buffer<'a> {
118108
if span.hi != next.span().lo {
119109
return;
120110
}
121-
let cur = self.get_cur_mut().take().unwrap();
122111
let next = self.next_mut().take().unwrap();
112+
let cur = self.get_cur();
123113
let cur_token = cur.token();
124114
let token = if cur_token.is_greater() {
125115
let next_token = next.token();
@@ -139,7 +129,6 @@ pub trait Buffer<'a> {
139129
// >>>=
140130
Self::Token::ZERO_FILL_RSHIFT_EQ
141131
} else {
142-
self.set_cur(cur);
143132
self.set_next(Some(next));
144133
return;
145134
}
@@ -155,12 +144,10 @@ pub trait Buffer<'a> {
155144
// <<=
156145
Self::Token::LSHIFT_EQ
157146
} else {
158-
self.set_cur(cur);
159147
self.set_next(Some(next));
160148
return;
161149
}
162150
} else {
163-
self.set_cur(cur);
164151
self.set_next(Some(next));
165152
return;
166153
};
@@ -170,8 +157,8 @@ pub trait Buffer<'a> {
170157
}
171158

172159
#[inline(always)]
173-
fn is(&mut self, expected: &Self::Token) -> bool {
174-
self.cur().is_some_and(|cur| cur == expected)
160+
fn is(&self, expected: &Self::Token) -> bool {
161+
self.cur() == expected
175162
}
176163

177164
#[inline(always)]
@@ -185,23 +172,13 @@ pub trait Buffer<'a> {
185172

186173
/// Returns start of current token.
187174
#[inline]
188-
fn cur_pos(&mut self) -> BytePos {
189-
let _ = self.cur();
190-
self.get_cur()
191-
.map(|item| item.span().lo)
192-
.unwrap_or_else(|| {
193-
// eof
194-
self.last_pos()
195-
})
175+
fn cur_pos(&self) -> BytePos {
176+
self.get_cur().span().lo
196177
}
197178

198179
#[inline]
199180
fn cur_span(&self) -> Span {
200-
let data = self
201-
.get_cur()
202-
.map(|item| item.span())
203-
.unwrap_or(self.prev_span());
204-
Span::new_with_checked(data.lo, data.hi)
181+
self.get_cur().span()
205182
}
206183

207184
/// Returns last byte position of previous token.

0 commit comments

Comments
 (0)