refactor(es/lexer): token eof

Author: bvanjoi

crates/swc_common/src/syntax_pos.rs

@@ -407,6 +407,7 @@ impl Span {
     }
 
     #[inline]
+    #[track_caller]
     pub fn new_with_checked(lo: BytePos, hi: BytePos) -> Self {
         debug_assert!(lo <= hi, "lo: {lo:#?}, hi: {hi:#?}");
         Span { lo, hi }

crates/swc_ecma_lexer/src/common/lexer/mod.rs

@@ -1730,7 +1730,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
     }
 
     /// `#`
-    fn read_token_number_sign(&mut self) -> LexResult<Option<Self::Token>> {
+    fn read_token_number_sign(&mut self) -> LexResult<Self::Token> {
         debug_assert!(self.cur().is_some_and(|c| c == '#'));
 
         self.bump(); // '#'
@@ -1741,7 +1741,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
             !self.input().is_at_start() || self.cur() != Some('!'),
             "#! should have already been handled by read_shebang()"
         );
-        Ok(Some(Self::Token::HASH))
+        Ok(Self::Token::HASH)
     }
 
     /// Read a token given `.`.
@@ -1936,14 +1936,14 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
     }
 
     #[inline(never)]
-    fn read_slash(&mut self) -> LexResult<Option<Self::Token>> {
+    fn read_slash(&mut self) -> LexResult<Self::Token> {
         debug_assert_eq!(self.cur(), Some('/'));
         self.bump(); // '/'
-        Ok(Some(if self.eat(b'=') {
+        Ok(if self.eat(b'=') {
             Self::Token::DIV_EQ
         } else {
             Self::Token::DIV
-        }))
+        })
     }
 
     /// This can be used if there's no keyword starting with the first
@@ -2084,7 +2084,7 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
     fn read_keyword_with(
         &mut self,
         convert: &dyn Fn(&str) -> Option<Self::Token>,
-    ) -> LexResult<Option<Self::Token>> {
+    ) -> LexResult<Self::Token> {
         debug_assert!(self.cur().is_some());
 
         let start = self.cur_pos();
@@ -2100,11 +2100,11 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
                     SyntaxError::EscapeInReservedWord { word: Atom::new(s) },
                 )
             } else {
-                Ok(Some(word))
+                Ok(word)
             }
         } else {
             let atom = self.atom(s);
-            Ok(Some(Self::Token::unknown_ident(atom, self)))
+            Ok(Self::Token::unknown_ident(atom, self))
         }
     }
 

crates/swc_ecma_lexer/src/common/lexer/token.rs

@@ -149,6 +149,7 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
     const CASE: Self;
     const DEFAULT: Self;
     const DEBUGGER: Self;
+    const EOF: Self;
 
     fn jsx_name(name: &'a str, lexer: &mut Self::Lexer) -> Self;
     fn is_jsx_name(&self) -> bool;
@@ -586,6 +587,10 @@ pub trait TokenFactory<'a, TokenAndSpan, I: Tokens<TokenAndSpan>>: Sized + Parti
     fn is_exp(&self) -> bool {
         Self::EXP.eq(self)
     }
+    #[inline(always)]
+    fn is_eof(&self) -> bool {
+        Self::EOF.eq(self)
+    }
     fn is_no_substitution_template_literal(&self) -> bool;
     fn is_template_head(&self) -> bool;
 }

crates/swc_ecma_lexer/src/common/parser/buffer.rs

@@ -1,10 +1,13 @@
-use debug_unreachable::debug_unreachable;
+use swc_atoms::Atom;
 use swc_common::{BytePos, Span};
 use swc_ecma_ast::EsVersion;
 
 use super::token_and_span::TokenAndSpan as TokenAndSpanTrait;
 use crate::common::{
-    context::Context, input::Tokens, lexer::token::TokenFactory, syntax::SyntaxFlags,
+    context::Context,
+    input::Tokens,
+    lexer::{token::TokenFactory, LexResult},
+    syntax::SyntaxFlags,
 };
 
 pub trait NextTokenAndSpan {
@@ -30,9 +33,9 @@ pub trait Buffer<'a> {
     fn set_next(&mut self, token: Option<Self::Next>);
     fn next_mut(&mut self) -> &mut Option<Self::Next>;
 
-    fn cur(&mut self) -> Option<&Self::Token>;
-    fn get_cur(&self) -> Option<&Self::TokenAndSpan>;
-    fn get_cur_mut(&mut self) -> &mut Option<Self::TokenAndSpan>;
+    fn cur(&self) -> &Self::Token;
+    fn get_cur(&self) -> &Self::TokenAndSpan;
+    fn get_cur_mut(&mut self) -> &mut Self::TokenAndSpan;
 
     fn prev_span(&self) -> Span;
     fn set_prev_span(&mut self, span: Span);
@@ -43,47 +46,38 @@ pub trait Buffer<'a> {
 
     fn store(&mut self, token: Self::Token) {
         debug_assert!(self.next().is_none());
-        debug_assert!(self.get_cur().is_none());
+        debug_assert!(!self.cur().is_eof());
         let span = self.prev_span();
         let token = Self::TokenAndSpan::new(token, span, false);
         self.set_cur(token);
     }
 
-    #[allow(dead_code)]
-    fn cur_debug<'b>(&'b self) -> Option<&'b Self::Token>
-    where
-        Self::TokenAndSpan: 'b,
-    {
-        self.get_cur().map(|it| it.token())
+    #[cold]
+    #[inline(never)]
+    fn dump_cur(&self) -> String {
+        format!("{:?}", self.cur())
     }
 
-    fn dump_cur(&mut self) -> String;
-
-    /// Returns current token.
-    fn bump(&mut self) -> Self::Token {
-        let prev = match self.get_cur_mut().take() {
-            Some(t) => t,
-            None => unsafe {
-                debug_unreachable!(
-                    "Current token is `None`. Parser should not call bump() without knowing \
-                     current token"
-                )
-            },
-        };
-        self.set_prev_span(prev.span());
-        prev.take_token()
-    }
+    /// find next token.
+    fn bump(&mut self);
+    fn expect_word_token_and_bump(&mut self) -> Atom;
+    fn expect_number_token_and_bump(&mut self) -> (f64, Atom);
+    fn expect_string_token_and_bump(&mut self) -> (Atom, Atom);
+    fn expect_bigint_token_and_bump(&mut self) -> (Box<num_bigint::BigInt>, Atom);
+    fn expect_regex_token_and_bump(&mut self) -> (Atom, Atom);
+    fn expect_template_token_and_bump(&mut self) -> (LexResult<Atom>, Atom);
+    fn expect_error_token_and_bump(&mut self) -> crate::error::Error;
+    fn expect_jsx_name_token_and_bump(&mut self) -> Atom;
+    fn expect_jsx_text_token_and_bump(&mut self) -> (Atom, Atom);
+    fn expect_shebang_token_and_bump(&mut self) -> Atom;
 
     #[inline]
     fn knows_cur(&self) -> bool {
-        self.get_cur().is_some()
+        !self.cur().is_eof()
     }
 
-    fn had_line_break_before_cur(&mut self) -> bool {
-        self.cur();
-        self.get_cur()
-            .map(|it| it.had_line_break())
-            .unwrap_or_else(|| true)
+    fn had_line_break_before_cur(&self) -> bool {
+        self.get_cur().had_line_break()
     }
 
     /// This returns true on eof.
@@ -118,8 +112,8 @@ pub trait Buffer<'a> {
         if span.hi != next.span().lo {
             return;
         }
-        let cur = self.get_cur_mut().take().unwrap();
         let next = self.next_mut().take().unwrap();
+        let cur = self.get_cur();
         let cur_token = cur.token();
         let token = if cur_token.is_greater() {
             let next_token = next.token();
@@ -139,7 +133,6 @@ pub trait Buffer<'a> {
                 // >>>=
                 Self::Token::ZERO_FILL_RSHIFT_EQ
             } else {
-                self.set_cur(cur);
                 self.set_next(Some(next));
                 return;
             }
@@ -155,12 +148,10 @@ pub trait Buffer<'a> {
                 // <<=
                 Self::Token::LSHIFT_EQ
             } else {
-                self.set_cur(cur);
                 self.set_next(Some(next));
                 return;
             }
         } else {
-            self.set_cur(cur);
             self.set_next(Some(next));
             return;
         };
@@ -170,8 +161,8 @@ pub trait Buffer<'a> {
     }
 
     #[inline(always)]
-    fn is(&mut self, expected: &Self::Token) -> bool {
-        self.cur().is_some_and(|cur| cur == expected)
+    fn is(&self, expected: &Self::Token) -> bool {
+        self.cur() == expected
     }
 
     #[inline(always)]
@@ -185,23 +176,13 @@ pub trait Buffer<'a> {
 
     /// Returns start of current token.
     #[inline]
-    fn cur_pos(&mut self) -> BytePos {
-        let _ = self.cur();
-        self.get_cur()
-            .map(|item| item.span().lo)
-            .unwrap_or_else(|| {
-                // eof
-                self.last_pos()
-            })
+    fn cur_pos(&self) -> BytePos {
+        self.get_cur().span().lo
     }
 
     #[inline]
     fn cur_span(&self) -> Span {
-        let data = self
-            .get_cur()
-            .map(|item| item.span())
-            .unwrap_or(self.prev_span());
-        Span::new_with_checked(data.lo, data.hi)
+        self.get_cur().span()
     }
 
     /// Returns last byte position of previous token.

crates/swc_ecma_lexer/src/common/parser/class_and_fn.rs

@@ -187,25 +187,19 @@ pub fn parse_super_class<'a, P: Parser<'a>>(
 }
 
 pub fn is_class_method<'a, P: Parser<'a>>(p: &mut P) -> bool {
-    p.input_mut().is(&P::Token::LPAREN)
-        || (p.input().syntax().typescript()
-            && p.input_mut()
-                .cur()
-                .is_some_and(|cur| cur.is_less() || cur.is_jsx_tag_start()))
+    let cur = p.input().cur();
+    cur == &P::Token::LPAREN
+        || (p.input().syntax().typescript() && (cur.is_less() || cur.is_jsx_tag_start()))
 }
 
 pub fn is_class_property<'a, P: Parser<'a>>(p: &mut P, asi: bool) -> bool {
-    (p.input().syntax().typescript()
-        && p.input_mut()
-            .cur()
-            .is_some_and(|cur| cur.is_bang() || cur.is_colon()))
-        || p.input_mut()
-            .cur()
-            .is_some_and(|cur| cur.is_equal() || cur.is_rbrace())
+    let cur = p.input().cur();
+    (p.input().syntax().typescript() && (cur.is_bang() || cur.is_colon()))
+        || (cur.is_equal() || cur.is_rbrace())
         || if asi {
             p.is_general_semi()
         } else {
-            p.input_mut().is(&P::Token::SEMI)
+            p.input().is(&P::Token::SEMI)
         }
 }
 
@@ -986,11 +980,7 @@ fn parse_class_member_with_is_static<'a, P: Parser<'a>>(
     }
 
     trace_cur!(p, parse_class_member_with_is_static__normal_class_member);
-    let key = if readonly.is_some()
-        && p.input_mut()
-            .cur()
-            .is_some_and(|cur| cur.is_bang() || cur.is_colon())
-    {
+    let key = if readonly.is_some() && (p.input().cur().is_bang() || p.input().cur().is_colon()) {
         Key::Public(PropName::Ident(IdentName::new(
             atom!("readonly"),
             readonly.unwrap(),
@@ -1657,7 +1647,7 @@ fn parse_class_inner<'a, P: Parser<'a>>(
             p.do_outside_of_context(Context::HasSuperClass, parse_class_body)?
         };
 
-        if p.input_mut().cur().is_none() {
+        if p.input_mut().cur().is_eof() {
             let eof_text = p.input_mut().dump_cur();
             p.emit_err(
                 p.input().cur_span(),