perf(es/parser): Remove OneDirectionalList and reduce allocation of Vec (#11000)

**Description:**

1. Separate `CommentsBuffer` into legacy and new ones with
`CommentsBufferTrait`.
2. For the new one, remove `OneDirectionalList` and use `Vec`. Record
positions in `LexerCheckpoint` for recovery.

There's regression for legacy lexer. I'm not sure the reason.

Author: CPunisher

crates/swc_ecma_lexer/src/common/lexer/comments_buffer.rs

@@ -1,93 +1,21 @@
-use std::{iter::Rev, rc::Rc, vec::IntoIter};
-
 use swc_common::{comments::Comment, BytePos};
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct BufferedComment {
     pub kind: BufferedCommentKind,
     pub pos: BytePos,
     pub comment: Comment,
 }
 
-#[derive(Clone)]
+#[derive(Debug, Clone, Copy)]
 pub enum BufferedCommentKind {
     Leading,
     Trailing,
 }
 
-#[derive(Clone)]
-pub struct CommentsBuffer {
-    comments: OneDirectionalList<BufferedComment>,
-    pending_leading: OneDirectionalList<Comment>,
-}
-
-impl Default for CommentsBuffer {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl CommentsBuffer {
-    pub fn new() -> Self {
-        Self {
-            comments: OneDirectionalList::new(),
-            pending_leading: OneDirectionalList::new(),
-        }
-    }
-
-    pub fn push(&mut self, comment: BufferedComment) {
-        self.comments.push(comment);
-    }
-
-    pub fn push_pending_leading(&mut self, comment: Comment) {
-        self.pending_leading.push(comment);
-    }
-
-    pub fn take_comments(&mut self) -> Rev<IntoIter<BufferedComment>> {
-        self.comments.take_all()
-    }
-
-    pub fn take_pending_leading(&mut self) -> Rev<IntoIter<Comment>> {
-        self.pending_leading.take_all()
-    }
-}
-
-/// A one direction linked list that can be cheaply
-/// cloned with the clone maintaining its position in the list.
-#[derive(Clone)]
-struct OneDirectionalList<T: Clone> {
-    last_node: Option<Rc<OneDirectionalListNode<T>>>,
-}
-
-impl<T: Clone> OneDirectionalList<T> {
-    pub fn new() -> Self {
-        Self { last_node: None }
-    }
-
-    pub fn take_all(&mut self) -> Rev<IntoIter<T>> {
-        // these are stored in reverse, so we need to reverse them back
-        let mut items = Vec::new();
-        let mut current_node = self.last_node.take();
-        while let Some(node) = current_node {
-            let mut node = match Rc::try_unwrap(node) {
-                Ok(n) => n,
-                Err(n) => n.as_ref().clone(),
-            };
-            items.push(node.item);
-            current_node = node.previous.take();
-        }
-        items.into_iter().rev()
-    }
-
-    pub fn push(&mut self, item: T) {
-        let previous = self.last_node.take();
-        let new_item = OneDirectionalListNode { item, previous };
-        self.last_node = Some(Rc::new(new_item));
-    }
-}
-
-#[derive(Clone)]
-struct OneDirectionalListNode<T: Clone> {
-    item: T,
-    previous: Option<Rc<OneDirectionalListNode<T>>>,
+pub trait CommentsBufferTrait {
+    fn push_comment(&mut self, comment: BufferedComment);
+    fn push_pending(&mut self, comment: Comment);
+    fn pending_to_comment(&mut self, kind: BufferedCommentKind, pos: BytePos);
+    fn take_comments(&mut self) -> impl Iterator<Item = BufferedComment> + '_;
 }

crates/swc_ecma_lexer/src/common/lexer/mod.rs

@@ -1,7 +1,6 @@
 use std::borrow::Cow;
 
 use char::{Char, CharExt};
-use comments_buffer::{BufferedComment, BufferedCommentKind};
 use either::Either::{self, Left, Right};
 use num_bigint::BigInt as BigIntValue;
 use smartstring::{LazyCompact, SmartString};
@@ -18,7 +17,7 @@ use self::jsx::xhtml;
 use super::{context::Context, input::Tokens};
 use crate::{
     common::lexer::{
-        comments_buffer::CommentsBuffer,
+        comments_buffer::{BufferedComment, BufferedCommentKind, CommentsBufferTrait},
         number::{parse_integer, LazyInteger},
     },
     error::SyntaxError,
@@ -78,14 +77,15 @@ fn remove_underscore(s: &str, has_underscore: bool) -> Cow<'_, str> {
 pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
     type State: self::state::State;
     type Token: token::TokenFactory<'a, TokenAndSpan, Self, Lexer = Self>;
+    type CommentsBuffer: CommentsBufferTrait;
 
     fn input(&self) -> &StringInput<'a>;
     fn input_mut(&mut self) -> &mut StringInput<'a>;
     fn state(&self) -> &Self::State;
     fn state_mut(&mut self) -> &mut Self::State;
     fn comments(&self) -> Option<&'a dyn swc_common::comments::Comments>;
-    fn comments_buffer(&self) -> Option<&CommentsBuffer>;
-    fn comments_buffer_mut(&mut self) -> Option<&mut CommentsBuffer>;
+    fn comments_buffer(&self) -> Option<&Self::CommentsBuffer>;
+    fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer>;
     /// # Safety
     ///
     /// We know that the start and the end are valid
@@ -272,10 +272,10 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
                     };
 
                     if is_for_next {
-                        self.comments_buffer_mut().unwrap().push_pending_leading(cmt);
+                        self.comments_buffer_mut().unwrap().push_pending(cmt);
                     } else {
                         let pos = self.state().prev_hi();
-                        self.comments_buffer_mut().unwrap().push(BufferedComment {
+                        self.comments_buffer_mut().unwrap().push_comment(BufferedComment {
                             kind: BufferedCommentKind::Trailing,
                             pos,
                             comment: cmt,
@@ -303,16 +303,16 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
             };
 
             if is_for_next {
-                self.comments_buffer_mut()
-                    .unwrap()
-                    .push_pending_leading(cmt);
+                self.comments_buffer_mut().unwrap().push_pending(cmt);
             } else {
                 let pos = self.state().prev_hi();
-                self.comments_buffer_mut().unwrap().push(BufferedComment {
-                    kind: BufferedCommentKind::Trailing,
-                    pos,
-                    comment: cmt,
-                });
+                self.comments_buffer_mut()
+                    .unwrap()
+                    .push_comment(BufferedComment {
+                        kind: BufferedCommentKind::Trailing,
+                        pos,
+                        comment: cmt,
+                    });
             }
         }
 
@@ -412,16 +412,16 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
                             };
 
                             if is_for_next {
-                                self.comments_buffer_mut()
-                                    .unwrap()
-                                    .push_pending_leading(cmt);
+                                self.comments_buffer_mut().unwrap().push_pending(cmt);
                             } else {
                                 let pos = self.state().prev_hi();
-                                self.comments_buffer_mut().unwrap().push(BufferedComment {
-                                    kind: BufferedCommentKind::Trailing,
-                                    pos,
-                                    comment: cmt,
-                                });
+                                self.comments_buffer_mut()
+                                    .unwrap()
+                                    .push_comment(BufferedComment {
+                                        kind: BufferedCommentKind::Trailing,
+                                        pos,
+                                        comment: cmt,
+                                    });
                             }
                         }
 
@@ -887,25 +887,16 @@ pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
             let start_pos = self.start_pos();
             let comments_buffer = self.comments_buffer_mut().unwrap();
 
+            // if the file had no tokens and no shebang, then treat any
+            // comments in the leading comments buffer as leading.
+            // Otherwise treat them as trailing.
+            let kind = if last == start_pos {
+                BufferedCommentKind::Leading
+            } else {
+                BufferedCommentKind::Trailing
+            };
             // move the pending to the leading or trailing
-            for c in comments_buffer.take_pending_leading() {
-                // if the file had no tokens and no shebang, then treat any
-                // comments in the leading comments buffer as leading.
-                // Otherwise treat them as trailing.
-                if last == start_pos {
-                    comments_buffer.push(BufferedComment {
-                        kind: BufferedCommentKind::Leading,
-                        pos: last,
-                        comment: c,
-                    });
-                } else {
-                    comments_buffer.push(BufferedComment {
-                        kind: BufferedCommentKind::Trailing,
-                        pos: last,
-                        comment: c,
-                    });
-                }
-            }
+            comments_buffer.pending_to_comment(kind, last);
 
             // now fill the user's passed in comments
             for comment in comments_buffer.take_comments() {

crates/swc_ecma_lexer/src/lexer/comments_buffer.rs

@@ -0,0 +1,93 @@
+use std::{iter::Rev, rc::Rc, vec::IntoIter};
+
+use swc_common::{comments::Comment, BytePos};
+
+use crate::common::lexer::comments_buffer::{
+    BufferedComment, BufferedCommentKind, CommentsBufferTrait,
+};
+
+#[derive(Clone)]
+pub struct CommentsBuffer {
+    comments: OneDirectionalList<BufferedComment>,
+    pending_leading: OneDirectionalList<Comment>,
+}
+
+impl Default for CommentsBuffer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl CommentsBuffer {
+    pub fn new() -> Self {
+        Self {
+            comments: OneDirectionalList::new(),
+            pending_leading: OneDirectionalList::new(),
+        }
+    }
+}
+
+impl CommentsBufferTrait for CommentsBuffer {
+    #[inline(always)]
+    fn push_comment(&mut self, comment: BufferedComment) {
+        self.comments.push(comment);
+    }
+
+    #[inline(always)]
+    fn push_pending(&mut self, comment: Comment) {
+        self.pending_leading.push(comment);
+    }
+
+    #[inline(always)]
+    fn take_comments(&mut self) -> impl Iterator<Item = BufferedComment> + '_ {
+        self.comments.take_all()
+    }
+
+    #[inline(always)]
+    fn pending_to_comment(&mut self, kind: BufferedCommentKind, pos: BytePos) {
+        for comment in self.pending_leading.take_all() {
+            let comment = BufferedComment { kind, pos, comment };
+            self.comments.push(comment);
+        }
+    }
+}
+
+/// A one direction linked list that can be cheaply
+/// cloned with the clone maintaining its position in the list.
+#[derive(Clone)]
+struct OneDirectionalList<T: Clone> {
+    last_node: Option<Rc<OneDirectionalListNode<T>>>,
+}
+
+impl<T: Clone> OneDirectionalList<T> {
+    pub fn new() -> Self {
+        Self { last_node: None }
+    }
+
+    pub fn take_all(&mut self) -> Rev<IntoIter<T>> {
+        // these are stored in reverse, so we need to reverse them back
+        let mut items = Vec::new();
+        let mut current_node = self.last_node.take();
+        while let Some(node) = current_node {
+            let mut node = match Rc::try_unwrap(node) {
+                Ok(n) => n,
+                Err(n) => n.as_ref().clone(),
+            };
+            items.push(node.item);
+            current_node = node.previous.take();
+        }
+        items.into_iter().rev()
+    }
+
+    pub fn push(&mut self, item: T) {
+        let previous = self.last_node.take();
+        let new_item = OneDirectionalListNode { item, previous };
+        self.last_node = Some(Rc::new(new_item));
+    }
+}
+
+#[derive(Clone)]
+struct OneDirectionalListNode<T: Clone> {
+    item: T,
+    previous: Option<Rc<OneDirectionalListNode<T>>>,
+}

crates/swc_ecma_lexer/src/lexer/mod.rs

@@ -14,18 +14,17 @@ pub use self::state::{TokenContext, TokenContexts, TokenFlags, TokenType};
 use self::table::{ByteHandler, BYTE_HANDLERS};
 use crate::{
     common::{
-        lexer::{
-            char::CharExt, comments_buffer::CommentsBuffer, fixed_len_span, pos_span, LexResult,
-            Lexer as LexerTrait,
-        },
+        lexer::{char::CharExt, fixed_len_span, pos_span, LexResult, Lexer as LexerTrait},
         syntax::{Syntax, SyntaxFlags},
     },
     error::{Error, SyntaxError},
+    lexer::comments_buffer::CommentsBuffer,
     tok,
     token::{BinOpToken, Token, TokenAndSpan},
     Context,
 };
 
+mod comments_buffer;
 mod jsx;
 mod number;
 mod state;
@@ -56,6 +55,7 @@ pub struct Lexer<'a> {
 impl FusedIterator for Lexer<'_> {}
 
 impl<'a> crate::common::lexer::Lexer<'a, TokenAndSpan> for Lexer<'a> {
+    type CommentsBuffer = CommentsBuffer;
     type State = self::state::State;
     type Token = self::Token;
 
@@ -90,14 +90,12 @@ impl<'a> crate::common::lexer::Lexer<'a, TokenAndSpan> for Lexer<'a> {
     }
 
     #[inline(always)]
-    fn comments_buffer(&self) -> Option<&crate::common::lexer::comments_buffer::CommentsBuffer> {
+    fn comments_buffer(&self) -> Option<&Self::CommentsBuffer> {
         self.comments_buffer.as_ref()
     }
 
     #[inline(always)]
-    fn comments_buffer_mut(
-        &mut self,
-    ) -> Option<&mut crate::common::lexer::comments_buffer::CommentsBuffer> {
+    fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer> {
         self.comments_buffer.as_mut()
     }
 

crates/swc_ecma_lexer/src/lexer/state.rs

@@ -11,7 +11,7 @@ use crate::{
         input::Tokens,
         lexer::{
             char::CharExt,
-            comments_buffer::{BufferedComment, BufferedCommentKind},
+            comments_buffer::{BufferedCommentKind, CommentsBufferTrait},
             state::{
                 State as StateTrait, TokenKind as TokenKindTrait, TokenType as TokenTypeTrait,
             },
@@ -870,13 +870,7 @@ impl Iterator for Lexer<'_> {
         let span = self.span(start);
         if !matches!(token, Token::Eof) {
             if let Some(comments) = self.comments_buffer.as_mut() {
-                for comment in comments.take_pending_leading() {
-                    comments.push(BufferedComment {
-                        kind: BufferedCommentKind::Leading,
-                        pos: start,
-                        comment,
-                    });
-                }
+                comments.pending_to_comment(BufferedCommentKind::Leading, start);
             }
 
             self.state.update(start, token.kind());