Add support for the new f-string tokens per PEP 701

crates/ruff_python_parser/Cargo.toml

@@ -18,6 +18,7 @@ ruff_python_ast = { path = "../ruff_python_ast" }
 ruff_text_size = { path = "../ruff_text_size" }
 
 anyhow = { workspace = true }
+bitflags = { workspace = true }
 is-macro = { workspace = true }
 itertools = { workspace = true }
 lalrpop-util = { version = "0.20.0", default-features = false }

crates/ruff_python_parser/src/lexer/cursor.rs

@@ -96,6 +96,18 @@ impl<'a> Cursor<'a> {
         }
     }
 
+    pub(super) fn eat_char3(&mut self, c1: char, c2: char, c3: char) -> bool {
+        let mut chars = self.chars.clone();
+        if chars.next() == Some(c1) && chars.next() == Some(c2) && chars.next() == Some(c3) {
+            self.bump();
+            self.bump();
+            self.bump();
+            true
+        } else {
+            false
+        }
+    }
+
     pub(super) fn eat_if<F>(&mut self, mut predicate: F) -> Option<char>
     where
         F: FnMut(char) -> bool,

crates/ruff_python_parser/src/lexer/fstring.rs

@@ -0,0 +1,161 @@
+use bitflags::bitflags;
+
+use ruff_text_size::TextSize;
+
+bitflags! {
+    #[derive(Debug)]
+    pub(crate) struct FStringContextFlags: u32 {
+        /// The current f-string is a triple-quoted f-string i.e., the number of
+        /// opening quotes is 3. If this flag is not set, the number of opening
+        /// quotes is 1.
+        const TRIPLE = 1 << 0;
+
+        /// The current f-string is a double-quoted f-string. If this flag is not
+        /// set, the current f-string is a single-quoted f-string.
+        const DOUBLE = 1 << 1;
+
+        /// The current f-string is a raw f-string i.e., prefixed with `r`/`R`.
+        /// If this flag is not set, the current f-string is a normal f-string.
+        const RAW = 1 << 2;
+    }
+}
+
+/// The context representing the current f-string that the lexer is in.
+#[derive(Debug)]
+pub(crate) struct FStringContext {
+    flags: FStringContextFlags,
+
+    /// The number of open parentheses for the current f-string. This includes all
+    /// three types of parentheses: round (`(`), square (`[`), and curly (`{`).
+    open_parentheses_count: u32,
+
+    /// The current depth of format spec for the current f-string. This is because
+    /// there can be multiple format specs nested for the same f-string.
+    /// For example, `{a:{b:{c}}}` has 3 format specs.
+    format_spec_depth: u32,
+}
+
+impl FStringContext {
+    pub(crate) fn new(flags: FStringContextFlags) -> Self {
+        Self {
+            flags,
+            open_parentheses_count: 0,
+            format_spec_depth: 0,
+        }
+    }
+
+    /// Returns the quote character for the current f-string.
+    pub(crate) fn quote_char(&self) -> char {
+        if self.flags.contains(FStringContextFlags::DOUBLE) {
+            '"'
+        } else {
+            '\''
+        }
+    }
+
+    /// Returns the number of quotes for the current f-string.
+    pub(crate) fn quote_size(&self) -> TextSize {
+        if self.is_triple_quoted() {
+            TextSize::from(3)
+        } else {
+            TextSize::from(1)
+        }
+    }
+
+    /// Returns the triple quotes for the current f-string if it is a triple-quoted
+    /// f-string, `None` otherwise.
+    pub(crate) fn triple_quotes(&self) -> Option<&'static str> {
+        if self.is_triple_quoted() {
+            if self.flags.contains(FStringContextFlags::DOUBLE) {
+                Some(r#"""""#)
+            } else {
+                Some("'''")
+            }
+        } else {
+            None
+        }
+    }
+
+    /// Returns `true` if the current f-string is a raw f-string.
+    pub(crate) fn is_raw_string(&self) -> bool {
+        self.flags.contains(FStringContextFlags::RAW)
+    }
+
+    /// Returns `true` if the current f-string is a triple-quoted f-string.
+    pub(crate) fn is_triple_quoted(&self) -> bool {
+        self.flags.contains(FStringContextFlags::TRIPLE)
+    }
+
+    /// Returns `true` if the current f-string has open parentheses.
+    pub(crate) fn has_open_parentheses(&mut self) -> bool {
+        self.open_parentheses_count > 0
+    }
+
+    /// Increments the number of parentheses for the current f-string.
+    pub(crate) fn increment_opening_parentheses(&mut self) {
+        self.open_parentheses_count += 1;
+    }
+
+    /// Decrements the number of parentheses for the current f-string. If the
+    /// lexer is in a format spec, also decrements the number of format specs.
+    pub(crate) fn decrement_closing_parentheses(&mut self) {
+        if self.is_in_format_spec() {
+            self.format_spec_depth = self.format_spec_depth.saturating_sub(1);
+        }
+        self.open_parentheses_count = self.open_parentheses_count.saturating_sub(1);
+    }
+
+    /// Returns `true` if the lexer is in a f-string expression i.e., between
+    /// two curly braces.
+    pub(crate) fn is_in_expression(&self) -> bool {
+        self.open_parentheses_count > self.format_spec_depth
+    }
+
+    /// Returns `true` if the lexer is in a f-string format spec i.e., after a colon.
+    pub(crate) fn is_in_format_spec(&self) -> bool {
+        self.format_spec_depth > 0 && !self.is_in_expression()
+    }
+
+    /// Returns `true` if the context is in a valid position to start format spec
+    /// i.e., at the same level of nesting as the opening parentheses token.
+    /// Increments the number of format specs if it is.
+    ///
+    /// This assumes that the current character for the lexer is a colon (`:`).
+    pub(crate) fn try_start_format_spec(&mut self) -> bool {
+        if self
+            .open_parentheses_count
+            .saturating_sub(self.format_spec_depth)
+            == 1
+        {
+            self.format_spec_depth += 1;
+            true
+        } else {
+            false
+        }
+    }
+}
+
+/// The f-strings stack is used to keep track of all the f-strings that the
+/// lexer encounters. This is necessary because f-strings can be nested.
+#[derive(Debug, Default)]
+pub(crate) struct FStrings {
+    stack: Vec<FStringContext>,
+}
+
+impl FStrings {
+    pub(crate) fn push(&mut self, context: FStringContext) {
+        self.stack.push(context);
+    }
+
+    pub(crate) fn pop(&mut self) -> Option<FStringContext> {
+        self.stack.pop()
+    }
+
+    pub(crate) fn current(&self) -> Option<&FStringContext> {
+        self.stack.last()
+    }
+
+    pub(crate) fn current_mut(&mut self) -> Option<&mut FStringContext> {
+        self.stack.last_mut()
+    }
+}

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__empty_fstrings.snap

@@ -0,0 +1,27 @@
+---
+source: crates/ruff_python_parser/src/lexer.rs
+expression: lex_source(source)
+---
+[
+    FStringStart,
+    FStringEnd,
+    String {
+        value: "",
+        kind: String,
+        triple_quoted: false,
+    },
+    FStringStart,
+    FStringEnd,
+    FStringStart,
+    FStringEnd,
+    String {
+        value: "",
+        kind: String,
+        triple_quoted: false,
+    },
+    FStringStart,
+    FStringEnd,
+    FStringStart,
+    FStringEnd,
+    Newline,
+]

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring.snap

@@ -0,0 +1,40 @@
+---
+source: crates/ruff_python_parser/src/lexer.rs
+expression: lex_source(source)
+---
+[
+    FStringStart,
+    FStringMiddle {
+        value: "normal ",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "foo",
+    },
+    Rbrace,
+    FStringMiddle {
+        value: " {another} ",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "bar",
+    },
+    Rbrace,
+    FStringMiddle {
+        value: " {",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "three",
+    },
+    Rbrace,
+    FStringMiddle {
+        value: "}",
+        is_raw: false,
+    },
+    FStringEnd,
+    Newline,
+]

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap

@@ -0,0 +1,27 @@
+---
+source: crates/ruff_python_parser/src/lexer.rs
+expression: lex_source(source)
+---
+[
+    FStringStart,
+    FStringMiddle {
+        value: "\n# not a comment ",
+        is_raw: false,
+    },
+    Lbrace,
+    Comment(
+        "# comment {",
+    ),
+    NonLogicalNewline,
+    Name {
+        name: "x",
+    },
+    NonLogicalNewline,
+    Rbrace,
+    FStringMiddle {
+        value: " # not a comment\n",
+        is_raw: false,
+    },
+    FStringEnd,
+    Newline,
+]

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_conversion.snap

@@ -0,0 +1,50 @@
+---
+source: crates/ruff_python_parser/src/lexer.rs
+expression: lex_source(source)
+---
+[
+    FStringStart,
+    Lbrace,
+    Name {
+        name: "x",
+    },
+    Exclamation,
+    Name {
+        name: "s",
+    },
+    Rbrace,
+    FStringMiddle {
+        value: " ",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "x",
+    },
+    Equal,
+    Exclamation,
+    Name {
+        name: "r",
+    },
+    Rbrace,
+    FStringMiddle {
+        value: " ",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "x",
+    },
+    Colon,
+    FStringMiddle {
+        value: ".3f!r",
+        is_raw: false,
+    },
+    Rbrace,
+    FStringMiddle {
+        value: " {x!r}",
+        is_raw: false,
+    },
+    FStringEnd,
+    Newline,
+]

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_escape.snap

@@ -0,0 +1,32 @@
+---
+source: crates/ruff_python_parser/src/lexer.rs
+expression: lex_source(source)
+---
+[
+    FStringStart,
+    FStringMiddle {
+        value: "\\",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "x",
+    },
+    Colon,
+    FStringMiddle {
+        value: "\\\"\\",
+        is_raw: false,
+    },
+    Lbrace,
+    Name {
+        name: "x",
+    },
+    Rbrace,
+    Rbrace,
+    FStringMiddle {
+        value: " \\\"\\\"\\\n end",
+        is_raw: false,
+    },
+    FStringEnd,
+    Newline,
+]