Fix f-string formatting in traceback of Python 3.12

In Python 3.12, new tokens were added to "tokenize" module in order to
differentiate simple strings and f-string. Additionally, the expressions
inside the f-string are properly parsed as well. The unit tests have
been updated consequently.

Author: Delgan

loguru/_better_exceptions.py

@@ -46,6 +46,12 @@ class SyntaxHighlighter:
     _builtins = set(dir(builtins))
     _constants = {"True", "False", "None"}
     _punctation = {"(", ")", "[", "]", "{", "}", ":", ",", ";"}
+    _strings = {tokenize.STRING}
+    _fstring_middle = None
+
+    if sys.version_info >= (3, 12):
+        _strings.update({tokenize.FSTRING_START, tokenize.FSTRING_MIDDLE, tokenize.FSTRING_END})
+        _fstring_middle = tokenize.FSTRING_MIDDLE
 
     def __init__(self, style=None):
         self._style = style or self._default_style
@@ -56,7 +62,12 @@ def highlight(self, source):
         output = ""
 
         for token in self.tokenize(source):
-            type_, string, start, end, line = token
+            type_, string, (start_row, start_column), (_, end_column), line = token
+
+            if type_ == self._fstring_middle:
+                # When an f-string contains "{{" or "}}", they appear as "{" or "}" in the "string"
+                # attribute of the token. However, they do not count in the column position.
+                end_column += string.count("{") + string.count("}")
 
             if type_ == tokenize.NAME:
                 if string in self._constants:
@@ -74,23 +85,20 @@ def highlight(self, source):
                     color = style["operator"]
             elif type_ == tokenize.NUMBER:
                 color = style["number"]
-            elif type_ == tokenize.STRING:
+            elif type_ in self._strings:
                 color = style["string"]
             elif type_ == tokenize.COMMENT:
                 color = style["comment"]
             else:
                 color = style["other"]
 
-            start_row, start_column = start
-            _, end_column = end
-
             if start_row != row:
                 source = source[column:]
                 row, column = start_row, 0
 
             if type_ != tokenize.ENCODING:
                 output += line[column:start_column]
-                output += color.format(string)
+                output += color.format(line[start_column:end_column])
 
             column = end_column
 

tests/exceptions/output/modern/f_string.txt

@@ -1,11 +1,18 @@
 
 Traceback (most recent call last):
 
-  File "tests/exceptions/source/modern/f_string.py", line 17, in <module>
+  File "tests/exceptions/source/modern/f_string.py", line 21, in <module>
     hello()
     └ <function hello at 0xDEADBEEF>
 
-  File "tests/exceptions/source/modern/f_string.py", line 13, in hello
-    f"{name}" and f'{{ {f / 0} }}'
+  File "tests/exceptions/source/modern/f_string.py", line 11, in hello
+    output = f"Hello" + f' ' + f"""World""" and world()
+                                                └ <function world at 0xDEADBEEF>
+
+  File "tests/exceptions/source/modern/f_string.py", line 17, in world
+    f"{name} -> { f }" and {} or f'{{ {f / 0} }}'
+       │          │                    └ 1
+       │          └ 1
+       └ 'world'
 
 ZeroDivisionError: division by zero

tests/exceptions/source/modern/f_string.py

@@ -8,9 +8,13 @@
 
 
 def hello():
+    output = f"Hello" + f' ' + f"""World""" and world()
+
+
+def world():
     name = "world"
     f = 1
-    f"{name}" and f'{{ {f / 0} }}'
+    f"{name} -> { f }" and {} or f'{{ {f / 0} }}'
 
 
 with logger.catch():

tests/test_exceptions_formatting.py

@@ -230,7 +230,6 @@ def test_exception_others(filename):
     "filename, minimum_python_version",
     [
         ("type_hints", (3, 6)),
-        ("f_string", (3, 6)),
         ("positional_only_argument", (3, 8)),
         ("walrus_operator", (3, 8)),
         ("match_statement", (3, 10)),
@@ -242,6 +241,7 @@ def test_exception_others(filename):
         ("grouped_as_cause_and_context", (3, 11)),
         ("grouped_max_length", (3, 11)),
         ("grouped_max_depth", (3, 11)),
+        ("f_string", (3, 12)),  # Available since 3.6 but in 3.12 the lexer for f-string changed.
     ],
 )
 def test_exception_modern(filename, minimum_python_version):