[Python] Improve named backref regexp highlighting (#4362)

This commit prepares named backreferences for correctly handling interpolation.

Primarily required to drive raw f-strings, but also helps with raw unicode strings, 
which act as format-strings.

It prevents placeholders `%s` or string repelacements `{name}` from breaking 
syntax highlighting or causing unwanted illegal highlighting.

Author: deathaxe

Python/Embeddings/RegExp (for Python).sublime-syntax

@@ -13,10 +13,6 @@ variables:
   deactivate_x_mode: (?:\?[imsLua]*-[imsLua]*x[imxsLua]*)
   other_modifiers: \?(?:[ixmsLua]*-)?[ixmsLua]+
 
-  # capture group identifiers
-  capture_group_name: (?:[[:alpha:]_]\w*)
-  capture_group: (?:[1-9][0-9]*|{{capture_group_name}})
-
   # quantifiers
   ranged_quantifier: '{{brace_start}}\d+(?:,\d*)?{{brace_end}}'
   brace_start: \{
@@ -34,33 +30,64 @@ contexts:
 
   group-start:
     - meta_prepend: true
-    - match: (\?P=)({{capture_group_name}})(?=\))
-      captures:
-        1: keyword.other.back-reference.named.regexp
-        2: variable.other.backref-and-recursion.regexp
-    - match: (\?P)(<)({{capture_group_name}})(>)
+    # named capture group back-reference assertion (lookahead)
+    - match: \?P=
+      scope: keyword.other.backref-and-recursion.regexp
+      set: group-assertion-name
+    # named capture group back-reference match (for: regex package)
+    - match: \?&
+      scope: keyword.other.backref-and-recursion.regexp
+      set: group-capturing-name
+    # conditional capture group back-reference
+    - match: \?(\()
       captures:
-        1: keyword.other.backref-and-recursion.regexp
-        2: punctuation.definition.capture-group-name.begin.regexp
-        3: entity.name.capture-group.regexp
-        4: punctuation.definition.capture-group-name.end.regexp
-    # We can make this more sophisticated to match the | character that separates
-    # yes-pattern from no-pattern, but it's not really necessary.
-    - match: (\?)(\()({{capture_group}})(\))
+        1: punctuation.definition.capture-group-name.begin.regexp
+      set: group-conditional-name
+    # named capture group definition
+    - match: \?P(<)
       captures:
-        1: keyword.other.backref-and-recursion.conditional.regexp
-        2: punctuation.definition.group.begin.assertion.conditional.regexp
-        3: variable.other.back-reference.regexp
-        4: punctuation.definition.group.end.assertion.conditional.regexp
+        1: punctuation.definition.capture-group-name.begin.regexp
+      set: group-definition-name
+
+  group-assertion-name:
+    - meta_content_scope: keyword.other.backref-and-recursion.regexp variable.other.capture-group.regexp
+    - match: \)
+      scope: punctuation.section.group.end.regexp
+      pop: 3
+
+  group-capturing-name:
+    - meta_content_scope: keyword.other.backref-and-recursion.regexp variable.other.capture-group.regexp
+    - match: \)
+      scope: punctuation.section.group.end.regexp
+      pop: 3
+
+  group-conditional-name:
+    - meta_scope: keyword.other.backref-and-recursion.regexp
+    - meta_content_scope: variable.other.capture-group.regexp
+    - match: \)
+      scope: punctuation.definition.capture-group-name.end.regexp
+      pop: 1
+
+  group-definition-name:
+    - meta_scope: keyword.other.backref-and-recursion.regexp
+    - meta_content_scope: entity.name.capture-group.regexp
+    - match: \>
+      scope: punctuation.definition.capture-group-name.end.regexp
+      pop: 1
 
   backrefs:
     - meta_prepend: true
-    - match: \\g(<)({{capture_group}})(>)
-      scope: keyword.other.backref-and-recursion.regexp
+    - match: \\g(<)
       captures:
         1: punctuation.definition.capture-group-name.begin.regexp
-        2: variable.other.backref-and-recursion.regexp
-        3: punctuation.definition.capture-group-name.end.regexp
+      push: backref-name
+
+  backref-name:
+    - meta_scope: keyword.other.backref-and-recursion.regexp
+    - meta_content_scope: variable.other.capture-group.regexp
+    - match: \>
+      scope: punctuation.definition.capture-group-name.end.regexp
+      pop: 1
 
   literals:
     - meta_prepend: true

Python/tests/syntax_test_python_strings.py

@@ -120,37 +120,48 @@
 #                    ^ variable.other.backref-and-recursion.regexp
 #                     ^ - keyword
 
+regex = r'(?P<quote>[\'"]).*?(?&quote)'   # `?&` is for the regex package
+#          ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#            ^ punctuation.definition.capture-group-name.begin.regexp
+#             ^^^^^ entity.name.capture-group.regexp - invalid
+#                  ^ punctuation.definition.capture-group-name.end.regexp
+#                             ^^^^^^^ keyword.other.backref-and-recursion.regexp
+#                               ^^^^^ variable.other.capture-group.regexp
+#                                    ^ - keyword - variable
+
 regex = r'(?P<quote>[\'"]).*?(?P=quote)'
-#          ^^ keyword.other.backref-and-recursion.regexp
+#          ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
 #            ^ punctuation.definition.capture-group-name.begin.regexp
 #             ^^^^^ entity.name.capture-group.regexp - invalid
 #                  ^ punctuation.definition.capture-group-name.end.regexp
-#                             ^^^ keyword.other.back-reference.named.regexp
-#                                ^^^^^ variable.other.backref-and-recursion.regexp - invalid
+#                             ^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#                                ^^^^^ variable.other.capture-group.regexp - invalid
+#                                     ^ - keyword - variable
 
 regex = r'(?P<Quote>[\'"]).*?(?P=Quote)'
-#          ^^ keyword.other.backref-and-recursion.regexp
+#          ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
 #            ^ punctuation.definition.capture-group-name.begin.regexp
 #             ^^^^^ entity.name.capture-group.regexp - invalid
 #                  ^ punctuation.definition.capture-group-name.end.regexp
-#                             ^^^ keyword.other.back-reference.named.regexp
-#                                ^^^^^ variable.other.backref-and-recursion.regexp - invalid
+#                             ^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#                                ^^^^^ variable.other.capture-group.regexp - invalid
+#                                     ^ - keyword - variable
 
 regex = r'(?P<quote>[\'"]).*?\g<quote>'
-#          ^^ keyword.other.backref-and-recursion.regexp
+#          ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
 #            ^ punctuation.definition.capture-group-name.begin.regexp
 #             ^^^^^ entity.name.capture-group.regexp - invalid
 #                  ^ punctuation.definition.capture-group-name.end.regexp
-#                            ^^ keyword.other.backref-and-recursion.regexp
-#                               ^^^^^ variable.other.backref-and-recursion.regexp - invalid
+#                            ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#                               ^^^^^ variable.other.capture-group.regexp - invalid
 
 regex = r'(?P<Quote>[\'"]).*?\g<Quote>'
-#          ^^ keyword.other.backref-and-recursion.regexp
+#          ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
 #            ^ punctuation.definition.capture-group-name.begin.regexp
 #             ^^^^^ entity.name.capture-group.regexp - invalid
 #                  ^ punctuation.definition.capture-group-name.end.regexp
-#                            ^^ keyword.other.backref-and-recursion.regexp
-#                               ^^^^^ variable.other.backref-and-recursion.regexp - invalid
+#                            ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#                               ^^^^^ variable.other.capture-group.regexp - invalid
 
 regex = r'''\b ([fobar]*){1}(?:a|b)?'''
 #           ^^^^^^^^^^^^^^^^^^^^^^^^ meta.mode.extended.regexp
@@ -1263,19 +1274,19 @@
 #                                           ^^ constant.character.escape.regexp constant.character.escape.python
 
 match = re.match(r'(?P<test>a)?b(?(test)c|d)', line)
-#                  ^^^^^^^^^^ meta.group.regexp
+#                  ^^^^^^^^^^^ meta.group.regexp
 #                  ^ punctuation.section.group.begin
-#                   ^^ keyword.other.backref-and-recursion
+#                   ^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
 #                     ^ punctuation.definition.capture-group-name.begin
 #                      ^^^^ entity.name.capture-group
 #                          ^ punctuation.definition.capture-group-name.end
 #                            ^ punctuation.section.group.end
 #                             ^ keyword.operator.quantifier
 #                               ^ punctuation.section.group.begin
-#                                ^ keyword.other.backref-and-recursion.conditional
-#                                 ^ punctuation.definition.group.begin.assertion.conditional
-#                                  ^^^^ variable.other.back-reference
-#                                      ^ punctuation.definition.group.end.assertion.conditional
+#                                ^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#                                 ^ punctuation.definition.capture-group-name.begin.regexp
+#                                  ^^^^ variable.other.capture-group.regexp
+#                                      ^ punctuation.definition.capture-group-name.end.regexp
 #                                        ^ keyword.operator.alternation
 #                                          ^ punctuation.section.group.end
 match = re.match(r'(a)?b(?(1)c|d)', line)
@@ -1284,16 +1295,16 @@
 #                    ^ punctuation.section.group.end
 #                     ^ keyword.operator.quantifier
 #                       ^ punctuation.section.group.begin
-#                        ^ keyword.other.backref-and-recursion.conditional
-#                         ^ punctuation.definition.group.begin.assertion.conditional
-#                          ^ variable.other.back-reference - punctuation - keyword
-#                           ^ punctuation.definition.group.end.assertion.conditional
+#                        ^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
+#                         ^ punctuation.definition.capture-group-name.begin.regexp
+#                          ^ variable.other.capture-group.regexp
+#                           ^ punctuation.definition.capture-group-name.end.regexp
 #                             ^ keyword.operator.alternation
 #                               ^ punctuation.section.group.end
 match = re.search(r'''(?P<quote>['"]).*?(?P=quote)''', line)
 #                     ^^^^^^^^^^^^^^^ meta.group.regexp
 #                     ^ punctuation.section.group.begin
-#                      ^^ keyword.other.backref-and-recursion
+#                      ^^^^^^^^^ keyword.other.backref-and-recursion.regexp - keyword keyword
 #                        ^ punctuation.definition.capture-group-name.begin
 #                         ^^^^^ entity.name.capture-group
 #                              ^ punctuation.definition.capture-group-name.end
@@ -1303,8 +1314,8 @@
 #                                   ^ punctuation.section.group.end
 #                                    ^ keyword.other.any - meta.group
 #                                     ^^ keyword.operator.quantifier
-#                                        ^^^ keyword.other.back-reference.named
-#                                           ^^^^^ variable.other.backref-and-recursion - keyword
+#                                        ^^^ keyword.other.backref-and-recursion.regexp - keyword keyword - variable
+#                                           ^^^^^ keyword.other.backref-and-recursion.regexp variable.other.capture-group.regexp - keyword keyword
 match = re.search(r'''(?ix)some text(?-i)''', line)
 #                     ^ punctuation.definition.modifier.begin
 #                       ^^ storage.modifier.mode
@@ -1465,6 +1476,96 @@
 #                                                                         ^^^^^ - keyword.operator
 #                                                                              ^^ keyword.operator.quantifier.regexp
 
+fr"(?P<{name!s}>.*(?&{name})"
+#   ^^^ keyword.other.backref-and-recursion.regexp
+#     ^ punctuation.definition.capture-group-name.begin.regexp
+#      ^^^^^^^^ meta.string.python meta.interpolation.python
+#      ^ punctuation.section.interpolation.begin.python
+#       ^^^^ meta.generic-name.python
+#           ^^ storage.modifier.conversion.python
+#             ^ punctuation.section.interpolation.end.python
+#              ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.end.regexp
+#                  ^^ keyword.other.backref-and-recursion.regexp
+#                    ^^^^^^ meta.string.python meta.interpolation.python
+#                    ^ punctuation.section.interpolation.begin.python
+#                     ^^^^ meta.generic-name.python
+#                         ^ punctuation.section.interpolation.end.python
+#                          ^ punctuation.section.group.end.regexp
+
+fr'(?P<{name!s}>.*(?&{name})'
+#   ^^^ keyword.other.backref-and-recursion.regexp
+#     ^ punctuation.definition.capture-group-name.begin.regexp
+#      ^^^^^^^^ meta.string.python meta.interpolation.python
+#      ^ punctuation.section.interpolation.begin.python
+#       ^^^^ meta.generic-name.python
+#           ^^ storage.modifier.conversion.python
+#             ^ punctuation.section.interpolation.end.python
+#              ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.end.regexp
+#                  ^^ keyword.other.backref-and-recursion.regexp
+#                    ^^^^^^ meta.string.python meta.interpolation.python
+#                    ^ punctuation.section.interpolation.begin.python
+#                     ^^^^ meta.generic-name.python
+#                         ^ punctuation.section.interpolation.end.python
+#                          ^ punctuation.section.group.end.regexp
+
+fr"(?P={name!s})"
+#   ^^^ keyword.other.backref-and-recursion.regexp
+#      ^^^^^^^^ meta.string.python meta.interpolation.python
+#      ^ punctuation.section.interpolation.begin.python
+#       ^^^^ meta.generic-name.python
+#           ^^ storage.modifier.conversion.python
+#             ^ punctuation.section.interpolation.end.python
+#              ^ punctuation.section.group.end.regexp
+
+fr'(?P={name!s})'
+#   ^^^ keyword.other.backref-and-recursion.regexp
+#      ^^^^^^^^ meta.string.python meta.interpolation.python
+#      ^ punctuation.section.interpolation.begin.python
+#       ^^^^ meta.generic-name.python
+#           ^^ storage.modifier.conversion.python
+#             ^ punctuation.section.interpolation.end.python
+#              ^ punctuation.section.group.end.regexp
+
+fr"(?({name!s})yes|no)"
+#   ^ keyword.other.backref-and-recursion.regexp
+#    ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.begin.regexp
+#     ^^^^^^^^ meta.string.python meta.interpolation.python
+#     ^ punctuation.section.interpolation.begin.python
+#      ^^^^ meta.generic-name.python
+#          ^^ storage.modifier.conversion.python
+#            ^ punctuation.section.interpolation.end.python
+#             ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.end.regexp
+
+fr'(?({name!s})yes|no)'
+#   ^ keyword.other.backref-and-recursion.regexp
+#    ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.begin.regexp
+#     ^^^^^^^^ meta.string.python meta.interpolation.python
+#     ^ punctuation.section.interpolation.begin.python
+#      ^^^^ meta.generic-name.python
+#          ^^ storage.modifier.conversion.python
+#            ^ punctuation.section.interpolation.end.python
+#             ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.end.regexp
+
+fr"\g<{name!s}>"
+#   ^^ keyword.other.backref-and-recursion.regexp
+#    ^ punctuation.definition.capture-group-name.begin.regexp
+#     ^^^^^^^^ meta.string.python meta.interpolation.python
+#     ^ punctuation.section.interpolation.begin.python
+#      ^^^^ meta.generic-name.python
+#          ^^ storage.modifier.conversion.python
+#            ^ punctuation.section.interpolation.end.python
+#             ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.end.regexp
+
+fr'\g<{name!s}>'
+#   ^^ keyword.other.backref-and-recursion.regexp
+#    ^ punctuation.definition.capture-group-name.begin.regexp
+#     ^^^^^^^^ meta.string.python meta.interpolation.python
+#     ^ punctuation.section.interpolation.begin.python
+#      ^^^^ meta.generic-name.python
+#          ^^ storage.modifier.conversion.python
+#            ^ punctuation.section.interpolation.end.python
+#             ^ keyword.other.backref-and-recursion.regexp punctuation.definition.capture-group-name.end.regexp
+
 # Most of these were inspired by
 # https://github.com/python/cpython/commit/9a4135e939bc223f592045a38e0f927ba170da32
 f'{x=:}'