feat(robot): unify variable handling and prepare for RF 7.3 type parsing

- Added support for parse_type parameter in Robot Framework 7.3+
- Unified variable processing across different RF versions

Author: d-biehl

packages/robot/src/robotcode/robot/diagnostics/entities.py

@@ -187,6 +187,7 @@ class VariableDefinition(SourceEntity):
 
     value: Any = field(default=None, compare=False)
     value_is_native: bool = field(default=False, compare=False)
+    value_type: Optional[str] = field(default=None, compare=False)
 
     @functools.cached_property
     def matcher(self) -> VariableMatcher:

packages/robot/src/robotcode/robot/diagnostics/library_doc.py

@@ -67,7 +67,7 @@
 from ..utils import get_robot_version
 from ..utils.ast import (
     cached_isinstance,
-    get_variable_token,
+    get_first_variable_token,
     range_from_token,
     strip_variable_token,
 )
@@ -2785,7 +2785,7 @@ def _get_argument_definitions_from_line(
 
     for argument_token in (cast(RobotToken, e) for e in arguments):
         try:
-            argument = get_variable_token(argument_token)
+            argument = get_first_variable_token(argument_token)
 
             if argument is not None and argument.value != "@{}":
                 if argument.value not in args:

packages/robot/src/robotcode/robot/diagnostics/namespace.py

@@ -326,10 +326,12 @@ def iter_over_keyword_names_and_owners(
             yield ".".join(tokens[:i]), ".".join(tokens[i:])
 
 
-def strip_variable_token(token: Token, identifiers: str = "$@&%*", matcher: Optional[VariableMatcher] = None) -> Token:
-    if token.type == Token.VARIABLE:
+def strip_variable_token(
+    token: Token, identifiers: str = "$@&%*", parse_type: bool = False, matcher: Optional[VariableMatcher] = None
+) -> Token:
+    if token.type in [Token.VARIABLE, Token.ASSIGN]:
         if matcher is None:
-            matcher = search_variable(token.value, identifiers, ignore_errors=True)
+            matcher = search_variable(token.value, identifiers, parse_type=parse_type, ignore_errors=True)
 
         if matcher.is_variable():
             value = matcher.base
@@ -346,13 +348,13 @@ def strip_variable_token(token: Token, identifiers: str = "$@&%*", matcher: Opti
     return token
 
 
-def get_variable_token(token: Token) -> Optional[Token]:
+def get_first_variable_token(token: Token, extra_types: Optional[Set[str]] = None) -> Optional[Token]:
     return next(
         (
             v
             for v in itertools.dropwhile(
                 lambda t: t.type in Token.NON_DATA_TOKENS,
-                tokenize_variables(token, ignore_errors=True),
+                tokenize_variables(token, ignore_errors=True, extra_types=extra_types),
             )
             if v.type == Token.VARIABLE
         ),

packages/robot/src/robotcode/robot/diagnostics/namespace_analyzer.py

@@ -8,30 +8,63 @@
 from robot.variables.search import search_variable as robot_search_variable
 from robotcode.robot.utils.match import normalize
 
+from . import get_robot_version
+
 
 class InvalidVariableError(Exception):
     pass
 
 
 class VariableMatcher:
-    def __init__(self, string: str, identifiers: str = "$@&%", ignore_errors: bool = True) -> None:
-        self.string = string
+    if get_robot_version() >= (7, 3):
+
+        def __init__(
+            self, string: str, identifiers: str = "$@&%", parse_type: bool = False, ignore_errors: bool = True
+        ) -> None:
+            self.string = string
+
+            self.match = robot_search_variable(
+                string, identifiers=identifiers, parse_type=parse_type, ignore_errors=ignore_errors
+            )
+
+            if not ignore_errors and self.match.base is None:
+                raise InvalidVariableError(f"Invalid variable '{string}'")
+
+            self.base = self.match.base
+            self.identifier = self.match.identifier
+            self.name = "%s{%s}" % (self.identifier, self.base.strip()) if self.base else None
+            self.type = self.match.type
+            self.items = self.match.items
+            self.start = self.match.start
+            self.end = self.match.end
+            self.after = self.match.after
+            self.before = self.match.before
+
+            self.normalized_name = normalize(self.base) if self.base else None
+
+    else:
+
+        def __init__(
+            self, string: str, identifiers: str = "$@&%", parse_type: bool = False, ignore_errors: bool = True
+        ) -> None:
+            self.string = string
 
-        self.match = robot_search_variable(string, identifiers=identifiers, ignore_errors=ignore_errors)
+            self.match = robot_search_variable(string, identifiers=identifiers, ignore_errors=ignore_errors)
 
-        if not ignore_errors and self.match.base is None:
-            raise InvalidVariableError(f"Invalid variable '{string}'")
+            if not ignore_errors and self.match.base is None:
+                raise InvalidVariableError(f"Invalid variable '{string}'")
 
-        self.base = self.match.base
-        self.identifier = self.match.identifier
-        self.name = "%s{%s}" % (self.identifier, self.base.strip()) if self.base else None
-        self.items = self.match.items
-        self.start = self.match.start
-        self.end = self.match.end
-        self.after = self.match.after
-        self.before = self.match.before
+            self.base = self.match.base
+            self.identifier = self.match.identifier
+            self.name = "%s{%s}" % (self.identifier, self.base.strip()) if self.base else None
+            self.type = None
+            self.items = self.match.items
+            self.start = self.match.start
+            self.end = self.match.end
+            self.after = self.match.after
+            self.before = self.match.before
 
-        self.normalized_name = normalize(self.base) if self.base else None
+            self.normalized_name = normalize(self.base) if self.base else None
 
     def __eq__(self, o: object) -> bool:
         if self.normalized_name is None:
@@ -75,17 +108,57 @@ def is_list_variable(self) -> bool:
     def is_dict_variable(self) -> bool:
         return bool(self.match.is_dict_variable())
 
-    def is_assign(self, allow_assign_mark: bool = False) -> bool:
-        return bool(self.match.is_assign(allow_assign_mark))
+    if get_robot_version() >= (6, 1):
+
+        def is_assign(
+            self, allow_assign_mark: bool = False, allow_nested: bool = False, allow_items: bool = False
+        ) -> bool:
+            return bool(
+                self.match.is_assign(
+                    allow_assign_mark=allow_assign_mark, allow_nested=allow_nested, allow_items=allow_items
+                )
+            )
+    else:
+
+        def is_assign(
+            self, allow_assign_mark: bool = False, allow_nested: bool = False, allow_items: bool = False
+        ) -> bool:
+            return bool(self.match.is_assign(allow_assign_mark=allow_assign_mark))
+
+    if get_robot_version() >= (6, 1):
+
+        def is_scalar_assign(self, allow_assign_mark: bool = False, allow_nested: bool = False) -> bool:
+            return bool(self.match.is_scalar_assign(allow_assign_mark=allow_assign_mark, allow_nested=allow_nested))
+    else:
+
+        def is_scalar_assign(self, allow_assign_mark: bool = False, allow_nested: bool = False) -> bool:
+            return bool(self.match.is_scalar_assign(allow_assign_mark=allow_assign_mark))
+
+    if get_robot_version() >= (6, 1):
+
+        def is_list_assign(
+            self,
+            allow_assign_mark: bool = False,
+            allow_nested: bool = False,
+        ) -> bool:
+            return bool(self.match.is_list_assign(allow_assign_mark=allow_assign_mark, allow_nested=allow_nested))
+    else:
+
+        def is_list_assign(
+            self,
+            allow_assign_mark: bool = False,
+            allow_nested: bool = False,
+        ) -> bool:
+            return bool(self.match.is_list_assign(allow_assign_mark=allow_assign_mark))
 
-    def is_scalar_assign(self, allow_assign_mark: bool = False) -> bool:
-        return bool(self.match.is_scalar_assign(allow_assign_mark))
+    if get_robot_version() >= (6, 1):
 
-    def is_list_assign(self, allow_assign_mark: bool = False) -> bool:
-        return bool(self.match.is_list_assign(allow_assign_mark))
+        def is_dict_assign(self, allow_assign_mark: bool = False, allow_nested: bool = False) -> bool:
+            return bool(self.match.is_dict_assign(allow_assign_mark=allow_assign_mark, allow_nested=allow_nested))
+    else:
 
-    def is_dict_assign(self, allow_assign_mark: bool = False) -> bool:
-        return bool(self.match.is_dict_assign(allow_assign_mark))
+        def is_dict_assign(self, allow_assign_mark: bool = False, allow_nested: bool = False) -> bool:
+            return bool(self.match.is_dict_assign(allow_assign_mark=allow_assign_mark))
 
 
 BUILTIN_VARIABLES = [
@@ -143,8 +216,10 @@ def is_variable(string: str, identifiers: str = "$@&") -> bool:
 
 
 @functools.lru_cache(maxsize=8192)
-def search_variable(string: str, identifiers: str = "$@&%*", ignore_errors: bool = False) -> VariableMatcher:
-    return VariableMatcher(string, identifiers, ignore_errors)
+def search_variable(
+    string: str, identifiers: str = "$@&%*", parse_type: bool = False, ignore_errors: bool = False
+) -> VariableMatcher:
+    return VariableMatcher(string, identifiers, parse_type, ignore_errors)
 
 
 @functools.lru_cache(maxsize=8192)

packages/robot/src/robotcode/robot/utils/ast.py

@@ -13,10 +13,10 @@ result:
       line: 7
   target_range:
     end:
-      character: 22
+      character: 8
       line: 20
     start:
-      character: 0
+      character: 2
       line: 20
   target_selection_range:
     end:

packages/robot/src/robotcode/robot/utils/variables.py

@@ -13,10 +13,10 @@ result:
       line: 7
   target_range:
     end:
-      character: 22
+      character: 8
       line: 20
     start:
-      character: 0
+      character: 2
       line: 20
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-007-020-var_in_variables_import].out

@@ -13,10 +13,10 @@ result:
       line: 7
   target_range:
     end:
-      character: 22
+      character: 8
       line: 20
     start:
-      character: 0
+      character: 2
       line: 20
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-007-023-var_in_variables_import].out

@@ -13,10 +13,10 @@ result:
       line: 10
   target_range:
     end:
-      character: 22
+      character: 8
       line: 20
     start:
-      character: 0
+      character: 2
       line: 20
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-007-025-var_in_variables_import].out

@@ -13,10 +13,10 @@ result:
       line: 10
   target_range:
     end:
-      character: 22
+      character: 8
       line: 20
     start:
-      character: 0
+      character: 2
       line: 20
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-010-020-var_in_resource_import].out

@@ -13,10 +13,10 @@ result:
       line: 10
   target_range:
     end:
-      character: 22
+      character: 8
       line: 20
     start:
-      character: 0
+      character: 2
       line: 20
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-010-023-var_in_resource_import].out

@@ -13,10 +13,10 @@ result:
       line: 14
   target_range:
     end:
-      character: 11
+      character: 5
       line: 25
     start:
-      character: 0
+      character: 2
       line: 25
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-010-025-var_in_resource_import].out

@@ -13,10 +13,10 @@ result:
       line: 14
   target_range:
     end:
-      character: 11
+      character: 5
       line: 25
     start:
-      character: 0
+      character: 2
       line: 25
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-014-020-var_in_Libary_import_path].out

@@ -13,10 +13,10 @@ result:
       line: 14
   target_range:
     end:
-      character: 11
+      character: 5
       line: 25
     start:
-      character: 0
+      character: 2
       line: 25
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-014-021-var_in_Libary_import_path].out

@@ -13,10 +13,10 @@ result:
       line: 14
   target_range:
     end:
-      character: 22
+      character: 9
       line: 23
     start:
-      character: 0
+      character: 2
       line: 23
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-014-022-var_in_Libary_import_path].out

@@ -13,10 +13,10 @@ result:
       line: 14
   target_range:
     end:
-      character: 22
+      character: 9
       line: 23
     start:
-      character: 0
+      character: 2
       line: 23
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-014-057-var_in_library_parameters].out

@@ -13,10 +13,10 @@ result:
       line: 14
   target_range:
     end:
-      character: 22
+      character: 9
       line: 23
     start:
-      character: 0
+      character: 2
       line: 23
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-014-060-var_in_library_parameters].out

@@ -13,10 +13,10 @@ result:
       line: 23
   target_range:
     end:
-      character: 22
+      character: 9
       line: 23
     start:
-      character: 0
+      character: 2
       line: 23
   target_selection_range:
     end:

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_goto_definition.test_definition[goto.robot-014-063-var_in_library_parameters].out

@@ -13,10 +13,10 @@ result:
       line: 23
   target_range:
     end:
-      character: 22
+      character: 9
       line: 23
     start:
-      character: 0
+      character: 2
       line: 23
   target_selection_range:
     end: