New semantic analyzer: Enable namedtuple tests and fix self-check error (#6452)

This fixes the `Name 'cls' already defined error` in self-check with new analyzer. The error is caused by the fact that now we actually analyze generated methods. I solve them simply by using underscored versions of `_self` and `_cls` (this is btw the same what runtime implementation does for some generated methods).

I also enable both named tuple test files. This needs the same temporary redundant variable hack as I did for typed dicts, it can be removed when we will not need to be compatible with old analyzer, I will create an issue so we will not forget about this.

I skip a bunch of test cases for recursive named tuples (there is already an issue for this). I also skip one test because it is tricky to fix, I will open a follow-up issue.

Author: ilevkivskyi

mypy/newsemanal/semanal.py

@@ -1915,6 +1915,16 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool:
         # Yes, it's a valid namedtuple, but defer if it is not ready.
         if not info:
             self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
+        else:
+            # TODO: This is needed for one-to-one compatibility with old analyzer, otherwise
+            # type checker will try to infer Any for the l.h.s. causing named tuple class
+            # object to have type Any when it appears in runtime context.
+            # Remove this and update the checker after new analyzer is the default one!
+            # See also #6458.
+            lvalue.fullname = self.qualified_name(name)
+            lvalue.is_inferred_def = True
+            lvalue.kind = kind = self.current_symbol_kind()
+            lvalue.node = self.make_name_lvalue_var(lvalue, kind, inferred=True)
         return True
 
     def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool:

mypy/newsemanal/semanal_namedtuple.py

@@ -407,9 +407,9 @@ def add_method(funcname: str,
                        is_new: bool = False,
                        ) -> None:
             if is_classmethod or is_new:
-                first = [Argument(Var('cls'), TypeType.make_normalized(selftype), None, ARG_POS)]
+                first = [Argument(Var('_cls'), TypeType.make_normalized(selftype), None, ARG_POS)]
             else:
-                first = [Argument(Var('self'), selftype, None, ARG_POS)]
+                first = [Argument(Var('_self'), selftype, None, ARG_POS)]
             args = first + args
 
             types = [arg.type_annotation for arg in args]

mypy/test/hacks.py

@@ -12,7 +12,6 @@
     'check-callable.test',
     'check-classes.test',
     'check-classvar.test',
-    'check-class-namedtuple.test',
     'check-custom-plugin.test',
     'check-dataclasses.test',
     'check-default-plugin.test',
@@ -28,7 +27,6 @@
     'check-literal.test',
     'check-modules.test',
     'check-multiple-inheritance.test',
-    'check-namedtuple.test',
     'check-newtype.test',
     'check-optional.test',
     'check-overloading.test',

test-data/unit/check-class-namedtuple.test

@@ -587,7 +587,9 @@ reveal_type(takes_base(Base(1)))  # E: Revealed type is 'builtins.int'
 reveal_type(takes_base(Child(1)))  # E: Revealed type is 'builtins.int'
 [builtins fixtures/tuple.pyi]
 
+-- Depends on collecting functions via AST, not symbol tables.
 [case testNewNamedTupleIllegalNames]
+# flags: --no-new-semantic-analyzer
 from typing import Callable, NamedTuple
 
 class XMethBad(NamedTuple):
@@ -612,16 +614,16 @@ class AnnotationsAsAMethod(NamedTuple):
 
 class ReuseNames(NamedTuple):
     x: int
-    def x(self) -> str:  # E: Name 'x' already defined on line 22
+    def x(self) -> str:  # E: Name 'x' already defined on line 23
         return ''
 
     def y(self) -> int:
         return 0
-    y: str  # E: Name 'y' already defined on line 26
+    y: str  # E: Name 'y' already defined on line 27
 
 class ReuseCallableNamed(NamedTuple):
     z: Callable[[ReuseNames], int]
-    def z(self) -> int:  # E: Name 'z' already defined on line 31
+    def z(self) -> int:  # E: Name 'z' already defined on line 32
         return 0
 
 [builtins fixtures/dict.pyi]

test-data/unit/check-namedtuple.test

@@ -346,12 +346,13 @@ x._replace(5)  # E: Too many positional arguments for "_replace" of "X"
 [builtins fixtures/list.pyi]
 
 [case testNamedTupleReplaceAsClass]
+# flags: --new-semantic-analyzer --no-strict-optional
 from collections import namedtuple
 
 X = namedtuple('X', ['x', 'y'])
 x = None  # type: X
 X._replace(x, x=1, y=2)
-X._replace(x=1, y=2)  # E: Missing positional argument "self" in call to "_replace" of "X"
+X._replace(x=1, y=2)  # E: Missing positional argument "_self" in call to "_replace" of "X"
 
 [builtins fixtures/list.pyi]
 
@@ -497,7 +498,9 @@ b = B._make([''])  # type: B
 
 [builtins fixtures/list.pyi]
 
+-- Depends on tuple fallback PR
 [case testNamedTupleIncompatibleRedefinition]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple
 class Crash(NamedTuple):
     count: int  # E: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], Any], int]")
@@ -573,6 +576,7 @@ tmp/b.py:4: error: Revealed type is 'Tuple[Any, fallback=a.N]'
 tmp/b.py:7: error: Revealed type is 'Tuple[Any, fallback=a.N]'
 
 [case testSimpleSelfReferentialNamedTuple]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple
 class MyNamedTuple(NamedTuple):
     parent: 'MyNamedTuple'
@@ -583,8 +587,8 @@ def bar(nt: MyNamedTuple) -> MyNamedTuple:
 x: MyNamedTuple
 reveal_type(x.parent)
 [out]
-main:2: error: Recursive types not fully supported yet, nested types replaced with "Any"
-main:9: error: Revealed type is 'Tuple[Any, fallback=__main__.MyNamedTuple]'
+main:3: error: Recursive types not fully supported yet, nested types replaced with "Any"
+main:10: error: Revealed type is 'Tuple[Any, fallback=__main__.MyNamedTuple]'
 
 -- Some crazy self-referential named tuples and types dicts
 -- to be sure that everything works
@@ -611,6 +615,7 @@ class B:
 [out]
 
 [case testSelfRefNT1]
+# flags: --no-new-semantic-analyzer
 from typing import Tuple, NamedTuple
 
 Node = NamedTuple('Node', [ # E: Recursive types not fully supported yet, nested types replaced with "Any"
@@ -623,6 +628,7 @@ reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[b
 
 
 [case testSelfRefNT2]
+# flags: --no-new-semantic-analyzer
 from typing import Tuple, NamedTuple
 
 A = NamedTuple('A', [ # E
@@ -637,10 +643,11 @@ n: A
 reveal_type(n) # E: Revealed type is 'Tuple[builtins.str, builtins.tuple[Tuple[Tuple[builtins.str, builtins.tuple[Any], fallback=__main__.A], builtins.int, fallback=__main__.B]], fallback=__main__.A]'
 [builtins fixtures/tuple.pyi]
 [out]
-main:3: error: Recursive types not fully supported yet, nested types replaced with "Any"
-main:7: error: Recursive types not fully supported yet, nested types replaced with "Any"
+main:4: error: Recursive types not fully supported yet, nested types replaced with "Any"
+main:8: error: Recursive types not fully supported yet, nested types replaced with "Any"
 
 [case testSelfRefNT3]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple, Tuple
 
 class B(NamedTuple): # E
@@ -659,9 +666,10 @@ lst = [m, n]
 reveal_type(lst[0]) # E: Revealed type is 'Tuple[builtins.object, builtins.object]'
 [builtins fixtures/tuple.pyi]
 [out]
-main:3: error: Recursive types not fully supported yet, nested types replaced with "Any"
+main:4: error: Recursive types not fully supported yet, nested types replaced with "Any"
 
 [case testSelfRefNT4]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple
 
 class B(NamedTuple): # E
@@ -676,10 +684,11 @@ n: A
 reveal_type(n.y[0]) # E: Revealed type is 'Tuple[builtins.str, Tuple[Any, builtins.int, fallback=__main__.B], fallback=__main__.A]'
 [builtins fixtures/tuple.pyi]
 [out]
-main:3: error: Recursive types not fully supported yet, nested types replaced with "Any"
-main:7: error: Recursive types not fully supported yet, nested types replaced with "Any"
+main:4: error: Recursive types not fully supported yet, nested types replaced with "Any"
+main:8: error: Recursive types not fully supported yet, nested types replaced with "Any"
 
 [case testSelfRefNT5]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple
 
 B = NamedTuple('B', [ # E: Recursive types not fully supported yet, nested types replaced with "Any"
@@ -697,6 +706,7 @@ reveal_type(f) # E: Revealed type is 'def (m: Tuple[Tuple[builtins.str, Tuple[An
 [builtins fixtures/tuple.pyi]
 
 [case testRecursiveNamedTupleInBases]
+# flags: --no-new-semantic-analyzer
 from typing import List, NamedTuple, Union
 
 Exp = Union['A', 'B']  # E: Recursive types not fully supported yet, nested types replaced with "Any"
@@ -716,6 +726,7 @@ my_eval(A([B(1), B(2)])) # OK
 [out]
 
 [case testNamedTupleImportCycle]
+# flags: --new-semantic-analyzer
 import b
 [file a.py]
 class C:
@@ -725,10 +736,8 @@ from b import tp
 x: tp
 reveal_type(x.x)  # E: Revealed type is 'builtins.int'
 
-# Unfortunately runtime part doesn't work yet, see docstring in SemanticAnalyzerPass3.update_imported_vars()
-reveal_type(tp)  # E: Revealed type is 'Any' \
-                 # E: Cannot determine type of 'tp'
-tp('x')  # E: Cannot determine type of 'tp'
+reveal_type(tp)  # E: Revealed type is 'def (x: builtins.int) -> Tuple[builtins.int, fallback=b.tp]'
+tp('x')  # E: Argument 1 to "tp" has incompatible type "str"; expected "int"
 
 [file b.py]
 from a import C
@@ -738,6 +747,7 @@ tp = NamedTuple('tp', [('x', int)])
 [out]
 
 [case testSubclassOfRecursiveNamedTuple]
+# flags: --no-new-semantic-analyzer
 from typing import List, NamedTuple
 
 class Command(NamedTuple):  # type: ignore