New semantic analyzer: generalize placeholders to all symbol nodes

mypy/checkexpr.py

@@ -31,7 +31,7 @@
     ConditionalExpr, ComparisonExpr, TempNode, SetComprehension,
     DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr,
     YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr,
-    TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderTypeInfo,
+    TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderNode,
     ARG_POS, ARG_OPT, ARG_NAMED, ARG_STAR, ARG_STAR2, LITERAL_TYPE, REVEAL_TYPE
 )
 from mypy.literals import literal
@@ -212,8 +212,8 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
                                                         alias_definition=e.is_alias_rvalue
                                                         or lvalue)
         else:
-            if isinstance(node, PlaceholderTypeInfo):
-                assert False, 'PlaceholderTypeInfo %r leaked to checker' % node.fullname()
+            if isinstance(node, PlaceholderNode):
+                assert False, 'PlaceholderNode %r leaked to checker' % node.fullname()
             # Unknown reference; use any type implicitly to avoid
             # generating extra type errors.
             result = AnyType(TypeOfAny.from_error)

mypy/newsemanal/semanal_main.py

@@ -66,19 +66,22 @@ def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
 
     worklist = scc[:]
     iteration = 0
+    final_iteration = False
     while worklist:
         iteration += 1
         if iteration == MAX_ITERATIONS:
             # Give up. Likely it's impossible to bind all names.
             state.manager.incomplete_namespaces.clear()
+            final_iteration = True
         elif iteration > MAX_ITERATIONS:
             assert False, 'Max iteration count reached in semantic analysis'
         all_deferred = []  # type: List[str]
         while worklist:
             next_id = worklist.pop()
             state = graph[next_id]
             assert state.tree is not None
-            deferred, incomplete = semantic_analyze_target(next_id, state, state.tree, None)
+            deferred, incomplete = semantic_analyze_target(next_id, state, state.tree, None,
+                                                           final_iteration)
             all_deferred += deferred
             if not incomplete:
                 state.manager.incomplete_namespaces.discard(next_id)
@@ -97,11 +100,13 @@ def process_functions(graph: 'Graph', scc: List[str]) -> None:
         targets = get_all_leaf_targets(symtable, module, None)
         for target, node, active_type in targets:
             assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator))
-            process_top_level_function(analyzer, graph[module], module, node, active_type)
+            process_top_level_function(analyzer, graph[module], module, target, node, active_type)
 
 
 def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
-                               state: 'State', module: str,
+                               state: 'State',
+                               module: str,
+                               target: str,
                                node: Union[FuncDef, OverloadedFuncDef, Decorator],
                                active_type: Optional[TypeInfo]) -> None:
     """Analyze single top-level function or method.
@@ -122,8 +127,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
             # OK, this is one last pass, now missing names will be reported.
             more_iterations = False
             analyzer.incomplete_namespaces.discard(module)
-        deferred, incomplete = semantic_analyze_target(module, state, node,
-                                                       active_type)
+        deferred, incomplete = semantic_analyze_target(target, state, node, active_type, False)
 
     # After semantic analysis is done, discard local namespaces
     # to avoid memory hoarding.
@@ -152,7 +156,8 @@ def get_all_leaf_targets(symtable: SymbolTable,
 def semantic_analyze_target(target: str,
                             state: 'State',
                             node: Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator],
-                            active_type: Optional[TypeInfo]) -> Tuple[List[str], bool]:
+                            active_type: Optional[TypeInfo],
+                            final_iteration: bool) -> Tuple[List[str], bool]:
     tree = state.tree
     assert tree is not None
     analyzer = state.manager.new_semantic_analyzer
@@ -168,7 +173,7 @@ def semantic_analyze_target(target: str,
             if isinstance(node, Decorator):
                 # Decorator expressions will be processed as part of the module top level.
                 node = node.func
-            analyzer.refresh_partial(node, [])
+            analyzer.refresh_partial(node, [], final_iteration)
     if analyzer.deferred:
         return [target], analyzer.incomplete
     else:

mypy/newsemanal/typeanal.py

@@ -23,7 +23,7 @@
     UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression,
     IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED,
     ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr,
-    Decorator, ImportedName, TypeAlias, MypyFile, PlaceholderTypeInfo
+    Decorator, ImportedName, TypeAlias, MypyFile, PlaceholderNode
 )
 from mypy.tvar_scope import TypeVarScope
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
@@ -214,12 +214,18 @@ def visit_unbound_type_nonoptional(self, t: UnboundType) -> Type:
                 #
                 # TODO: Remove this special case.
                 return AnyType(TypeOfAny.implementation_artifact)
-            if isinstance(node, PlaceholderTypeInfo):
-                if self.allow_placeholder:
-                    self.api.defer()
+            if isinstance(node, PlaceholderNode):
+                if node.becomes_typeinfo:
+                    # Reference to placeholder type.
+                    if self.allow_placeholder:
+                        self.api.defer()
+                    else:
+                        self.api.record_incomplete_ref()
+                    return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
                 else:
+                    # Reference to an unknown placeholder node.
                     self.api.record_incomplete_ref()
-                return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
+                    return AnyType(TypeOfAny.special_form)
             if node is None:
                 # UNBOUND_IMPORTED can happen if an unknown name was imported.
                 if sym.kind != UNBOUND_IMPORTED:
@@ -545,7 +551,7 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type:
 
     def visit_placeholder_type(self, t: PlaceholderType) -> Type:
         n = self.api.lookup_fully_qualified(t.fullname)
-        if isinstance(n.node, PlaceholderTypeInfo):
+        if isinstance(n.node, PlaceholderNode):
             self.api.defer()  # Still incomplete
             return t
         else:

mypy/nodes.py

@@ -2668,30 +2668,58 @@ def deserialize(cls, data: JsonDict) -> 'TypeAlias':
                    no_args=no_args, normalized=normalized)
 
 
-class PlaceholderTypeInfo(SymbolNode):
-    """Temporary node that will later become a type but is incomplete.
+class PlaceholderNode(SymbolNode):
+    """Temporary symbol node that will later become a real SymbolNode.
 
     These are only present during semantic analysis when using the new
-    semantic analyzer. These are created if some dependencies of a type
-    definition are not yet complete. They are used to create
-    PlaceholderType instances for types that refer to incomplete types.
-    Example where this can be used:
+    semantic analyzer. These are created if some essential dependencies
+    of a definition are not yet complete.
+
+    A typical use is for names imported from a module which is still
+    incomplete (within an import cycle):
+
+      from m import f  # Initially may create PlaceholderNode
+
+    This is particularly important if the imported shadows a name from
+    an enclosing scope or builtins:
+
+      from m import int  # Placeholder avoids mixups with builtins.int
+
+    Another case where this is useful is when there is another definition
+    or assignment:
+
+      from m import f
+      def f() -> None: ...
+
+    In the above example, the presence of PlaceholderNode allows us to
+    handle the second definition as a redefinition.
+
+    They are also used to create PlaceholderType instances for types
+    that refer to incomplete types. Example:
 
       class C(Sequence[C]): ...
 
-    We create a PlaceholderTypeInfo for C so that the type C in
-    Sequence[C] can be bound. (The long-term purpose of placeholder
-    types is to evolve into something that can support general
-    recursive types. The base class use case could be supported
-    through other means as well.)
+    We create a PlaceholderNode (with becomes_typeinfo=True) for C so
+    that the type C in Sequence[C] can be bound.
+
+    Attributes:
+
+      fullname: Full name of of the PlaceholderNode.
+      node: AST node that contains the definition that caused this to
+          be created. This is only useful for debugging.
+      becomes_typeinfo: If True, this refers something that will later
+          become a TypeInfo. It can't be used with type variables, in
+          particular, as this would cause issues with class type variable
+          detection.
 
-    PlaceholderTypeInfo can only refer to something that will become
-    a TypeInfo. It can't be used with type variables, in particular,
-    as this would cause issues with class type variable detection.
+    The long-term purpose of placeholder nodes/types is to evolve into
+    something that can support general recursive types.
     """
 
-    def __init__(self, fullname: str) -> None:
+    def __init__(self, fullname: str, node: Node, becomes_typeinfo: bool = False) -> None:
         self._fullname = fullname
+        self.node = node
+        self.becomes_typeinfo = becomes_typeinfo
         self.line = -1
 
     def name(self) -> str:
@@ -2701,10 +2729,10 @@ def fullname(self) -> str:
         return self._fullname
 
     def serialize(self) -> JsonDict:
-        assert False, "PlaceholderTypeInfo can't be serialized"
+        assert False, "PlaceholderNode can't be serialized"
 
     def accept(self, visitor: NodeVisitor[T]) -> T:
-        return visitor.visit_placeholder_type_info(self)
+        return visitor.visit_placeholder_node(self)
 
 
 class SymbolTableNode:

mypy/visitor.py

@@ -358,7 +358,7 @@ def visit_var(self, o: 'mypy.nodes.Var') -> T:
     def visit_type_alias(self, o: 'mypy.nodes.TypeAlias') -> T:
         pass
 
-    def visit_placeholder_type_info(self, o: 'mypy.nodes.PlaceholderTypeInfo') -> T:
+    def visit_placeholder_node(self, o: 'mypy.nodes.PlaceholderNode') -> T:
         pass
 
     # Statements