New semantic analyzer: generalize placeholders to all symbol nodes (#…

…6336)

This generalizes the concept of placeholder nodes to also cover
assignment statements and imports. Previously they were only
created for class definitions. The main motivation is to prevent 
access to definitions in outer namespaces if there is an 
incomplete definition that should take precedence.

Fixes #6299.

This also does a few other updates:

* During the final iteration, some references to placeholder nodes
  generate an error instead of producing placeholder nodes. This
  allows the analysis to terminate in certain cases of import
  cycles at least.
* Major refactoring of `analyze_name_lvalue`. The motivation is
  that some early experiments made the old structure unwieldy,
  but the refactoring may not be as important for the final design.
  This seems like a code quality improvement so I'm including it 
  here.
* If a name lvalue was bound during an earlier iteration, we don't 
  rebind it. I'd like to gradually move to this approach more 
  generally.
* Some forward references to names aren't treated as undefined 
  any more. I think that these worked by accident. Now these 
  generally generate "cannot determine type" errors.
* Most definitions won't generate incomplete namespaces any
  more, since placeholders count as definitions in this context. 
  Star imports still generate incomplete namespaces.
* Remove redundant flags from some test cases.

mypy/checkexpr.py

@@ -31,7 +31,7 @@
     ConditionalExpr, ComparisonExpr, TempNode, SetComprehension,
     DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr,
     YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr,
-    TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderTypeInfo,
+    TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderNode,
     ARG_POS, ARG_OPT, ARG_NAMED, ARG_STAR, ARG_STAR2, LITERAL_TYPE, REVEAL_TYPE
 )
 from mypy.literals import literal
@@ -212,8 +212,8 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
                                                         alias_definition=e.is_alias_rvalue
                                                         or lvalue)
         else:
-            if isinstance(node, PlaceholderTypeInfo):
-                assert False, 'PlaceholderTypeInfo %r leaked to checker' % node.fullname()
+            if isinstance(node, PlaceholderNode):
+                assert False, 'PlaceholderNode %r leaked to checker' % node.fullname()
             # Unknown reference; use any type implicitly to avoid
             # generating extra type errors.
             result = AnyType(TypeOfAny.from_error)

mypy/newsemanal/semanal_main.py

@@ -66,19 +66,22 @@ def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
 
     worklist = scc[:]
     iteration = 0
+    final_iteration = False
     while worklist:
         iteration += 1
         if iteration == MAX_ITERATIONS:
             # Give up. Likely it's impossible to bind all names.
             state.manager.incomplete_namespaces.clear()
+            final_iteration = True
         elif iteration > MAX_ITERATIONS:
             assert False, 'Max iteration count reached in semantic analysis'
         all_deferred = []  # type: List[str]
         while worklist:
             next_id = worklist.pop()
             state = graph[next_id]
             assert state.tree is not None
-            deferred, incomplete = semantic_analyze_target(next_id, state, state.tree, None)
+            deferred, incomplete = semantic_analyze_target(next_id, state, state.tree, None,
+                                                           final_iteration)
             all_deferred += deferred
             if not incomplete:
                 state.manager.incomplete_namespaces.discard(next_id)
@@ -97,11 +100,13 @@ def process_functions(graph: 'Graph', scc: List[str]) -> None:
         targets = get_all_leaf_targets(symtable, module, None)
         for target, node, active_type in targets:
             assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator))
-            process_top_level_function(analyzer, graph[module], module, node, active_type)
+            process_top_level_function(analyzer, graph[module], module, target, node, active_type)
 
 
 def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
-                               state: 'State', module: str,
+                               state: 'State',
+                               module: str,
+                               target: str,
                                node: Union[FuncDef, OverloadedFuncDef, Decorator],
                                active_type: Optional[TypeInfo]) -> None:
     """Analyze single top-level function or method.
@@ -122,8 +127,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
             # OK, this is one last pass, now missing names will be reported.
             more_iterations = False
             analyzer.incomplete_namespaces.discard(module)
-        deferred, incomplete = semantic_analyze_target(module, state, node,
-                                                       active_type)
+        deferred, incomplete = semantic_analyze_target(target, state, node, active_type, False)
 
     # After semantic analysis is done, discard local namespaces
     # to avoid memory hoarding.
@@ -152,7 +156,8 @@ def get_all_leaf_targets(symtable: SymbolTable,
 def semantic_analyze_target(target: str,
                             state: 'State',
                             node: Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator],
-                            active_type: Optional[TypeInfo]) -> Tuple[List[str], bool]:
+                            active_type: Optional[TypeInfo],
+                            final_iteration: bool) -> Tuple[List[str], bool]:
     tree = state.tree
     assert tree is not None
     analyzer = state.manager.new_semantic_analyzer
@@ -168,7 +173,7 @@ def semantic_analyze_target(target: str,
             if isinstance(node, Decorator):
                 # Decorator expressions will be processed as part of the module top level.
                 node = node.func
-            analyzer.refresh_partial(node, [])
+            analyzer.refresh_partial(node, [], final_iteration)
     if analyzer.deferred:
         return [target], analyzer.incomplete
     else:

mypy/newsemanal/typeanal.py

@@ -23,7 +23,7 @@
     UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression,
     IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED,
     ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr,
-    Decorator, ImportedName, TypeAlias, MypyFile, PlaceholderTypeInfo
+    Decorator, ImportedName, TypeAlias, MypyFile, PlaceholderNode
 )
 from mypy.tvar_scope import TypeVarScope
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
@@ -214,12 +214,18 @@ def visit_unbound_type_nonoptional(self, t: UnboundType) -> Type:
                 #
                 # TODO: Remove this special case.
                 return AnyType(TypeOfAny.implementation_artifact)
-            if isinstance(node, PlaceholderTypeInfo):
-                if self.allow_placeholder:
-                    self.api.defer()
+            if isinstance(node, PlaceholderNode):
+                if node.becomes_typeinfo:
+                    # Reference to placeholder type.
+                    if self.allow_placeholder:
+                        self.api.defer()
+                    else:
+                        self.api.record_incomplete_ref()
+                    return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
                 else:
+                    # Reference to an unknown placeholder node.
                     self.api.record_incomplete_ref()
-                return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
+                    return AnyType(TypeOfAny.special_form)
             if node is None:
                 # UNBOUND_IMPORTED can happen if an unknown name was imported.
                 if sym.kind != UNBOUND_IMPORTED:
@@ -545,7 +551,7 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type:
 
     def visit_placeholder_type(self, t: PlaceholderType) -> Type:
         n = self.api.lookup_fully_qualified(t.fullname)
-        if isinstance(n.node, PlaceholderTypeInfo):
+        if isinstance(n.node, PlaceholderNode):
             self.api.defer()  # Still incomplete
             return t
         else:

mypy/nodes.py

@@ -2668,30 +2668,58 @@ def deserialize(cls, data: JsonDict) -> 'TypeAlias':
                    no_args=no_args, normalized=normalized)
 
 
-class PlaceholderTypeInfo(SymbolNode):
-    """Temporary node that will later become a type but is incomplete.
+class PlaceholderNode(SymbolNode):
+    """Temporary symbol node that will later become a real SymbolNode.
 
     These are only present during semantic analysis when using the new
-    semantic analyzer. These are created if some dependencies of a type
-    definition are not yet complete. They are used to create
-    PlaceholderType instances for types that refer to incomplete types.
-    Example where this can be used:
+    semantic analyzer. These are created if some essential dependencies
+    of a definition are not yet complete.
+
+    A typical use is for names imported from a module which is still
+    incomplete (within an import cycle):
+
+      from m import f  # Initially may create PlaceholderNode
+
+    This is particularly important if the imported shadows a name from
+    an enclosing scope or builtins:
+
+      from m import int  # Placeholder avoids mixups with builtins.int
+
+    Another case where this is useful is when there is another definition
+    or assignment:
+
+      from m import f
+      def f() -> None: ...
+
+    In the above example, the presence of PlaceholderNode allows us to
+    handle the second definition as a redefinition.
+
+    They are also used to create PlaceholderType instances for types
+    that refer to incomplete types. Example:
 
       class C(Sequence[C]): ...
 
-    We create a PlaceholderTypeInfo for C so that the type C in
-    Sequence[C] can be bound. (The long-term purpose of placeholder
-    types is to evolve into something that can support general
-    recursive types. The base class use case could be supported
-    through other means as well.)
+    We create a PlaceholderNode (with becomes_typeinfo=True) for C so
+    that the type C in Sequence[C] can be bound.
+
+    Attributes:
+
+      fullname: Full name of of the PlaceholderNode.
+      node: AST node that contains the definition that caused this to
+          be created. This is only useful for debugging.
+      becomes_typeinfo: If True, this refers something that will later
+          become a TypeInfo. It can't be used with type variables, in
+          particular, as this would cause issues with class type variable
+          detection.
 
-    PlaceholderTypeInfo can only refer to something that will become
-    a TypeInfo. It can't be used with type variables, in particular,
-    as this would cause issues with class type variable detection.
+    The long-term purpose of placeholder nodes/types is to evolve into
+    something that can support general recursive types.
     """
 
-    def __init__(self, fullname: str) -> None:
+    def __init__(self, fullname: str, node: Node, becomes_typeinfo: bool = False) -> None:
         self._fullname = fullname
+        self.node = node
+        self.becomes_typeinfo = becomes_typeinfo
         self.line = -1
 
     def name(self) -> str:
@@ -2701,10 +2729,10 @@ def fullname(self) -> str:
         return self._fullname
 
     def serialize(self) -> JsonDict:
-        assert False, "PlaceholderTypeInfo can't be serialized"
+        assert False, "PlaceholderNode can't be serialized"
 
     def accept(self, visitor: NodeVisitor[T]) -> T:
-        return visitor.visit_placeholder_type_info(self)
+        return visitor.visit_placeholder_node(self)
 
 
 class SymbolTableNode:

mypy/visitor.py

@@ -358,7 +358,7 @@ def visit_var(self, o: 'mypy.nodes.Var') -> T:
     def visit_type_alias(self, o: 'mypy.nodes.TypeAlias') -> T:
         pass
 
-    def visit_placeholder_type_info(self, o: 'mypy.nodes.PlaceholderTypeInfo') -> T:
+    def visit_placeholder_node(self, o: 'mypy.nodes.PlaceholderNode') -> T:
         pass
 
     # Statements