Skip to content

Commit 15e9b5b

Browse files
authored
New semantic analyzer: generalize placeholders to all symbol nodes (#6336)
This generalizes the concept of placeholder nodes to also cover assignment statements and imports. Previously they were only created for class definitions. The main motivation is to prevent access to definitions in outer namespaces if there is an incomplete definition that should take precedence. Fixes #6299. This also does a few other updates: * During the final iteration, some references to placeholder nodes generate an error instead of producing placeholder nodes. This allows the analysis to terminate in certain cases of import cycles at least. * Major refactoring of `analyze_name_lvalue`. The motivation is that some early experiments made the old structure unwieldy, but the refactoring may not be as important for the final design. This seems like a code quality improvement so I'm including it here. * If a name lvalue was bound during an earlier iteration, we don't rebind it. I'd like to gradually move to this approach more generally. * Some forward references to names aren't treated as undefined any more. I think that these worked by accident. Now these generally generate "cannot determine type" errors. * Most definitions won't generate incomplete namespaces any more, since placeholders count as definitions in this context. Star imports still generate incomplete namespaces. * Remove redundant flags from some test cases.
1 parent e9dc189 commit 15e9b5b

File tree

7 files changed

+238
-157
lines changed

7 files changed

+238
-157
lines changed

mypy/checkexpr.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
ConditionalExpr, ComparisonExpr, TempNode, SetComprehension,
3232
DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr,
3333
YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr,
34-
TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderTypeInfo,
34+
TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, SymbolNode, PlaceholderNode,
3535
ARG_POS, ARG_OPT, ARG_NAMED, ARG_STAR, ARG_STAR2, LITERAL_TYPE, REVEAL_TYPE
3636
)
3737
from mypy.literals import literal
@@ -212,8 +212,8 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
212212
alias_definition=e.is_alias_rvalue
213213
or lvalue)
214214
else:
215-
if isinstance(node, PlaceholderTypeInfo):
216-
assert False, 'PlaceholderTypeInfo %r leaked to checker' % node.fullname()
215+
if isinstance(node, PlaceholderNode):
216+
assert False, 'PlaceholderNode %r leaked to checker' % node.fullname()
217217
# Unknown reference; use any type implicitly to avoid
218218
# generating extra type errors.
219219
result = AnyType(TypeOfAny.from_error)

mypy/newsemanal/semanal.py

Lines changed: 122 additions & 101 deletions
Large diffs are not rendered by default.

mypy/newsemanal/semanal_main.py

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,19 +66,22 @@ def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
6666

6767
worklist = scc[:]
6868
iteration = 0
69+
final_iteration = False
6970
while worklist:
7071
iteration += 1
7172
if iteration == MAX_ITERATIONS:
7273
# Give up. Likely it's impossible to bind all names.
7374
state.manager.incomplete_namespaces.clear()
75+
final_iteration = True
7476
elif iteration > MAX_ITERATIONS:
7577
assert False, 'Max iteration count reached in semantic analysis'
7678
all_deferred = [] # type: List[str]
7779
while worklist:
7880
next_id = worklist.pop()
7981
state = graph[next_id]
8082
assert state.tree is not None
81-
deferred, incomplete = semantic_analyze_target(next_id, state, state.tree, None)
83+
deferred, incomplete = semantic_analyze_target(next_id, state, state.tree, None,
84+
final_iteration)
8285
all_deferred += deferred
8386
if not incomplete:
8487
state.manager.incomplete_namespaces.discard(next_id)
@@ -97,11 +100,13 @@ def process_functions(graph: 'Graph', scc: List[str]) -> None:
97100
targets = get_all_leaf_targets(symtable, module, None)
98101
for target, node, active_type in targets:
99102
assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator))
100-
process_top_level_function(analyzer, graph[module], module, node, active_type)
103+
process_top_level_function(analyzer, graph[module], module, target, node, active_type)
101104

102105

103106
def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
104-
state: 'State', module: str,
107+
state: 'State',
108+
module: str,
109+
target: str,
105110
node: Union[FuncDef, OverloadedFuncDef, Decorator],
106111
active_type: Optional[TypeInfo]) -> None:
107112
"""Analyze single top-level function or method.
@@ -122,8 +127,7 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
122127
# OK, this is one last pass, now missing names will be reported.
123128
more_iterations = False
124129
analyzer.incomplete_namespaces.discard(module)
125-
deferred, incomplete = semantic_analyze_target(module, state, node,
126-
active_type)
130+
deferred, incomplete = semantic_analyze_target(target, state, node, active_type, False)
127131

128132
# After semantic analysis is done, discard local namespaces
129133
# to avoid memory hoarding.
@@ -152,7 +156,8 @@ def get_all_leaf_targets(symtable: SymbolTable,
152156
def semantic_analyze_target(target: str,
153157
state: 'State',
154158
node: Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator],
155-
active_type: Optional[TypeInfo]) -> Tuple[List[str], bool]:
159+
active_type: Optional[TypeInfo],
160+
final_iteration: bool) -> Tuple[List[str], bool]:
156161
tree = state.tree
157162
assert tree is not None
158163
analyzer = state.manager.new_semantic_analyzer
@@ -168,7 +173,7 @@ def semantic_analyze_target(target: str,
168173
if isinstance(node, Decorator):
169174
# Decorator expressions will be processed as part of the module top level.
170175
node = node.func
171-
analyzer.refresh_partial(node, [])
176+
analyzer.refresh_partial(node, [], final_iteration)
172177
if analyzer.deferred:
173178
return [target], analyzer.incomplete
174179
else:

mypy/newsemanal/typeanal.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression,
2424
IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED,
2525
ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr,
26-
Decorator, ImportedName, TypeAlias, MypyFile, PlaceholderTypeInfo
26+
Decorator, ImportedName, TypeAlias, MypyFile, PlaceholderNode
2727
)
2828
from mypy.tvar_scope import TypeVarScope
2929
from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
@@ -214,12 +214,18 @@ def visit_unbound_type_nonoptional(self, t: UnboundType) -> Type:
214214
#
215215
# TODO: Remove this special case.
216216
return AnyType(TypeOfAny.implementation_artifact)
217-
if isinstance(node, PlaceholderTypeInfo):
218-
if self.allow_placeholder:
219-
self.api.defer()
217+
if isinstance(node, PlaceholderNode):
218+
if node.becomes_typeinfo:
219+
# Reference to placeholder type.
220+
if self.allow_placeholder:
221+
self.api.defer()
222+
else:
223+
self.api.record_incomplete_ref()
224+
return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
220225
else:
226+
# Reference to an unknown placeholder node.
221227
self.api.record_incomplete_ref()
222-
return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
228+
return AnyType(TypeOfAny.special_form)
223229
if node is None:
224230
# UNBOUND_IMPORTED can happen if an unknown name was imported.
225231
if sym.kind != UNBOUND_IMPORTED:
@@ -545,7 +551,7 @@ def visit_forwardref_type(self, t: ForwardRef) -> Type:
545551

546552
def visit_placeholder_type(self, t: PlaceholderType) -> Type:
547553
n = self.api.lookup_fully_qualified(t.fullname)
548-
if isinstance(n.node, PlaceholderTypeInfo):
554+
if isinstance(n.node, PlaceholderNode):
549555
self.api.defer() # Still incomplete
550556
return t
551557
else:

mypy/nodes.py

Lines changed: 45 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2668,30 +2668,58 @@ def deserialize(cls, data: JsonDict) -> 'TypeAlias':
26682668
no_args=no_args, normalized=normalized)
26692669

26702670

2671-
class PlaceholderTypeInfo(SymbolNode):
2672-
"""Temporary node that will later become a type but is incomplete.
2671+
class PlaceholderNode(SymbolNode):
2672+
"""Temporary symbol node that will later become a real SymbolNode.
26732673
26742674
These are only present during semantic analysis when using the new
2675-
semantic analyzer. These are created if some dependencies of a type
2676-
definition are not yet complete. They are used to create
2677-
PlaceholderType instances for types that refer to incomplete types.
2678-
Example where this can be used:
2675+
semantic analyzer. These are created if some essential dependencies
2676+
of a definition are not yet complete.
2677+
2678+
A typical use is for names imported from a module which is still
2679+
incomplete (within an import cycle):
2680+
2681+
from m import f # Initially may create PlaceholderNode
2682+
2683+
This is particularly important if the imported shadows a name from
2684+
an enclosing scope or builtins:
2685+
2686+
from m import int # Placeholder avoids mixups with builtins.int
2687+
2688+
Another case where this is useful is when there is another definition
2689+
or assignment:
2690+
2691+
from m import f
2692+
def f() -> None: ...
2693+
2694+
In the above example, the presence of PlaceholderNode allows us to
2695+
handle the second definition as a redefinition.
2696+
2697+
They are also used to create PlaceholderType instances for types
2698+
that refer to incomplete types. Example:
26792699
26802700
class C(Sequence[C]): ...
26812701
2682-
We create a PlaceholderTypeInfo for C so that the type C in
2683-
Sequence[C] can be bound. (The long-term purpose of placeholder
2684-
types is to evolve into something that can support general
2685-
recursive types. The base class use case could be supported
2686-
through other means as well.)
2702+
We create a PlaceholderNode (with becomes_typeinfo=True) for C so
2703+
that the type C in Sequence[C] can be bound.
2704+
2705+
Attributes:
2706+
2707+
fullname: Full name of of the PlaceholderNode.
2708+
node: AST node that contains the definition that caused this to
2709+
be created. This is only useful for debugging.
2710+
becomes_typeinfo: If True, this refers something that will later
2711+
become a TypeInfo. It can't be used with type variables, in
2712+
particular, as this would cause issues with class type variable
2713+
detection.
26872714
2688-
PlaceholderTypeInfo can only refer to something that will become
2689-
a TypeInfo. It can't be used with type variables, in particular,
2690-
as this would cause issues with class type variable detection.
2715+
The long-term purpose of placeholder nodes/types is to evolve into
2716+
something that can support general recursive types.
26912717
"""
26922718

2693-
def __init__(self, fullname: str) -> None:
2719+
def __init__(self, fullname: str, node: Node, becomes_typeinfo: bool = False) -> None:
26942720
self._fullname = fullname
2721+
self.node = node
2722+
self.becomes_typeinfo = becomes_typeinfo
26952723
self.line = -1
26962724

26972725
def name(self) -> str:
@@ -2701,10 +2729,10 @@ def fullname(self) -> str:
27012729
return self._fullname
27022730

27032731
def serialize(self) -> JsonDict:
2704-
assert False, "PlaceholderTypeInfo can't be serialized"
2732+
assert False, "PlaceholderNode can't be serialized"
27052733

27062734
def accept(self, visitor: NodeVisitor[T]) -> T:
2707-
return visitor.visit_placeholder_type_info(self)
2735+
return visitor.visit_placeholder_node(self)
27082736

27092737

27102738
class SymbolTableNode:

mypy/visitor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ def visit_var(self, o: 'mypy.nodes.Var') -> T:
358358
def visit_type_alias(self, o: 'mypy.nodes.TypeAlias') -> T:
359359
pass
360360

361-
def visit_placeholder_type_info(self, o: 'mypy.nodes.PlaceholderTypeInfo') -> T:
361+
def visit_placeholder_node(self, o: 'mypy.nodes.PlaceholderNode') -> T:
362362
pass
363363

364364
# Statements

0 commit comments

Comments
 (0)