Don't turn collections containing None into partial types (#2289)

This means mypy will no longer give Need type annotation for variable errors for assignments with values of type Generator[str, None, None] or List[None] or similar. However, lists, sets, and dicts that are initialized with None values that were previously inferred may now need type annotations. I.e. constructs that look like:

x = [None]
x.append(0)

will now be type errors unless they have explicit annotations.

Supplants #2225. Fixes #2246. Fixes #2195. Fixes #2258.

Author: ddfisher

mypy/checker.py

@@ -2698,22 +2698,38 @@ def infer_operator_assignment_method(type: Type, operator: str) -> Tuple[bool, s
 def is_valid_inferred_type(typ: Type) -> bool:
     """Is an inferred type valid?
 
-    Examples of invalid types include the None type or a type with a None component.
+    Examples of invalid types include the None type or List[<uninhabited>].
+
+    When not doing strict Optional checking, all types containing None are
+    invalid.  When doing strict Optional checking, only None and types that are
+    incompletely defined (i.e. contain UninhabitedType) are invalid.
     """
     if is_same_type(typ, NoneTyp()):
         # With strict Optional checking, we *may* eventually infer NoneTyp, but
         # we only do that if we can't infer a specific Optional type.  This
         # resolution happens in leave_partial_types when we pop a partial types
         # scope.
         return False
+    return is_valid_inferred_type_component(typ)
+
+
+def is_valid_inferred_type_component(typ: Type) -> bool:
+    """Is this part of a type a valid inferred type?
+
+    In strict Optional mode this excludes bare None types, as otherwise every
+    type containing None would be invalid.
+    """
+    if not experiments.STRICT_OPTIONAL:
+        if is_same_type(typ, NoneTyp()):
+            return False
     if is_same_type(typ, UninhabitedType()):
         return False
     elif isinstance(typ, Instance):
         for arg in typ.args:
-            if not is_valid_inferred_type(arg):
+            if not is_valid_inferred_type_component(arg):
                 return False
     elif isinstance(typ, TupleType):
         for item in typ.items:
-            if not is_valid_inferred_type(item):
+            if not is_valid_inferred_type_component(item):
                 return False
     return True

test-data/unit/check-optional.test

@@ -167,8 +167,7 @@ reveal_type(x)  # E: Revealed type is 'Union[builtins.int, builtins.None]'
 
 [case testInferOptionalListType]
 x = [None]
-x.append(1)
-reveal_type(x)  # E: Revealed type is 'builtins.list[Union[builtins.int, builtins.None]]'
+x.append(1)  # E: Argument 1 to "append" of "list" has incompatible type "int"; expected None
 [builtins fixtures/list.pyi]
 
 [case testInferNonOptionalListType]
@@ -180,8 +179,10 @@ x()  # E: List[int] not callable
 [case testInferOptionalDictKeyValueTypes]
 x = {None: None}
 x["bar"] = 1
-reveal_type(x)  # E: Revealed type is 'builtins.dict[Union[builtins.str, builtins.None], Union[builtins.int, builtins.None]]'
 [builtins fixtures/dict.pyi]
+[out]
+main:2: error: Invalid index type "str" for "dict"
+main:2: error: Incompatible types in assignment (expression has type "int", target has type None)
 
 [case testInferNonOptionalDictType]
 x = {}
@@ -437,3 +438,16 @@ def g() -> Dict[None, None]:
 [case testRaiseFromNone]
 raise BaseException from None
 [builtins fixtures/exception.pyi]
+
+[case testOptionalNonPartialTypeWithNone]
+from typing import Generator
+def f() -> Generator[str, None, None]: pass
+x = f()
+reveal_type(x)  # E: Revealed type is 'typing.Generator[builtins.str, builtins.None, builtins.None]'
+l = [f()]
+reveal_type(l)  # E: Revealed type is 'builtins.list[typing.Generator*[builtins.str, builtins.None, builtins.None]]'
+[builtins fixtures/list.pyi]
+
+[case testNoneListTernary]
+x = [None] if "" else [1]  # E: List item 0 has incompatible type "int"
+[builtins fixtures/list.pyi]