Issue python#3816

This PR does the following:
- Fixes failing tests from PR 6792
- Adds suugested annotation note
- Adds annotation checker for sets
- Adds tests

mypy/build.py

@@ -908,7 +908,7 @@ def write_deps_cache(rdeps: Dict[str, Dict[str, Set[str]]],
             hash = st.meta.hash
         meta_snapshot[id] = hash
 
-    meta = {'snapshot': meta_snapshot, 'deps_meta': fg_deps_meta}
+    meta = {'snapshot': meta_snapshot, 'deps_meta': fg_deps_meta}  # type: Dict[str, object]
 
     if not metastore.write(DEPS_META_FILE, json.dumps(meta)):
         manager.log("Error writing fine-grained deps meta JSON file {}".format(DEPS_META_FILE))

mypy/checker.py

@@ -15,7 +15,7 @@
     SymbolTable, Statement, MypyFile, Var, Expression, Lvalue, Node,
     OverloadedFuncDef, FuncDef, FuncItem, FuncBase, TypeInfo,
     ClassDef, Block, AssignmentStmt, NameExpr, MemberExpr, IndexExpr,
-    TupleExpr, ListExpr, ExpressionStmt, ReturnStmt, IfStmt,
+    TupleExpr, ListExpr, DictExpr, SetExpr, ExpressionStmt, ReturnStmt, IfStmt,
     WhileStmt, OperatorAssignmentStmt, WithStmt, AssertStmt,
     RaiseStmt, TryStmt, ForStmt, DelStmt, CallExpr, IntExpr, StrExpr,
     UnicodeExpr, OpExpr, UnaryExpr, LambdaExpr, TempNode, SymbolTableNode,
@@ -2146,8 +2146,119 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type
                 rvalue_type = self.expr_checker.accept(rvalue)
                 if not inferred.is_final:
                     rvalue_type = remove_instance_last_known_values(rvalue_type)
+                    self.check_forbidden_inference_types(inferred, rvalue, rvalue_type)
                 self.infer_variable_type(inferred, lvalue, rvalue_type, rvalue)
 
+    def check_forbidden_inference_types(self, var: Var, rvalue: Context,
+                                        rvalue_type: Type) -> None:
+        """Check for any list or dict literals that are inferred ambiguously and require annotation.
+        List and Dict literals with multiple object types require annotation:
+        >>> x = [1, 'a']  # Error
+        >>> x = {'a': 1, 'b': 'c'}  # Error
+        >>> x: List[object] = [1, 'a']  # OK
+        Unannotated List and Dict literals that contain an object of type `object` are OK:
+        >>> x = [1, 'a', object()]  # OK
+        Dict keys may be inferred to type `object` without annotation:
+        >>> x = ['a': 1, 2: 2]  # OK
+        """
+        if not self.scope.top_function() and self.options.allow_untyped_globals:
+            return
+
+        rvalue_type = get_proper_type(rvalue_type)
+        if isinstance(rvalue_type, Instance):
+            if isinstance(rvalue, ListExpr):
+                list_arg_type = rvalue_type.args[0]
+                list_arg_type = get_proper_type(list_arg_type)
+                if isinstance(list_arg_type, Instance):
+                    if list_arg_type.type.fullname != 'builtins.object':
+                        # This list already has a specific type, no further checks needed
+                        return
+
+                    if self.iterable_has_object_type(rvalue.items):
+                        # A list that contains an an instance must
+                        # necessarily be of type List[object], since
+                        # there is no more specific type that can be
+                        # assigned
+                        return
+
+                    list_arg_type_names = self.get_arg_type_names(rvalue.items)
+                    if list_arg_type_names and len(list_arg_type_names) > 1:
+                        msg = 'Suggested annotation: List[Any] or List[Union[' + ', '.join(
+                            sorted(list_arg_type_names)) + ']]'
+                    else:
+                        msg = 'Suggested annotation: List[Any]'
+                    self.msg.need_annotation_for_var(var, rvalue)
+                    self.msg.note(msg, context=rvalue, code=codes.OVERRIDE)
+
+            elif isinstance(rvalue, SetExpr):
+                set_arg_type = rvalue_type.args[0]
+                set_arg_type = get_proper_type(set_arg_type)
+                if isinstance(set_arg_type, Instance):
+                    if set_arg_type.type.fullname != 'builtins.object':
+                        return
+
+                    if self.iterable_has_object_type(rvalue.items):
+                        return
+
+                    set_arg_type_names = self.get_arg_type_names(rvalue.items)
+                    if set_arg_type_names and len(set_arg_type_names) > 1:
+                        msg = 'Suggested annotation: Set[Any] or Set[Union[' + ', '.join(
+                            sorted(set_arg_type_names)) + ']]'
+                    else:
+                        msg = 'Suggested annotation: Set[Any]'
+                    self.msg.need_annotation_for_var(var, rvalue)
+                    self.msg.note(msg, context=rvalue, code=codes.OVERRIDE)
+
+            elif isinstance(rvalue, DictExpr):
+                dict_arg_type = rvalue_type.args[1]
+                dict_arg_type = get_proper_type(dict_arg_type)
+                if isinstance(dict_arg_type, Instance):
+                    if dict_arg_type.type.fullname != 'builtins.object':
+                        # The values in this dict have a specific type, no further checks
+                        return
+
+                    if self.iterable_has_object_type([v for k, v in rvalue.items]):
+                        return
+
+                    dict_items_type_names = self.get_arg_type_names(
+                        [v for k, v in rvalue.items]
+                    )
+                    dict_key = get_proper_type(rvalue_type.args[0])
+                    dict_keys_type_name = 'Any'
+                    if isinstance(dict_key, Instance):
+                        dict_keys_type_name = dict_key.type.name
+                    if dict_items_type_names and len(dict_items_type_names) > 1:
+                        msg = 'Suggested annotation: Dict[{}, Any] or Dict[{}, Union['.format(
+                            dict_keys_type_name, dict_keys_type_name) + ', '.join(
+                            sorted(dict_items_type_names)) + ']]'
+                    else:
+                        msg = 'Suggested annotation: Dict[{}, Any]'.format(
+                            dict_keys_type_name)
+                    self.msg.need_annotation_for_var(var, rvalue)
+                    self.msg.note(msg, context=rvalue, code=codes.OVERRIDE)
+
+    def iterable_has_object_type(self, items: Iterable[Expression]) -> bool:
+        """Check for any instance that has type object
+        """
+        for i in items:
+            i_type = self.type_map[i]
+            i_type = get_proper_type(i_type)
+            if isinstance(i_type, Instance):
+                if i_type.type.fullname == 'builtins.object':
+                    return True
+        return False
+
+    def get_arg_type_names(self, items: Iterable[Expression]) -> Set[str]:
+        """Returns a set of names of the type of args in an iterable
+        """
+        arg_type_names = set()
+        for i in items:
+            i_type = self.type_map[i]
+            i_type = get_proper_type(i_type)
+            if isinstance(i_type, Instance):
+                arg_type_names.add(i_type.type.name)
+        return arg_type_names
+
     # (type, operator) tuples for augmented assignments supported with partial types
     partial_type_augmented_ops = {
         ('builtins.list', '+'),

mypy/strconv.py

@@ -153,7 +153,7 @@ def visit_overloaded_func_def(self, o: 'mypy.nodes.OverloadedFuncDef') -> str:
         return self.dump(a, o)
 
     def visit_class_def(self, o: 'mypy.nodes.ClassDef') -> str:
-        a = [o.name, o.defs.body]
+        a = [o.name, o.defs.body]  # type: List[object]
         # Display base types unless they are implicitly just builtins.object
         # (in this case base_type_exprs is empty).
         if o.base_type_exprs:

mypy/suggestions.py

@@ -670,7 +670,7 @@ def json_suggestion(self, mod: str, func_name: str, node: FuncDef,
             'path': path,
             'func_name': func_name,
             'samples': 0
-        }
+            }  # type: Dict[str, object]
         return json.dumps([obj], sort_keys=True)
 
     def pyannotate_signature(

test-data/stdlib-samples/3.2/test/test_pprint.py

@@ -6,7 +6,7 @@
 import collections
 import itertools
 
-from typing import List, Any, Dict, Tuple, cast, Callable
+from typing import List, Any, Dict, Tuple, cast, Callable, Union
 
 # list, tuple and dict subclasses that do or don't overwrite __repr__
 class list2(list):
@@ -175,7 +175,7 @@ def test_basic_line_wrap(self) -> None:
     def test_nested_indentations(self) -> None:
         o1 = list(range(10))
         o2 = {'first':1, 'second':2, 'third':3}
-        o = [o1, o2]
+        o = [o1, o2]  # type: List[object]
         expected = """\
 [   [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
     {   'first': 1,
@@ -439,7 +439,7 @@ def test_set_reprs(self) -> None:
     def test_depth(self) -> None:
         nested_tuple = (1, (2, (3, (4, (5, 6)))))
         nested_dict = {1: {2: {3: {4: {5: {6: 6}}}}}}
-        nested_list = [1, [2, [3, [4, [5, [6, []]]]]]]
+        nested_list = [1, [2, [3, [4, [5, [6, []]]]]]]  # type: List[Union[int, List]]
         self.assertEqual(pprint.pformat(nested_tuple), repr(nested_tuple))
         self.assertEqual(pprint.pformat(nested_dict), repr(nested_dict))
         self.assertEqual(pprint.pformat(nested_list), repr(nested_list))

test-data/unit/check-classes.test

@@ -6394,6 +6394,7 @@ class Child(Base):  # E: Too few arguments for "__init_subclass__" of "Base"
 [builtins fixtures/object_with_init_subclass.pyi]
 
 [case testInitSubclassTooFewArgs2]
+from typing import Dict, Union
 class Base:
     default_name: str
 
@@ -6402,7 +6403,7 @@ class Base:
         cls.default_name = default_name
         return
 # TODO implement this, so that no error is raised?
-d = {"default_name": "abc", "thing": 0}
+d: Dict[str, Union[str, int]] = {"default_name": "abc", "thing": 0}
 class Child(Base, **d):  # E: Too few arguments for "__init_subclass__" of "Base"
     pass
 [builtins fixtures/object_with_init_subclass.pyi]

test-data/unit/check-functions.test

@@ -2421,16 +2421,18 @@ def g() -> List[Union[str, int]]:
 [case testInferredTypeIsObjectMismatch]
 from typing import Union, Dict, List
 def f() -> Dict[str, Union[str, int]]:
-    x = {'a': 'a', 'b': 2}
-    return x # E: Incompatible return value type (got "Dict[str, object]", expected "Dict[str, Union[str, int]]")
+    x = {'a': 'a', 'b': 2}  # E: Need type annotation for 'x' \
+                            # N: Suggested annotation: Dict[str, Any] or Dict[str, Union[int, str]]
+    return x  # E: Incompatible return value type (got "Dict[str, object]", expected "Dict[str, Union[str, int]]")
 
 def g() -> Dict[str, Union[str, int]]:
     x: Dict[str, Union[str, int]] = {'a': 'a', 'b': 2}
     return x
 
 def h() -> List[Union[str, int]]:
-    x = ['a', 2]
-    return x # E: Incompatible return value type (got "List[object]", expected "List[Union[str, int]]")
+    x = ['a', 2]  # E: Need type annotation for 'x' \
+                  # N: Suggested annotation: List[Any] or List[Union[int, str]]
+    return x  # E: Incompatible return value type (got "List[object]", expected "List[Union[str, int]]")
 
 def i() -> List[Union[str, int]]:
     x: List[Union[str, int]] = ['a', 2]

test-data/unit/check-incremental.test

@@ -584,7 +584,8 @@ my_dict = {
 }
 
 [file mod1_private.py.2]
-my_dict = {
+from typing import Dict, List, Union
+my_dict: Dict[str, List[Union[int, str]]] = {
     'a': [1, 2, 3],
     'b': [4, 5, 'a']
 }

test-data/unit/check-inference.test

@@ -3176,3 +3176,27 @@ reveal_type(f(lambda: 1))  # N: Revealed type is 'builtins.int*'
 def g() -> None: pass
 
 reveal_type(f(g))  # N: Revealed type is 'None'
+
+-- Tests for checking if annotation is needed
+-- --------------
+
+[case testexplicitAnnotationNeededForLists]
+x = [1, 'two', 3.5]  # E: Need type annotation for 'x' \
+                     # N: Suggested annotation: List[Any] or List[Union[float, int, str]]
+
+y = [2, 'three', object()]
+
+class C:
+    def __init__(self) -> None:
+        self.x = [object()]
+
+c: C
+c.x = [1, 'two']
+[builtins fixtures/list.pyi]
+
+[case testexplicitAnnotationNeededForDict]
+x = {'a': 1, 'b': 'two'}  # E: Need type annotation for 'x' \
+                          # N: Suggested annotation: Dict[str, Any] or Dict[str, Union[int, str]]
+
+y = {'a': 1, 'two': 2}
+[builtins fixtures/dict.pyi]

test-data/unit/check-literal.test

@@ -1475,7 +1475,8 @@ b = [1, 1, 1]
 c: List[Literal[1, 2, 3]] = [1, 2, 3]
 d = [1, 2, 3]
 e: List[Literal[1, "x"]] = [1, "x"]
-f = [1, "x"]
+f = [1, "x"] # E: Need type annotation for 'f' \
+             # N: Suggested annotation: List[Any] or List[Union[int, str]]
 g: List[List[List[Literal[1, 2, 3]]]] = [[[1, 2, 3], [3]]]
 h: List[Literal[1]] = []
 
@@ -1495,7 +1496,8 @@ lit3: Literal["foo"]
 arr1 = [lit1, lit1, lit1]
 arr2 = [lit1, lit2]
 arr3 = [lit1, 4, 5]
-arr4 = [lit1, lit2, lit3]
+arr4 = [lit1, lit2, lit3] # E: Need type annotation for 'arr4' \
+                          # N: Suggested annotation: List[Any]
 arr5 = [object(), lit1]
 
 reveal_type(arr1)  # N: Revealed type is 'builtins.list[Literal[1]]'

test-data/unit/check-typeddict.test

@@ -527,8 +527,10 @@ from typing import Mapping
 Cell = TypedDict('Cell', {'value': int})
 left = Cell(value=42)
 right = 42
-joined1 = [left, right]
-joined2 = [right, left]
+joined1 = [left, right]  # E: Need type annotation for 'joined1' \
+                         # N: Suggested annotation: List[Any]
+joined2 = [right, left]  # E: Need type annotation for 'joined2' \
+                         # N: Suggested annotation: List[Any]
 reveal_type(joined1)  # N: Revealed type is 'builtins.list[builtins.object*]'
 reveal_type(joined2)  # N: Revealed type is 'builtins.list[builtins.object*]'
 [builtins fixtures/dict.pyi]