Generator fixup

mypy/checkexpr.py

@@ -1192,12 +1192,6 @@ def visit_dict_expr(self, e: DictExpr) -> Type:
                                args,
                                [nodes.ARG_POS] * len(args), e)[0]
 
-    def visit_yield_expr(self, e: YieldExpr) -> Type:
-        # TODO: Implement proper type checking of yield expressions.
-        if e.expr:
-            self.accept(e.expr)
-        return AnyType()
-
     def visit_func_expr(self, e: FuncExpr) -> Type:
         """Type check lambda expression."""
         inferred_type = self.infer_lambda_type_using_context(e)

mypy/messages.py

@@ -29,10 +29,11 @@
 BOOLEAN_EXPECTED_FOR_NOT = 'Boolean value expected for not operand'
 INVALID_EXCEPTION = 'Exception must be derived from BaseException'
 INVALID_EXCEPTION_TYPE = 'Exception type must be derived from BaseException'
-INVALID_RETURN_TYPE_FOR_YIELD = \
+INVALID_RETURN_TYPE_FOR_GENERATOR = \
     'The return type of a generator function should be "Generator" or one of its supertypes'
-INVALID_RETURN_TYPE_FOR_YIELD_FROM = \
-    'Iterable function return type expected for "yield from"'
+INVALID_GENERATOR_RETURN_ITEM_TYPE = \
+    'The return type of a generator function must be None in its third type parameter in Python 2'
+YIELD_VALUE_EXPECTED = 'Yield value expected'
 INCOMPATIBLE_TYPES = 'Incompatible types'
 INCOMPATIBLE_TYPES_IN_ASSIGNMENT = 'Incompatible types in assignment'
 INCOMPATIBLE_REDEFINITION = 'Incompatible redefinition'

mypy/nodes.py

@@ -323,7 +323,6 @@ class FuncItem(FuncBase):
     # Is this an overload variant of function with more than one overload variant?
     is_overload = False
     is_generator = False   # Contains a yield statement?
-    is_coroutine = False   # Contains @coroutine or yield from Future
     is_static = False      # Uses @staticmethod?
     is_class = False       # Uses @classmethod?
     # Variants of function with type variables with values expanded
@@ -634,26 +633,6 @@ def accept(self, visitor: NodeVisitor[T]) -> T:
         return visitor.visit_assert_stmt(self)
 
 
-class YieldStmt(Node):
-    expr = None  # type: Node
-
-    def __init__(self, expr: Node) -> None:
-        self.expr = expr
-
-    def accept(self, visitor: NodeVisitor[T]) -> T:
-        return visitor.visit_yield_stmt(self)
-
-
-class YieldFromStmt(Node):
-    expr = None  # type: Node
-
-    def __init__(self, expr: Node) -> None:
-        self.expr = expr
-
-    def accept(self, visitor: NodeVisitor[T]) -> T:
-        return visitor.visit_yield_from_stmt(self)
-
-
 class DelStmt(Node):
     expr = None  # type: Node
 

mypy/parse.py

@@ -19,13 +19,13 @@
     MypyFile, Import, Node, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef,
     ClassDef, Decorator, Block, Var, OperatorAssignmentStmt,
     ExpressionStmt, AssignmentStmt, ReturnStmt, RaiseStmt, AssertStmt,
-    YieldStmt, DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl,
+    DelStmt, BreakStmt, ContinueStmt, PassStmt, GlobalDecl,
     WhileStmt, ForStmt, IfStmt, TryStmt, WithStmt, CastExpr,
     TupleExpr, GeneratorExpr, ListComprehension, ListExpr, ConditionalExpr,
     DictExpr, SetExpr, NameExpr, IntExpr, StrExpr, BytesExpr, UnicodeExpr,
     FloatExpr, CallExpr, SuperExpr, MemberExpr, IndexExpr, SliceExpr, OpExpr,
     UnaryExpr, FuncExpr, TypeApplication, PrintStmt, ImportBase, ComparisonExpr,
-    StarExpr, YieldFromStmt, YieldFromExpr, NonlocalDecl, DictionaryComprehension,
+    StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension,
     SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, ExecStmt, Argument,
     BackquoteExpr
 )
@@ -879,8 +879,6 @@ def parse_statement(self) -> Tuple[Node, bool]:
             stmt = self.parse_nonlocal_decl()
         elif ts == 'assert':
             stmt = self.parse_assert_stmt()
-        elif ts == 'yield':
-            stmt = self.parse_yield_stmt()
         elif ts == 'del':
             stmt = self.parse_del_stmt()
         elif ts == 'with':
@@ -963,34 +961,21 @@ def parse_assert_stmt(self) -> AssertStmt:
         node = AssertStmt(expr)
         return node
 
-    def parse_yield_stmt(self) -> Union[YieldStmt, YieldFromStmt]:
-        self.expect('yield')
+    def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]:
+        self.expect("yield")
         expr = None
-        node = YieldStmt(expr)
+        node = YieldExpr(expr)  # type: Union[YieldFromExpr, YieldExpr]
         if not isinstance(self.current(), Break):
             if self.current_str() == "from":
                 self.expect("from")
-                expr = self.parse_expression()  # Here comes when yield from is not assigned
-                node_from = YieldFromStmt(expr)
-                return node_from  # return here, we've gotted the type
+                expr = self.parse_expression()  # when yield from is assigned to a variable
+                node = YieldFromExpr(expr)
             else:
-                expr = self.parse_expression()
-                node = YieldStmt(expr)
-        return node
-
-    def parse_yield_or_yield_from_expr(self) -> Union[YieldFromExpr, YieldExpr]:
-        self.expect("yield")
-        node = None  # type: Union[YieldFromExpr, YieldExpr]
-        if self.current_str() == "from":
-            self.expect("from")
-            expr = self.parse_expression()  # Here comes when yield from is assigned to a variable
-            node = YieldFromExpr(expr)
-        else:
-            if self.current_str() == ')':
-                node = YieldExpr(None)
-            else:
-                expr = self.parse_expression()
-                node = YieldExpr(expr)
+                if self.current_str() == ')':
+                    node = YieldExpr(None)
+                else:
+                    expr = self.parse_expression()
+                    node = YieldExpr(expr)
         return node
 
     def parse_ellipsis(self) -> EllipsisExpr:

mypy/semanal.py

@@ -49,15 +49,15 @@
     ClassDef, Var, GDEF, MODULE_REF, FuncItem, Import,
     ImportFrom, ImportAll, Block, LDEF, NameExpr, MemberExpr,
     IndexExpr, TupleExpr, ListExpr, ExpressionStmt, ReturnStmt,
-    RaiseStmt, YieldStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt,
+    RaiseStmt, AssertStmt, OperatorAssignmentStmt, WhileStmt,
     ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt,
     GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr,
     SliceExpr, CastExpr, TypeApplication, Context, SymbolTable,
     SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr,
     FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr,
     StrExpr, PrintStmt, ConditionalExpr, PromoteExpr,
     ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases,
-    YieldFromStmt, YieldFromExpr, NamedTupleExpr, NonlocalDecl,
+    YieldFromExpr, NamedTupleExpr, NonlocalDecl,
     SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr,
     YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, COVARIANT, CONTRAVARIANT,
     INVARIANT, UNBOUND_IMPORTED
@@ -1447,7 +1447,6 @@ def visit_decorator(self, dec: Decorator) -> None:
                 self.check_decorated_function_is_method('abstractmethod', dec)
             elif refers_to_fullname(d, 'asyncio.tasks.coroutine'):
                 removed.append(i)
-                dec.func.is_coroutine = True
             elif refers_to_fullname(d, 'builtins.staticmethod'):
                 removed.append(i)
                 dec.func.is_static = True
@@ -1504,20 +1503,6 @@ def visit_raise_stmt(self, s: RaiseStmt) -> None:
         if s.from_expr:
             s.from_expr.accept(self)
 
-    def visit_yield_stmt(self, s: YieldStmt) -> None:
-        if not self.is_func_scope():
-            self.fail("'yield' outside function", s)
-        else:
-            self.function_stack[-1].is_generator = True
-        if s.expr:
-            s.expr.accept(self)
-
-    def visit_yield_from_stmt(self, s: YieldFromStmt) -> None:
-        if not self.is_func_scope():
-            self.fail("'yield from' outside function", s)
-        if s.expr:
-            s.expr.accept(self)
-
     def visit_assert_stmt(self, s: AssertStmt) -> None:
         if s.expr:
             s.expr.accept(self)
@@ -1681,6 +1666,8 @@ def visit_star_expr(self, expr: StarExpr) -> None:
     def visit_yield_from_expr(self, e: YieldFromExpr) -> None:
         if not self.is_func_scope():  # not sure
             self.fail("'yield from' outside function", e)
+        else:
+            self.function_stack[-1].is_generator = True
         if e.expr:
             e.expr.accept(self)
 
@@ -1909,6 +1896,10 @@ def visit__promote_expr(self, expr: PromoteExpr) -> None:
         expr.type = self.anal_type(expr.type)
 
     def visit_yield_expr(self, expr: YieldExpr) -> None:
+        if not self.is_func_scope():
+            self.fail("'yield' outside function", expr)
+        else:
+            self.function_stack[-1].is_generator = True
         if expr.expr:
             expr.expr.accept(self)
 

mypy/test/data/check-expressions.test

@@ -1322,8 +1322,10 @@ class str: pass
 def f(x: int) -> None:
     x = yield f('')
     x = 1
+[builtins fixtures/for.py]
 [out]
 main: note: In function "f":
+main:1: error: The return type of a generator function should be "Generator" or one of its supertypes
 main:2: error: Argument 1 to "f" has incompatible type "str"; expected "int"
 
 [case testYieldExpressionWithNone]
@@ -1334,6 +1336,33 @@ def f(x: int) -> Iterator[None]:
 [out]
 
 
+-- Yield from expression
+-- ----------------
+
+
+
+[case testYieldFromIteratorHasNoValue]
+from typing import Iterator
+def f() -> Iterator[int]:
+    yield 5
+def g() -> Iterator[int]:
+    a = yield from f()
+[out]
+main: note: In function "g":
+main:5: error: Function does not return a value
+
+[case testYieldFromGeneratorHasValue]
+from typing import Iterator, Generator
+def f() -> Generator[int, None, str]:
+    yield 5
+    return "ham"
+def g() -> Iterator[int]:
+    a = "string"
+    a = yield from f()
+[out]
+
+
+
 -- dict(x=y, ...)
 -- --------------
 

mypy/test/data/check-statements.test

@@ -51,6 +51,42 @@ def g() -> None:
 [out]
 main: note: In function "g":
 
+[case testReturnInGenerator]
+from typing import Generator
+def f() -> Generator[int, None, str]:
+    yield 1
+    return "foo"
+[out]
+
+[case testEmptyReturnInGenerator]
+from typing import Generator
+def f() -> Generator[int, None, str]:
+    yield 1
+    return  # E: Return value expected
+[out]
+main: note: In function "f":
+
+[case testEmptyReturnInNoneTypedGenerator]
+from typing import Generator
+def f() -> Generator[int, None, None]:
+    yield 1
+    return
+[out]
+
+[case testNonEmptyReturnInNoneTypedGenerator]
+from typing import Generator
+def f() -> Generator[int, None, None]:
+    yield 1
+    return 42  # E: No return value expected
+[out]
+main: note: In function "f":
+
+[case testReturnInIterator]
+from typing import Iterator
+def f() -> Iterator[int]:
+    yield 1
+    return "foo"
+[out]
 
 -- If statement
 -- ------------
@@ -675,8 +711,8 @@ def f() -> Any:
 
 [case testYieldInFunctionReturningFunction]
 from typing import Callable
-def f() -> Callable[[], None]:
-    yield object() # E: The return type of a generator function should be "Generator" or one of its supertypes
+def f() -> Callable[[], None]: # E: The return type of a generator function should be "Generator" or one of its supertypes
+    yield object()
 [out]
 main: note: In function "f":
 
@@ -687,8 +723,8 @@ def f():
 
 [case testWithInvalidInstanceReturnType]
 import typing
-def f() -> int:
-    yield 1 # E: The return type of a generator function should be "Generator" or one of its supertypes
+def f() -> int: # E: The return type of a generator function should be "Generator" or one of its supertypes
+    yield 1
 [builtins fixtures/for.py]
 [out]
 main: note: In function "f":
@@ -715,6 +751,22 @@ def f() -> Iterator[None]:
     yield
 [builtins fixtures/for.py]
 
+[case testYieldWithNoValueWhenValueRequired]
+from typing import Iterator
+def f() -> Iterator[int]:
+    yield  # E: Yield value expected
+[builtins fixtures/for.py]
+[out]
+main: note: In function "f":
+
+[case testYieldWithExplicitNone]
+from typing import Iterator
+def f() -> Iterator[None]:
+    yield None  # E: Incompatible types in yield (actual type None, expected type None)
+[builtins fixtures/for.py]
+[out]
+main: note: In function "f":
+
 
 -- Yield from statement
 -- --------------------
@@ -746,17 +798,17 @@ def f() -> Any:
 from typing import Iterator, Callable
 def g() -> Iterator[int]:
     yield 42
-def f() -> Callable[[], None]:
-    yield from g()  # E: Iterable function return type expected for "yield from"
+def f() -> Callable[[], None]:  # E: The return type of a generator function should be "Generator" or one of its supertypes
+    yield from g()
 [out]
 main: note: In function "f":
 
 [case testYieldFromNotIterableReturnType]
 from typing import Iterator
 def g() -> Iterator[int]:
     yield 42
-def f() -> int:
-    yield from g()  # E: Iterable function return type expected for "yield from"
+def f() -> int:  # E: The return type of a generator function should be "Generator" or one of its supertypes
+    yield from g()
 [out]
 main: note: In function "f":
 

mypy/test/data/lib-stub/typing.py

@@ -53,6 +53,9 @@ def throw(self, typ: Any, val: Any=None, tb=None) -> None: pass
     @abstractmethod
     def close(self) -> None: pass
 
+    @abstractmethod
+    def __iter__(self) -> 'Generator[T, U, V]': pass
+
 class Sequence(Generic[T]):
     @abstractmethod
     def __getitem__(self, n: Any) -> T: pass