Properly handle exceptions in multiprocessing tasks

Fix pytest-dev#1984

CHANGELOG.rst

@@ -6,7 +6,7 @@
 * Import errors when collecting test modules now display the full traceback (`#1976`_).
   Thanks `@cwitty`_ for the report and `@nicoddemus`_ for the PR.
 
-* Fix confusing command-line help message for custom options with two or more `metavar` properties (`#2004`_).
+* Fix confusing command-line help message for custom options with two or more ``metavar`` properties (`#2004`_).
   Thanks `@okulynyak`_ and `@davehunt`_ for the report and `@nicoddemus`_ for the PR.
 
 * When loading plugins, import errors which contain non-ascii messages are now properly handled in Python 2 (`#1998`_).
@@ -19,15 +19,18 @@
 
 *
 
-*
+* Properly handle exceptions in ``multiprocessing`` tasks (`#1984`_).
+  Thanks `@adborden`_ for the report and `@nicoddemus`_ for the PR.
 
 *
 
 
+.. _@adborden: https://github.com/adborden
 .. _@cwitty: https://github.com/cwitty
 .. _@okulynyak: https://github.com/okulynyak
 
 .. _#1976: https://github.com/pytest-dev/pytest/issues/1976
+.. _#1984: https://github.com/pytest-dev/pytest/issues/1984
 .. _#1998: https://github.com/pytest-dev/pytest/issues/1998
 .. _#2004: https://github.com/pytest-dev/pytest/issues/2004
 .. _#2005: https://github.com/pytest-dev/pytest/issues/2005

_pytest/_code/code.py

@@ -623,16 +623,23 @@ def repr_excinfo(self, excinfo):
             e = excinfo.value
             descr = None
             while e is not None:
-                reprtraceback = self.repr_traceback(excinfo)
-                reprcrash = excinfo._getreprcrash()
+                if excinfo:
+                    reprtraceback = self.repr_traceback(excinfo)
+                    reprcrash = excinfo._getreprcrash()
+                else:
+                    # fallback to native repr if the exception doesn't have a traceback:
+                    # ExceptionInfo objects require a full traceback to work
+                    reprtraceback = ReprTracebackNative(py.std.traceback.format_exception(type(e), e, None))
+                    reprcrash = None
+
                 repr_chain += [(reprtraceback, reprcrash, descr)]
                 if e.__cause__ is not None:
                     e = e.__cause__
-                    excinfo = ExceptionInfo((type(e), e, e.__traceback__))
+                    excinfo = ExceptionInfo((type(e), e, e.__traceback__)) if e.__traceback__ else None
                     descr = 'The above exception was the direct cause of the following exception:'
                 elif e.__context__ is not None:
                     e = e.__context__
-                    excinfo = ExceptionInfo((type(e), e, e.__traceback__))
+                    excinfo = ExceptionInfo((type(e), e, e.__traceback__)) if e.__traceback__ else None
                     descr = 'During handling of the above exception, another exception occurred:'
                 else:
                     e = None

testing/code/test_excinfo.py

@@ -1050,6 +1050,49 @@ def h():
         assert line.endswith('mod.py')
         assert tw.lines[47] == ":15: AttributeError"
 
+    @pytest.mark.skipif("sys.version_info[0] < 3")
+    @pytest.mark.parametrize('reason, description', [
+        ('cause', 'The above exception was the direct cause of the following exception:'),
+        ('context', 'During handling of the above exception, another exception occurred:'),
+    ])
+    def test_exc_chain_repr_without_traceback(self, importasmod, reason, description):
+        """
+        Handle representation of exception chains where one of the exceptions doesn't have a
+        real traceback, such as those raised in a subprocess submitted by the multiprocessing
+        module (#1984).
+        """
+        from _pytest.pytester import LineMatcher
+        exc_handling_code = ' from e' if reason == 'cause' else ''
+        mod = importasmod("""
+            def f():
+                try:
+                    g()
+                except Exception as e:
+                    raise RuntimeError('runtime problem'){exc_handling_code}
+            def g():
+                raise ValueError('invalid value')
+        """.format(exc_handling_code=exc_handling_code))
+
+        with pytest.raises(RuntimeError) as excinfo:
+            mod.f()
+
+        attr = '__%s__' % reason
+        getattr(excinfo.value, attr).__traceback__ = None
+
+        r = excinfo.getrepr()
+        tw = py.io.TerminalWriter(stringio=True)
+        tw.hasmarkup = False
+        r.toterminal(tw)
+
+        matcher = LineMatcher(tw.stringio.getvalue().splitlines())
+        matcher.fnmatch_lines([
+            "ValueError: invalid value",
+            description,
+            "* except Exception as e:",
+            "> * raise RuntimeError('runtime problem')" + exc_handling_code,
+            "E *RuntimeError: runtime problem",
+        ])
+
 
 @pytest.mark.parametrize("style", ["short", "long"])
 @pytest.mark.parametrize("encoding", [None, "utf8", "utf16"])

testing/test_assertion.py

@@ -749,6 +749,37 @@ def test_onefails():
         "*test_traceback_failure.py:4: AssertionError"
     ])
 
+
+@pytest.mark.skipif(sys.version_info[:2] <= (3, 3), reason='Python 3.4+ shows chained exceptions on multiprocess')
+def test_exception_handling_no_traceback(testdir):
+    """
+    Handle chain exceptions in tasks submitted by the multiprocess module (#1984).
+    """
+    p1 = testdir.makepyfile("""
+        from multiprocessing import Pool
+
+        def process_task(n):
+            assert n == 10
+
+        def multitask_job():
+            tasks = [1]
+            with Pool(processes=1) as pool:
+                pool.map(process_task, tasks)
+
+        def test_multitask_job():
+            multitask_job()
+    """)
+    result = testdir.runpytest(p1, "--tb=long")
+    result.stdout.fnmatch_lines([
+        "====* FAILURES *====",
+        "*multiprocessing.pool.RemoteTraceback:*",
+        "Traceback (most recent call last):",
+        "*assert n == 10",
+        "The above exception was the direct cause of the following exception:",
+        "> * multitask_job()",
+    ])
+
+
 @pytest.mark.skipif("'__pypy__' in sys.builtin_module_names or sys.platform.startswith('java')" )
 def test_warn_missing(testdir):
     testdir.makepyfile("")