ENH: Continue running doctests on failure

CHANGES.rst

@@ -1,6 +1,8 @@
 0.12.2 (unreleased)
 ===================
 
+- Respect ``--doctest-continue-on-failure`` flag. [#197]
+
 - Report doctests raising skip exceptions as skipped. [#196]
 
 0.12.1 (2022-09-26)

README.rst

@@ -94,6 +94,12 @@ reason both ``--doctest-modules`` and ``--doctest-plus`` are given, the
 ``pytest-doctestplus`` plugin will be used, regardless of the contents of
 ``setup.cfg``.
 
+``pytest-doctestplus`` respects the ``--doctest-continue-on-failure`` flag.
+If set, doctests will report all failing lines, which may be useful to detect
+independent errors within the same doctest. However, it is likely to generate
+false positives when an early failure causes a variable later lines access to
+remain unset or have an unexpected value.
+
 This plugin respects the doctest options that are used by the built-in doctest
 plugin and are set in ``doctest_optionflags`` in ``setup.cfg``. By default,
 ``ELLIPSIS`` and ``NORMALIZE_WHITESPACE`` are used. For a description of all

pytest_doctestplus/plugin.py

@@ -15,6 +15,7 @@
 
 import pytest
 from _pytest.outcomes import OutcomeException  # Private API, but been around since 3.7
+from _pytest.doctest import _get_continue_on_failure  # Since 3.5, still in 7.3
 from packaging.version import Version
 
 from pytest_doctestplus.utils import ModuleChecker
@@ -261,7 +262,12 @@ def collect(self):
             # uses internal doctest module parsing mechanism
             finder = DocTestFinderPlus(doctest_ufunc=use_doctest_ufunc)
             runner = DebugRunnerPlus(
-                verbose=False, optionflags=options, checker=OutputChecker())
+                verbose=False,
+                optionflags=options,
+                checker=OutputChecker(),
+                # Helper disables continue-on-failure when debugging is enabled
+                continue_on_failure=_get_continue_on_failure(config),
+            )
 
             for test in finder.find(module):
                 if test.examples:  # skip empty doctests
@@ -323,7 +329,9 @@ def collect(self):
             optionflags = get_optionflags(self) | FIX
 
             runner = DebugRunnerPlus(
-                verbose=False, optionflags=optionflags, checker=OutputChecker())
+                verbose=False, optionflags=optionflags, checker=OutputChecker(),
+                continue_on_failure=_get_continue_on_failure(self.config),
+            )
 
             parser = DocTestParserPlus()
             test = parser.get_doctest(text, globs, filepath, filename, 0)
@@ -673,9 +681,10 @@ def find(self, obj, name=None, module=None, globs=None, extraglobs=None):
         if name is None and hasattr(obj, '__name__'):
             name = obj.__name__
         else:
-            raise ValueError("DocTestFinder.find: name must be given "
-                                "when obj.__name__ doesn't exist: {!r}"
-                                .format((type(obj),)))
+            raise ValueError(
+                "DocTestFinder.find: name must be given when obj.__name__ doesn't exist: "
+                f"{type(obj)!r}"
+            )
 
         if self._doctest_ufunc:
             for ufunc_name, ufunc_method in obj.__dict__.items():
@@ -712,8 +721,23 @@ def test_filter(test):
 
 
 class DebugRunnerPlus(doctest.DebugRunner):
+    def __init__(self, checker=None, verbose=None, optionflags=0, continue_on_failure=True):
+        super().__init__(checker=checker, verbose=verbose, optionflags=optionflags)
+        self.continue_on_failure = continue_on_failure
+
+    def report_failure(self, out, test, example, got):
+        failure = doctest.DocTestFailure(test, example, got)
+        if self.continue_on_failure:
+            out.append(failure)
+        else:
+            raise failure
+
     def report_unexpected_exception(self, out, test, example, exc_info):
         cls, exception, traceback = exc_info
         if isinstance(exception, (OutcomeException, SkipTest)):
             raise exception
-        super().report_unexpected_exception(out, test, example, exc_info)
+        failure = doctest.UnexpectedException(test, example, exc_info)
+        if self.continue_on_failure:
+            out.append(failure)
+        else:
+            raise failure

tests/test_doctestplus.py

@@ -987,6 +987,58 @@ class MyClass:
     reprec.assertoutcome(skipped=1, failed=0)
 
 
+@pytest.mark.parametrize('cont_on_fail', [False, True])
+def test_fail_two_tests(testdir, cont_on_fail):
+    p = testdir.makepyfile(
+        """
+        class MyClass:
+            '''
+            .. doctest::
+
+                >>> print(2)
+                1
+
+            .. doctest::
+
+                >>> print(3)
+                1
+            '''
+            pass
+    """
+    )
+    arg = ("--doctest-continue-on-failure",) if cont_on_fail else ()
+    reprec = testdir.inline_run(p, "--doctest-plus", *arg)
+    reprec.assertoutcome(skipped=0, failed=1)
+    _, _, failed = reprec.listoutcomes()
+    report = failed[0]
+    assert "Expected:\n    1\nGot:\n    2" in report.longreprtext
+    assert ("Expected:\n    1\nGot:\n    3" in report.longreprtext) is cont_on_fail
+
+
+@pytest.mark.parametrize('cont_on_fail', [False, True])
+def test_fail_data_dependency(testdir, cont_on_fail):
+    p = testdir.makepyfile(
+        """
+        class MyClass:
+            '''
+            .. doctest::
+
+                >>> import nonexistentmodule as nem
+                >>> a = nem.calculate_something()
+            '''
+            pass
+    """
+    )
+    arg = ("--doctest-continue-on-failure",) if cont_on_fail else ()
+    reprec = testdir.inline_run(p, "--doctest-plus", *arg)
+    reprec.assertoutcome(skipped=0, failed=1)
+    _, _, failed = reprec.listoutcomes()
+    # Both lines fail in a single error
+    report = failed[0]
+    assert " as nem\nUNEXPECTED EXCEPTION: ModuleNotFoundError" in report.longreprtext
+    assert ("something()\nUNEXPECTED EXCEPTION: NameError" in report.longreprtext) is cont_on_fail
+
+
 def test_ufunc(testdir):
     pytest.importorskip('numpy')