Fix reporting for teardown errors

AUTHORS

@@ -7,6 +7,7 @@ Aaron Coleman
 Abdeali JK
 Abdelrahman Elbehery
 Abhijeet Kasurde
+ace2016
 Adam Johnson
 Adam Stewart
 Adam Uhlir

changelog/1004.bugfix.rst

@@ -0,0 +1,2 @@
+Tests which pass their call phase but fail during teardown are now reported only
+as errors instead of being reported as both passed and errored.

src/_pytest/terminal.py

@@ -409,6 +409,7 @@ def __init__(self, config: Config, file: TextIO | None = None) -> None:
         self.isatty = compat.CallableBool(file.isatty())
         self._progress_nodeids_reported: set[str] = set()
         self._timing_nodeids_reported: set[str] = set()
+        self._pending_passed_reports: dict[str, list[tuple[str, TestReport]]] = {}
         self._show_progress_info = self._determine_show_progress_info()
         self._collect_report_last_write = timing.Instant()
         self._already_displayed_warnings: int | None = None
@@ -634,6 +635,26 @@ def pytest_runtest_logstart(
 
     def pytest_runtest_logreport(self, report: TestReport) -> None:
         self._tests_ran = True
+
+        if report.when == "call" and report.passed:
+            category = self._process_test_report(report)
+            self._pending_passed_reports.setdefault(report.nodeid, []).append(
+                (category, report)
+            )
+            return
+
+        if report.when == "teardown":
+            pending_passed = self._pending_passed_reports.pop(report.nodeid, [])
+            if report.failed:
+                for category, passed_report in pending_passed:
+                    reports = self.stats.get(category, [])
+                    self.stats[category] = [
+                        rep for rep in reports if rep is not passed_report
+                    ]
+
+        self._process_test_report(report)
+
+    def _process_test_report(self, report: TestReport) -> str:
         rep = report
 
         res = TestShortLogReport(
@@ -647,7 +668,7 @@ def pytest_runtest_logreport(self, report: TestReport) -> None:
         self._add_stats(category, [rep])
         if not letter and not word:
             # Probably passed setup/teardown.
-            return
+            return category
         if markup is None:
             was_xfail = hasattr(report, "wasxfail")
             if rep.passed and not was_xfail:
@@ -707,6 +728,7 @@ def pytest_runtest_logreport(self, report: TestReport) -> None:
                 self._tw.write(" " + line)
                 self.currentfspath = -2
         self.flush()
+        return category
 
     @property
     def _is_last_item(self) -> bool:

testing/acceptance_test.py

@@ -1623,13 +1623,13 @@ def test_fail_call():
     )
     result = pytester.runpytest()
     assert result.ret == ExitCode.TESTS_FAILED
-    result.assert_outcomes(failed=1, passed=1, errors=2)
+    result.assert_outcomes(failed=1, errors=2)
     result.stdout.fnmatch_lines(
         [
             "=* short test summary info =*",
             "FAILED test_it.py::test_fail_call - StopIteration: 3",
             "ERROR test_it.py::test_fail_setup - StopIteration: 1",
             "ERROR test_it.py::test_fail_teardown - StopIteration: 2",
-            "=* 1 failed, 1 passed, 2 errors in * =*",
+            "=* 1 failed, 2 errors in * =*",
         ]
     )

testing/python/fixtures.py

@@ -949,7 +949,7 @@ def test_it(fix):
             """
         )
         result = pytester.runpytest()
-        result.assert_outcomes(passed=1, errors=1)
+        result.assert_outcomes(errors=1)
         result.stdout.fnmatch_lines(
             [
                 (
@@ -971,7 +971,7 @@ def test_it(request):
             """
         )
         result = pytester.runpytest()
-        result.assert_outcomes(passed=1, errors=1)
+        result.assert_outcomes(errors=1)
         result.stdout.fnmatch_lines(
             [
                 (
@@ -1134,7 +1134,7 @@ def test_second():
         """
         )
         result = pytester.runpytest()
-        result.assert_outcomes(passed=2, errors=1)
+        result.assert_outcomes(passed=1, errors=1)
         result.stdout.fnmatch_lines(
             [
                 '  | *ExceptionGroup: errors while tearing down fixture "subrequest" of <Function test_first> (2 sub-exceptions)',  # noqa: E501
@@ -3759,7 +3759,7 @@ def test_3():
             *KeyError*
             *ERROR*teardown*test_2*
             *KeyError*
-            *3 pass*2 errors*
+            *1 pass*2 errors*
         """
         )
 
@@ -4185,7 +4185,7 @@ def test_1(arg1):
         result.stdout.fnmatch_lines(
             """
             *pytest.fail*teardown*
-            *1 passed*1 error*
+            *1 error*
         """
         )
 
@@ -4511,7 +4511,7 @@ def test_second(my_fixture):
         """
     )
     result = pytester.runpytest("-v", "--setup-show")
-    result.assert_outcomes(passed=2, errors=1)
+    result.assert_outcomes(passed=1, errors=1)
     result.stdout.fnmatch_lines(
         [
             "*test_first*PASSED",

testing/test_terminal.py

@@ -693,9 +693,10 @@ def teardown_function(function):
                 "*assert 0*",
                 "*Captured stdout*",
                 "*teardown func*",
-                "*1 passed*1 error*",
+                "*1 error*",
             ]
         )
+        result.stdout.no_fnmatch_line("*1 passed*")
 
     def test_teardown_fixture_error_and_test_failure(self, pytester: Pytester) -> None:
         pytester.makepyfile(
@@ -2453,7 +2454,7 @@ def test_capture_no_progress_enabled(
 
 
 class TestProgressWithTeardown:
-    """Ensure we show the correct percentages for tests that fail during teardown (#3088)"""
+    """Test progress for tests that fail during teardown (#1004, #3088)."""
 
     @pytest.fixture
     def contest_with_teardown_fixture(self, pytester: Pytester) -> None:
@@ -2531,9 +2532,10 @@ def test_teardown_many_verbose(
                     "test_bar.py::test_bar[0] PASSED  * [  5%]",
                     "test_bar.py::test_bar[0] ERROR   * [  5%]",
                     "test_bar.py::test_bar[4] PASSED  * [ 25%]",
+                    "test_bar.py::test_bar[4] ERROR   * [ 25%]",
                     "test_foo.py::test_foo[14] PASSED * [100%]",
                     "test_foo.py::test_foo[14] ERROR  * [100%]",
-                    "=* 20 passed, 20 errors in *",
+                    "=* 20 errors in *",
                 ]
             )
         )

testing/test_unittest.py

@@ -1629,7 +1629,7 @@ def test(self):
         """
         )
         result = pytester.runpytest("-s", testpath)
-        result.assert_outcomes(passed=1, errors=1)
+        result.assert_outcomes(errors=1)
         result.stdout.fnmatch_lines(
             [
                 "*Unittest class cleanup errors *2 sub-exceptions*",
@@ -1653,7 +1653,7 @@ def test(self):
         """
         )
         result = pytester.runpytest("-s", testpath)
-        result.assert_outcomes(passed=1, errors=1)
+        result.assert_outcomes(errors=1)
         result.stdout.fnmatch_lines(
             [
                 "*ERROR at teardown of MyTestCase.test*",