#7938 - [Plugin: Stepwise][Enhancements] Refactoring, smarter registration & --sw-skip functionality (#7939)

* adding --sw-skip shorthand for stepwise skip

* be explicit rather than implicit with default args for stepwise

* add constant for sw cache dir; only register plugin if necessary rather check check activity always;

* use str format; remove unused args in hooks

* assert cache upfront, allow stepwise to have a reference to the cache

* type hinting lf, skip, move literal strings into module constants

* convert parametrized option into a list

* add a sessionfinish hook for stepwise to keep backwards behaviour the same

* add changelog for #7938

* Improve performance of stepwise modifyitems & address PR feedback

* add test for stepwise deselected based on performance enhancements

* Apply suggestions from code review

* delete from items, account for edge case where failed_index = 0

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>

Author: symonk

changelog/7938.improvement.rst

@@ -0,0 +1 @@
+New ``--sw-skip`` argument which is a shorthand for ``--stepwise-skip``.

src/_pytest/stepwise.py

@@ -1,5 +1,6 @@
 from typing import List
 from typing import Optional
+from typing import TYPE_CHECKING
 
 import pytest
 from _pytest import nodes
@@ -8,83 +9,84 @@
 from _pytest.main import Session
 from _pytest.reports import TestReport
 
+if TYPE_CHECKING:
+    from _pytest.cacheprovider import Cache
+
+STEPWISE_CACHE_DIR = "cache/stepwise"
+
 
 def pytest_addoption(parser: Parser) -> None:
     group = parser.getgroup("general")
     group.addoption(
         "--sw",
         "--stepwise",
         action="store_true",
+        default=False,
         dest="stepwise",
         help="exit on test failure and continue from last failing test next time",
     )
     group.addoption(
+        "--sw-skip",
         "--stepwise-skip",
         action="store_true",
+        default=False,
         dest="stepwise_skip",
         help="ignore the first failing test but stop on the next failing test",
     )
 
 
 @pytest.hookimpl
 def pytest_configure(config: Config) -> None:
-    config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin")
+    # We should always have a cache as cache provider plugin uses tryfirst=True
+    if config.getoption("stepwise"):
+        config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin")
+
+
+def pytest_sessionfinish(session: Session) -> None:
+    if not session.config.getoption("stepwise"):
+        assert session.config.cache is not None
+        # Clear the list of failing tests if the plugin is not active.
+        session.config.cache.set(STEPWISE_CACHE_DIR, [])
 
 
 class StepwisePlugin:
     def __init__(self, config: Config) -> None:
         self.config = config
-        self.active = config.getvalue("stepwise")
         self.session: Optional[Session] = None
         self.report_status = ""
-
-        if self.active:
-            assert config.cache is not None
-            self.lastfailed = config.cache.get("cache/stepwise", None)
-            self.skip = config.getvalue("stepwise_skip")
+        assert config.cache is not None
+        self.cache: Cache = config.cache
+        self.lastfailed: Optional[str] = self.cache.get(STEPWISE_CACHE_DIR, None)
+        self.skip: bool = config.getoption("stepwise_skip")
 
     def pytest_sessionstart(self, session: Session) -> None:
         self.session = session
 
     def pytest_collection_modifyitems(
-        self, session: Session, config: Config, items: List[nodes.Item]
+        self, config: Config, items: List[nodes.Item]
     ) -> None:
-        if not self.active:
-            return
         if not self.lastfailed:
             self.report_status = "no previously failed tests, not skipping."
             return
 
-        already_passed = []
-        found = False
-
-        # Make a list of all tests that have been run before the last failing one.
-        for item in items:
+        # check all item nodes until we find a match on last failed
+        failed_index = None
+        for index, item in enumerate(items):
             if item.nodeid == self.lastfailed:
-                found = True
+                failed_index = index
                 break
-            else:
-                already_passed.append(item)
 
         # If the previously failed test was not found among the test items,
         # do not skip any tests.
-        if not found:
+        if failed_index is None:
             self.report_status = "previously failed test not found, not skipping."
-            already_passed = []
         else:
-            self.report_status = "skipping {} already passed items.".format(
-                len(already_passed)
-            )
-
-        for item in already_passed:
-            items.remove(item)
-
-        config.hook.pytest_deselected(items=already_passed)
+            self.report_status = f"skipping {failed_index} already passed items."
+            deselected = items[:failed_index]
+            del items[:failed_index]
+            config.hook.pytest_deselected(items=deselected)
 
     def pytest_runtest_logreport(self, report: TestReport) -> None:
-        if not self.active:
-            return
-
         if report.failed:
             if self.skip:
                 # Remove test from the failed ones (if it exists) and unset the skip option
@@ -109,14 +111,9 @@ def pytest_runtest_logreport(self, report: TestReport) -> None:
                     self.lastfailed = None
 
     def pytest_report_collectionfinish(self) -> Optional[str]:
-        if self.active and self.config.getoption("verbose") >= 0 and self.report_status:
-            return "stepwise: %s" % self.report_status
+        if self.config.getoption("verbose") >= 0 and self.report_status:
+            return f"stepwise: {self.report_status}"
         return None
 
-    def pytest_sessionfinish(self, session: Session) -> None:
-        assert self.config.cache is not None
-        if self.active:
-            self.config.cache.set("cache/stepwise", self.lastfailed)
-        else:
-            # Clear the list of failing tests if the plugin is not active.
-            self.config.cache.set("cache/stepwise", [])
+    def pytest_sessionfinish(self) -> None:
+        self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed)

testing/test_stepwise.py

@@ -93,6 +93,23 @@ def test_run_without_stepwise(stepwise_testdir):
     result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])
 
 
+def test_stepwise_output_summary(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+        @pytest.mark.parametrize("expected", [True, True, True, True, False])
+        def test_data(expected):
+            assert expected
+        """
+    )
+    result = testdir.runpytest("-v", "--stepwise")
+    result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."])
+    result = testdir.runpytest("-v", "--stepwise")
+    result.stdout.fnmatch_lines(
+        ["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"]
+    )
+
+
 def test_fail_and_continue_with_stepwise(stepwise_testdir):
     # Run the tests with a failing second test.
     result = stepwise_testdir.runpytest(
@@ -117,14 +134,10 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir):
     assert "test_success_after_fail PASSED" in stdout
 
 
-def test_run_with_skip_option(stepwise_testdir):
+@pytest.mark.parametrize("stepwise_skip", ["--stepwise-skip", "--sw-skip"])
+def test_run_with_skip_option(stepwise_testdir, stepwise_skip):
     result = stepwise_testdir.runpytest(
-        "-v",
-        "--strict-markers",
-        "--stepwise",
-        "--stepwise-skip",
-        "--fail",
-        "--fail-last",
+        "-v", "--strict-markers", "--stepwise", stepwise_skip, "--fail", "--fail-last",
     )
     assert _strip_resource_warnings(result.stderr.lines) == []