Use sorting by hashes to shuffle items

Fixes #210.

Author: adamchainz

HISTORY.rst

@@ -2,6 +2,13 @@
 History
 =======
 
+* Rework shuffling algorithm to use hashing. This means that running a subset
+  of tests with the same seed will now produce the same ordering as running the
+  full set of tests. This allows narrowing down ordering-related failures.
+
+  Thanks to Tom Grainger for the suggestion in `Issue #210
+  <https://github.com/pytest-dev/pytest-randomly/issues/210>`__.
+
 3.9.0 (2021-08-12)
 ------------------
 
@@ -24,7 +31,7 @@ History
 * Fix deprecation warning from importlib-metadata 3.9.0+.
 
   Thanks to Dominic Davis-Foster for report in `Issue #333
-  <https://github.com/pytest-dev/pytest-randomly/issue/333>`__.
+  <https://github.com/pytest-dev/pytest-randomly/issues/333>`__.
 
 * Stop distributing tests to reduce package size. Tests are not intended to be
   run outside of the tox setup in the repository. Repackagers can use GitHub's

src/pytest_randomly/__init__.py

@@ -4,7 +4,7 @@
 import sys
 from itertools import groupby
 from types import ModuleType
-from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
 
 from _pytest.config import Config
 from _pytest.config.argparsing import Parser
@@ -134,7 +134,7 @@ def pytest_configure_node(self, node: Item) -> None:
 entrypoint_reseeds: Optional[List[Callable[[int], None]]] = None
 
 
-def _reseed(config: Config, offset: int = 0) -> None:
+def _reseed(config: Config, offset: int = 0) -> int:
     global entrypoint_reseeds
     seed = config.getoption("randomly_seed") + offset
     if seed not in random_states:
@@ -164,6 +164,8 @@ def _reseed(config: Config, offset: int = 0) -> None:
     for reseed in entrypoint_reseeds:
         reseed(seed)
 
+    return seed
+
 
 def _truncate_seed_for_numpy(seed: int) -> int:
     seed = abs(seed)
@@ -199,17 +201,26 @@ def pytest_collection_modifyitems(config: Config, items: List[Item]) -> None:
     if not config.getoption("randomly_reorganize"):
         return
 
-    _reseed(config)
+    seed = _reseed(config)
 
-    module_items: List[List[Item]] = [
-        list(group) for _key, group in groupby(items, _get_module)
-    ]
+    modules_items: List[Tuple[Optional[ModuleType], List[Item]]] = []
+    for module, group in groupby(items, _get_module):
+        modules_items.append(
+            (
+                module,
+                _shuffle_by_class(list(group), seed),
+            )
+        )
 
-    for sub_items in module_items:
-        sub_items[:] = shuffle_by_class(sub_items)
-    random.shuffle(module_items)
+    def _module_key(module_item: Tuple[Optional[ModuleType], List[Item]]) -> bytes:
+        module, _items = module_item
+        if module is None:
+            return _md5(f"{seed}::None")
+        return _md5(f"{seed}::{module.__name__}")
 
-    items[:] = reduce_list_of_lists(module_items)
+    modules_items.sort(key=_module_key)
+
+    items[:] = reduce_list_of_lists([subitems for module, subitems in modules_items])
 
 
 def _get_module(item: Item) -> Optional[ModuleType]:
@@ -219,16 +230,28 @@ def _get_module(item: Item) -> Optional[ModuleType]:
         return None
 
 
-def shuffle_by_class(items: List[Item]) -> List[Item]:
-    class_items: List[List[Item]] = [
-        list(group) for _key, group in groupby(items, _get_cls)
-    ]
+def _shuffle_by_class(items: List[Item], seed: int) -> List[Item]:
+    klasses_items: List[Tuple[Optional[Type[Any]], List[Item]]] = []
 
-    for sub_items in class_items:
-        random.shuffle(sub_items)
-    random.shuffle(class_items)
+    for klass, group in groupby(items, _get_cls):
+        klass_items = [(_md5(f"{seed}::{item.nodeid}"), item) for item in group]
+        klass_items.sort()
+        klasses_items.append(
+            (
+                klass,
+                [item for _key, item in klass_items],
+            )
+        )
 
-    return reduce_list_of_lists(class_items)
+    def _cls_key(klass_items: Tuple[Optional[Type[Any]], List[Item]]) -> bytes:
+        klass, items = klass_items
+        if klass is None:
+            return _md5(f"{seed}::None")
+        return _md5(f"{seed}::{klass.__module__}.{klass.__qualname__}")
+
+    klasses_items.sort(key=_cls_key)
+
+    return reduce_list_of_lists([subitems for klass, subitems in klasses_items])
 
 
 def _get_cls(item: Item) -> Optional[Type[Any]]:
@@ -245,6 +268,12 @@ def reduce_list_of_lists(lists: List[List[T]]) -> List[T]:
     return new_list
 
 
+def _md5(string: str) -> bytes:
+    hasher = hashlib.md5()
+    hasher.update(string.encode())
+    return hasher.digest()
+
+
 if have_faker:
 
     @fixture(autouse=True)

tests/test_pytest_randomly.py

@@ -240,10 +240,10 @@ def test_it():
 
     out.assert_outcomes(passed=4, failed=0)
     assert out.outlines[8:12] == [
+        "test_b.py::test_it PASSED",
+        "test_a.py::test_it PASSED",
         "test_d.py::test_it PASSED",
         "test_c.py::test_it PASSED",
-        "test_a.py::test_it PASSED",
-        "test_b.py::test_it PASSED",
     ]
 
 
@@ -260,10 +260,10 @@ def test_it():
 
     out.assert_outcomes(passed=4, failed=0)
     assert out.outlines[8:12] == [
+        "test_b.py::test_it PASSED",
+        "test_a.py::test_it PASSED",
         "test_d.py::test_it PASSED",
         "test_c.py::test_it PASSED",
-        "test_a.py::test_it PASSED",
-        "test_b.py::test_it PASSED",
     ]
 
 
@@ -300,9 +300,9 @@ def test_d(self):
     out.assert_outcomes(passed=4, failed=0)
     assert out.outlines[8:12] == [
         "test_one.py::D::test_d PASSED",
+        "test_one.py::B::test_b PASSED",
         "test_one.py::C::test_c PASSED",
         "test_one.py::A::test_a PASSED",
-        "test_one.py::B::test_b PASSED",
     ]
 
 
@@ -331,10 +331,10 @@ def test_d(self):
 
     out.assert_outcomes(passed=4, failed=0)
     assert out.outlines[8:12] == [
-        "test_one.py::T::test_d PASSED",
         "test_one.py::T::test_c PASSED",
-        "test_one.py::T::test_a PASSED",
         "test_one.py::T::test_b PASSED",
+        "test_one.py::T::test_a PASSED",
+        "test_one.py::T::test_d PASSED",
     ]
 
 
@@ -360,10 +360,10 @@ def test_d():
 
     out.assert_outcomes(passed=4, failed=0)
     assert out.outlines[8:12] == [
-        "test_one.py::test_d PASSED",
         "test_one.py::test_c PASSED",
         "test_one.py::test_a PASSED",
         "test_one.py::test_b PASSED",
+        "test_one.py::test_d PASSED",
     ]
 
 
@@ -394,10 +394,10 @@ def test_d():
 
     out.assert_outcomes(passed=4, failed=0)
     assert out.outlines[8:12] == [
-        "test_one.py::test_d PASSED",
         "test_one.py::test_c PASSED",
         "test_one.py::test_a PASSED",
         "test_one.py::test_b PASSED",
+        "test_one.py::test_d PASSED",
     ]
 
 
@@ -419,7 +419,7 @@ def bar():
             return 9002
         """
     )
-    args = ["-v", "--doctest-modules", "--randomly-seed=5"]
+    args = ["-v", "--doctest-modules", "--randomly-seed=1"]
 
     out = ourtestdir.runpytest(*args)
     out.assert_outcomes(passed=2)
@@ -432,6 +432,8 @@ def bar():
 def test_it_works_with_the_simplest_test_items(ourtestdir):
     ourtestdir.makepyfile(
         conftest="""
+        import sys
+
         import pytest
 
 
@@ -463,7 +465,8 @@ def pytest_collect_file(path, parent):
                 items=[
                     NoOpItem.from_parent(
                         name=str(path) + "1",
-                        parent=parent, module="foo"
+                        parent=parent,
+                        module=sys.modules[__name__],
                     ),
                     NoOpItem.from_parent(
                         name=str(path) + "2",
@@ -492,7 +495,7 @@ def test_doctests_in_txt_files_reordered(ourtestdir):
         0
         """
     )
-    args = ["-v", "--randomly-seed=1"]
+    args = ["-v", "--randomly-seed=2"]
 
     out = ourtestdir.runpytest(*args)
     out.assert_outcomes(passed=2)