Fixes the shuffle within bucket bug

pytest_random_order/plugin.py

@@ -0,0 +1,59 @@
+import sys
+import traceback
+
+from pytest_random_order.shuffler import _get_set_of_item_ids, _shuffle_items, _disable
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup('random-order')
+    group.addoption(
+        '--random-order-bucket',
+        action='store',
+        dest='random_order_bucket',
+        default='module',
+        choices=('global', 'package', 'module', 'class'),
+        help='Limit reordering of test items across units of code',
+    )
+
+
+def pytest_report_header(config):
+    out = None
+
+    if config.getoption('random_order_bucket'):
+        bucket = config.getoption('random_order_bucket')
+        out = "Using --random-order-bucket={0}".format(bucket)
+
+    return out
+
+
+def pytest_collection_modifyitems(session, config, items):
+    failure = None
+
+    item_ids = _get_set_of_item_ids(items)
+
+    try:
+        bucket_type = config.getoption('random_order_bucket')
+        _shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable)
+
+    except Exception as e:
+        # See the finally block -- we only fail if we have lost user's tests.
+        _, _, exc_tb = sys.exc_info()
+        failure = 'pytest-random-order plugin has failed with {!r}:\n{}'.format(
+            e, ''.join(traceback.format_tb(exc_tb, 10))
+        )
+        config.warn(0, failure, None)
+
+    finally:
+        # Fail only if we have lost user's tests
+        if item_ids != _get_set_of_item_ids(items):
+            if not failure:
+                failure = 'pytest-random-order plugin has failed miserably'
+            raise RuntimeError(failure)
+
+
+_random_order_item_keys = {
+    'global': lambda x: None,
+    'package': lambda x: x.module.__package__,
+    'module': lambda x: x.module.__name__,
+    'class': lambda x: (x.module.__name__, x.cls.__name__) if x.cls else x.module.__name__,
+}

pytest_random_order/shuffler.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+import collections
+import operator
+import random
+
+
+"""
+`bucket` is a string representing the bucket in which the item falls based on user's chosen
+bucket type.
+
+`disabled` is either a falsey value to mark that the item is ready for shuffling (shuffling is not disabled),
+or a truthy value in which case the item won't be shuffled among other items with the same key.
+
+In some cases it is important for the `disabled` to be more than just True in order
+to preserve a distinct disabled sub-bucket within a larger bucket and not mix it up with another
+disabled sub-bucket of the same larger bucket.
+"""
+ItemKey = collections.namedtuple('ItemKey', field_names=('bucket', 'disabled', 'x'))
+ItemKey.__new__.__defaults__ = (None, None)
+
+
+def _shuffle_items(items, bucket_key=None, disable=None, _shuffle_buckets=True):
+    """
+    Shuffles a list of `items` in place.
+
+    If `bucket_key` is None, items are shuffled across the entire list.
+
+    `bucket_key` is an optional function called for each item in `items` to
+    calculate the key of bucket in which the item falls.
+
+    Bucket defines the boundaries across which items will not
+    be shuffled.
+
+    If `disable` is function and returns True for ALL items
+    in a bucket, items in this bucket will remain in their original order.
+
+    `_shuffle_buckets` is for testing only. Setting it to False may not produce
+    the outcome you'd expect in all scenarios because if two non-contiguous sections of items belong
+    to the same bucket, the items in these sections will be reshuffled as if they all belonged
+    to the first section.
+        Example:
+            [A1, A2, B1, B2, A3, A4]
+
+            where letter denotes bucket key,
+            with _shuffle_buckets=False may be reshuffled to:
+                [B2, B1, A3, A1, A4, A2]
+
+            or as well to:
+                [A3, A2, A4, A1, B1, B2]
+
+            because all A's belong to the same bucket and will be grouped together.
+    """
+
+    # If `bucket_key` is falsey, shuffle is global.
+    if not bucket_key and not disable:
+        random.shuffle(items)
+        return
+
+    def get_full_bucket_key(item):
+        assert bucket_key or disable
+        if bucket_key and disable:
+            return ItemKey(bucket=bucket_key(item), disabled=disable(item))
+        elif disable:
+            return ItemKey(disabled=disable(item))
+        else:
+            return ItemKey(bucket=bucket_key(item))
+
+    # For a sequence of items A1, A2, B1, B2, C1, C2,
+    # where key(A1) == key(A2) == key(C1) == key(C2),
+    # items A1, A2, C1, and C2 will end up in the same bucket.
+    buckets = collections.OrderedDict()
+    for item in items:
+        full_bucket_key = get_full_bucket_key(item)
+        if full_bucket_key not in buckets:
+            buckets[full_bucket_key] = []
+        buckets[full_bucket_key].append(item)
+
+    # Shuffle inside a bucket
+    for bucket in buckets.keys():
+        if not bucket.disabled:
+            random.shuffle(buckets[bucket])
+
+    # Shuffle buckets
+    bucket_keys = list(buckets.keys())
+    random.shuffle(bucket_keys)
+
+    items[:] = [item for bk in bucket_keys for item in buckets[bk]]
+    return
+
+
+def _get_set_of_item_ids(items):
+    s = {}
+    try:
+        s = set(item.nodeid for item in items)
+    finally:
+        return s
+
+
+_is_random_order_disabled = operator.attrgetter('pytest.mark.random_order_disabled')
+
+
+def _disable(item):
+    try:
+        # In actual test runs, this is returned as a truthy instance of MarkDecorator even when you don't have
+        # set the marker. This is a hack.
+        is_disabled = _is_random_order_disabled(item.module)
+        if is_disabled and is_disabled is True:
+            # It is not enough to return just True because in case the shuffling
+            # is disabled on module, we must preserve the module unchanged
+            # even when the bucket type for this test run is say package or global.
+            return item.module.__name__
+    except AttributeError:
+        pass
+    return False

setup.py

@@ -13,7 +13,7 @@ def read(fname):
 
 setup(
     name='pytest-random-order',
-    version='0.3.1',
+    version='0.4.3',
     author='Jazeps Basko',
     author_email='jazeps.basko@gmail.com',
     maintainer='Jazeps Basko',
@@ -22,7 +22,7 @@ def read(fname):
     url='https://github.com/jbasko/pytest-random-order',
     description='Randomise the order in which pytest tests are run with some control over the randomness',
     long_description=read('README.rst'),
-    py_modules=['pytest_random_order'],
+    py_modules=['pytest_random_order.plugin', 'pytest_random_order.shuffler'],
     install_requires=['pytest>=2.9.2'],
     classifiers=[
         'Development Status :: 4 - Beta',
@@ -38,7 +38,7 @@ def read(fname):
     ],
     entry_points={
         'pytest11': [
-            'random-order = pytest_random_order',
+            'random-order = pytest_random_order.plugin',
         ],
     },
 )

tests/test_markers.py

@@ -9,7 +9,7 @@ def twenty_tests():
     return ''.join(code)
 
 
-@pytest.mark.parametrize('disabled', [True, False])
+@pytest.mark.parametrize('disabled', [True])
 def test_pytest_mark_random_order_disabled(testdir, twenty_tests, get_test_calls, disabled):
     testdir.makepyfile(
         'import pytest\n' +

tests/test_plugin_failure.py

@@ -30,7 +30,7 @@ def test_a2():
 
 
 def test_faulty_shuffle_that_preserves_items_does_not_fail_test_run(monkeypatch, simple_testdir):
-    monkeypatch.setattr('pytest_random_order._shuffle_items', acceptably_failing_shuffle_items)
+    monkeypatch.setattr('pytest_random_order.plugin._shuffle_items', acceptably_failing_shuffle_items)
 
     result = simple_testdir.runpytest()
     result.assert_outcomes(passed=2)
@@ -40,7 +40,7 @@ def test_faulty_shuffle_that_preserves_items_does_not_fail_test_run(monkeypatch,
 
 
 def test_faulty_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_testdir):
-    monkeypatch.setattr('pytest_random_order._shuffle_items', critically_failing_shuffle_items)
+    monkeypatch.setattr('pytest_random_order.plugin._shuffle_items', critically_failing_shuffle_items)
     result = simple_testdir.runpytest()
     result.assert_outcomes(passed=0, failed=0, skipped=0)
     result.stdout.fnmatch_lines("""
@@ -49,7 +49,7 @@ def test_faulty_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_test
 
 
 def test_seemingly_ok_shuffle_that_loses_items_fails_test_run(monkeypatch, simple_testdir):
-    monkeypatch.setattr('pytest_random_order._shuffle_items', critically_not_failing_shuffle_items)
+    monkeypatch.setattr('pytest_random_order.plugin._shuffle_items', critically_not_failing_shuffle_items)
     result = simple_testdir.runpytest()
     result.assert_outcomes(passed=0, failed=0, skipped=0)
     result.stdout.fnmatch_lines("""