v0.8.0 --failed-first works (pytest-dev#30)

Closes pytest-dev#28 - stops breaking `--failed-first` flag.

Author: jbasko

README.rst

@@ -156,6 +156,13 @@ You can now use the ``--random-order-seed=...`` bit as an argument to the next r
     $ pytest -v --random-order-seed=24775
 
 
+Run Last Failed Tests First
++++++++++++++++++++++++++++
+
+Since v0.8.0 pytest cache plugin's ``--failed-first`` flag is supported -- tests that failed in the last run
+will be run before tests that passed irrespective of shuffling bucket type.
+
+
 Disable Randomisation or the Plugin
 +++++++++++++++++++++++++++++++++++
 

pytest_random_order/bucket_types.py

@@ -0,0 +1,71 @@
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+import functools
+
+bucket_type_keys = OrderedDict()
+
+
+def bucket_type_key(bucket_type):
+    """
+    Registers a function that calculates test item key for the specified bucket type.
+    """
+
+    def decorator(f):
+
+        @functools.wraps(f)
+        def wrapped(item, session):
+            key = f(item)
+
+            if session is not None:
+                for handler in session.pytest_random_order_bucket_type_key_handlers:
+                    key = handler(item, key)
+
+            return key
+
+        bucket_type_keys[bucket_type] = wrapped
+        return wrapped
+
+    return decorator
+
+
+@bucket_type_key('global')
+def get_global_key(item):
+    return None
+
+
+@bucket_type_key('package')
+def get_package_key(item):
+    return item.module.__package__
+
+
+@bucket_type_key('module')
+def get_module_key(item):
+    return item.module.__name__
+
+
+@bucket_type_key('class')
+def get_class_key(item):
+    if item.cls:
+        return item.module.__name__, item.cls.__name__
+    else:
+        return item.module.__name__
+
+
+@bucket_type_key('parent')
+def get_parent_key(item):
+    return item.parent
+
+
+@bucket_type_key('grandparent')
+def get_grandparent_key(item):
+    return item.parent.parent
+
+
+@bucket_type_key('none')
+def get_none_key(item):
+    raise RuntimeError('When shuffling is disabled (bucket_type=none), item key should not be calculated')
+
+
+bucket_types = bucket_type_keys.keys()

pytest_random_order/cache.py

@@ -0,0 +1,36 @@
+"""
+This module is called "cache" because it builds on the "cache" plugin:
+
+    https://docs.pytest.org/en/latest/cache.html
+
+"""
+
+FAILED_FIRST_LAST_FAILED_BUCKET_KEY = '<failed_first_last_failed>'
+
+
+def process_failed_first_last_failed(session, config, items):
+    if not config.getoption('failedfirst'):
+        return
+
+    last_failed_raw = config.cache.get('cache/lastfailed', None)
+    if not last_failed_raw:
+        return
+
+    # Get the names of last failed tests
+    last_failed = []
+    for key in last_failed_raw.keys():
+        parts = key.split('::')
+        if len(parts) == 3:
+            last_failed.append(tuple(parts))
+        elif len(parts) == 2:
+            last_failed.append((parts[0], None, parts[1]))
+        else:
+            raise NotImplementedError()
+
+    def assign_last_failed_to_same_bucket(item, key):
+        if item.nodeid in last_failed_raw:
+            return FAILED_FIRST_LAST_FAILED_BUCKET_KEY
+        else:
+            return key
+
+    session.pytest_random_order_bucket_type_key_handlers.append(assign_last_failed_to_same_bucket)

pytest_random_order/plugin.py

@@ -2,6 +2,8 @@
 import sys
 import traceback
 
+from pytest_random_order.bucket_types import bucket_type_keys, bucket_types
+from pytest_random_order.cache import process_failed_first_last_failed
 from pytest_random_order.shuffler import _get_set_of_item_ids, _shuffle_items, _disable
 
 
@@ -12,7 +14,7 @@ def pytest_addoption(parser):
         action='store',
         dest='random_order_bucket',
         default='module',
-        choices=('global', 'package', 'module', 'class', 'parent', 'grandparent', 'none'),
+        choices=bucket_types,
         help='Limit reordering of test items across units of code',
     )
     group.addoption(
@@ -25,7 +27,10 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    config.addinivalue_line("markers", "random_order(disabled=True): disable reordering of tests within a module or class")
+    config.addinivalue_line(
+        'markers',
+        'random_order(disabled=True): disable reordering of tests within a module or class'
+    )
 
 
 def pytest_report_header(config):
@@ -43,13 +48,22 @@ def pytest_report_header(config):
 def pytest_collection_modifyitems(session, config, items):
     failure = None
 
+    session.pytest_random_order_bucket_type_key_handlers = []
+    process_failed_first_last_failed(session, config, items)
+
     item_ids = _get_set_of_item_ids(items)
 
     try:
         seed = str(config.getoption('random_order_seed'))
         bucket_type = config.getoption('random_order_bucket')
         if bucket_type != 'none':
-            _shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable, seed=seed)
+            _shuffle_items(
+                items,
+                bucket_key=bucket_type_keys[bucket_type],
+                disable=_disable,
+                seed=seed,
+                session=session,
+            )
 
     except Exception as e:
         # See the finally block -- we only fail if we have lost user's tests.
@@ -65,13 +79,3 @@ def pytest_collection_modifyitems(session, config, items):
             if not failure:
                 failure = 'pytest-random-order plugin has failed miserably'
             raise RuntimeError(failure)
-
-
-_random_order_item_keys = {
-    'global': lambda x: None,
-    'package': lambda x: x.module.__package__,
-    'module': lambda x: x.module.__name__,
-    'class': lambda x: (x.module.__name__, x.cls.__name__) if x.cls else x.module.__name__,
-    'parent': lambda x: x.parent,
-    'grandparent': lambda x: x.parent.parent,
-}

pytest_random_order/shuffler.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 
 from collections import namedtuple
+
+from pytest_random_order.cache import FAILED_FIRST_LAST_FAILED_BUCKET_KEY
+
 try:
     from collections import OrderedDict
 except ImportError:
@@ -23,7 +26,7 @@
 ItemKey.__new__.__defaults__ = (None, None)
 
 
-def _shuffle_items(items, bucket_key=None, disable=None, seed=None):
+def _shuffle_items(items, bucket_key=None, disable=None, seed=None, session=None):
     """
     Shuffles a list of `items` in place.
 
@@ -52,11 +55,11 @@ def _shuffle_items(items, bucket_key=None, disable=None, seed=None):
     def get_full_bucket_key(item):
         assert bucket_key or disable
         if bucket_key and disable:
-            return ItemKey(bucket=bucket_key(item), disabled=disable(item))
+            return ItemKey(bucket=bucket_key(item, session), disabled=disable(item, session))
         elif disable:
-            return ItemKey(disabled=disable(item))
+            return ItemKey(disabled=disable(item, session))
         else:
-            return ItemKey(bucket=bucket_key(item))
+            return ItemKey(bucket=bucket_key(item, session))
 
     # For a sequence of items A1, A2, B1, B2, C1, C2,
     # where key(A1) == key(A2) == key(C1) == key(C2),
@@ -69,15 +72,29 @@ def get_full_bucket_key(item):
         buckets[full_bucket_key].append(item)
 
     # Shuffle inside a bucket
-    for bucket in buckets.keys():
-        if not bucket.disabled:
-            random.shuffle(buckets[bucket])
 
-    # Shuffle buckets
     bucket_keys = list(buckets.keys())
-    random.shuffle(bucket_keys)
 
-    items[:] = [item for bk in bucket_keys for item in buckets[bk]]
+    for full_bucket_key in buckets.keys():
+        if full_bucket_key.bucket == FAILED_FIRST_LAST_FAILED_BUCKET_KEY:
+            # Do not shuffle the last failed bucket
+            continue
+
+        if not full_bucket_key.disabled:
+            random.shuffle(buckets[full_bucket_key])
+
+    # Shuffle buckets
+
+    # Only the first bucket can be FAILED_FIRST_LAST_FAILED_BUCKET_KEY
+    if bucket_keys and bucket_keys[0].bucket == FAILED_FIRST_LAST_FAILED_BUCKET_KEY:
+        new_bucket_keys = list(buckets.keys())[1:]
+        random.shuffle(new_bucket_keys)
+        new_bucket_keys.insert(0, bucket_keys[0])
+    else:
+        new_bucket_keys = list(buckets.keys())
+        random.shuffle(new_bucket_keys)
+
+    items[:] = [item for bk in new_bucket_keys for item in buckets[bk]]
     return
 
 
@@ -89,7 +106,7 @@ def _get_set_of_item_ids(items):
         return s
 
 
-def _disable(item):
+def _disable(item, session):
     marker = item.get_marker('random_order')
     if marker:
         is_disabled = marker.kwargs.get('disabled', False)

setup.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import os
 import codecs
+import os
+
 from setuptools import setup
 
 
@@ -13,7 +14,7 @@ def read(fname):
 
 setup(
     name='pytest-random-order',
-    version='0.7.0',
+    version='0.8.0',
     author='Jazeps Basko',
     author_email='jazeps.basko@gmail.com',
     maintainer='Jazeps Basko',
@@ -22,7 +23,12 @@ def read(fname):
     url='https://github.com/jbasko/pytest-random-order',
     description='Randomise the order in which pytest tests are run with some control over the randomness',
     long_description=read('README.rst'),
-    py_modules=['pytest_random_order.plugin', 'pytest_random_order.shuffler'],
+    py_modules=[
+        'pytest_random_order.bucket_types',
+        'pytest_random_order.cache',
+        'pytest_random_order.plugin',
+        'pytest_random_order.shuffler',
+    ],
     install_requires=['pytest>=2.9.2'],
     classifiers=[
         'Development Status :: 5 - Production/Stable',

tests/test_actual_test_runs.py

@@ -206,3 +206,24 @@ def test_generated_seed_is_reported_and_run_can_be_reproduced(testdir, twenty_te
     result2.assert_outcomes(passed=20)
     calls2 = get_test_calls(result2)
     assert calls == calls2
+
+
+@pytest.mark.parametrize('bucket', [
+    'global',
+    'package',
+    'module',
+    'class',
+    'parent',
+    'grandparent',
+    'none',
+])
+def test_failed_first(tmp_tree_of_tests, get_test_calls, bucket):
+    result1 = tmp_tree_of_tests.runpytest('--random-order-bucket={0}'.format(bucket), '--verbose')
+    result1.assert_outcomes(passed=14, failed=3)
+
+    result2 = tmp_tree_of_tests.runpytest('--random-order-bucket={0}'.format(bucket), '--failed-first', '--verbose')
+    result2.assert_outcomes(passed=14, failed=3)
+
+    calls2 = get_test_calls(result2)
+    first_three_tests = set(c.name for c in calls2[:3])
+    assert set(['test_a1', 'test_b2', 'test_ee2']) == first_three_tests

tests/test_shuffle.py

@@ -4,30 +4,30 @@
 from pytest_random_order.shuffler import _shuffle_items
 
 
-def identity_key(x):
-    return x
+def identity_key(item, session):
+    return item
 
 
-def modulus_2_key(x):
-    return x % 2
+def modulus_2_key(item, session):
+    return item % 2
 
 
-def lt_10_key(x):
-    return x < 10
+def lt_10_key(item, session):
+    return item < 10
 
 
-def disable_if_gt_1000(x):
+def disable_if_gt_1000(item, session):
     # if disable returns a truthy value, it must also be usable as a key.
-    if x > 1000:
-        return x // 1000
+    if item > 1000:
+        return item // 1000
     else:
         return False
 
 
 @pytest.mark.parametrize('key', [
     None,
-    lambda x: None,
-    lambda x: x % 2,
+    lambda item, session: None,
+    lambda item, session: item % 2,
 ])
 def test_shuffles_empty_list_in_place(key):
     items = []
@@ -39,8 +39,8 @@ def test_shuffles_empty_list_in_place(key):
 
 @pytest.mark.parametrize('key', [
     None,
-    lambda x: None,
-    lambda x: x % 2,
+    lambda item, session: None,
+    lambda item, session: item % 2,
 ])
 def test_shuffles_one_item_list_in_place(key):
     items = [42]
@@ -67,10 +67,10 @@ def test_two_bucket_reshuffle():
     _shuffle_items(items, bucket_key=lt_10_key)
     assert items != items_copy
     for i, item in enumerate(items):
-        if lt_10_key(i):
-            assert lt_10_key(item) == lt_10_key(items[0]), items
+        if lt_10_key(i, None):
+            assert lt_10_key(item, None) == lt_10_key(items[0], None), items
         else:
-            assert lt_10_key(item) == lt_10_key(items[10]), items
+            assert lt_10_key(item, None) == lt_10_key(items[10], None), items
 
 
 def test_eight_bucket_reshuffle():