pytest-dev#23 - Allow disabling shuffling with --random-order-bucket=none (pytest-dev#24)

Author: jbasko

docs/index.rst

@@ -26,7 +26,7 @@ test modules (which is the default behaviour of the plugin), just run pytest as
     $ pytest -v
 
 To change the level of randomness allowed, run pytest with ``--random-order-bucket=<bucket-type>`` option
-where ``<bucket-type>`` can be ``class``, ``module``, ``package``, or ``global``:
+where ``<bucket-type>`` can be ``class``, ``module``, ``package``, ``global``, or ``none``:
 
 ::
 
@@ -67,6 +67,9 @@ package
 global
     All tests fall in the same bucket, full randomness, tests probably take longer to run.
 
+none
+    Disable shuffling.
+
 If you have three buckets of tests ``A``, ``B``, and ``C`` with three tests ``1`` and ``2``, and ``3`` in each of them,
 then one of many potential orderings that non-global randomisation can produce could be:
 
@@ -148,8 +151,8 @@ You can now use the ``--random-order-seed=...`` bit as an argument to the next r
     $ pytest -v --random-order-seed=24775
 
 
-Disable the Plugin
-++++++++++++++++++
+Disable Randomisation or the Plugin
++++++++++++++++++++++++++++++++++++
 
 If the plugin misbehaves or you just want to assure yourself that it is not the plugin making your tests fail or
 pass undeservedly, you can disable it:
@@ -158,3 +161,9 @@ pass undeservedly, you can disable it:
 
     $ pytest -p no:random-order -v
 
+To disable just the shuffling, but let the plugin exist:
+
+::
+
+    $ pytest --random-order-bucket=none
+

pytest_random_order/plugin.py

@@ -12,7 +12,7 @@ def pytest_addoption(parser):
         action='store',
         dest='random_order_bucket',
         default='module',
-        choices=('global', 'package', 'module', 'class'),
+        choices=('global', 'package', 'module', 'class', 'none'),
         help='Limit reordering of test items across units of code',
     )
     group.addoption(
@@ -55,7 +55,8 @@ def pytest_collection_modifyitems(session, config, items):
     try:
         seed = getattr(config, 'random_order_seed', None)
         bucket_type = config.getoption('random_order_bucket')
-        _shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable, seed=seed)
+        if bucket_type != 'none':
+            _shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable, seed=seed)
 
     except Exception as e:
         # See the finally block -- we only fail if we have lost user's tests.

tests/test_actual_test_runs.py

@@ -136,8 +136,14 @@ def inspect_attr(this_call, prev_call, attr_name):
         assert num_modules <= num_module_switches <= num_modules + 1
 
 
-@pytest.mark.parametrize('bucket', ['class', 'module', 'package', 'global'])
-def test_it_works_with_actual_tests(tmp_tree_of_tests, get_test_calls, bucket):
+@pytest.mark.parametrize('bucket,min_sequences,max_sequences', [
+    ('class', 2, 5),
+    ('module', 2, 5),
+    ('package', 2, 5),
+    ('global', 2, 5),
+    ('none', 1, 1),
+])
+def test_it_works_with_actual_tests(tmp_tree_of_tests, get_test_calls, bucket, min_sequences, max_sequences):
     sequences = set()
 
     for x in range(5):
@@ -148,7 +154,7 @@ def test_it_works_with_actual_tests(tmp_tree_of_tests, get_test_calls, bucket):
         assert len(seq) == 17
         sequences.add(seq)
 
-    assert 1 < len(sequences) <= 5
+    assert min_sequences <= len(sequences) <= max_sequences
 
 
 def test_random_order_seed_is_respected(testdir, twenty_tests, get_test_calls):

tests/test_cli.py

@@ -4,7 +4,7 @@ def test_help_message(testdir):
     )
     result.stdout.fnmatch_lines([
         'random-order:',
-        '*--random-order-bucket={global,package,module,class}*',
+        '*--random-order-bucket={global,package,module,class,none}*',
         '*--random-order-seed=*',
     ])
 

tests/test_plugin_failure.py

@@ -35,7 +35,7 @@ def test_faulty_shuffle_that_preserves_items_does_not_fail_test_run(monkeypatch,
     result = simple_testdir.runpytest()
     result.assert_outcomes(passed=2)
     result.stdout.fnmatch_lines("""
-        *W0 None pytest-random-order plugin has failed with ValueError*
+        *pytest-random-order plugin has failed with ValueError*
     """)