[tests] add pytest -m 'not slow' option

docs/src/developer_guide/testing.rst

@@ -10,6 +10,14 @@ To run the test suite, simply invoke ``pytest``:
     # or to run a single test file
     pytest src/bsdfs/tests/test_diffuse.py
 
+Note that running the full test suite can take a long time, especially on machines
+with less cores. It is possible to skip the execution of the slower tests in the
+following way:
+
+.. code-block:: bash
+
+    pytest -m 'not slow'
+
 The build system also exposes a ``pytest`` target that executes ``setpath`` and
 parallelizes the test execution.
 
@@ -125,28 +133,28 @@ to a test failure.
 For more information, see :py:class:`mitsuba.python.chi2.ChiSquareTest`.
 
 
-.. Rendering test suite and Student-T test
-.. ---------------------------------------
+Rendering test suite and Z-test
+---------------------------------------
 
-.. On top of test *unit tests*, the framework implements a mechanism that automatically renders a set
-.. of test scenes and applies the *Student-T test* to compare the resulting images and some reference
-.. images.
+On top of test *unit tests*, the framework implements a mechanism that automatically renders a set
+of test scenes and applies the `Z-test <https://en.wikipedia.org/wiki/Z-test>`_ to compare the
+resulting images and some reference images.
 
-.. Those tests are really useful to reveal bugs at the interaction between the individual
-.. components of the renderer.
+Those tests are really useful to reveal bugs at the interaction between the individual
+components of the renderer.
 
-.. The test scenes are rendered using all the different enabled variants of the renderer, ensuring for
-.. instance that the ``scalar_rgb`` renders match the ``gpu_rgb`` renders.
+The test scenes are rendered using all the different enabled variants of the renderer, ensuring for
+instance that the ``scalar_rgb`` renders match the ``gpu_rgb`` renders.
 
-.. To only run the rendering test suite, use the following command:
+To only run the rendering test suite, use the following command:
 
-.. .. code-block:: bash
+.. code-block:: bash
 
-..     pytest src/librender/tests/test_renders.py
+    pytest src/librender/tests/test_renders.py
 
-.. One can easily add a scene to the ``resources/data/tests/scenes/`` folder to add it to the rendering
-.. test suite. Then, the missing reference images can be generated using the following command:
+One can easily add a scene to the ``resources/data/tests/scenes/`` folder to add it to the rendering
+test suite. Then, the missing reference images can be generated using the following command:
 
-.. .. code-block:: bash
+.. code-block:: bash
 
-..     python src/librender/tests/test_renders.py --spp 512
+    python src/librender/tests/test_renders.py

src/conftest.py

@@ -78,3 +78,13 @@ def variants_all_rgb(request):
     except Exception:
         pytest.skip('Mitsuba variant "%s" is not enabled!' % request.param)
     return request.param
+
+
+def pytest_configure(config):
+    markexpr = config.getoption("markexpr", 'False')
+    if not 'not slow' in markexpr:
+        print("""\033[93mRunning the full test suite. To skip slow tests, please run 'pytest -m 'not slow' \033[0m""")
+
+    config.addinivalue_line(
+        "markers", "slow: marks tests as slow (deselect with -m 'not slow')"
+    )

src/librender/tests/test_renders.py

@@ -24,7 +24,7 @@ def variants_all(request):
     __file__), '../../../resources/data/tests/scenes'))
 scenes = glob.glob(join(TEST_SCENE_DIR, '*', '*.xml'))
 
-# Exclude certain tests
+# List of test scene folders to exclude
 EXCLUDE_FOLDERS = []
 
 
@@ -76,6 +76,7 @@ def stdnormal_cdf(x):
     return p_value
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(*['scene_fname', scenes])
 def test_render(variants_all, scene_fname):
     from mitsuba.core import Bitmap, Struct, Thread, set_thread_count
@@ -88,7 +89,7 @@ def test_render(variants_all, scene_fname):
     Thread.thread().file_resolver().prepend(scene_dir)
 
     ref_fname, ref_var_fname = get_ref_fname(scene_fname)
-    if not (exists(ref_fname) and exists(ref_var_fname)) :
+    if not (exists(ref_fname) and exists(ref_var_fname)):
         pytest.skip("Non-existent reference data.")
 
     ref_bmp = read_rgb_bmp_to_xyz(ref_fname)
@@ -123,7 +124,6 @@ def test_render(variants_all, scene_fname):
     success = (p_value > alpha)
 
     if (np.count_nonzero(success) / 3) >= (0.9975 * pixel_count):
-    # if np.all(success):
         print('Accepted the null hypothesis (min(p-value) = %f, significance level = %f)' %
               (np.min(p_value), alpha))
     else:
@@ -135,7 +135,8 @@ def test_render(variants_all, scene_fname):
         if not exists(output_dir):
             os.makedirs(output_dir)
 
-        output_prefix = join(output_dir, splitext(basename(scene_fname))[0] + '_' + mitsuba.variant())
+        output_prefix = join(output_dir, splitext(
+            basename(scene_fname))[0] + '_' + mitsuba.variant())
 
         img_rgb_bmp = xyz_to_rgb_bmp(img)
 
@@ -160,12 +161,11 @@ def test_render(variants_all, scene_fname):
         assert False
 
 
-def main():
+if __name__ == '__main__':
     """
     Generate reference images for all the scenes contained within the TEST_SCENE_DIR directory,
     and for all the color mode having their `scalar_*` mode enabled.
     """
-
     parser = argparse.ArgumentParser(prog='RenderReferenceImages')
     parser.add_argument('--overwrite', action='store_true',
                         help='Force rerendering of all reference images. Otherwise, only missing references will be rendered.')
@@ -208,7 +208,3 @@ def main():
             # Write variance image to a file
             xyz_to_rgb_bmp(var_img).write(var_fname)
             print('Saved variance image to: ' + var_fname)
-
-
-if __name__ == '__main__':
-    main()