Use new SCANCODE_REGEN_TEST_FIXTURES env var

SCANCODE_REGEN_TEST_FIXTURES sets the flag regen=True in tests and
will regenerate the test data fixtures (typically expected YAML or JSON
outputs)

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

CHANGELOG.rst

@@ -48,14 +48,6 @@ Important API changes:
   column to "path". The "copyright_holder" has been ranmed to "holder"
 
 
-Development environment changes:
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
- - The license cache consistency is not checked anymore when you are using a Git
-   checkout. The SCANCODE_DEV_MODE tag file has been removed entirely. Use
-   instead the --reindex-licenses option to rebuild the license index.
-
-
 Copyright detection:
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -122,7 +114,6 @@ License detection:
 - Small, two-words matches that overlap the previous or next match by
   by the word "license" and assimilated are now filtered as false matches.
 
-
 - The new --licenses-reference option adds a new "licenses_reference" top
   level attribute to a scan when using the JSON and YAML outputs. This contains
   all the details and the full text of every licenses seen in a file or
@@ -251,13 +242,26 @@ Changes:
 - add resource-level attribute `for_packages`.
 - remove `package-data` attribute `root_path`.
 
+
 Documentation Update
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Various documentations have been updated to reflects API changes and
   correct minor documentation issues.
 
 
+Development environment changes:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The license cache consistency is not checked anymore when you are using a Git
+  checkout. The SCANCODE_DEV_MODE tag file has been removed entirely. Use
+  instead the --reindex-licenses option to rebuild the license index.
+
+- We can now regenerate updated test fixtures using the new SCANCODE_REGEN_TEST_FIXTURES
+  environemnt variable. There is no need to replace the regen=False with regen=True
+  in the code.
+
+
 30.1.0 - 2021-09-25
 --------------------
 

src/scancode_config.py

@@ -170,3 +170,7 @@ def _create_dir(location):
 _create_dir(__scancode_temp_base_dir)
 _prefix = 'scancode-tk-' + __version__ + '-'
 scancode_temp_dir = tempfile.mkdtemp(prefix=_prefix, dir=__scancode_temp_base_dir)
+
+
+# Used for tests to regenerate fixtures with regen=True
+REGEN_TEST_FIXTURES = os.getenv('SCANCODE_REGEN_TEST_FIXTURES', False)

tests/cluecode/cluecode_test_utils.py

@@ -21,6 +21,8 @@
 
 from cluecode.copyrights import detect_copyrights
 from cluecode.copyrights import Detection
+from scancode_config import REGEN_TEST_FIXTURES
+
 
 """
 Data-driven Copyright test utilities.
@@ -166,7 +168,7 @@ def make_copyright_test_functions(
     test,
     index,
     test_data_dir=test_env.test_data_dir,
-    regen=False,
+    regen=REGEN_TEST_FIXTURES,
 ):
     """
     Build and return a test function closing on tests arguments and the function
@@ -234,7 +236,7 @@ def build_tests(
     copyright_tests,
     clazz,
     test_data_dir=test_env.test_data_dir,
-    regen=False,
+    regen=REGEN_TEST_FIXTURES,
 ):
     """
     Dynamically build test methods from a sequence of CopyrightTest and attach

tests/cluecode/test_copyrights.py

@@ -10,9 +10,12 @@
 
 import pytest
 
+from commoncode.testcase import FileBasedTesting
+
 from cluecode_test_utils import build_tests
 from cluecode_test_utils import load_copyright_tests
-from commoncode.testcase import FileBasedTesting
+from scancode_config import REGEN_TEST_FIXTURES
+
 
 pytestmark = pytest.mark.scanslow
 
@@ -28,4 +31,4 @@ class TestCopyrightDataDriven(FileBasedTesting):
     pass
 
 
-build_tests(copyright_tests=load_copyright_tests(), clazz=TestCopyrightDataDriven, regen=False)
+build_tests(copyright_tests=load_copyright_tests(), clazz=TestCopyrightDataDriven, regen=REGEN_TEST_FIXTURES)

tests/cluecode/test_plugin_email_url.py

@@ -9,12 +9,12 @@
 
 import os
 
-import click
-
 from commoncode.testcase import FileDrivenTesting
 
 from scancode.cli_test_utils import check_json_scan
 from scancode.cli_test_utils import run_scan_click
+from scancode_config import REGEN_TEST_FIXTURES
+
 
 test_env = FileDrivenTesting()
 test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
@@ -25,28 +25,28 @@ def test_scan_email():
     result_file = test_env.get_temp_file('json')
     args = ['--email', '--strip-root', test_dir, '--json', result_file]
     run_scan_click(args)
-    check_json_scan(test_env.get_test_loc('plugin_email_url/emails.expected.json'), result_file, regen=False)
+    check_json_scan(test_env.get_test_loc('plugin_email_url/emails.expected.json'), result_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_scan_email_with_threshold():
     test_dir = test_env.get_test_loc('plugin_email_url/files')
     result_file = test_env.get_temp_file('json')
     args = ['--email', '--strip-root', '--max-email', '2', test_dir, '--json', result_file]
     run_scan_click(args)
-    check_json_scan(test_env.get_test_loc('plugin_email_url/emails-threshold.expected.json'), result_file, regen=False)
+    check_json_scan(test_env.get_test_loc('plugin_email_url/emails-threshold.expected.json'), result_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_scan_url():
     test_dir = test_env.get_test_loc('plugin_email_url/files')
     result_file = test_env.get_temp_file('json')
     args = ['--url', '--strip-root', test_dir, '--json', result_file]
     run_scan_click(args)
-    check_json_scan(test_env.get_test_loc('plugin_email_url/urls.expected.json'), result_file, regen=False)
+    check_json_scan(test_env.get_test_loc('plugin_email_url/urls.expected.json'), result_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_scan_url_with_threshold():
     test_dir = test_env.get_test_loc('plugin_email_url/files')
     result_file = test_env.get_temp_file('json')
     args = ['--url', '--strip-root', '--max-url', '2', test_dir, '--json', result_file]
     run_scan_click(args)
-    check_json_scan(test_env.get_test_loc('plugin_email_url/urls-threshold.expected.json'), result_file, regen=False)
+    check_json_scan(test_env.get_test_loc('plugin_email_url/urls-threshold.expected.json'), result_file, regen=REGEN_TEST_FIXTURES)

tests/cluecode/test_plugin_filter_clues.py

@@ -18,6 +18,9 @@
 from scancode.cli_test_utils import check_json_scan
 from scancode.cli_test_utils import run_scan_click
 
+from scancode_config import REGEN_TEST_FIXTURES
+
+
 test_env = FileDrivenTesting()
 test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
@@ -41,7 +44,7 @@ def test_scan_plugin_filter_clues_for_rule():
     args = ['-clieu', '--filter-clues', test_dir, '--json', result_file]
     run_scan_click(args)
     expected = test_env.get_test_loc('plugin_filter_clues/filtered-expected.json')
-    check_json_scan(expected, result_file, remove_file_date=True, regen=False)
+    check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES)
 
 
 def test_scan_plugin_filter_clues_does_not_filter_incorrectly():
@@ -53,7 +56,7 @@ def test_scan_plugin_filter_clues_does_not_filter_incorrectly():
     args = ['-clieu', '--filter-clues', test_dir, '--json', result_file]
     run_scan_click(args)
     expected = test_env.get_test_loc('plugin_filter_clues/filtered-expected2.json')
-    check_json_scan(expected, result_file, remove_file_date=True, regen=False)
+    check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES)
 
 
 # Regression on types tracked in https://github.com/nexB/typecode/issues/21
@@ -66,4 +69,4 @@ def test_scan_plugin_filter_clues_for_license():
     args = ['-clieu', '--filter-clues', test_dir, '--json', result_file]
     run_scan_click(args)
     expected = test_env.get_test_loc('plugin_filter_clues/filtered-expected3.json')
-    check_json_scan(expected, result_file, remove_file_date=True, regen=False)
+    check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES)

tests/cluecode/test_plugin_ignore_copyrights.py

@@ -11,9 +11,13 @@
 from os.path import join
 
 from commoncode.testcase import FileDrivenTesting
+
+from cluecode.plugin_ignore_copyrights import is_ignored
 from scancode.cli_test_utils import run_scan_click
 from scancode.cli_test_utils import check_json_scan
-from cluecode.plugin_ignore_copyrights import is_ignored
+from scancode_config import REGEN_TEST_FIXTURES
+
+
 
 
 class TestIgnoreCopyrights(FileDrivenTesting):
@@ -37,11 +41,11 @@ def test_ignore_holders(self):
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_ignore_copyrights/holders.expected.json')
         run_scan_click(['-c', '--ignore-copyright-holder', 'Regents', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES)
 
     def test_ignore_authors(self):
         test_dir = self.extract_test_tar('plugin_ignore_copyrights/basic.tgz')
         result_file = self.get_temp_file('json')
         expected_file = self.get_test_loc('plugin_ignore_copyrights/authors.expected.json')
         run_scan_click(['-c', '--ignore-author', 'Berkeley', '--json-pp', result_file, test_dir])
-        check_json_scan(expected_file, result_file, remove_file_date=True, regen=False)
+        check_json_scan(expected_file, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES)

tests/formattedcode/test_output_csv.py

@@ -14,11 +14,12 @@
 import os
 
 import pytest
-
 from commoncode.testcase import FileDrivenTesting
+
 from formattedcode.output_csv import flatten_scan
 from scancode.cli_test_utils import run_scan_click
 from scancode.cli_test_utils import run_scan_plain
+from scancode_config import REGEN_TEST_FIXTURES
 
 
 test_env = FileDrivenTesting()
@@ -38,7 +39,7 @@ def load_scan(json_input):
     return scan_results
 
 
-def check_json(result, expected_file, regen=False):
+def check_json(result, expected_file, regen=REGEN_TEST_FIXTURES):
     if regen:
         with io.open(expected_file, 'w', encoding='utf-8') as reg:
             reg.write(json.dumps(result, indent=4, separators=(',', ': ')))
@@ -49,7 +50,7 @@ def check_json(result, expected_file, regen=False):
 
 def check_csvs(result_file, expected_file,
                ignore_keys=('date', 'file_type', 'mime_type', 'package_uuid'),
-               regen=False):
+               regen=REGEN_TEST_FIXTURES):
     """
     Load and compare two CSVs.
     `ignore_keys` is a tuple of keys that will be ignored in the comparisons.
@@ -93,7 +94,7 @@ def test_flatten_scan_minimal():
         ])
     result = list(flatten_scan(scan, headers))
     expected_file = test_env.get_test_loc('csv/flatten_scan/minimal.json-expected')
-    check_json(result, expected_file, regen=False)
+    check_json(result, expected_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_flatten_scan_can_process_path_with_and_without_leading_slash():
@@ -119,7 +120,7 @@ def test_can_process_live_scan_for_packages_with_root():
     args = ['--package', test_dir, '--csv', result_file]
     run_scan_plain(args)
     expected_file = test_env.get_test_loc('csv/packages/expected.csv')
-    check_csvs(result_file, expected_file, regen=False)
+    check_csvs(result_file, expected_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_output_can_handle_non_ascii_paths():
@@ -234,7 +235,7 @@ def test_can_process_live_scan_with_all_options():
     args = ['-clip', '--email', '--url', '--strip-root', test_dir, '--csv', result_file]
     run_scan_plain(args)
     expected_file = test_env.get_test_loc('csv/livescan/expected.csv')
-    check_csvs(result_file, expected_file, regen=False)
+    check_csvs(result_file, expected_file, regen=REGEN_TEST_FIXTURES)
 
 
 @pytest.mark.scanslow
@@ -244,7 +245,7 @@ def test_can_process_live_scan_for_packages_strip_root():
     args = ['--package', '--strip-root', test_dir, '--csv', result_file]
     run_scan_plain(args)
     expected_file = test_env.get_test_loc('csv/packages/expected-no-root.csv')
-    check_csvs(result_file, expected_file, regen=False)
+    check_csvs(result_file, expected_file, regen=REGEN_TEST_FIXTURES)
 
 
 @pytest.mark.scanslow
@@ -254,7 +255,7 @@ def test_output_contains_license_expression():
     args = ['--from-json', test_file, '--csv', result_file]
     run_scan_plain(args)
     expected_file = test_env.get_test_loc('csv/expressions/expected.csv')
-    check_csvs(result_file, expected_file, regen=False)
+    check_csvs(result_file, expected_file, regen=REGEN_TEST_FIXTURES)
 
 
 @pytest.mark.scanslow
@@ -264,4 +265,4 @@ def test_output_handles_non_standard_data():
     args = ['--from-json', test_file, '--csv', result_file]
     run_scan_plain(args)
     expected_file = test_env.get_test_loc('csv/non-standard/identified.csv')
-    check_csvs(result_file, expected_file, regen=False)
+    check_csvs(result_file, expected_file, regen=REGEN_TEST_FIXTURES)

tests/formattedcode/test_output_cyclonedx.py

@@ -14,15 +14,17 @@
 
 import saneyaml
 from lxml import etree
-
 from commoncode.testcase import FileDrivenTesting
+
 from formattedcode.output_cyclonedx import CycloneDxComponent
 from formattedcode.output_cyclonedx import CycloneDxExternalRef
 from formattedcode.output_cyclonedx import CycloneDxHashObject
 from formattedcode.output_cyclonedx import CycloneDxLicenseExpression
 from formattedcode.output_cyclonedx import CycloneDxMetadata
 from formattedcode.output_cyclonedx import get_author_from_parties
 from scancode.cli_test_utils import run_scan_click
+from scancode_config import REGEN_TEST_FIXTURES
+
 
 test_env = FileDrivenTesting()
 test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
@@ -36,7 +38,7 @@ def load_and_clean_json(result_file):
     return result
 
 
-def check_cyclone_output(expected_file, result_file, regen=False):
+def check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES):
     """
     Check that expected and result_file are equal. Ignore headers.
     If `regen` is True the expected_file is overwritten with `results_file`.
@@ -57,7 +59,7 @@ def check_cyclone_output(expected_file, result_file, regen=False):
         assert result == expected
 
 
-def check_cyclone_xml_output(expected_file, result_file, regen=False):
+def check_cyclone_xml_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES):
     """
     Check that expected and result_file are equal. Ignore headers.
     If `regen` is True the expected_file is overwritten with `results_file`.
@@ -220,12 +222,12 @@ def test_cyclonedx_plugin_json():
     result_file = test_env.get_temp_file('cyclonedx.json')
     run_scan_click(['-p', test_dir, '--cyclonedx', result_file])
     expected_file = test_env.get_test_loc('cyclonedx/expected.json')
-    check_cyclone_output(expected_file, result_file, regen=False)
+    check_cyclone_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_cyclonedx_plugin_xml_components_and_dependencies_are_serialized_correctly():
     test_dir = test_env.get_test_loc('cyclonedx/simple')
     result_file = test_env.get_temp_file('cyclonedx.xml')
     run_scan_click(['-p', test_dir, '--cyclonedx-xml', result_file])
     expected_file = test_env.get_test_loc('cyclonedx/expected.xml')
-    check_cyclone_xml_output(expected_file, result_file, regen=False)
+    check_cyclone_xml_output(expected_file, result_file, regen=REGEN_TEST_FIXTURES)

tests/formattedcode/test_output_debian.py

@@ -12,13 +12,16 @@
 import shutil
 
 from commoncode.testcase import FileDrivenTesting
+
 from scancode.cli_test_utils import run_scan_plain
+from scancode_config import REGEN_TEST_FIXTURES
+
 
 test_env = FileDrivenTesting()
 test_env.test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
 
-def check_debian_copyright_output(test_dir, expected_file, regen=False):
+def check_debian_copyright_output(test_dir, expected_file, regen=REGEN_TEST_FIXTURES):
     """
     Rrun a scan on ``test_dir`` with a debian output and check if the created
     file matches ``expected_file``.
@@ -49,10 +52,10 @@ def check_debian_copyright_output(test_dir, expected_file, regen=False):
 def test_debian_basic():
     test_dir = test_env.get_test_loc('debian/basic/scan')
     expected_file = test_env.get_test_loc('debian/basic/expected.copyright')
-    check_debian_copyright_output(test_dir, expected_file, regen=False)
+    check_debian_copyright_output(test_dir, expected_file, regen=REGEN_TEST_FIXTURES)
 
 
 def test_debian_multiple_files():
     test_dir = test_env.get_test_loc('debian/multiple_files/scan')
     expected_file = test_env.get_test_loc('debian/multiple_files/expected.copyright')
-    check_debian_copyright_output(test_dir, expected_file, regen=False)
+    check_debian_copyright_output(test_dir, expected_file, regen=REGEN_TEST_FIXTURES)