Apply LicenseDetection everywhere

Modify code to align to the LicenseDetection model everywhere.

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

src/formattedcode/output_debian.py

@@ -16,6 +16,7 @@
 from formattedcode import FileOptionType
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
+from licensedcode.detection import get_matches_from_detections
 
 from scancode import notice
 
@@ -179,8 +180,8 @@ def get_texts(detected_licenses):
 
     # set of (start line, end line, matched_rule identifier)
     seen = set()
-    for lic in detected_licenses:
-        key = lic['start_line'], lic['end_line'], lic['matched_rule']['identifier']
+    for lic in get_matches_from_detections(detected_licenses):
+        key = lic['start_line'], lic['end_line'], lic['licensedb_identifier']
         if key not in seen:
             yield lic['matched_text']
             seen.add(key)

src/formattedcode/output_html.py

@@ -29,6 +29,7 @@
 from commoncode.cliutils import OUTPUT_GROUP
 from plugincode.output import output_impl
 from plugincode.output import OutputPlugin
+from licensedcode.detection import get_matches_from_detections
 
 """
 Output plugins to write scan results using templates such as HTML.
@@ -146,6 +147,7 @@ def generate_output(results, version, template):
     # support adding new scans at all
 
     from licensedcode.cache import get_licenses_db
+    licenses_db = get_licenses_db()
 
     converted = {}
     converted_infos = {}
@@ -170,22 +172,22 @@ def generate_output(results, version, template):
                     'value': entry['copyright'],
                 })
         if LICENSES in scanned_file:
-            for entry in scanned_file[LICENSES]:
+            for match in get_matches_from_detections(scanned_file[LICENSES]):
                 # make copy
-                entry = dict(entry)
-                entry_key = entry['key']
+                match = dict(match)
+                license_expression = match['license_expression']
                 results.append({
-                    'start': entry['start_line'],
-                    'end': entry['end_line'],
+                    'start': match['start_line'],
+                    'end': match['end_line'],
                     'what': 'license',
-                    'value': entry_key,
+                    'value': license_expression,
                 })
 
                 # FIXME: we should NOT rely on license objects: only use what is in the JSON instead
-                if entry_key not in licenses:
-                    licenses[entry_key] = entry
+                if license_expression not in licenses:
+                    licenses[license_expression] = match
                     # we were modifying the scan data in place ....
-                    entry['object'] = get_licenses_db().get(entry_key)
+                    match['object'] = licenses_db.get(license_expression)
         if results:
             converted[path] = sorted(results, key=itemgetter('start'))
 

src/licensedcode/detection.py

@@ -759,6 +759,19 @@ def get_matches_from_detections(license_detections):
     return license_matches
 
 
+def get_license_keys_from_detections(license_detections):
+    """
+    Return a list of unique license key strings from a list of LicenseDetections.
+    """
+    license_keys = set()
+
+    matches = get_matches_from_detections(license_detections)
+    for match in matches:
+        licenses = match.get('licenses')
+        license_keys.update([entry.get('key') for entry in licenses])
+    return list(license_keys)
+
+
 def analyze_detection(license_matches):
     """
     Analyse a list of LicenseMatch objects, and determine if the license detection
@@ -919,5 +932,8 @@ def detect_licenses(location, min_score, deadline, **kwargs):
         **kwargs,
     )
 
+    if not matches:
+        return
+
     for group_of_matches in group_matches(matches):
         yield LicenseDetection.from_matches(matches=group_of_matches)

src/licensedcode/plugin_license.py

@@ -31,8 +31,6 @@
 
 TRACE = os.environ.get('SCANCODE_DEBUG_PLUGIN_LICENSE', False)
 
-def logger_debug(*args): pass
-
 
 def logger_debug(*args):
     pass

src/licensedcode/plugin_license_policy.py

@@ -7,16 +7,39 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from inspect import trace
 from os.path import exists
 from os.path import isdir
 
 import attr
+import os
+import logging
 import saneyaml
 
 from plugincode.post_scan import PostScanPlugin
 from plugincode.post_scan import post_scan_impl
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import POST_SCAN_GROUP
+from licensedcode.detection import get_license_keys_from_detections
+
+
+TRACE = os.environ.get('SCANCODE_DEBUG_LICENSE_POLICY', False)
+
+
+def logger_debug(*args):
+    pass
+
+
+logger = logging.getLogger(__name__)
+
+
+import sys
+
+logging.basicConfig(stream=sys.stdout)
+logger.setLevel(logging.DEBUG)
+
+def logger_debug(*args):
+    return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
 @post_scan_impl
@@ -63,7 +86,7 @@ def process_codebase(self, codebase, license_policy, **kwargs):
                 continue
 
             try:
-                resource_license_keys = set([entry.get('key') for entry in resource.licenses])
+                resource_license_keys = get_license_keys_from_detections(resource.licenses)
 
             except AttributeError:
                 # add license_policy regardless if there is license info or not

src/licensedcode/plugin_license_text.py

@@ -13,6 +13,7 @@
 from plugincode.post_scan import post_scan_impl
 from commoncode.cliutils import PluggableCommandLineOption
 from commoncode.cliutils import POST_SCAN_GROUP
+from licensedcode.detection import get_matches_from_detections
 
 # Set to True to enable debug tracing
 TRACE = False
@@ -72,12 +73,13 @@ def process_codebase(self, codebase, is_license_text, **kwargs):
                 continue
             # keep unique texts/line ranges since we repeat this for each matched licenses
             license_texts = set()
-            for lic in resource.licenses:
+            matches = get_matches_from_detections(resource.licenses)
+            for match in matches:
                 license_texts.add(
-                    (lic.get('matched_text'),
-                     lic.get('start_line', 0),
-                     lic.get('end_line', 0),
-                     lic.get('matched_rule', {}).get('match_coverage', 0))
+                    (match.get('matched_text'),
+                     match.get('start_line', 0),
+                     match.get('end_line', 0),
+                     match.get('match_coverage', 0))
                 )
 
             # use coverage to weight and estimate of the the actual matched length

src/summarycode/score.py

@@ -16,6 +16,7 @@
 from plugincode.post_scan import post_scan_impl
 
 from packagedcode.utils import combine_expressions
+from licensedcode.detection import get_matches_from_detections
 
 # Tracing flags
 TRACE = False
@@ -124,11 +125,12 @@ def compute_license_score(codebase):
     """
 
     scoring_elements = ScoringElements()
-    declared_licenses = get_field_values_from_codebase_resources(
+    license_detections = get_field_values_from_codebase_resources(
         codebase=codebase,
         field_name='licenses',
         key_files_only=True,
     )
+    declared_licenses = get_matches_from_detections(license_detections)
     declared_license_expressions = get_field_values_from_codebase_resources(
         codebase=codebase, field_name='license_expressions', key_files_only=True
     )
@@ -140,9 +142,10 @@ def compute_license_score(codebase):
         codebase=codebase, field_name='copyrights', key_files_only=True
     )
 
-    other_licenses = get_field_values_from_codebase_resources(
+    other_license_detections = get_field_values_from_codebase_resources(
         codebase=codebase, field_name='licenses', key_files_only=False
     )
+    other_licenses = get_matches_from_detections(other_license_detections)
 
     scoring_elements.declared_license = bool(declared_licenses)
     if scoring_elements.declared_license:
@@ -244,20 +247,19 @@ class LicenseFilter(object):
 
 def is_good_license(detected_license):
     """
-    Return True if a `detected license` mapping is consider to a high quality
+    Return True if a `detected license` mapping is considered to be a high quality
     conclusive match.
     """
     score = detected_license['score']
-    rule = detected_license['matched_rule']
-    coverage = rule.get('match_coverage') or 0
-    relevance = rule.get('rule_relevance') or 0
+    coverage = detected_license.get('match_coverage') or 0
+    relevance = detected_license.get('rule_relevance') or 0
     match_types = dict(
         [
-            ('is_license_text', rule['is_license_text']),
-            ('is_license_notice', rule['is_license_notice']),
-            ('is_license_reference', rule['is_license_reference']),
-            ('is_license_tag', rule['is_license_tag']),
-            ('is_license_intro', rule['is_license_intro']),
+            ('is_license_text', detected_license['is_license_text']),
+            ('is_license_notice', detected_license['is_license_notice']),
+            ('is_license_reference', detected_license['is_license_reference']),
+            ('is_license_tag', detected_license['is_license_tag']),
+            ('is_license_intro', detected_license['is_license_intro']),
         ]
     )
     matched = False
@@ -320,15 +322,24 @@ def get_field_values_from_codebase_resources(codebase, field_name, key_files_onl
     return values
 
 
-def get_license_categories(license_infos):
+def get_categories_from_match(license_match):
+    """
+    Return a list of license category strings from a single LicenseMatch mapping.
+    """
+    licenses = license_match.get('licenses')
+    return [license_info.get('category') for license_info in licenses]
+
+
+def get_license_categories(declared_licenses):
     """
     Return a list of license category strings from `license_infos`
     """
     license_categories = []
-    for license_info in license_infos:
-        category = license_info.get('category', '')
-        if category not in license_categories:
-            license_categories.append(category)
+
+    for match in declared_licenses:
+        for category in get_categories_from_match(match):
+            if category not in license_categories:
+                license_categories.append(category)
     return license_categories
 
 
@@ -339,11 +350,10 @@ def check_for_license_texts(declared_licenses):
     If so, return True. Otherwise, return False.
     """
     for declared_license in declared_licenses:
-        matched_rule = declared_license.get('matched_rule', {})
         if any(
             [
-                matched_rule.get('is_license_text', False),
-                matched_rule.get('is_license_notice', False),
+                declared_license.get('is_license_text', False),
+                declared_license.get('is_license_notice', False),
             ]
         ):
             return True
@@ -379,9 +389,10 @@ def check_for_conflicting_licenses(other_licenses):
 
     If so, return True. Otherwise, return False.
     """
-    for license_info in other_licenses:
-        if license_info.get('category', '') in CONFLICTING_LICENSE_CATEGORIES:
-            return True
+    for license_match in other_licenses:
+        for category in get_categories_from_match(license_match):
+            if category in CONFLICTING_LICENSE_CATEGORIES:
+                return True
     return False
 
 

tests/licensedcode/test_plugin_license_detection.py

@@ -144,10 +144,10 @@ def test_license_match_referenced_filename():
 
 def test_get_referenced_filenames():
     license_matches = [
-        {'matched_rule': {'referenced_filenames' : ['LICENSE.txt', 'COPYING']}},
-        {'matched_rule': {'referenced_filenames' : ['COPYING', 'LICENSE.txt']}},
-        {'matched_rule': {'referenced_filenames' : ['copying']}},
-        {'matched_rule': {'referenced_filenames' : []}},
+        {'referenced_filenames' : ['LICENSE.txt', 'COPYING']},
+        {'referenced_filenames' : ['COPYING', 'LICENSE.txt']},
+        {'referenced_filenames' : ['copying']},
+        {'referenced_filenames' : []},
     ]
     expected = ['LICENSE.txt', 'COPYING', 'copying']
     assert get_referenced_filenames(license_matches) == expected

tests/licensedcode/test_plugin_license_policy.py

@@ -15,12 +15,31 @@
 from licensedcode.plugin_license_policy import load_license_policy
 from scancode.cli_test_utils import load_json_result
 from scancode.cli_test_utils import run_scan_click
-
+from scancode.cli_test_utils import check_json_scan
+from scancode_config import REGEN_TEST_FIXTURES
 
 class TestLicensePolicy(FileDrivenTesting):
 
     test_data_dir = join(dirname(__file__), 'data')
 
+    def test_end_to_end_scan_with_license_policy(self):
+        test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz')
+        policy_file = self.get_test_loc('plugin_license_policy/process_codebase_info_license_valid_policy_file.yml')
+        result_file = self.get_temp_file('json')
+        args = [
+            '--info',
+            '--license',
+            '--license-policy',
+            policy_file,
+            test_dir,
+            '--json-pp',
+            result_file
+        ]
+        run_scan_click(args)
+        test_loc = self.get_test_loc('plugin_license_policy/policy-codebase.expected.json')
+        check_json_scan(test_loc, result_file, regen=REGEN_TEST_FIXTURES)
+
+
     def test_process_codebase_info_license_duplicate_key_policy_file(self):
         test_dir = self.extract_test_tar('plugin_license_policy/policy-codebase.tgz')
         policy_file = self.get_test_loc('plugin_license_policy/process_codebase_info_license_duplicate_key_policy_file.yml')

tests/scancode/test_api.py

@@ -129,8 +129,7 @@ def test_get_license_with_expression(self):
         test_file = self.get_test_loc('api/license/apache-1.0.txt')
         results = api.get_licenses(test_file)
         expected = [
-            'apache-1.0',
-            'gpl-2.0 WITH linux-syscall-exception-gpl OR linux-openib'
+            'apache-1.0 AND (gpl-2.0 WITH linux-syscall-exception-gpl OR linux-openib)'
         ]
         assert results['license_expressions'] == expected
 
@@ -144,5 +143,5 @@ def test_get_license_returns_correct_lines(self):
         test_file = self.get_test_loc('api/license/correct_lines2')
         results = api.get_licenses(test_file)
         assert results['license_expressions'] == ['mit']
-        assert results['licenses'][0]['start_line'] == 2
-        assert results['licenses'][0]['end_line'] == 4
+        assert results['licenses'][0]['matches'][0]['start_line'] == 2
+        assert results['licenses'][0]['matches'][0]['end_line'] == 4