Scancode: Fix false positive reported by scancode output analyser script

ScanCode can possibly return many licenses found for a single file scanned.
This commit ensures that the file is not reported as lacking a permissive license
if at least one license found in it is permissive.
Previously the script was reporting an issue if it found at least one license
in a file that was not permissive.

Additionally catch more errors and provide specific details about failures.
Provide unitest.

Author: hugueskamba

tools/test/travis-ci/scancode-evaluate.py

@@ -13,22 +13,30 @@
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations
+limitations 
 """
 
-# Asumptions for this script:
+# Asumptions for this script: 
 # 1. directory_name is scanned directory.
 #  Files are copied to this directory with full tree. As result, if we find
 #  license offender, we can have full path (just scrape directory_name). We do this
 #  magic because scancode allows to scan directories/one file.
 # 2. SPDX and license text is a must for all code files
 
-import json
 import argparse
-import sys
-import os.path
+import json
 import logging
+import os.path
 import re
+import sys
+from enum import Enum
+
+
+class ReturnCode(Enum):
+    """Return codes."""
+
+    SUCCESS = 0
+    ERROR = -1
 
 userlog = logging.getLogger("scancode-evaluate")
 userlog.setLevel(logging.INFO)
@@ -37,95 +45,113 @@
 userlog.addHandler(log_file_handler)
 
 MISSING_LICENSE_TEXT = "Missing license header"
-MISSING_PERMISIVE_LICENSE_TEXT = "Non-permissive license"
+MISSING_PERMISSIVE_LICENSE_TEXT = "Non-permissive license"
 MISSING_SPDX_TEXT = "Missing SPDX license identifier"
 
-def license_check(directory_name, file):
-    """ Check licenses in the scancode json file for specified directory
+
+def path_leaf(path):
+    """Return the leaf of a path."""
+    head, tail = os.path.split(path)
+    # Ensure the correct file name is returned if the file ends with a slash
+    return tail or os.path.basename(head)
+
+def has_permissive_text_in_scancode_output(scancode_output_data_file_licenses):
+    """Returns true if at list one license in the scancode output is permissive."""
+    return any(
+        scancode_output_data_file_license['category'] == 'Permissive'
+        for scancode_output_data_file_license in scancode_output_data_file_licenses
+    )
+
+def has_spdx_text_in_scancode_output(scancode_output_data_file_licenses):
+    """Returns true if at least one license in the scancode output has the spdx identifier."""
+    return any(
+        'spdx' in scancode_output_data_file_license['matched_rule']['identifier']
+        for scancode_output_data_file_license in scancode_output_data_file_licenses
+    )
+
+def has_spdx_text_in_analysed_file(scanned_file_content):
+    """Returns true if the file analysed by ScanCode contains SPDX identifier."""
+    return bool(re.findall("SPDX-License-Identifier:?", scanned_file_content))
+
+def license_check(scancode_output):
+    """Check licenses in the scancode json file for specified directory.
 
     This function does not verify if file exists, should be done prior the call.
 
     Args:
-    directory_name - where scancode was run, used to scrape this from paths
     file - scancode json output file (output from scancode --license --json-pp)
 
-    Returns:
+    Returns:    
     0 if nothing found
     >0 - count how many license isses found
-    -1 if any error in file licenses found
+    ReturnCode.ERROR.value if any error in file licenses found
     """
 
     offenders = []
     try:
-        # find all licenses in the files, must be licensed and permissive
-        with open(file, 'r') as scancode_output:
-            results = json.load(scancode_output)
-    except ValueError:
-        userlog.warning("JSON could not be decoded")
-        return -1
-
-    try:
-        for file in results['files']:
-            license_offender = {}
-            license_offender['file'] = file
-            # ignore directory, not relevant here
-            if license_offender['file']['type'] == 'directory':
+        with open(scancode_output, 'r') as read_file:
+            scancode_output_data = json.load(read_file)
+    except json.JSONDecodeError as jex:
+        userlog.warning("JSON could not be decoded, Invalid JSON in body: %s", jex)
+        return ReturnCode.ERROR.value
+
+    if 'files' not in scancode_output_data:
+        userlog.warning("Missing `files` attribute in %s" % (scancode_output))
+        return ReturnCode.ERROR.value
+
+    for scancode_output_data_file in scancode_output_data['files']:
+        try:
+            if scancode_output_data_file['type'] != 'file':
                 continue
-            if not license_offender['file']['licenses']:
-                license_offender['reason'] = MISSING_LICENSE_TEXT
-                offenders.append(license_offender.copy())
+        except KeyError as e:
+            userlog.warning("Could not find %s attribute in %s" % (str(e), scancode_output))
+            return ReturnCode.ERROR.value
+
+        try:
+            if not scancode_output_data_file['licenses']:
+                scancode_output_data_file['fail_reason'] = MISSING_LICENSE_TEXT
+                offenders.append(scancode_output_data_file)
+                # check the next file in the scancode output
                 continue
-
-            found_spdx = spdx_check(offenders, license_offender)
-
-            if not found_spdx:
+        except KeyError as e:
+            userlog.warning("Could not find %s attribute in %s" % (str(e), scancode_output))
+            return ReturnCode.ERROR.value
+
+        try:
+            if not has_permissive_text_in_scancode_output(scancode_output_data_file['licenses']):
+                scancode_output_data_file['fail_reason'] = MISSING_PERMISSIVE_LICENSE_TEXT
+                offenders.append(scancode_output_data_file)
+        except KeyError as e:
+            userlog.warning("Could not find %s attribute in %s" % (str(e), scancode_output))
+            return ReturnCode.ERROR.value
+
+        try:
+            if not has_spdx_text_in_scancode_output(scancode_output_data_file['licenses']):
+                # Scancode does not recognize license notice in Python file headers.
+                # Issue: https://github.com/nexB/scancode-toolkit/issues/1913
+                # Therefore check if the file tested by ScanCode actually has a licence notice.
+                file_path = os.path.abspath(scancode_output_data_file['path'])
                 try:
-                    # Issue reported here https://github.com/nexB/scancode-toolkit/issues/1913
-                    # We verify here if SPDX is not really there as SDPX is part of the license text
-                    # scancode has some problems detecting it properly
-                    with open(os.path.join(os.path.abspath(license_offender['file']['path'])), 'r') as spdx_file_check:
-                        filetext = spdx_file_check.read()
-                    matches = re.findall("SPDX-License-Identifier:?", filetext)
-                    if matches:
-                        continue
-                    license_offender['reason'] = MISSING_SPDX_TEXT
-                    offenders.append(license_offender.copy())
+                    with open(file_path, 'r') as read_file:
+                        scanned_file_content = read_file.read()
                 except UnicodeDecodeError:
-                    # not valid file for license check
+                    userlog.warning("Unable to look for SPDX text in `{}`:".format(file))
+                    # Ignore files that cannot be decoded
+                    # check the next file in the scancode output
                     continue
-    except KeyError:
-        userlog.warning("Invalid scancode json file")
-        return -1
+                if not has_spdx_text_in_analysed_file(scanned_file_content):
+                    scancode_output_data_file['fail_reason'] = MISSING_SPDX_TEXT
+                    offenders.append(scancode_output_data_file)
+        except KeyError as e:
+            userlog.warning("Could not find %s attribute in %s" % (str(e), scancode_output))
+            return ReturnCode.ERROR.value
 
     if offenders:
         userlog.warning("Found files with missing license details, please review and fix")
         for offender in offenders:
-            userlog.warning("File: " + offender['file']['path'][len(directory_name):] + " " + "reason: " + offender['reason'])
+            userlog.warning("File: %s reason: %s" % (path_leaf(offender['path']), offender['fail_reason']))
     return len(offenders)
 
-
-def spdx_check(offenders, license_offender):
-        """ Parse through list of licenses to determine whether licenses are permissive
-                @input list of offender, individual offender dict
-                @output none
-        """
-        found_spdx = False
-        # iterate through licenses, stop once permissive license has been found
-        for i in range(len(license_offender['file']['licenses'])):
-                # is any of the licenses permissive ?
-                if license_offender['file']['licenses'][i]['category'] == 'Permissive':
-                        # confirm that it has spdx license key
-                        if license_offender['file']['licenses'][i]['matched_rule']['identifier'].find("spdx") != -1:
-                                found_spdx = True
-                        # if no spdx found return anyway
-                        return found_spdx
-        # otherwise file is missing permissive license
-        license_offender['reason'] = MISSING_PERMISIVE_LICENSE_TEXT
-        offenders.append(license_offender.copy())
-
-        # missing spdx and permissive license
-        return found_spdx
-
 def parse_args():
     parser = argparse.ArgumentParser(
         description="License check.")
@@ -135,15 +161,15 @@ def parse_args():
                         help='Directory name where are files being checked')
     return parser.parse_args()
 
-
 if __name__ == "__main__":
+
     args = parse_args()
     if args.file and os.path.isfile(args.file):
-        count = license_check(args.directory_name, args.file)
-        if count == 0:
-            sys.exit(0)
-        else:
-            sys.exit(-1)
+        sys.exit(
+            ReturnCode.SUCCESS.value
+            if license_check(args.file) == 0
+            else ReturnCode.ERROR.value
+        )
     else:
         userlog.warning("Could not find the scancode json file")
-        sys.exit(-1)
+        sys.exit(ReturnCode.ERROR.value)