411 toolkit upgrade 2

.gitignore

@@ -40,6 +40,7 @@ local
 policies.yml
 *.rdb
 *.aof
+.vscode
 
 # This is only created when packaging for external redistribution
 /thirdparty/

CHANGELOG.rst

@@ -7,6 +7,10 @@ v31.0.0 (next)
 - WARNING: Drop support for Python 3.6 and 3.7. Add support for Python 3.10.
   Upgrade Django to version 4.x series.
 
+- Upgrade ScanCode-toolkit to version v31.
+  See https://github.com/nexB/scancode-toolkit/blob/develop/CHANGELOG.rst for an
+  overview of the changes in v31 compared to v30.
+
 - Implement run status auto-refresh using the htmx JavaScript library.
   The statuses of queued and running pipeline are now automatically refreshed
   in the project list and project details views every 10 seconds.

scanpipe/migrations/0016_discoveredpackage_package_uid.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-06-09 18:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0015_alter_codebaseresource_project_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='discoveredpackage',
+            name='package_uid',
+            field=models.CharField(blank=True, help_text='Unique identifier for this package.', max_length=1024),
+        ),
+    ]

scanpipe/models.py

@@ -1726,6 +1726,11 @@ class DiscoveredPackage(
         blank=True,
         help_text=_("A list of dependencies for this package."),
     )
+    package_uid = models.CharField(
+        max_length=1024,
+        blank=True,
+        help_text=_("Unique identifier for this package."),
+    )
 
     # `AbstractPackage` model overrides:
     keywords = models.JSONField(default=list, blank=True)
@@ -1769,7 +1774,7 @@ def create_from_data(cls, project, package_data):
         If one of the values of the required fields is not available, a "ProjectError"
         is created instead of a new DiscoveredPackage instance.
         """
-        required_fields = ["type", "name", "version"]
+        required_fields = ["type", "name"]
         missing_values = [
             field_name
             for field_name in required_fields

scanpipe/pipelines/docker.py

@@ -20,12 +20,12 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-from scanpipe.pipelines import root_filesystems
+from scanpipe.pipelines.root_filesystems import RootFS
 from scanpipe.pipes import docker
 from scanpipe.pipes import rootfs
 
 
-class Docker(root_filesystems.RootFS):
+class Docker(RootFS):
     """
     A pipeline to analyze Docker images.
     """

scanpipe/pipelines/load_inventory.py

@@ -42,15 +42,14 @@ def get_scan_json_input(self):
         Locates a JSON scan input from a project's input/ directory.
         """
         inputs = list(self.project.inputs(pattern="*.json"))
+
         if len(inputs) != 1:
             raise Exception("Only 1 JSON input file supported")
+
         self.input_location = str(inputs[0].absolute())
 
     def build_inventory_from_scan(self):
         """
-        Processes a given JSON scan input to populate codebase resources and packages.
+        Processes a JSON Scan results file to populate codebase resources and packages.
         """
-        project = self.project
-        scanned_codebase = scancode.get_virtual_codebase(project, self.input_location)
-        scancode.create_codebase_resources(project, scanned_codebase)
-        scancode.create_discovered_packages(project, scanned_codebase)
+        scancode.create_inventory_from_scan(self.project, self.input_location)

scanpipe/pipelines/scan_codebase.py

@@ -22,7 +22,6 @@
 
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
-from scanpipe.pipes import output
 from scanpipe.pipes import rootfs
 from scanpipe.pipes import scancode
 from scanpipe.pipes.input import copy_inputs

scanpipe/pipelines/scan_package.py

@@ -56,13 +56,9 @@ def steps(cls):
         "--license-text",
         "--package",
         "--url",
-    ] + [
         "--classify",
-        "--consolidate",
         "--is-license-text",
-        "--license-clarity-score",
         "--summary",
-        "--summary-key-files",
     ]
 
     def get_package_archive_input(self):
@@ -102,33 +98,31 @@ def run_scancode(self):
         """
         Scans extracted codebase/ content.
         """
-        self.scan_output = self.project.get_output_file_path("scancode", "json")
+        scan_output_path = self.project.get_output_file_path("scancode", "json")
+        self.scan_output_location = str(scan_output_path.absolute())
 
         with self.save_errors(scancode.ScancodeError):
             scancode.run_scancode(
                 location=str(self.project.codebase_path),
-                output_file=str(self.scan_output),
+                output_file=self.scan_output_location,
                 options=self.scancode_options,
                 raise_on_error=True,
             )
 
-        if not self.scan_output.exists():
+        if not scan_output_path.exists():
             raise FileNotFoundError("ScanCode output not available.")
 
     def build_inventory_from_scan(self):
         """
-        Processes the JSON scan results to determine resources and packages.
+        Processes a JSON Scan results file to populate codebase resources and packages.
         """
-        project = self.project
-        scanned_codebase = scancode.get_virtual_codebase(project, str(self.scan_output))
-        scancode.create_codebase_resources(project, scanned_codebase)
-        scancode.create_discovered_packages(project, scanned_codebase)
+        scancode.create_inventory_from_scan(self.project, self.scan_output_location)
 
     def make_summary_from_scan_results(self):
         """
         Builds a summary in JSON format from the generated scan results.
         """
-        summary = scancode.make_results_summary(self.project, str(self.scan_output))
+        summary = scancode.make_results_summary(self.project, self.scan_output_location)
         output_file = self.project.get_output_file_path("summary", "json")
 
         with output_file.open("w") as summary_file:

scanpipe/pipes/__init__.py

@@ -29,8 +29,6 @@
 
 from django.db.models import Count
 
-from packageurl import normalize_qualifiers
-
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredPackage
 from scanpipe.pipes import scancode
@@ -73,12 +71,19 @@ def update_or_create_package(project, package_data, codebase_resource=None):
     """
     Gets, updates or creates a DiscoveredPackage then returns it.
     Uses the `project` and `package_data` mapping to lookup and creates the
-    DiscoveredPackage using its Package URL as a unique key.
+    DiscoveredPackage using its Package URL and package_uid as a unique key.
     """
     purl_data = DiscoveredPackage.extract_purl_data(package_data)
+    package_uid = package_data.get("package_uid")
+    purl_data_and_package_uid = {
+        **purl_data,
+        "package_uid": package_uid,
+    }
 
     try:
-        package = DiscoveredPackage.objects.get(project=project, **purl_data)
+        package = DiscoveredPackage.objects.get(
+            project=project, **purl_data_and_package_uid
+        )
     except DiscoveredPackage.DoesNotExist:
         package = None
 

scanpipe/pipes/codebase.py

@@ -53,6 +53,8 @@ def get_tree(resource, fields, codebase=None):
     return resource_dict
 
 
+# TODO: Walking the ProjectCodebase is broken as we do not have a consistent way
+# to get the root of a codebase.
 class ProjectCodebase:
     """
     Represents the codebase of a project stored in the database.

scanpipe/pipes/docker.py

@@ -22,7 +22,6 @@
 
 import logging
 import posixpath
-from functools import partial
 from pathlib import Path
 
 from container_inspector.image import Image
@@ -152,23 +151,21 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
         raise rootfs.DistroNotFound(f"Distro not found.")
 
     distro_id = image.distro.identifier
-    if distro_id not in rootfs.PACKAGE_GETTER_BY_DISTRO:
+    if distro_id not in rootfs.SUPPORTED_DISTROS:
         raise rootfs.DistroNotSupported(f'Distro "{distro_id}" is not supported.')
 
-    package_getter = partial(
-        rootfs.PACKAGE_GETTER_BY_DISTRO[distro_id],
-        distro=distro_id,
-        detect_licenses=detect_licenses,
-    )
-
-    installed_packages = image.get_installed_packages(package_getter)
+    installed_packages = image.get_installed_packages(rootfs.package_getter)
 
     for i, (purl, package, layer) in enumerate(installed_packages):
         logger.info(f"Creating package #{i}: {purl}")
         created_package = pipes.update_or_create_package(project, package.to_dict())
 
+        installed_files = []
+        if hasattr(package, "resources"):
+            installed_files = package.resources
+
         # We have no files for this installed package, we cannot go further.
-        if not package.installed_files:
+        if not installed_files:
             logger.info(f"  No installed_files for: {purl}")
             continue
 
@@ -177,8 +174,9 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
 
         codebase_resources = project.codebaseresources.all()
 
-        for install_file in package.installed_files:
-            install_file_path = pipes.normalize_path(install_file.path)
+        for install_file in installed_files:
+            install_file_path = install_file.get_path(strip_root=True)
+            install_file_path = pipes.normalize_path(install_file_path)
             layer_rootfs_path = posixpath.join(
                 layer.layer_id,
                 install_file_path.strip("/"),

scanpipe/pipes/output.py

@@ -168,7 +168,12 @@ def get_headers(self, project):
     def get_packages(self, project):
         from scanpipe.api.serializers import DiscoveredPackageSerializer
 
-        packages = project.discoveredpackages.all()
+        packages = project.discoveredpackages.all().order_by(
+            "type",
+            "namespace",
+            "name",
+            "version",
+        )
 
         for obj in packages.iterator():
             yield self.encode(DiscoveredPackageSerializer(obj).data)
@@ -280,9 +285,9 @@ def _add_xlsx_worksheet(workbook, worksheet_name, rows, fields):
 # https://github.com/nexB/scancode-toolkit/pull/2381
 # https://github.com/nexB/scancode-toolkit/issues/2350
 mappings_key_by_fieldname = {
-    "copyrights": "value",
-    "holders": "value",
-    "authors": "value",
+    "copyrights": "copyright",
+    "holders": "holder",
+    "authors": "author",
     "emails": "email",
     "urls": "url",
 }