Add a "Load Packages from SBOMs" Product action in the REST API #59 (#62)

* Add a "Load Packages from SBOMs" Product action in the REST API #59

Signed-off-by: tdruez <tdruez@nexb.com>

* Add a "Import scan results" Product action in the REST API #59

Signed-off-by: tdruez <tdruez@nexb.com>

* Add "Pull data from a ScanCode.io" Product action in the REST API #59

Signed-off-by: tdruez <tdruez@nexb.com>

---------

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

CHANGELOG.rst

@@ -24,6 +24,13 @@ Release notes
 - Add dark theme support in UI.
   https://github.com/nexB/dejacode/issues/25
 
+- Add "Load Packages from SBOMs", "Import scan results", and
+  "Pull ScanCode.io project data" feature as Product action in the REST API.
+  https://github.com/nexB/dejacode/issues/59
+
+- Refactor the "Import manifest" feature as "Load SBOMs".
+  https://github.com/nexB/dejacode/issues/61
+
 ### Version 5.0.1
 
 - Improve the stability of the "Check for new Package versions" feature.

product_portfolio/api.py

@@ -11,7 +11,10 @@
 import django_filters
 from rest_framework import permissions
 from rest_framework import serializers
+from rest_framework import status
+from rest_framework.decorators import action
 from rest_framework.permissions import SAFE_METHODS
+from rest_framework.response import Response
 
 from component_catalog.api import KeywordsField
 from component_catalog.api import PackageEmbeddedSerializer
@@ -31,6 +34,9 @@
 from dje.filters import NameVersionFilter
 from dje.permissions import assign_all_object_permissions
 from product_portfolio.filters import ComponentCompletenessAPIFilter
+from product_portfolio.forms import ImportFromScanForm
+from product_portfolio.forms import LoadSBOMsForm
+from product_portfolio.forms import PullProjectDataForm
 from product_portfolio.models import CodebaseResource
 from product_portfolio.models import Product
 from product_portfolio.models import ProductComponent
@@ -191,6 +197,57 @@ class Meta:
         )
 
 
+class LoadSBOMsFormSerializer(serializers.Serializer):
+    """Serializer equivalent of LoadSBOMsForm, used for API documentation."""
+
+    input_file = serializers.FileField(
+        required=True,
+        help_text=LoadSBOMsForm.base_fields["input_file"].label,
+    )
+    update_existing_packages = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=LoadSBOMsForm.base_fields["update_existing_packages"].help_text,
+    )
+    scan_all_packages = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=LoadSBOMsForm.base_fields["scan_all_packages"].help_text,
+    )
+
+
+class ImportFromScanSerializer(serializers.Serializer):
+    """Serializer equivalent of ImportFromScanForm, used for API documentation."""
+
+    upload_file = serializers.FileField(
+        required=True,
+    )
+    create_codebase_resources = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=ImportFromScanForm.base_fields["create_codebase_resources"].help_text,
+    )
+    stop_on_error = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=ImportFromScanForm.base_fields["stop_on_error"].help_text,
+    )
+
+
+class PullProjectDataSerializer(serializers.Serializer):
+    """Serializer equivalent of PullProjectDataForm, used for API documentation."""
+
+    project_name_or_uuid = serializers.CharField(
+        required=True,
+        help_text=PullProjectDataForm.base_fields["project_name_or_uuid"].label,
+    )
+    update_existing_packages = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=PullProjectDataForm.base_fields["update_existing_packages"].help_text,
+    )
+
+
 class ProductViewSet(CreateRetrieveUpdateListViewSet):
     queryset = Product.objects.none()
     serializer_class = ProductSerializer
@@ -240,6 +297,72 @@ def perform_create(self, serializer):
         super().perform_create(serializer)
         assign_all_object_permissions(self.request.user, serializer.instance)
 
+    @action(detail=True, methods=["post"], serializer_class=LoadSBOMsFormSerializer)
+    def load_sboms(self, request, *args, **kwargs):
+        """
+        Load Packages from SBOMs.
+
+        DejaCode supports the following SBOM formats:
+        * CycloneDX BOM as JSON bom.json and .cdx.json,
+        * SPDX document as JSON .spdx.json,
+        * AboutCode .ABOUT files,
+
+        Multiple SBOMs: You can provide multiple SBOMs by packaging them into a zip
+        archive. DejaCode will handle and process them accordingly.
+        """
+        product = self.get_object()
+
+        form = LoadSBOMsForm(data=request.POST, files=request.FILES)
+        if not form.is_valid():
+            return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        form.submit(product=product, user=request.user)
+        return Response({"status": "SBOM file submitted to ScanCode.io for inspection."})
+
+    @action(detail=True, methods=["post"], serializer_class=ImportFromScanSerializer)
+    def import_from_scan(self, request, *args, **kwargs):
+        """
+        Import the scan results in the Product.
+
+        Upload a ScanCode.io or ScanCode-toolkit JSON output file.
+        """
+        product = self.get_object()
+
+        form = ImportFromScanForm(user=request.user, data=request.POST, files=request.FILES)
+        if not form.is_valid():
+            return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        try:
+            warnings, created_counts = form.save(product=product)
+        except ValidationError as error:
+            return Response(error.messages, status=status.HTTP_400_BAD_REQUEST)
+
+        if not created_counts:
+            msg = "Nothing imported."
+        else:
+            msg = "Imported from Scan: "
+            msg += ", ".join([f"{value} {key}" for key, value in created_counts.items()])
+        return Response({"status": msg})
+
+    @action(detail=True, methods=["post"], serializer_class=PullProjectDataSerializer)
+    def pull_scancodeio_project_data(self, request, *args, **kwargs):
+        """
+        Pull data from a ScanCode.io Project to import all its Discovered Packages.
+        Imported Packages will be assigned to this Product.
+        """
+        product = self.get_object()
+
+        form = PullProjectDataForm(data=request.POST)
+        if not form.is_valid():
+            return Response(form.errors, status=status.HTTP_400_BAD_REQUEST)
+
+        try:
+            form.submit(product=product, user=request.user)
+        except ValidationError as error:
+            return Response(error.messages, status=status.HTTP_400_BAD_REQUEST)
+
+        return Response({"status": "Packages import from ScanCode.io in progress..."})
+
 
 class BaseProductRelationSerializer(ValidateLicenseExpressionMixin, DataspacedSerializer):
     product = NameVersionHyperlinkedRelatedField(

product_portfolio/forms.py

@@ -7,6 +7,8 @@
 #
 
 from django import forms
+from django.core.exceptions import ValidationError
+from django.db import transaction
 from django.forms import BaseModelFormSet
 from django.forms.formsets import DELETION_FIELD_NAME
 from django.urls import reverse_lazy
@@ -28,6 +30,8 @@
 from component_catalog.license_expression_dje import LicenseExpressionFormMixin
 from component_catalog.models import Component
 from component_catalog.programming_languages import PROGRAMMING_LANGUAGES
+from dejacode_toolkit.scancodeio import ScanCodeIO
+from dje import tasks
 from dje.fields import SmartFileField
 from dje.forms import ColorCodeFormMixin
 from dje.forms import DataspacedAdminForm
@@ -47,6 +51,7 @@
 from product_portfolio.models import Product
 from product_portfolio.models import ProductComponent
 from product_portfolio.models import ProductPackage
+from product_portfolio.models import ScanCodeProject
 
 
 class NameVersionValidationFormMixin:
@@ -560,6 +565,27 @@ def helper(self):
         helper.add_input(Submit("import", "Import"))
         return helper
 
+    def save(self, product):
+        from product_portfolio.importers import ImportFromScan
+
+        sid = transaction.savepoint()
+        importer = ImportFromScan(
+            product,
+            self.user,
+            upload_file=self.cleaned_data.get("upload_file"),
+            create_codebase_resources=self.cleaned_data.get("create_codebase_resources"),
+            stop_on_error=self.cleaned_data.get("stop_on_error"),
+        )
+
+        try:
+            warnings, created_counts = importer.save()
+        except ValidationError:
+            transaction.savepoint_rollback(sid)
+            raise
+
+        transaction.savepoint_commit(sid)
+        return warnings, created_counts
+
 
 class LoadSBOMsForm(forms.Form):
     input_file = SmartFileField(
@@ -597,6 +623,24 @@ def helper(self):
         helper.add_input(Submit("submit", "Load Packages", css_class="btn-success"))
         return helper
 
+    def submit(self, product, user):
+        scancode_project = ScanCodeProject.objects.create(
+            product=product,
+            dataspace=product.dataspace,
+            type=ScanCodeProject.ProjectType.LOAD_SBOMS,
+            input_file=self.cleaned_data.get("input_file"),
+            update_existing_packages=self.cleaned_data.get("update_existing_packages"),
+            scan_all_packages=self.cleaned_data.get("scan_all_packages"),
+            created_by=user,
+        )
+
+        transaction.on_commit(
+            lambda: tasks.scancodeio_submit_load_sbom.delay(
+                scancodeproject_uuid=scancode_project.uuid,
+                user_uuid=user.uuid,
+            )
+        )
+
 
 class StrongTextWidget(forms.Widget):
     def render(self, name, value, attrs=None, renderer=None):
@@ -847,3 +891,35 @@ def helper(self):
         helper.form_id = "pull-project-data-form"
         helper.attrs = {"autocomplete": "off"}
         return helper
+
+    def get_project_data(self, project_name_or_uuid, user):
+        scancodeio = ScanCodeIO(user)
+        for field_name in ["name", "uuid"]:
+            project_data = scancodeio.find_project(**{field_name: project_name_or_uuid})
+            if project_data:
+                return project_data
+
+    def submit(self, product, user):
+        project_name_or_uuid = self.cleaned_data.get("project_name_or_uuid")
+        project_data = self.get_project_data(project_name_or_uuid, user)
+
+        if not project_data:
+            msg = f'Project "{project_name_or_uuid}" not found on ScanCode.io.'
+            raise ValidationError(msg)
+
+        scancode_project = ScanCodeProject.objects.create(
+            product=product,
+            dataspace=product.dataspace,
+            type=ScanCodeProject.ProjectType.PULL_FROM_SCANCODEIO,
+            project_uuid=project_data.get("uuid"),
+            update_existing_packages=self.cleaned_data.get("update_existing_packages"),
+            scan_all_packages=False,
+            status=ScanCodeProject.Status.SUBMITTED,
+            created_by=user,
+        )
+
+        transaction.on_commit(
+            lambda: tasks.pull_project_data_from_scancodeio.delay(
+                scancodeproject_uuid=scancode_project.uuid,
+            )
+        )

product_portfolio/templates/product_portfolio/import_from_scan.html

@@ -50,8 +50,11 @@ <h5>Option 2: From ScanCode.io pipeline results</h5>
       Upload a ScanCode.io JSON output file, <strong>generated with one of the following pipelines:</strong>
     </p>
     <p class="mb-0">
-      <code>docker</code>, <code>docker_windows</code>, <code>inspect_manifest</code>, <code>load_inventory</code>
-      <code>root_filesystems</code>, <code>scan_codebase</code>, <code>scan_package</code>
+      <code>analyze_docker_image</code>,
+      <code>analyze_windows_docker_image</code>,
+      <code>inspect_packages</code>,
+      <code>scan_codebase</code>,
+      <code>scan_single_package</code>
     </p>
   </div>