Update sync_vulnerablecode pipeline to import vulnerablecode Git repositories with new schema format

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

fedcode/pipelines/sync_vulnerablecode.py

@@ -8,11 +8,13 @@
 #
 
 import logging
-import os.path
 from itertools import zip_longest
+from pathlib import Path
 
 import saneyaml
+from django.db import transaction
 
+from aboutcode.hashid import get_core_purl
 from aboutcode.pipeline import LoopProgress
 from fedcode.activitypub import Activity
 from fedcode.activitypub import UpdateActivity
@@ -40,6 +42,10 @@ def get_git_repos(self):
         self.git_repos = Repository.objects.all()
 
     def sync_vulnerablecode_repositories(self):
+        """
+        Sync repositories
+        For vulnerablecode-data we have 3 files types vulnerabilities.yml, purls.yml, VCID-1ues-ahar-buaa.yml
+        """
         repositories_count = self.git_repos.count()
         self.log(f"Syncing vulnerability from {repositories_count:,d} repositories")
 
@@ -56,6 +62,7 @@ def sync_vulnerabilities(repository, logger):
     repo = repository.git_repo_obj
     latest_commit_hash = repo.head.commit.hexsha
     latest_commit = repo.commit(latest_commit_hash)
+
     if repository.last_imported_commit:
         last_imported_commit = repo.commit(repository.last_imported_commit)
         diffs = last_imported_commit.diff(latest_commit)
@@ -73,7 +80,7 @@ def sync_vulnerabilities(repository, logger):
     logger(f"Syncing {diff_count:,d} vulnerability scan from {repository.url}")
     progress = LoopProgress(total_iterations=diff_count, logger=logger)
     for diff in progress.iter(diffs):
-        if not diff.a_path.endswith(".yaml"):
+        if not diff.a_path.endswith(".yml"):
             continue
 
         if diff.a_path.startswith("."):
@@ -82,100 +89,139 @@ def sync_vulnerabilities(repository, logger):
         yaml_data_a_blob = saneyaml.load(diff.a_blob.data_stream.read()) if diff.a_blob else None
         yaml_data_b_blob = saneyaml.load(diff.b_blob.data_stream.read()) if diff.b_blob else None
 
-        if os.path.split(diff.a_path)[1].startswith("VCID") or os.path.split(diff.b_path)[
-            1
-        ].startswith("VCID"):
-            vul_handler(
-                diff.change_type,
-                repository,
-                yaml_data_a_blob,
-                yaml_data_b_blob,
-                logger,
+        a_name = Path(diff.a_path).name
+        b_name = Path(diff.b_path).name
+
+        if a_name == "vulnerabilities.yml" or b_name == "vulnerabilities.yml":
+            note_handler(
+                diff.change_type, repository.admin, yaml_data_a_blob, yaml_data_b_blob, logger
             )
-            continue
 
-        pkg_handler(
-            diff.change_type,
-            repository.admin,
-            yaml_data_a_blob,
-            yaml_data_b_blob,
-        )
+        if a_name == "purls.yml" or b_name == "purls.yml":
+            pkg_handler(
+                diff.change_type, repository.admin, yaml_data_a_blob, yaml_data_b_blob, logger
+            )
+
+        if a_name.startswith("VCID") or b_name.startswith("VCID"):
+            vul_handler(diff.change_type, repository, yaml_data_a_blob, yaml_data_b_blob, logger)
 
     repository.last_imported_commit = latest_commit_hash
     repository.save()
     logger("The Importer run successfully")
 
 
 def vul_handler(change_type, repo_obj, yaml_data_a_blob, yaml_data_b_blob, logger):
+    """
+    VCID-XXXX-XXXX-XXXX.yml
+    """
+    vulnerability_a_id = yaml_data_a_blob.get("vulnerability_id") if yaml_data_a_blob else None
+    vulnerability_b_id = yaml_data_b_blob.get("vulnerability_id") if yaml_data_b_blob else None
+
     if change_type == "A":  # A for added paths
         Vulnerability.objects.get_or_create(
-            id=yaml_data_b_blob.get("vulnerability_id"),
+            id=vulnerability_b_id,
             repo=repo_obj,
         )
     elif change_type in [
         "M",
         "R",
     ]:  # R for renamed paths , M for paths with modified data
-        vul = Vulnerability.objects.get(
-            id=yaml_data_a_blob.get("vulnerability_id"),
-            repo=repo_obj,
-        )
-        vul.filename = yaml_data_b_blob.get("vulnerability_id")
-        vul.save()
+        with transaction.atomic():
+            Vulnerability.objects.get(id=vulnerability_a_id, repo=repo_obj).delete()
+            Vulnerability.objects.create(id=vulnerability_b_id, repo=repo_obj)
+
     elif change_type == "D":  # D for deleted paths
-        vul = Vulnerability.objects.filter(
+        Vulnerability.objects.get(
             id=yaml_data_b_blob.get("vulnerability_id"),
             repo=repo_obj,
-        )
-        vul.delete()
+        ).delete()
     else:
         logger(f"Invalid Vulnerability File", level=logging.ERROR)
 
 
-def pkg_handler(change_type, default_service, yaml_data_a_blob, yaml_data_b_blob):
+def pkg_handler(change_type, default_service, yaml_data_a_blob, yaml_data_b_blob, logger):
+    """
+    purls.yml
+    """
+
+    if change_type == "A":
+        for purl in yaml_data_b_blob:
+            core_purl = get_core_purl(purl)
+            pkg, _ = Package.objects.get_or_create(purl=core_purl, service=default_service)
+
+    # elif change_type == "M":
+    #     pkg = Package.objects.get(purl=package_a, service=default_service)
+    #     pkg.purl = package_b
+    #     pkg.save()
+    #
+    #     for version_a, version_b in zip_longest(
+    #         yaml_data_a_blob, yaml_data_b_blob
+    #     ):
+    #         if version_b and not version_a:
+    #             utils.create_note(pkg, version_b)
+    #
+    #         if version_a and not version_b:
+    #             utils.delete_note(pkg, version_a)
+    #
+    #         if version_a and version_b:
+    #             note = Note.objects.get(acct=pkg.acct, content=saneyaml.dump(version_a))
+    #             if note.content == saneyaml.dump(version_b):
+    #                 continue
+    #
+    #             note.content = saneyaml.dump(version_b)
+    #             note.save()
+    #
+    #             update_activity = UpdateActivity(actor=pkg.to_ap, object=note.to_ap)
+    #             Activity.federate(
+    #                 targets=pkg.followers_inboxes,
+    #                 body=update_activity.to_ap(),
+    #                 key_id=pkg.key_id,
+    #             )
+    #
+    # elif change_type == "D":
+    #     pkg = Package.objects.get(purl=package_a, service=default_service)
+    #     for version in yaml_data_a_blob:
+    #         utils.delete_note(pkg, version)
+    #     pkg.delete()
+
+
+def note_handler(change_type, default_service, yaml_data_a_blob, yaml_data_b_blob, logger):
+    """
+    vulnerabilities.yml
+    """
     if change_type == "A":
-        package = yaml_data_b_blob.get("package")
-
-        pkg, _ = Package.objects.get_or_create(purl=package, service=default_service)
-
-        for version in yaml_data_b_blob.get("versions", []):
-            utils.create_note(pkg, version)
-
-    elif change_type == "M":
-        old_package = yaml_data_a_blob.get("package")
-        new_package = yaml_data_b_blob.get("package")
-
-        pkg = Package.objects.get(purl=old_package, service=default_service)
-        pkg.purl = new_package
-        pkg.save()
-
-        for version_a, version_b in zip_longest(
-            yaml_data_a_blob.get("versions", []), yaml_data_b_blob.get("versions", [])
-        ):
-            if version_b and not version_a:
-                utils.create_note(pkg, version_b)
-
-            if version_a and not version_b:
-                utils.delete_note(pkg, version_a)
-
-            if version_a and version_b:
-                note = Note.objects.get(acct=pkg.acct, content=saneyaml.dump(version_a))
-                if note.content == saneyaml.dump(version_b):
-                    continue
-
-                note.content = saneyaml.dump(version_b)
-                note.save()
-
-                update_activity = UpdateActivity(actor=pkg.to_ap, object=note.to_ap)
-                Activity.federate(
-                    targets=pkg.followers_inboxes,
-                    body=update_activity.to_ap(),
-                    key_id=pkg.key_id,
-                )
-
-    elif change_type == "D":
-        package = yaml_data_a_blob.get("package")
-        pkg = Package.objects.get(purl=package, service=default_service)
-        for version in yaml_data_a_blob.get("versions", []):
-            utils.delete_note(pkg, version)
-        pkg.delete()
+        for pkg_status in yaml_data_b_blob:
+            purl = pkg_status.get("purl")
+            if not purl:
+                logger(f"Invalid Vulnerability File", level=logging.ERROR)
+                return
+            core_purl = get_core_purl(purl)
+            pkg_b, _ = Package.objects.get_or_create(purl=core_purl, service=default_service)
+            temp = saneyaml.dump(pkg_status)
+            utils.create_note(pkg_b, temp)
+
+    # elif change_type == "M":
+    #     for pkg_status_a, pkg_status_b in zip_longest(
+    #         yaml_data_a_blob, yaml_data_b_blob
+    #     ):
+    #         if pkg_status_a and not pkg_status_b:
+    #             utils.create_note(pkg_a, pkg_status_b)
+    #
+    #         if pkg_status_a and not pkg_status_b:
+    #             utils.delete_note(pkg_a, pkg_status_b)
+    #
+    #         if pkg_status_a and pkg_status_b:
+    #             utils.update_note(pkg_a, saneyaml.dump(pkg_status_a), saneyaml.dump(pkg_status_b))
+    #
+    # elif change_type == "D":
+    #     for pkg_status in yaml_data_a_blob:
+    #         purl = pkg_status.get("purl")
+    #         if not purl:
+    #             logger(f"Invalid Vulnerability File", level=logging.ERROR)
+    #             return
+    #         core_purl = get_core_purl(purl)
+    #         pkg_a, _ = Package.objects.get_or_create(purl=core_purl, service=default_service)
+    #         temp = saneyaml.dump(pkg_status)
+    #         utils.delete_note(pkg_a, temp)
+    else:
+        logger(f"Invalid Vulnerability File", level=logging.ERROR)

fedcode/pipes/utils.py

@@ -15,6 +15,7 @@
 from fedcode.activitypub import Activity
 from fedcode.activitypub import CreateActivity
 from fedcode.activitypub import DeleteActivity
+from fedcode.activitypub import UpdateActivity
 from fedcode.models import Note
 
 
@@ -31,6 +32,23 @@ def create_note(pkg, note_dict):
     )
 
 
+def update_note(pkg, old_note_dict, new_note_dict):
+    if old_note_dict == new_note_dict:
+        return
+
+    note = Note.objects.get(acct=pkg.acct, content=saneyaml.dump(old_note_dict))
+
+    note.content = saneyaml.dump(new_note_dict)
+    note.save()
+
+    update_activity = UpdateActivity(actor=pkg.to_ap, object=note.to_ap)
+    Activity.federate(
+        targets=pkg.followers_inboxes,
+        body=update_activity.to_ap(),
+        key_id=pkg.key_id,
+    )
+
+
 def delete_note(pkg, note_dict):
     note = Note.objects.get(acct=pkg.acct, content=saneyaml.dump(note_dict))
     note_ap = note.to_ap

fedcode/utils.py

@@ -42,8 +42,8 @@ def parse_webfinger(subject):
         return tuple(subject.split("@"))
 
 
-def generate_webfinger(username, domain=FEDERATEDCODE_DOMAIN):
-    return username + "@" + domain
+def generate_webfinger(username: str, domain=FEDERATEDCODE_DOMAIN) -> str:
+    return str(username) + "@" + domain
 
 
 def clone_git_repo(repo_path, repo_url):