Drop project_kb_msr2019 V1 importer

Fix CI falling test
Resolve merge conflict and Update migration file
Remove duplication and create append_patch_classifications function
Update the project-kb test

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -24,7 +24,6 @@
 from vulnerabilities.importers import openssl
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
-from vulnerabilities.importers import project_kb_msr2019
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
 from vulnerabilities.importers import ruby
@@ -112,7 +111,6 @@
         mozilla.MozillaImporter,
         gentoo.GentooImporter,
         istio.IstioImporter,
-        project_kb_msr2019.ProjectKBMSRImporter,
         suse_scores.SUSESeverityScoreImporter,
         elixir_security.ElixirSecurityImporter,
         xen.XenImporter,

vulnerabilities/importers/project_kb_msr2019.py

@@ -20,9 +20,6 @@
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
 from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees
-from vulnerabilities.pipelines.v2_improvers import (
-    collect_commits_project_kb as collect_commits_project_kb_v2,
-)
 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
@@ -75,6 +72,5 @@
         unfurl_version_range_v2.UnfurlVersionRangePipeline,
         compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
-        collect_commits_project_kb_v2.CollectFixCommitsProjectKBPipeline,
     ]
 )

vulnerabilities/improvers/__init__.py

@@ -16,13 +16,9 @@
 from fetchcode.vcs import fetch_via_vcs
 
 from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackageV2
-from vulnerabilities.importer import PackageCommitPatchData
-from vulnerabilities.importer import PatchData
-from vulnerabilities.importer import ReferenceV2
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
-from vulnerabilities.pipes.advisory import classify_patch_source
+from vulnerabilities.pipes.advisory import append_patch_classifications
 from vulnerabilities.severity_systems import GENERIC
 
 
@@ -90,23 +86,14 @@ def collect_advisories(self):
                     patch_url = commit_data.get("patchUrl")
                     commit_id = commit_data.get("commitId")
 
-                    base_purl, patch_objs = classify_patch_source(
+                    append_patch_classifications(
                         url=patch_url,
                         commit_hash=commit_id,
                         patch_text=None,
+                        affected_packages=affected_packages,
+                        references=references,
+                        patches=patches,
                     )
-                    for patch_obj in patch_objs:
-                        if isinstance(patch_obj, PackageCommitPatchData):
-                            fixed_commit = patch_obj
-                            affected_package = AffectedPackageV2(
-                                package=base_purl,
-                                fixed_by_commit_patches=[fixed_commit],
-                            )
-                            affected_packages.append(affected_package)
-                        elif isinstance(patch_obj, PatchData):
-                            patches.append(patch_obj)
-                        elif isinstance(patch_obj, ReferenceV2):
-                            references.append(patch_obj)
 
                 url = (
                     "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/"

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -7,51 +7,66 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import json
+import csv
 from pathlib import Path
 from typing import Iterable
 
 import saneyaml
 from fetchcode.vcs import fetch_via_vcs
 from packageurl import PackageURL
-from univers.maven import VersionRange
+from univers.version_range import RANGE_CLASS_BY_SCHEMES
+from univers.versions import InvalidVersion
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackageV2
-from vulnerabilities.importer import ReferenceV2
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.pipes.advisory import append_patch_classifications
 from vulnerabilities.utils import get_advisory_url
+from vulnerabilities.utils import is_commit
 
 
 class ProjectKBPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
     ProjectKB Importer Pipeline
     Collect advisory from ProjectKB data:
     - YAML statements: https://github.com/SAP/project-kb/blob/vulnerability-data/statements/*/*.yaml
+    - CSV database https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv
     """
 
     pipeline_id = "project-kb_v2"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt"
-    repo_url = "git+https://github.com/SAP/project-kb@vulnerability-data"
+    main_branch = "git+https://github.com/SAP/project-kb"
+    vuln_data_branch = "git+https://github.com/SAP/project-kb@vulnerability-data"
 
     @classmethod
     def steps(cls):
         return (cls.clone_repo, cls.collect_and_store_advisories, cls.clean_downloads)
 
     def clone_repo(self):
-        self.log("Processing ProjectKB advisory data...")
-        self.vcs_response = fetch_via_vcs(self.repo_url)
+        self.log("Cloning ProjectKB advisory data...")
+        self.main_branch_vcs = fetch_via_vcs(self.main_branch)
+        self.vuln_data_branch_vcs = fetch_via_vcs(self.vuln_data_branch)
 
     def advisories_count(self):
-        base_path = Path(self.vcs_response.dest_dir) / "statements"
-        count = sum(1 for _ in base_path.rglob("*.yaml"))
+        base_path = Path(self.vuln_data_branch_vcs.dest_dir) / "statements"
+        csv_path = (
+            Path(self.main_branch_vcs.dest_dir) / "MSR2019/dataset/vulas_db_msr2019_release.csv"
+        )
+
+        count_files = sum(1 for _ in base_path.rglob("*.yaml"))
+        with open(csv_path, newline="", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)
+            count_rows = sum(1 for _ in reader)
+
+        count = count_files + count_rows
         self.log(f"Estimated advisories to process: {count}")
         return count
 
     def collect_advisories(self) -> Iterable[AdvisoryData]:
-        """Collect fix commits from YAML statements under /statements."""
-        base_path = Path(self.vcs_response.dest_dir) / "statements"
+        self.log("Collecting fix commits from YAML statements under /statements....")
+        base_path = Path(self.vuln_data_branch_vcs.dest_dir) / "statements"
 
         for yaml_file in base_path.rglob("*.yaml"):
             if yaml_file.name != "statement.yaml":
@@ -67,23 +82,30 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
             note_texts = []
             for note_entry in yaml_data.get("notes", []):
                 text_content = note_entry.get("text")
-                if text_content:
-                    note_texts.append(text_content)
+                if not text_content:
+                    continue
+                note_texts.append(text_content)
             description = "\n".join(note_texts)
 
             references = []
+            affected_packages = []
+            patches = []
             for fix in yaml_data.get("fixes", []):
                 for commit in fix.get("commits", []):
-                    commit_id = commit.get("id")
-                    repo_url = commit.get("repository")
-                    if not commit_id or not repo_url:
-                        continue
-
-                    commit_url = repo_url.replace(".git", "") + "/commit/" + commit_id
-                    ref = ReferenceV2.from_url(commit_url)
-                    references.append(ref)
+                    commit_hash = commit.get("id")
+                    if not is_commit(commit_hash):
+                        commit_hash = None
+
+                    vcs_url = commit.get("repository")
+                    append_patch_classifications(
+                        url=vcs_url,
+                        commit_hash=commit_hash,
+                        patch_text=None,
+                        affected_packages=affected_packages,
+                        references=references,
+                        patches=patches,
+                    )
 
-            affected_packages = []
             for artifact in yaml_data.get("artifacts", []):
                 affected = artifact.get("affected")
                 if not affected:
@@ -92,9 +114,20 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
                 purl_str = artifact.get("id")
                 purl = PackageURL.from_string(purl_str)
 
+                try:
+                    version_range_class = RANGE_CLASS_BY_SCHEMES.get(purl.type)
+                    version_class = (
+                        version_range_class.version_class if version_range_class else None
+                    )
+                    version_range = version_class(purl.version)
+                except InvalidVersion:
+                    self.log(f"Invalid Version: {purl.version!r} for purl type: {purl.type!r}")
+                    continue
+
                 affected_package = AffectedPackageV2(
                     package=PackageURL(type=purl.type, namespace=purl.namespace, name=purl.name),
-                    fixed_version_range=VersionRange.from_version(purl.version),
+                    fixed_version_range=version_range if not affected else None,
+                    affected_version_range=version_range if affected else None,
                 )
                 affected_packages.append(affected_package)
 
@@ -106,19 +139,55 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
 
             yield AdvisoryData(
                 advisory_id=vulnerability_id,
-                aliases=[],
-                summary=description or "",
+                summary=description,
                 affected_packages=affected_packages,
                 references_v2=references,
+                patches=patches,
                 url=advisory_url,
-                original_advisory_text=json.dumps(yaml_data, indent=2, ensure_ascii=False),
+            )
+
+        self.log("Collecting fix commits from ProjectKB ( vulas_db_msr2019_release )...")
+        csv_path = (
+            Path(self.main_branch_vcs.dest_dir) / "MSR2019/dataset/vulas_db_msr2019_release.csv"
+        )
+
+        with open(csv_path, newline="", encoding="utf-8") as f:
+            reader = csv.reader(f)
+            next(reader, None)  # skip header
+            rows = [r for r in reader if len(r) == 4 and r[0]]  # vuln_id, vcs_url, commit_hash, poc
+
+        for vuln_id, vcs_url, commit_hash, _ in rows:
+            if not vuln_id or not vcs_url or not commit_hash:
+                continue
+
+            patches = []
+            affected_packages = []
+            references = []
+            append_patch_classifications(
+                url=vcs_url,
+                commit_hash=commit_hash,
+                patch_text=None,
+                affected_packages=affected_packages,
+                references=references,
+                patches=patches,
+            )
+
+            yield AdvisoryData(
+                advisory_id=vuln_id,
+                affected_packages=affected_packages,
+                patches=patches,
+                references_v2=references,
+                url="https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv",
             )
 
     def clean_downloads(self):
         """Remove the cloned repository from disk."""
         self.log("Removing cloned repository...")
-        if self.vcs_response:
-            self.vcs_response.delete()
+        if self.main_branch_vcs:
+            self.main_branch_vcs.delete()
+
+        if self.vuln_data_branch_vcs:
+            self.vuln_data_branch_vcs.delete()
 
     def on_failure(self):
         """Ensure cleanup happens on pipeline failure."""