Add a test for the aosp importer

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -41,6 +41,7 @@
 from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
 from vulnerabilities.pipelines import pysec_importer
+from vulnerabilities.pipelines.v2_importers import aosp_importer
 from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2
 from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2
 from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2
@@ -81,6 +82,7 @@
         mozilla_importer_v2.MozillaImporterPipeline,
         github_osv_importer_v2.GithubOSVImporterPipeline,
         redhat_importer_v2.RedHatImporterPipeline,
+        aosp_importer.AospImporterPipeline,
         nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,
         gitlab_importer.GitLabImporterPipeline,

vulnerabilities/improvers/__init__.py

@@ -19,9 +19,6 @@
 from vulnerabilities.pipelines import flag_ghost_packages
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
-from vulnerabilities.pipelines.v2_improvers import (
-    collect_commits_aosp_dataset as collect_commits_aosp_v2,
-)
 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
@@ -71,6 +68,5 @@
         compute_version_rank_v2.ComputeVersionRankPipeline,
         compute_advisory_todo_v2.ComputeToDo,
         compute_advisory_todo.ComputeToDo,
-        collect_commits_aosp_v2.CollectFixCommitsAospDatasetPipeline,
     ]
 )

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -0,0 +1,97 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import shutil
+from pathlib import Path
+
+import dateparser
+from django.core.exceptions import ValidationError
+from fetchcode.vcs import fetch_via_vcs
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+
+
+class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    """
+    Pipeline to collect fix commits from Aosp Dataset:
+    """
+
+    pipeline_id = "aosp_dataset_fix_commits"
+    spdx_license_expression = "Apache-2.0"
+    license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE"
+    importer_name = "aosp_dataset"
+    qualified_name = "aosp_dataset_fix_commits"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone,
+            cls.collect_and_store_advisories,
+            cls.clean_downloads,
+        )
+
+    def clone(self):
+        self.repo_url = "git+https://github.com/quarkslab/aosp_dataset"
+        self.log(f"Cloning `{self.repo_url}`")
+        self.vcs_response = fetch_via_vcs(self.repo_url)
+
+    def advisories_count(self):
+        root = Path(self.vcs_response.dest_dir)
+        return sum(1 for _ in root.rglob("*.json"))
+
+    def collect_advisories(self):
+        self.log(f"Processing aosp_dataset fix commits.")
+        base_path = Path(self.vcs_response.dest_dir) / "cves"
+        for file_path in base_path.rglob("*.json"):
+            if not file_path.name.startswith("CVE-"):
+                continue
+
+            with open(file_path) as f:
+                vulnerability_data = json.load(f)
+
+            vulnerability_id = vulnerability_data.get("cveId", [])
+            if (
+                not vulnerability_id or "," in vulnerability_id
+            ):  # escape invalid multiple CVE-2017-13077, CVE-2017-13078
+                continue
+
+            summary = vulnerability_data.get("vulnerabilityType")
+            date_reported = vulnerability_data.get("dateReported")
+            date_published = dateparser.parse(date_reported) if date_reported else None
+
+            references = []
+            for commit_data in vulnerability_data.get("fixes", []):
+                vcs_url = commit_data.get("patchUrl")
+
+                if not vcs_url:
+                    continue
+
+                ref = ReferenceV2(reference_type="commit", url=vcs_url)
+                references.append(ref)
+
+            yield AdvisoryData(
+                advisory_id=vulnerability_id,
+                summary=summary,
+                references_v2=references,
+                date_published=date_published,
+                url=f"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/{file_path.name}",
+            )
+
+    def clean_downloads(self):
+        """Cleanup any temporary repository data."""
+        self.log("Cleaning up local repository resources.")
+        if hasattr(self, "repo") and self.repo.working_dir:
+            shutil.rmtree(path=self.repo.working_dir)
+
+    def on_failure(self):
+        """Ensure cleanup is always performed on failure."""
+        self.clean_downloads()

vulnerabilities/pipelines/v2_improvers/collect_commits_aosp_dataset.py

@@ -0,0 +1,28 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import os
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+
+from vulnerabilities.pipelines.v2_importers.aosp_importer import AospImporterPipeline
+from vulnerabilities.tests import util_tests
+
+TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "aosp"
+
+
+@pytest.mark.django_db
+def test_aosp_advisories1():
+    expected_file = os.path.join(TEST_DATA, "aosp_advisoryv2-expected.json")
+    pipeline = AospImporterPipeline()
+    pipeline.vcs_response = Mock(dest_dir=TEST_DATA)
+    result = [adv.to_dict() for adv in pipeline.collect_advisories()]
+    util_tests.check_results_against_json(result, expected_file)

vulnerabilities/tests/pipelines/v2_importers/test_commits_aosp_dataset.py

@@ -0,0 +1,36 @@
+[
+  {
+    "advisory_id": "CVE-2021-30294",
+    "aliases": [],
+    "summary": "Vulnerability",
+    "affected_packages": [],
+    "references_v2": [
+      {
+        "reference_id": "",
+        "reference_type": "",
+        "url": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8"
+      }
+    ],
+    "severities": [],
+    "date_published": null,
+    "weaknesses": [],
+    "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test1.json"
+  },
+  {
+    "advisory_id": "CVE-2017-13282",
+    "aliases": [],
+    "summary": "Remote Code Execution Vulnerability",
+    "affected_packages": [],
+    "references_v2": [
+      {
+        "reference_id": "",
+        "reference_type": "",
+        "url": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94"
+      }
+    ],
+    "severities": [],
+    "date_published": "2018-04-04T00:00:00",
+    "weaknesses": [],
+    "url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test2.json"
+  }
+]

vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json

@@ -0,0 +1,14 @@
+{
+  "cveId": "CVE-2021-30294",
+  "dateReported": null,
+  "vulnerabilityType": "Vulnerability",
+  "language": "c",
+  "fixes": [
+    {
+      "commitId": "",
+      "patchUrl": "https://source.codeaurora.org/quic/la/kernel/msm-5.4/commit/?id=d6876813add62f3cac7c429a41cc8710005d69e8"
+    }
+  ],
+  "severity": "High",
+  "component": "Qualcomm Display"
+}

vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test1.json

@@ -0,0 +1,14 @@
+{
+  "cveId": "CVE-2017-13282",
+  "dateReported": "2018-04-04",
+  "vulnerabilityType": "Remote Code Execution Vulnerability",
+  "language": "c",
+  "fixes": [
+    {
+      "commitId": "6ecbbc093f4383e90cbbf681cd55da1303a8ef94",
+      "patchUrl": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94"
+    }
+  ],
+  "severity": "Critical",
+  "component": "System"
+}