Handle vulnerabilities which don't have any vulnerability ids

requirements.txt

@@ -55,3 +55,4 @@ zipp==0.6.0
 requests==2.23.0
 toml==0.10.2
 PyYAML==5.3.1
+freezegun==1.1.0

vulnerabilities/api.py

@@ -63,7 +63,6 @@ class Meta:
 
 
 class HyperLinkedVulnerabilitySerializer(serializers.HyperlinkedModelSerializer):
-    vulnerability_id = serializers.CharField(source="cve_id")
 
     class Meta:
         model = Vulnerability
@@ -177,7 +176,6 @@ def bulk_search(self, request):
 
 
 class VulnerabilityFilterSet(filters.FilterSet):
-    vulnerability_id = filters.CharFilter(field_name="cve_id")
 
     class Meta:
         model = Vulnerability
@@ -208,13 +206,13 @@ def bulk_search(self, request):
                 },
             )
 
-        for cve_id in request.data["vulnerabilities"]:
-            filter_list.append(cve_id)
+        for vulnerability_id in request.data["vulnerabilities"]:
+            filter_list.append(vulnerability_id)
             # This handles the case when the said cve doesnt exist in db
-            response[cve_id] = {}
-        res = Vulnerability.objects.filter(cve_id__in=filter_list)
+            response[vulnerability_id] = {}
+        res = Vulnerability.objects.filter(vulnerability_id__in=filter_list)
         for vuln in res:
-            response[vuln.cve_id] = MinimalVulnerabilitySerializer(
+            response[vuln.vulnerability_id] = MinimalVulnerabilitySerializer(
                 vuln, context={"request": request}
             ).data
         return Response(response)

vulnerabilities/data_source.py

@@ -81,14 +81,14 @@ class Advisory:
     impacted_package_urls: Iterable[PackageURL]
     resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list)
     vuln_references: List[Reference] = dataclasses.field(default_factory=list)
-    cve_id: Optional[str] = None
+    vulnerability_id: Optional[str] = None
 
     def __hash__(self):
         s = "{}{}{}{}".format(
             self.summary,
-            "".join(sorted([str(p) for p in self.impacted_package_urls])),
-            "".join(sorted([str(p) for p in self.resolved_package_urls])),
-            self.cve_id,
+            ''.join(sorted([str(p) for p in self.impacted_package_urls])),
+            ''.join(sorted([str(p) for p in self.resolved_package_urls])),
+            self.vulnerability_id,
         )
         return hash(s)
 
@@ -539,8 +539,6 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
                     summary=description,
                     impacted_package_urls=affected_purls,
                     resolved_package_urls=safe_purls,
-                    cve_id=vuln_id,
-                    vuln_references=references,
-                )
-            )
+                    vulnerability_id=vuln_id,
+                    vuln_references=references))
         return all_adv

vulnerabilities/fixtures/debian.json

@@ -3,7 +3,7 @@
   "model": "vulnerabilities.vulnerability",
   "pk": 1,
   "fields": {
-    "cve_id": "CVE-2014-8242",
+    "vulnerability_id": "CVE-2014-8242",
     "summary": ""
 
   }
@@ -12,7 +12,7 @@
   "model": "vulnerabilities.vulnerability",
   "pk": 2,
   "fields": {
-    "cve_id": "CVE-2009-1382",
+    "vulnerability_id": "CVE-2009-1382",
     "summary": ""
 
   }
@@ -21,7 +21,7 @@
   "model": "vulnerabilities.vulnerability",
   "pk": 3,
   "fields": {
-    "cve_id": "CVE-2009-2459",
+    "vulnerability_id": "CVE-2009-2459",
     "summary": ""
 
   }

vulnerabilities/import_runner.py

@@ -153,12 +153,23 @@ def process_advisories(data_source: DataSource) -> None:
                         existing_ref = get_vuln_pkg_refs(vuln, pkg)
                         if not existing_ref:
                             bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
+                            # A vulnerability-package relationship does not exist already
+                            # if either the vulnerability or the package is just created.
 
                         else:
-                            # This handles conflicts between existing data and obtained data
-                            if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
-                                handle_conflicts([existing_ref[0], pkg_vuln_ref.to_model_object()])
-                                existing_ref.delete()
+                            # insert only if it there is no existing vulnerability-package relationship.  # nopep8
+                            existing_ref = get_vuln_pkg_refs(vuln, pkg)
+                            if not existing_ref:
+                                bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
+
+                            else:
+                                # This handles conflicts between existing data and obtained data
+                                if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
+                                    handle_conflicts(
+                                        [existing_ref[0], pkg_vuln_ref.to_model_object()]
+                                    )
+                                    existing_ref.delete()
+
             except Exception:
                 # TODO: store error but continue
                 logger.error(
@@ -223,26 +234,13 @@ def _get_or_create_vulnerability(
     advisory: Advisory,
 ) -> Tuple[models.Vulnerability, bool]:
 
-    if advisory.cve_id:
-        query_kwargs = {"cve_id": advisory.cve_id}
-    elif advisory.summary:
-        query_kwargs = {"summary": advisory.summary}
-    else:
-        return models.Vulnerability.objects.create(), True
-
-    try:
-        vuln, created = models.Vulnerability.objects.get_or_create(**query_kwargs)
-        # Eventually we only want to keep summary from NVD and ignore other descriptions.
-        if advisory.summary and vuln.summary != advisory.summary:
-            vuln.summary = advisory.summary
-            vuln.save()
-        return vuln, created
-
-    except Exception:
-        logger.error(
-            f"Failed to _get_or_create_vulnerability: {query_kwargs!r}:\n" + traceback.format_exc()
-        )
-        raise
+    vuln, created = models.Vulnerability.objects.get_or_create(vulnerability_id=advisory.vulnerability_id)  # nopep8
+    # Eventually we only want to keep summary from NVD and ignore other descriptions.
+    if advisory.summary and vuln.summary != advisory.summary:
+        vuln.summary = advisory.summary
+        vuln.save()
+
+    return vuln, created
 
 
 def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:

vulnerabilities/importer_yielder.py

@@ -58,16 +58,16 @@
             'debian_tracker_url': 'https://security-tracker.debian.org/tracker/data/json'
         },
     },
-#     {
-#         'name': 'safetydb',
-#         'license': 'cc-by-nc-4.0',
-#         'last_run': None,
-#         'data_source': 'SafetyDbDataSource',
-#         'data_source_cfg': {
-#             'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
-#             'etags': {}
-#         },
-#     },
+    {
+        'name': 'safetydb',
+        'license': 'cc-by-nc-4.0',
+        'last_run': None,
+        'data_source': 'SafetyDbDataSource',
+        'data_source_cfg': {
+            'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
+            'etags': {}
+        },
+    },
     {
         'name': 'npm',
         'license': 'mit',

vulnerabilities/importers/__init__.py

@@ -47,3 +47,4 @@
 from vulnerabilities.importers.suse_scores import SUSESeverityScoreDataSource
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource
+from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource

vulnerabilities/importers/alpine_linux.py

@@ -187,7 +187,7 @@ def _load_advisories(
                         impacted_package_urls=[],
                         resolved_package_urls=resolved_purls,
                         vuln_references=references,
-                        cve_id=vuln_ids[0].upper() if vuln_ids[0] != "CVE-????-?????" else None,
+                        vulnerability_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else None,
                     )
                 )
 

vulnerabilities/importers/apache_kafka.py

@@ -82,7 +82,7 @@ def to_advisory(self, advisory_page):
 
             advisories.append(
                 Advisory(
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     summary=cve_description_paragraph.text,
                     impacted_package_urls=affected_packages,
                     resolved_package_urls=fixed_packages,

vulnerabilities/importers/apache_tomcat.py

@@ -116,7 +116,7 @@ def to_advisories(self, apache_tomcat_advisory_html):
                         summary="",
                         impacted_package_urls=affected_packages,
                         resolved_package_urls=fixed_package,
-                        cve_id=cve_id,
+                        vulnerability_id=cve_id,
                         vuln_references=references,
                     )
                 )

vulnerabilities/importers/archlinux.py

@@ -130,7 +130,7 @@ def _parse(self, record) -> List[Advisory]:
 
             advisories.append(
                 Advisory(
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     summary="",
                     impacted_package_urls=impacted_purls,
                     resolved_package_urls=resolved_purls,

vulnerabilities/importers/debian.py

@@ -151,7 +151,7 @@ def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[Advisory]:
 
             advisories.append(
                 Advisory(
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     summary=record.get("description", ""),
                     impacted_package_urls=impacted_purls,
                     resolved_package_urls=resolved_purls,

vulnerabilities/importers/elixir_security.py

@@ -139,6 +139,6 @@ def process_file(self, path):
             summary=yaml_file["description"],
             impacted_package_urls=vuln_purls,
             resolved_package_urls=safe_purls,
-            cve_id=cve_id,
+            vulnerability_id=cve_id,
             vuln_references=vuln_references,
         )

vulnerabilities/importers/gentoo.py

@@ -77,7 +77,7 @@ def process_file(self, file):
         # this way, but there seems no alternative.
         for cve in xml_data["cves"]:
             advisory = Advisory(
-                cve_id=cve,
+                vulnerability_id=cve,
                 summary=xml_data["description"],
                 impacted_package_urls=xml_data["affected_purls"],
                 resolved_package_urls=xml_data["unaffected_purls"],

vulnerabilities/importers/github.py

@@ -229,7 +229,7 @@ def process_response(self) -> List[Advisory]:
                     for cve_id in cve_ids:
                         adv_list.append(
                             Advisory(
-                                cve_id=cve_id,
+                                vulnerability_id=cve_id,
                                 summary=vuln_desc,
                                 impacted_package_urls=affected_purls,
                                 resolved_package_urls=unaffected_purls,

vulnerabilities/importers/kaybee.py

@@ -66,7 +66,7 @@ def yaml_file_to_advisory(yaml_path):
             references.append(Reference(url=f"{commit['repository']}/{commit['id']}"))
 
     return Advisory(
-        cve_id=vuln_id,
+        vulnerability_id=vuln_id,
         summary=summary,
         impacted_package_urls=impacted_packages,
         resolved_package_urls=resolved_packages,

vulnerabilities/importers/nginx.py

@@ -109,7 +109,7 @@ def to_advisories(self, data):
 
             advisories.append(
                 Advisory(
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     summary=summary,
                     impacted_package_urls=vulnerable_packages,
                     resolved_package_urls=fixed_packages,

vulnerabilities/importers/npm.py

@@ -106,7 +106,7 @@ def process_file(self, file) -> List[Advisory]:
             advisories.append(
                 Advisory(
                     summary=record.get("overview", ""),
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     impacted_package_urls=impacted_purls,
                     resolved_package_urls=resolved_purls,
                     vuln_references=vuln_reference,

vulnerabilities/importers/nvd.py

@@ -89,7 +89,7 @@ def to_advisories(self, nvd_data):
             )
             summary = self.extract_summary(cve_item)
             yield Advisory(
-                cve_id=cve_id, summary=summary, vuln_references=references, impacted_package_urls=[]
+                vulnerability_id=cve_id, summary=summary, vuln_references=references, impacted_package_urls=[]  # nopep8
             )
 
     @staticmethod

vulnerabilities/importers/openssl.py

@@ -111,7 +111,7 @@ def to_advisories(xml_response: str) -> Set[Advisory]:
                 }
 
                 advisory = Advisory(
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     summary=summary,
                     impacted_package_urls=vuln_purls,
                     resolved_package_urls=safe_purls,

vulnerabilities/importers/postgresql.py

@@ -105,7 +105,7 @@ def to_advisories(data):
 
         advisories.append(
             Advisory(
-                cve_id=cve_id,
+                vulnerability_id=cve_id,
                 summary=summary,
                 vuln_references=references,
                 impacted_package_urls=affected_packages,

vulnerabilities/importers/redhat.py

@@ -138,7 +138,7 @@ def to_advisory(advisory_data):
 
     return Advisory(
         summary=advisory_data["bugzilla_description"],
-        cve_id=advisory_data["CVE"],
+        vulnerability_id=advisory_data["CVE"],
         impacted_package_urls=affected_purls,
         vuln_references=references,
     )

vulnerabilities/importers/retiredotnet.py

@@ -99,5 +99,5 @@ def process_file(self, path) -> List[Advisory]:
                 summary=json_doc['description'],
                 impacted_package_urls=affected_purls,
                 resolved_package_urls=fixed_purls,
-                cve_id=vuln_id,
+                vulnerability_id=vuln_id,
                 vuln_references=vuln_reference)

vulnerabilities/importers/ruby.py

@@ -80,8 +80,7 @@ def collect_packages(self):
 
     def process_file(self, path) -> List[Advisory]:
         record = load_yaml(path)
-        package_name = record.get(
-            'gem')
+        package_name = record.get('gem')
 
         if not package_name:
             return
@@ -131,7 +130,7 @@ def process_file(self, path) -> List[Advisory]:
             impacted_package_urls=impacted_purls,
             resolved_package_urls=resolved_purls,
             vuln_references=references,
-            cve_id=cve_id
+            vulnerability_id=cve_id
         )
 
     @staticmethod

vulnerabilities/importers/rust.py

@@ -135,7 +135,7 @@ def _load_advisory(self, path: str) -> Optional[Advisory]:
             summary=advisory.get("description", ""),
             impacted_package_urls=impacted_purls,
             resolved_package_urls=resolved_purls,
-            cve_id=cve_id,
+            vulnerability_id=cve_id,
             vuln_references=references,
         )