Make RedHat CVE import more robust

vulnerabilities/importers/redhat.py

@@ -20,9 +20,6 @@
 #  VulnerableCode is a free software code from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-
-import json
-
 from packageurl import PackageURL
 import requests
 
@@ -38,40 +35,56 @@ class RedhatDataSource(DataSource):
     CONFIG_CLASS = DataSourceConfiguration
 
     def __enter__(self):
-        self.redhat_response = fetch()
 
-    def updated_advisories(self):
-        processed_advisories = []
-        for advisory_data in self.redhat_response:
-            processed_advisories.append(to_advisory(advisory_data))
+        self.redhat_cves = fetch()
 
+    def updated_advisories(self):
+        processed_advisories = list(map(to_advisory, self.redhat_cves))
         return self.batch_advisories(processed_advisories)
 
 
 def fetch():
-
-    response = []
+    """
+    Return a list of CVE data mappings fetched from the RedHat API.
+    See:
+        https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/index
+    """
+    cves = []
     page_no = 1
-    url = "https://access.redhat.com/hydra/rest/securitydata/cve.json?page={}"
+    url_template = "https://access.redhat.com/hydra/rest/securitydata/cve.json?per_page=10000&page={}"  # nopep8
 
+    cve_data = None
     while True:
-        resp_json = requests.get(url.format(page_no)).json()
-        page_no += 1
-        if not resp_json:
+        current_url = url_template.format(page_no)
+        try:
+            print(f"Fetching: {current_url}")
+            response = requests.get(current_url)
+            if response.status_code != requests.codes.ok:
+                # TODO: log me
+                print(f"Failed to fetch results from {current_url}")
+                break
+            cve_data = response.json()
+        except Exception as e:
+            # TODO: log me
+            msg = f"Failed to fetch results from {current_url}:\n{e}"
+            print(msg)
             break
 
-        for advisory in resp_json:
-            response.append(advisory)
+        if not cve_data:
+            break
+        cves.extend(cve_data)
+        page_no += 1
 
-    return response
+    return cves
 
 
 def to_advisory(advisory_data):
     affected_purls = []
     if advisory_data.get("affected_packages"):
         for rpm in advisory_data["affected_packages"]:
-            if rpm_to_purl(rpm):
-                affected_purls.append(rpm_to_purl(rpm))
+            purl = rpm_to_purl(rpm)
+            if purl:
+                affected_purls.append(purl)
 
     references = []
     bugzilla = advisory_data.get("bugzilla")
@@ -135,16 +148,17 @@ def to_advisory(advisory_data):
         )
 
     references.append(Reference(severities=redhat_scores, url=advisory_data["resource_url"]))
-
     return Advisory(
-        summary=advisory_data["bugzilla_description"],
         vulnerability_id=advisory_data["CVE"],
+        summary=advisory_data["bugzilla_description"],
         impacted_package_urls=affected_purls,
         vuln_references=references,
     )
 
 
 def rpm_to_purl(rpm_string):
+    # FIXME: there is code in scancode to handle RPM conversion AND this should
+    # be all be part of the packageurl library
 
     # Red Hat uses `-:0` instead of just `-` to separate
     # package name and version
@@ -155,4 +169,4 @@ def rpm_to_purl(rpm_string):
     name, version = components
 
     if version[0].isdigit():
-        return PackageURL(name=name, type="rpm", version=version, namespace="redhat")
+        return PackageURL(namespace="redhat", name=name, type="rpm", version=version)