Merge pull request #322 from nexB/improve-ubuntu

Improve Ubuntu OVAL importer

Author: sbs2001

setup.cfg

@@ -54,4 +54,5 @@ addopts =
     -rfEsxXw
     --strict
     --ignore setup.py
+    --ignore vulnerabilities/lib_oval.py
     --doctest-modules

vulnerabilities/data_source.py

@@ -20,6 +20,7 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
+import pickle
 import dataclasses
 import logging
 import os
@@ -466,20 +467,19 @@ def _collect_pkgs(parsed_oval_data: Mapping) -> Set:
 
     def _fetch(self) -> Tuple[Mapping, Iterable[ET.ElementTree]]:
         """
-        This method  contains logic to fetch OVAL files and yield them into
-        a tuple of file's metadata and it's ET.ElementTree.
+        Return a two-tuple of ({mapping of Package URL data}, it's ET.ElementTree)
         Subclasses must implement this method.
 
-        Note: Mapping MUST INCLUDE "type" key. Example values of Mapping
-              {"type":"deb","qualifiers":{"distro":"buster"} }
+        Note:  Package URL data MUST INCLUDE a Package URL "type" key so
+        implement _fetch() accordingly.
+        For example::
 
+              {"type":"deb","qualifiers":{"distro":"buster"} }
         """
+        # TODO: enforce that we receive the proper data here
         raise NotImplementedError
 
     def updated_advisories(self) -> List[Advisory]:
-        """
-        Note: metadata MUST INCLUDE "type" key, implement _fetch accordingly.
-        """
         for metadata, oval_file in self._fetch():
             try:
                 oval_data = self.get_data_from_xml_doc(oval_file, metadata)
@@ -506,34 +506,40 @@ def set_api(self, all_pkgs: Iterable[str]):
 
     def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> List[Advisory]:
         """
-        The orchestration method of the OvalDataSource. This method breaks an OVAL xml
-        ElementTree into a list of `Advisory`.
+        The orchestration method of the OvalDataSource. This method breaks an
+        OVAL xml ElementTree into a list of `Advisory`.
+
+        Note: pkg_metadata is a mapping of Package URL data that MUST INCLUDE
+        "type" key.
 
-        Note: pkg_metadata MUST INCLUDE "type" key. Example value of pkg_metadata,
+        Example value of pkg_metadata:
               {"type":"deb","qualifiers":{"distro":"buster"} }
         """
+
         all_adv = []
         oval_doc = OvalParser(self.translations, xml_doc)
         raw_data = oval_doc.get_data()
         all_pkgs = self._collect_pkgs(raw_data)
         self.set_api(all_pkgs)
-        for definition_data in raw_data:  # definition_data -> Advisory
 
-            # These fields are definition level, i.e common for all
-            # elements connected/linked to an OvalDefinition
+        # convert definition_data to Advisory objects
+        for definition_data in raw_data:
+            # These fields are definition level, i.e common for all elements
+            # connected/linked to an OvalDefinition
             vuln_id = definition_data["vuln_id"]
             description = definition_data["description"]
             affected_purls = set()
             safe_purls = set()
             references = [Reference(url=url) for url in definition_data["reference_urls"]]
-
             for test_data in definition_data["test_data"]:
-                for package in test_data["package_list"]:
-                    pkg_name = package
-                    if package and len(pkg_name) >= 50:
+                for package_name in test_data["package_list"]:
+                    if package_name and len(package_name) >= 50:
                         continue
-                    aff_ver_range = test_data["version_ranges"]
-                    all_versions = self.pkg_manager_api.get(package)
+                    aff_ver_range = test_data["version_ranges"] or set()
+                    all_versions = self.pkg_manager_api.get(package_name)
+
+                    # FIXME: what is this 50 DB limit? that's too small for versions
+                    # FIXME: we should not drop data this way
                     # This filter is for filtering out long versions.
                     # 50 is limit because that's what db permits atm.
                     all_versions = set(filter(lambda x: len(x) < 50, all_versions))
@@ -543,22 +549,28 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
                     safe_versions = all_versions - affected_versions
 
                     for version in affected_versions:
-                        pkg_url = self.create_purl(
-                            pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata
+                        purl = self.create_purl(
+                            pkg_name=package_name,
+                            pkg_version=version,
+                            pkg_data=pkg_metadata,
                         )
-                        affected_purls.add(pkg_url)
+                        affected_purls.add(purl)
 
                     for version in safe_versions:
-                        pkg_url = self.create_purl(
-                            pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata
+                        purl = self.create_purl(
+                            pkg_name=package_name,
+                            pkg_version=version,
+                            pkg_data=pkg_metadata,
                         )
-                        safe_purls.add(pkg_url)
+                        safe_purls.add(purl)
 
             all_adv.append(
                 Advisory(
                     summary=description,
                     impacted_package_urls=affected_purls,
                     resolved_package_urls=safe_purls,
                     vulnerability_id=vuln_id,
-                    vuln_references=references))
+                    vuln_references=references,
+                )
+            )
         return all_adv

vulnerabilities/helpers.py

@@ -23,9 +23,9 @@
 import json
 import re
 
-import yaml
 import requests
 import toml
+import yaml
 
 # TODO add logging here
 
@@ -50,6 +50,13 @@ def fetch_yaml(url):
     return yaml.safe_load(response.content)
 
 
+# FIXME: this is NOT how etags work .
+# We should instead send the proper HTTP header
+# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match
+# and integrate this finely in the processing as this typically needs to use
+# streaming=True requests, and proper handling of the HTTP return code
+# In all cases this ends up being a single request, not a HEADD followed
+# by another real request
 def create_etag(data_src, url, etag_key):
     """
     Etags are like hashes of web responses. For a data source `data_src`,

vulnerabilities/importers/ubuntu.py

@@ -24,6 +24,7 @@
 import asyncio
 import bz2
 import dataclasses
+import logging
 from typing import Iterable
 from typing import List
 from typing import Mapping
@@ -38,6 +39,8 @@
 from vulnerabilities.package_managers import LaunchpadVersionAPI
 from vulnerabilities.helpers import create_etag
 
+logger = logging.getLogger(__name__)
+
 
 @dataclasses.dataclass
 class UbuntuConfiguration(DataSourceConfiguration):
@@ -58,22 +61,25 @@ def __init__(self, *args, **kwargs):
         self.pkg_manager_api = LaunchpadVersionAPI()
 
     def _fetch(self):
+        base_url = "https://people.canonical.com/~ubuntu-security/oval"
         releases = self.config.releases
-        for release in releases:
-            file_url = f"https://people.canonical.com/~ubuntu-security/oval/com.ubuntu.{release}.cve.oval.xml.bz2"  # nopep8
-            if not create_etag(data_src=self, url=file_url, etag_key="ETag"):
+        for i, release in enumerate(releases, 1):
+            file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2"  # nopep8
+            logger.info(f"Fetching Ubuntu Oval: {file_url}")
+            response = requests.get(file_url)
+            if response.status_code != requests.codes.ok:
+                logger.error(
+                    f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}"
+                )
                 continue
-            resp = requests.get(file_url)
-            extracted = bz2.decompress(resp.content)
+
+            extracted = bz2.decompress(response.content)
             yield (
                 {"type": "deb", "namespace": "ubuntu"},
                 ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
             )
-        # In case every file is latest, _fetch won't yield anything(due to checking for new etags),
-        # this would return None to added_advisories
-        # which will cause error, hence this
-        # function return an empty list
-        return []
+
+        logger.info(f"Fetched {i} Ubuntu Oval releases from {base_url}")
 
     def set_api(self, packages):
         asyncio.run(self.pkg_manager_api.load_api(packages))

vulnerabilities/lib_oval.py

@@ -1,13 +1,32 @@
 #!/usr/bin/env/ python3
-# Copyright© 2010 United States Government. All Rights Reserved.
+# Copyright (c) 2010 United States Government. All Rights Reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice, this
+# list of conditions and the following disclaimer in the documentation and/or
+# other materials provided with the distribution.
+#
+# * Neither the name of the Center for Internet Security, Inc. (CIS) nor the names
+# of its contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
 
-# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-# * Neither the name of the Center for Internet Security, Inc. (CIS) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """Library to simplify working with the OVAL XML structure
 
 
@@ -39,29 +58,29 @@
 
 1. Create an OvalDocument:
 
-    >> tree = ElementTree()
-    >> tree.parse("OvalTest.xml")
-    >> document = OvalDocument(tree)
+    >>> tree = ElementTree()
+    >>> tree.parse("OvalTest.xml")
+    >>> document = OvalDocument(tree)
 
 2. Find an oval element within the loaded document:
 
-    >> element = document.getElementByID("oval:org.mitre.oval:def:22382")
-    >> if element is not None:
-    >>    ....
+    >>> element = document.getElementByID("oval:org.mitre.oval:def:22382")
+    >>> if element is not None:
+    >>>    ....
 
 3. Read an XML file with a single OVAL Definition (error checking omitted for brevity):
 
-    >> tree = ElementTree()    
-    >> tree.parse('test-definition.xml')
-    >> root = tree.getroot()    
-    >> definition = lib_oval.OvalDefinition(root)
+    >>> tree = ElementTree()    
+    >>> tree.parse('test-definition.xml')
+    >>> root = tree.getroot()    
+    >>> definition = lib_oval.OvalDefinition(root)
     
 4. Change information in the definition from #3 and write the changes
 
-    >> meta = definition.getMetadata()
-    >> repo = meta.getOvalRepositoryInformation()
-    >> repo.setMinimumSchemaVersion("5.9")
-    >> tree.write("outfilename.xml", UTF-8", True)
+    >>> meta = definition.getMetadata()
+    >>> repo = meta.getOvalRepositoryInformation()
+    >>> repo.setMinimumSchemaVersion("5.9")
+    >>> tree.write("outfilename.xml", UTF-8", True)
         
 
 

vulnerabilities/lib_oval.py.ABOUT

@@ -1,14 +1,12 @@
 about_resource: lib_oval.py
-version: 6aaae0
-download_url: https://raw.githubusercontent.com/CISecurity/OVALRepo/6aaae00876ec716927e0ae5b9ccfa12f310427a0/scripts/lib_oval.py
-
-name: OVALRepo - lib_oval
+version: e74da20d7c1de91c098c564ce65d6b93656eb86f
+download_url: https://raw.githubusercontent.com/CISecurity/OVALRepo/e74da20d7c1de91c098c564ce65d6b93656eb86f/scripts/lib_oval.py
+package_url: pkg:github/CISecurity/OVALRepo@e74da20d7c1de91c098c564ce65d6b93656eb86f#scripts/lib_oval.py
 homepage_url: https://github.com/CISecurity/OVALRepo
 owner: Center for Internet Security
 author: Gunnar Engelbach <Gunnar.Engelbach@ThreatGuard.com>
 notes: This a single file extracted from OVALRepo that parses OVAL files.
 
-license: bsd-new
+copyright: Copyright (c) 2010 United States Government. All Rights Reserved.
+license_expression: bsd-new
 license_file: lib_oval.py.LICENSE
-
-copyright: Copyright (c) 2010 United States Government. All Rights Reserved.

vulnerabilities/lib_oval.py.LICENSE

@@ -1,9 +1,26 @@
-Copyright© 2010 United States Government. All Rights Reserved.
+Copyright (c) 2010 United States Government. All Rights Reserved.
 
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
 
-    Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-    Neither the name of the Center for Internet Security, Inc. (CIS) nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+    Redistributions of source code must retain the above copyright notice, this
+    list of conditions and the following disclaimer.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+    Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+    Neither the name of the Center for Internet Security, Inc. (CIS) nor the
+    names of its contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER, CIS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER, CIS OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.