Fix #85 with non-empty redhat security source and empty-list checks.

dagda/cli/command/start_cli_parser.py

@@ -31,7 +31,7 @@ class StartCLIParser:
     def __init__(self):
         super(StartCLIParser, self).__init__()
         self.parser = DagdaStartParser(prog='dagda.py start', usage=start_parser_text)
-        self.parser.add_argument('-d','--debug', action='store_true')
+        self.parser.add_argument('-d', '--debug', action='store_true')
         self.parser.add_argument('-s', '--server_host', type=str)
         self.parser.add_argument('-p', '--server_port', type=int)
         self.parser.add_argument('-m', '--mongodb_host', type=str)

dagda/driver/mongodb_driver.py

@@ -108,26 +108,34 @@ def bulk_insert_exploit_db_info(self, exploit_db_info_list):
     # Bulk insert the rhsa list
     def bulk_insert_rhsa(self, rhsa_list):
         # Bulk insert
-        self.db.rhsa.create_index([('product', pymongo.DESCENDING)])
-        self.db.rhsa.insert_many(rhsa_list)
+        if rhsa_list:
+            # Ensure the list is not empty.  See #85.
+            self.db.rhsa.create_index([('product', pymongo.DESCENDING)])
+            self.db.rhsa.insert_many(rhsa_list)
 
     # Bulk insert the rhba list
     def bulk_insert_rhba(self, rhba_list):
         # Bulk insert
-        self.db.rhba.create_index([('product', pymongo.DESCENDING)])
-        self.db.rhba.insert_many(rhba_list)
+        if rhba_list:
+            # Ensure the list is not empty.  See #85.
+            self.db.rhba.create_index([('product', pymongo.DESCENDING)])
+            self.db.rhba.insert_many(rhba_list)
 
     # Bulk insert the rhsa info list
     def bulk_insert_rhsa_info(self, rhsa_info_list):
         # Bulk insert
-        self.db.rhsa_info.create_index([('rhsa_id', pymongo.DESCENDING)])
-        self.db.rhsa_info.insert_many(rhsa_info_list)
+        if rhsa_info_list:
+            # Ensure the list is not empty.  See #85.
+            self.db.rhsa_info.create_index([('rhsa_id', pymongo.DESCENDING)])
+            self.db.rhsa_info.insert_many(rhsa_info_list)
 
     # Bulk insert the rhba info list
     def bulk_insert_rhba_info(self, rhba_info_list):
         # Bulk insert
-        self.db.rhba_info.create_index([('rhba_id', pymongo.DESCENDING)])
-        self.db.rhba_info.insert_many(rhba_info_list)
+        if rhba_info_list:
+            # Ensure the list is not empty.  See #85.
+            self.db.rhba_info.create_index([('rhba_id', pymongo.DESCENDING)])
+            self.db.rhba_info.insert_many(rhba_info_list)
 
     # Bulk insert the docker daemon events
     def bulk_insert_docker_daemon_events(self, events):

dagda/vulnDB/db_composer.py

@@ -94,7 +94,7 @@ def compose_vuln_db(self):
         self.mongoDbDriver.delete_rhba_info_collection()
         self.mongoDbDriver.delete_rhsa_collection()
         self.mongoDbDriver.delete_rhsa_info_collection()
-        bz2_file = get_http_resource_content('https://www.redhat.com/security/data/oval/rhsa.tar.bz2')
+        bz2_file = get_http_resource_content('https://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2')
         rhsa_list, rhba_list, rhsa_info_list, rhba_info_list = get_rhsa_and_rhba_lists_from_file(bz2_file)
         self.mongoDbDriver.bulk_insert_rhsa(rhsa_list)
         self.mongoDbDriver.bulk_insert_rhba(rhba_list)

dagda/vulnDB/ext_source_util.py

@@ -26,6 +26,7 @@
 from io import BytesIO
 import datetime
 import tarfile
+import bz2
 
 
 ACCESS_VECTOR = {'L': 'Local access', 'A': 'Adjacent Network', 'N': 'Network'}
@@ -228,9 +229,11 @@ def parse_bid_from_json(json_data, items):
 
 
 # Gets RHSA (Red Hat Security Advisory) and RHBA (Red Hat Bug Advisory) lists from bz2 file
+# This is expected to be a unified OVAL definitions XML file.
+# See https://www.redhat.com/security/data/metrics/ for details.
 def get_rhsa_and_rhba_lists_from_file(bz2_file):
     # Init
-    tar = tarfile.open(mode='r:bz2', fileobj=BytesIO(bz2_file))
+    xml_file_content = bz2.decompress(bz2_file)
     rhsa_list = []
     rhsa_id_list = []
     rhba_list = []
@@ -239,82 +242,79 @@ def get_rhsa_and_rhba_lists_from_file(bz2_file):
     rhsa_info_id_list = []
     rhba_info_list = []
     rhba_info_id_list = []
-    for xml_file in tar.getmembers():
-        if xml_file.size > 0:
-            xml_file_content = tar.extractfile(xml_file.name)
-            root = ET.parse(xml_file_content).getroot().find('{http://oval.mitre.org/XMLSchema/oval-definitions-5}definitions')
-            for entry in root.findall('{http://oval.mitre.org/XMLSchema/oval-definitions-5}definition'):
-                # Init
-                metadata = entry.find('{http://oval.mitre.org/XMLSchema/oval-definitions-5}metadata')
-                detail_info = {}
+    root = ET.fromstring(xml_file_content).find('{http://oval.mitre.org/XMLSchema/oval-definitions-5}definitions')
+    for entry in root.findall('{http://oval.mitre.org/XMLSchema/oval-definitions-5}definition'):
+        # Init
+        metadata = entry.find('{http://oval.mitre.org/XMLSchema/oval-definitions-5}metadata')
+        detail_info = {}
 
-                # Get IDs
-                rhsa_id = None
-                rhba_id = None
-                cves = []
-                for reference in metadata.findall("{http://oval.mitre.org/XMLSchema/oval-definitions-5}reference"):
-                    # Get RHSA (Red Hat Security Advisory)
-                    if 'RHSA' in reference.attrib['ref_id']:
-                        rhsa_id = reference.attrib['ref_id']
-                        if "-" in rhsa_id[5:]:
-                            rhsa_id = rhsa_id[:rhsa_id.index("-", 5)]
-                    # RHBA (Red Hat Bug Advisory)
-                    if 'RHBA' in reference.attrib['ref_id']:
-                        rhba_id = reference.attrib['ref_id']
-                        if "-" in rhba_id[5:]:
-                            rhba_id = rhba_id[:rhba_id.index("-", 5)]
-                    # Get related CVEs
-                    if reference.attrib['source'] == 'CVE':
-                        cves.append(reference.attrib['ref_id'])
+        # Get IDs
+        rhsa_id = None
+        rhba_id = None
+        cves = []
+        for reference in metadata.findall("{http://oval.mitre.org/XMLSchema/oval-definitions-5}reference"):
+            # Get RHSA (Red Hat Security Advisory)
+            if 'RHSA' in reference.attrib['ref_id']:
+                rhsa_id = reference.attrib['ref_id']
+                if "-" in rhsa_id[5:]:
+                    rhsa_id = rhsa_id[:rhsa_id.index("-", 5)]
+            # RHBA (Red Hat Bug Advisory)
+            if 'RHBA' in reference.attrib['ref_id']:
+                rhba_id = reference.attrib['ref_id']
+                if "-" in rhba_id[5:]:
+                    rhba_id = rhba_id[:rhba_id.index("-", 5)]
+            # Get related CVEs
+            if reference.attrib['source'] == 'CVE':
+                cves.append(reference.attrib['ref_id'])
 
-                detail_info['cve'] = cves
+        detail_info['cve'] = cves
 
-                # Get title and description
-                detail_info['title'] = metadata.findtext('{http://oval.mitre.org/XMLSchema/oval-definitions-5}title')
-                detail_info['description'] = metadata.findtext('{http://oval.mitre.org/XMLSchema/oval-definitions-5}description')
+        # Get title and description
+        detail_info['title'] = metadata.findtext('{http://oval.mitre.org/XMLSchema/oval-definitions-5}title')
+        detail_info['description'] = metadata.findtext('{http://oval.mitre.org/XMLSchema/oval-definitions-5}description')
 
-                # Get severity
-                detail_info['severity'] = metadata.find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}advisory") \
-                                                    .find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}severity").text
-                # Append detail info
-                if rhsa_id is not None:
-                    detail_info['rhsa_id'] = rhsa_id
-                    if rhsa_id not in rhsa_info_id_list:
-                        rhsa_info_id_list.append(rhsa_id)
-                        rhsa_info_list.append(detail_info)
-                if rhba_id is not None:
-                    detail_info['rhba_id'] = rhba_id
-                    if rhba_id not in rhba_info_id_list:
-                        rhba_info_id_list.append(rhba_id)
-                        rhba_info_list.append(detail_info)
+        # Get severity
+        detail_info['severity'] = metadata.find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}advisory") \
+                                            .find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}severity").text
+        # Append detail info
+        if rhsa_id is not None:
+            detail_info['rhsa_id'] = rhsa_id
+            if rhsa_id not in rhsa_info_id_list:
+                rhsa_info_id_list.append(rhsa_id)
+                rhsa_info_list.append(detail_info)
+        if rhba_id is not None:
+            detail_info['rhba_id'] = rhba_id
+            if rhba_id not in rhba_info_id_list:
+                rhba_info_id_list.append(rhba_id)
+                rhba_info_list.append(detail_info)
 
-                # Get vulnerable products
-                affected_cpe_list = metadata.find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}advisory") \
-                                            .find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}affected_cpe_list")
-                for cpe in affected_cpe_list:
-                  if cpe.text is not None:
-                    info_item = {}
-                    splitted_product = cpe.text.split(":")
-                    info_item['vendor'] = splitted_product[2]
-                    info_item['product'] = splitted_product[3]
-                    try:
-                        info_item['version'] = splitted_product[4]
-                    except IndexError:
-                        info_item['version'] = '-'
+        # Get vulnerable products
+        affected_cpe_list = metadata.find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}advisory") \
+                                    .find("{http://oval.mitre.org/XMLSchema/oval-definitions-5}affected_cpe_list")
+        for cpe in affected_cpe_list:
+          if cpe.text is not None:
+            info_item = {}
+            splitted_product = cpe.text.split(":")
+            info_item['vendor'] = splitted_product[2]
+            info_item['product'] = splitted_product[3]
+            try:
+                info_item['version'] = splitted_product[4]
+            except IndexError:
+                info_item['version'] = '-'
 
-                    tmp = '#' + info_item['vendor'] + '#' + info_item['product'] + '#' + info_item['version']
-                    if rhsa_id is not None:
-                        info_item['rhsa_id'] = rhsa_id
-                        tmp = rhsa_id + tmp
-                        if tmp not in rhsa_id_list:
-                            rhsa_id_list.append(tmp)
-                            rhsa_list.append(info_item)
-                    if rhba_id is not None:
-                        info_item['rhba_id'] = rhba_id
-                        tmp = rhba_id + tmp
-                        if tmp not in rhba_id_list:
-                            rhba_id_list.append(tmp)
-                            rhba_list.append(info_item)
+            tmp = '#' + info_item['vendor'] + '#' + info_item['product'] + '#' + info_item['version']
+            if rhsa_id is not None:
+                info_item['rhsa_id'] = rhsa_id
+                tmp = rhsa_id + tmp
+                if tmp not in rhsa_id_list:
+                    rhsa_id_list.append(tmp)
+                    rhsa_list.append(info_item)
+            if rhba_id is not None:
+                info_item['rhba_id'] = rhba_id
+                tmp = rhba_id + tmp
+                if tmp not in rhba_id_list:
+                    rhba_id_list.append(tmp)
+                    rhba_list.append(info_item)
 
     # Return
     return rhsa_list, rhba_list, rhsa_info_list, rhba_info_list

tests/analysis/static/dependencies/test_dep_info_extractor.py

@@ -20,6 +20,7 @@
 import unittest
 import json
 import os
+import tempfile
 from dagda.analysis.static.dependencies.dep_info_extractor import get_filtered_dependencies_info
 from dagda.analysis.static.dependencies.dep_info_extractor import read_4depcheck_output_file
 
@@ -29,7 +30,8 @@
 class DepInfoExtractorTestSuite(unittest.TestCase):
 
     def test_raw_info_to_json_array(self):
-        filtered_dep = get_filtered_dependencies_info(json.loads(mock_json_array), '/tmp/sdgas68kg')
+        filtered_dep = get_filtered_dependencies_info(
+            json.loads(mock_json_array), os.path.join(tempfile.gettempdir(), 'sdgas68kg'))
         self.assertEqual(len(filtered_dep), 3)
         self.assertTrue('python#lxml#1.0.1#/tmp/lxml.1.0.1.py' in filtered_dep)
         self.assertTrue('java#cxf#2.6.0#/tmp/cxf.2.6.0.jar' in filtered_dep)
@@ -41,21 +43,21 @@ def test_read_4depcheck_output_file_exception(self):
             read_4depcheck_output_file('no_image_name')
         except Exception as ex:
             msg = ex.get_message()
-        self.assertEqual(msg, '4depcheck output file [/tmp/4depcheck/no_image_name.json] not found.')
+        self.assertEqual(msg, '4depcheck output file [' + tempfile.gettempdir() + '/4depcheck/no_image_name.json] not found.')
 
     def test_read_4depcheck_empty_output_file(self):
         # Prepare test
         try:
-            os.makedirs('/tmp/4depcheck')
+            os.makedirs(os.path.join(tempfile.gettempdir(), '4depcheck'))
             created = True
         except OSError:
             created = False
-        with open('/tmp/4depcheck/empty_output_file.json', 'w') as f:
-            None
+        with open(os.path.join(tempfile.gettempdir(), '4depcheck', 'empty_output_file.json'), 'w') as f:
+            pass
         # Run
         raw_info = read_4depcheck_output_file('empty_output_file')
         # Clean up
-        os.remove('/tmp/4depcheck/empty_output_file.json')
+        os.remove(os.path.join(tempfile.gettempdir(), '4depcheck', 'empty_output_file.json'))
         if created:
             os.removedirs('/tmp/4depcheck')
         # Check

tests/analysis/static/util/test_utils.py

@@ -33,25 +33,29 @@ class UtilsTestSuite(unittest.TestCase):
 
     def test_extract_filesystem_bundle_and_clean_up(self):
         temporary_dir = extract_filesystem_bundle(MockDockerDriver(), container_id='asdfi76fdgauh')
-        self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'bin')))
-        self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'bin/bash')))
-        self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'etc')))
-        self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'etc/daemon.conf')))
-        self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'opt')))
-        clean_up(temporary_dir)
+        try:
+            self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'bin')))
+            self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'bin/bash')))
+            self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'etc')))
+            self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'etc/daemon.conf')))
+            self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'opt')))
+        finally:
+            clean_up(temporary_dir)
         self.assertFalse(os.path.isdir(os.path.join(temporary_dir, 'bin')))
         self.assertFalse(os.path.exists(os.path.join(temporary_dir, 'bin/bash')))
         self.assertFalse(os.path.isdir(os.path.join(temporary_dir, 'etc')))
         self.assertFalse(os.path.exists(os.path.join(temporary_dir, 'etc/daemon.conf')))
         self.assertFalse(os.path.isdir(os.path.join(temporary_dir, 'opt')))
 
         temporary_dir = extract_filesystem_bundle(MockDockerDriver(), image_name='alpine')
-        self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'bin')))
-        self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'bin/bash')))
-        self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'etc')))
-        self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'etc/daemon.conf')))
-        self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'opt')))
-        clean_up(temporary_dir)
+        try:
+            self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'bin')))
+            self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'bin/bash')))
+            self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'etc')))
+            self.assertTrue(os.path.exists(os.path.join(temporary_dir, 'etc/daemon.conf')))
+            self.assertTrue(os.path.isdir(os.path.join(temporary_dir, 'opt')))
+        finally:
+            clean_up(temporary_dir)
         self.assertFalse(os.path.isdir(os.path.join(temporary_dir, 'bin')))
         self.assertFalse(os.path.exists(os.path.join(temporary_dir, 'bin/bash')))
         self.assertFalse(os.path.isdir(os.path.join(temporary_dir, 'etc')))

tests/driver/test_mongodb_driver.py

@@ -231,6 +231,50 @@ def test_bulk_insert_sysdig_falco_events(self):
             "time": 1465256864.080226
         }])
 
+    def test_bulk_insert_rhsa(self):
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhsa([{"vendor": "redhat", "product": "enterprise_linux", "version": "4", "rhsa_id": "RHSA-2010:0002-01"}])
+        mock_driver.db.rhsa.insert_many.assert_called_once_with([{"vendor": "redhat", "product": "enterprise_linux", "version": "4", "rhsa_id": "RHSA-2010:0002-01"}])
+
+    def test_bulk_insert_rhsa_empty(self):
+        # Test bug #85
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhsa([])
+        mock_driver.db.rhsa.insert_many.assert_not_called()
+
+    def test_bulk_insert_rhba(self):
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhba([{"vendor": "redhat", "product": "enterprise_linux", "version": "4", "rhba_id": "RHBA-2010:0002-01"}])
+        mock_driver.db.rhba.insert_many.assert_called_once_with([{"vendor": "redhat", "product": "enterprise_linux", "version": "4", "rhba_id": "RHBA-2010:0002-01"}])
+
+    def test_bulk_insert_rhba_empty(self):
+        # Test bug #85
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhba([])
+        mock_driver.db.rhba.insert_many.assert_not_called()
+
+    def test_bulk_insert_rhsa_info(self):
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhsa_info([{"cve": ["CVE-20101"], "title": "Test", "description": "Test CVE", "severity": "high", "rhsa_id": "RHSA-2010:0002-01"}])
+        mock_driver.db.rhsa_info.insert_many.assert_called_once_with([{"cve": ["CVE-20101"], "title": "Test", "description": "Test CVE", "severity": "high", "rhsa_id": "RHSA-2010:0002-01"}])
+
+    def test_bulk_insert_rhsa_info_empty(self):
+        # Test bug #85
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhsa_info([])
+        mock_driver.db.rhsa_info.insert_many.assert_not_called()
+
+    def test_bulk_insert_rhba_info(self):
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhba_info([{"cve": ["CVE-20101"], "title": "Test", "description": "Test CVE", "severity": "high", "rhba_id": "RHBA-2010:0002-01"}])
+        mock_driver.db.rhba_info.insert_many.assert_called_once_with([{"cve": ["CVE-20101"], "title": "Test", "description": "Test CVE", "severity": "high", "rhba_id": "RHBA-2010:0002-01"}])
+
+    def test_bulk_insert_rhba_info_empty(self):
+        # Test bug #85
+        mock_driver = BulkInfoMongoDbDriver()
+        mock_driver.bulk_insert_rhba_info([])
+        mock_driver.db.rhba_info.insert_many.assert_not_called()
+
     def test_is_fp_false(self):
         mock_driver = IsFPMongoDbDriver()
         is_fp = mock_driver.is_fp('alpine', 'zlib')