Merge branch 'test/httplib2shim' into develop

ScoutSuite/providers/aws/utils.py

@@ -1,4 +1,5 @@
 import re
+from ScoutSuite.core.console import print_exception
 
 ec2_classic = 'EC2-Classic'
 
@@ -28,12 +29,16 @@ def is_throttled(e):
     :param e:                           Exception raised
     :return:                            True if it's a throttling exception else False
     """
-    return (hasattr(e, 'response')
-            and e.response
-            and 'Error' in e.response
-            and e.response['Error']['Code'] in ['Throttling',
-                                                'RequestLimitExceeded',
-                                                'ThrottlingException'])
+    try:
+        return (hasattr(e, 'response')
+                and e.response
+                and 'Error' in e.response
+                and e.response['Error']['Code'] in ['Throttling',
+                                                    'RequestLimitExceeded',
+                                                    'ThrottlingException'])
+    except Exception as e:
+        print_exception('Unable to validate exception for throttling: {}'.format(e))
+        return False
 
 
 def get_keys(src, dst, keys):

ScoutSuite/providers/gcp/facade/base.py

@@ -91,7 +91,7 @@ async def _get_projects_recursively(self, parent_type, parent_id):
             return None
 
         resourcemanager_client = self._get_client()
-        resourcemanager_client_v2 = self._build_arbitrary_client('cloudresourcemanager', 'v2')
+        resourcemanager_client_v2 = self._build_arbitrary_client('cloudresourcemanager', 'v2', force_new=True)
 
         projects = []
 

ScoutSuite/providers/gcp/facade/basefacade.py

@@ -1,3 +1,11 @@
+# Since the HTTP library used by the Google API Client library is not
+# thread-safe, we leverage https://github.com/GoogleCloudPlatform/httplib2shim
+# resolves the following:
+#   - https://github.com/nccgroup/ScoutSuite/issues/443
+#   - https://github.com/nccgroup/ScoutSuite/issues/665
+import httplib2shim
+httplib2shim.patch()
+
 from googleapiclient import discovery
 
 class GCPBaseFacade:
@@ -9,11 +17,20 @@ def __init__(self, client_name: str, client_version: str):
     def _build_client(self) -> discovery.Resource:
         return self._build_arbitrary_client(self._client_name, self._client_version)
 
-    def _build_arbitrary_client(self, client_name, client_version):
-        return discovery.build(client_name, client_version, cache_discovery=False, cache=MemoryCache())
+    def _build_arbitrary_client(self, client_name, client_version, force_new=False):
+        """
+        :param client_name: name of the service
+        :param client_version:  version of the client to create
+        :param force_new: whether to create a new client - useful to create arbitrary clients from facades
+        :return:
+        """
+        if force_new:
+            return discovery.build(client_name, client_version, cache_discovery=False, cache=MemoryCache())
+        else:
+            if not self._client:
+                self._client = discovery.build(client_name, client_version, cache_discovery=False, cache=MemoryCache())
+            return self._client
 
-    # Since the HTTP library used by the Google API Client library is not 
-    # thread-safe, we need to create a new client for each request.
     def _get_client(self) -> discovery.Resource:
         return self._build_client()
 

ScoutSuite/providers/utils.py

@@ -24,8 +24,7 @@ async def run_concurrently(function, backoff_seconds=15):
             return await run_function_concurrently(function)
     except Exception as e:
         # Determine whether the exception is due to API throttling
-        throttled = aws_is_throttled(e)  # FIXME - this only works for AWS
-        if throttled:
+        if is_throttled(e):
             print_info('Hitting API rate limiting, will retry in {}s'.format(backoff_seconds))
             await asyncio.sleep(backoff_seconds)
             return await run_concurrently(function, backoff_seconds + 15)
@@ -105,3 +104,15 @@ async def map_concurrently(coroutine, entities, **kwargs):
             results.append(result)
 
     return results
+
+
+def is_throttled(e):
+    """
+    Function that tries to determine if an exception was caused by throttling
+    TODO - this implementation is incomplete
+    """
+
+    if hasattr(e, 'message') and 'Google Cloud' in e.message:
+        return False
+    else:
+        return aws_is_throttled(e)

requirements.txt

@@ -26,6 +26,8 @@ google-cloud-kms>=1.0.0
 ## API Client Libraries
 google-api-python-client>=1.7.8
 oauth2client>=4.1.3
+## Necessary since API Client Libraries are not thread-safe
+httplib2shim>=0.0.3
 
 # Azure Provider
 requests>=2.22.0