Merge branch 'master' into SAAS-4562

celsogbezerra · Jul 29, 2022 · 22dde38 · 22dde38
2 parents 37d8e30 + f599d75
commit 22dde38
Show file tree

Hide file tree

Showing 472 changed files with 43,660 additions and 923 deletions.
diff --git a/README.md b/README.md
@@ -39,7 +39,7 @@ $ docker run -e AWS_ACCESS_KEY_ID=XX -e AWS_SECRET_ACCESS_KEY=YY cloudsploit:0.0
   + [CloudSploit Config File](#cloudsploit-config-file)
   + [Credential Files](#credential-files)
     + [AWS](#aws)
-    + [Azure](#azur)
+    + [Azure](#azure)
     + [GCP](#gcp)
     + [Oracle OCI](#oracle-oci)
   + [Environment Variables](#environment-variables)

diff --git a/collectors/aws/apigateway/getIntegration.js b/collectors/aws/apigateway/getIntegration.js
@@ -0,0 +1,52 @@
+var AWS = require('aws-sdk');
+var async = require('async');
+var helpers = require(__dirname + '/../../../helpers/aws');
+
+module.exports = function(AWSConfig, collection, retries, callback) {
+    var apigateway = new AWS.APIGateway(AWSConfig);
+
+    async.eachLimit(collection.apigateway.getRestApis[AWSConfig.region].data, 5, function(api, cb){
+        if (!collection.apigateway.getResources ||
+            !collection.apigateway.getResources[AWSConfig.region] ||
+            !collection.apigateway.getResources[AWSConfig.region][api.id] ||
+            !collection.apigateway.getResources[AWSConfig.region][api.id].data ||
+            !collection.apigateway.getResources[AWSConfig.region][api.id].data.items) {
+            return cb();
+        }
+
+        collection.apigateway.getIntegration[AWSConfig.region][api.id] = {};
+        async.eachLimit(collection.apigateway.getResources[AWSConfig.region][api.id].data.items, 3, function(resource, pCb){
+
+            collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id] = {};
+
+            async.eachOfLimit(resource.resourceMethods, 3, function(methodVal,methodKey, mCb){
+
+                collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id][methodKey] = {};
+
+                let params = {
+                    resourceId: resource.id,
+                    httpMethod: methodKey,
+                    restApiId : api.id,
+                };
+
+                helpers.makeCustomCollectorCall(apigateway, 'getIntegration', params, retries, null, null, null, function(err, data) {
+                    if (err) {
+                        collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id][methodKey].err = err;
+                        return mCb();
+                    }
+
+                    collection.apigateway.getIntegration[AWSConfig.region][api.id][resource.id][methodKey].data = data;
+                    mCb();
+                });
+            }, function(){
+                pCb();
+            });
+
+        }, function(){
+            cb();
+        });
+
+    }, function(){
+        callback();
+    });
+};
diff --git a/collectors/aws/collector.js b/collectors/aws/collector.js
@@ -82,7 +82,6 @@ var collect = function(AWSConfig, settings, callback) {
         async.eachOfLimit(helpers.calls, 10, function(call, service, serviceCb) {
             var serviceName = service;
             var serviceLower = service.toLowerCase();
-            var serviceIntegration = helpers.calls[serviceName] && helpers.calls[serviceName].sendIntegration && helpers.calls[serviceName].sendIntegration.enabled ? true : false;
             if (!collection[serviceLower]) collection[serviceLower] = {};
 
             // Loop through each of the service's functions
@@ -230,27 +229,18 @@ var collect = function(AWSConfig, settings, callback) {
                     callCb();
                 });
             }, function() {
-                if (serviceIntegration &&
-                    settings.identifier &&
-                    collection[serviceLower] &&
-                    Object.keys(collection[serviceLower]) &&
-                    Object.keys(collection[serviceLower]).length &&
-                    helpers.callsCollected(serviceName, collection, helpers.calls, helpers.postcalls)
-                ) {
-                    helpers.processIntegration(serviceName, settings, collection, helpers.calls, helpers.postcalls, debugMode, function() {
-                        return serviceCb();
-                    });
-                } else {
-                    return serviceCb();
-                }
+                return serviceCb();
             });
         }, function() {
             // Now loop through the follow up calls
             async.eachSeries(helpers.postcalls, function(postcallObj, postcallCb) {
                 async.eachOfLimit(postcallObj, 10, function(serviceObj, service, serviceCb) {
                     var serviceName = service;
                     var serviceLower = service.toLowerCase();
-                    var serviceIntegration = postcallObj && postcallObj[serviceName] && postcallObj[serviceName].sendIntegration && postcallObj[serviceName].sendIntegration.enabled ? true : false;
+                    var serviceIntegration = {
+                        enabled : postcallObj && postcallObj[serviceName] && postcallObj[serviceName].sendIntegration && postcallObj[serviceName].sendIntegration.enabled ? true : false,
+                        sendLast : postcallObj && postcallObj[serviceName] && postcallObj[serviceName].sendIntegration && postcallObj[serviceName].sendIntegration.sendLast ? true : false
+                    };
 
                     if (!collection[serviceLower]) collection[serviceLower] = {};
 
@@ -376,16 +366,21 @@ var collect = function(AWSConfig, settings, callback) {
                             callCb();
                         });
                     }, function() {
-                        if (serviceIntegration &&
+                        if (serviceIntegration.enabled &&
+                            !serviceIntegration.sendLast &&
                             settings.identifier &&
                             collection[serviceLower] &&
                             Object.keys(collection[serviceLower]) &&
                             Object.keys(collection[serviceLower]).length &&
                             helpers.callsCollected(serviceName, collection, helpers.calls, helpers.postcalls)
                         ) {
-                            helpers.processIntegration(serviceName, settings, collection, helpers.calls, helpers.postcalls, debugMode,function() {
+                            try {
+                                helpers.processIntegration(serviceName, settings, collection, helpers.calls, helpers.postcalls, debugMode,function() {
+                                    return serviceCb();
+                                });
+                            } catch (e) {
                                 return serviceCb();
-                            });
+                            }
                         } else {
                             return serviceCb();
                         }
@@ -394,7 +389,36 @@ var collect = function(AWSConfig, settings, callback) {
                     postcallCb();
                 });
             }, function() {
-                callback(null, collection, runApiCalls, errorSummary, errorTypeSummary, errors, retries);
+                if (settings.identifier) {
+                    async.each(helpers.integrationSendLast, function(serv, cb) {
+                        settings.identifier.service = serv.toLowerCase();
+
+                        if (collection[serv.toLowerCase()] &&
+                            Object.keys(collection[serv.toLowerCase()]) &&
+                            Object.keys(collection[serv.toLowerCase()]).length &&
+                            helpers.callsCollected(serv, collection, helpers.calls, helpers.postcalls)
+                        ) {
+                            try {
+                                helpers.processIntegration(serv, settings, collection, helpers.calls, helpers.postcalls, debugMode, function() {
+                                    console.log(`Integration for service ${serv} processed.`);
+                                    cb();
+                                });
+                            } catch (e) {
+                                cb();
+                            }
+
+                        } else {
+                            cb();
+                        }
+                    }, function() {
+                        callback(null, collection, runApiCalls, errorSummary, errorTypeSummary, errors, retries);
+                    });
+
+                } else {
+                    callback(null, collection, runApiCalls, errorSummary, errorTypeSummary, errors, retries);
+
+                }
+
             });
         });
     });

diff --git a/collectors/aws/collector_multipart.js b/collectors/aws/collector_multipart.js
@@ -27,9 +27,9 @@ var agent = new https.Agent({maxSockets: 100});
 AWS.config.update({httpOptions: {agent: agent}});
 
 var CALLS_CONFIG = {
-    TOTAL_PARTS: 13,
+    TOTAL_PARTS: 14,
     CALLS_PARTS: 4,
-    POSTCALLS_PARTS: 9
+    POSTCALLS_PARTS: 10
 };
 
 var rateError = {message: 'rate', statusCode: 429};
@@ -98,7 +98,6 @@ var collect = function(AWSConfig, settings, callback) {
             if (callsPart >= CALLS_CONFIG.CALLS_PARTS) return serviceCb();
             var serviceName = service;
             var serviceLower = service.toLowerCase();
-            var serviceIntegration = helpers.callsMultipart[callsPart] && helpers.callsMultipart[callsPart][serviceName] && helpers.callsMultipart[callsPart][serviceName].sendIntegration && helpers.callsMultipart[callsPart][serviceName].sendIntegration.enabled ? true : false;
             if (!collection[serviceLower]) collection[serviceLower] = {};
 
             // Loop through each of the service's functions
@@ -246,19 +245,7 @@ var collect = function(AWSConfig, settings, callback) {
                     callCb();
                 });
             }, function() {
-                // Note: We intentionally verify against the full api calls and postcalls array below instead of the multipart
-                if (serviceIntegration &&
-                    settings.identifier &&
-                    collection[serviceLower] &&
-                    Object.keys(collection[serviceLower]) &&
-                    Object.keys(collection[serviceLower]).length &&
-                    helpers.callsCollected(serviceName, collection, helpers.calls, helpers.postcalls)) {
-                    helpers.processIntegration(serviceName, settings, collection, helpers.calls, helpers.postcalls, debugMode, function() {
-                        return serviceCb();
-                    });
-                } else {
-                    return serviceCb();
-                }
+                return serviceCb();
             });
         }, function() {
             // Now loop through the follow up calls
@@ -271,7 +258,11 @@ var collect = function(AWSConfig, settings, callback) {
             async.eachOfLimit(helpers.postcallsMultipart[callsPart], 10, function(serviceObj, service, serviceCb) {
                 var serviceName = service;
                 var serviceLower = service.toLowerCase();
-                var serviceIntegration = helpers.postcallsMultipart[callsPart] && helpers.postcallsMultipart[callsPart][serviceName] && helpers.postcallsMultipart[callsPart][serviceName].sendIntegration && helpers.postcallsMultipart[callsPart][serviceName].sendIntegration.enabled ? true : false;
+                var sendIntegration = helpers.postcallsMultipart[callsPart] && helpers.postcallsMultipart[callsPart][serviceName] && helpers.postcallsMultipart[callsPart][serviceName].sendIntegration ? helpers.postcallsMultipart[callsPart][serviceName].sendIntegration : false;
+                var serviceIntegration = {
+                    enabled : sendIntegration && sendIntegration.enabled ? true : false,
+                    sendLast : sendIntegration && sendIntegration.sendLast ? true : false
+                };
 
                 if (!collection[serviceLower]) collection[serviceLower] = {};
 
@@ -399,7 +390,8 @@ var collect = function(AWSConfig, settings, callback) {
                     });
                 }, function() {
                     // Note: We intentionally verify against the full api calls and postcalls array below instead of the multipart
-                    if (serviceIntegration &&
+                    if (serviceIntegration.enabled &&
+                        !serviceIntegration.sendLast &&
                         settings.identifier &&
                         collection[serviceLower] &&
                         Object.keys(collection[serviceLower]) &&
@@ -413,6 +405,22 @@ var collect = function(AWSConfig, settings, callback) {
                     }
                 });
             }, function() {
+                if (settings.identifier &&
+                    settings.part == CALLS_CONFIG.TOTAL_PARTS) {
+                    for (let serv of helpers.integrationSendLast) {
+                        settings.identifier.service = serv.toLowerCase();
+
+                        if (collection[serv.toLowerCase()] &&
+                            Object.keys(collection[serv.toLowerCase()]) &&
+                            Object.keys(collection[serv.toLowerCase()]).length &&
+                            helpers.callsCollected(serv, collection, helpers.calls, helpers.postcalls)
+                        ) {
+                            helpers.processIntegration(serv, settings, collection, helpers.calls, helpers.postcalls, debugMode, function() {
+                                console.log(`Integration for service ${serv} processed.`);
+                            });
+                        }
+                    }
+                }
                 callback(null, collection, runApiCalls, errorSummary, errorTypeSummary, errors, retries);
             });
         });