Refactoring/aws/iam

ScoutSuite/providers/aws/configs/services.py

@@ -10,8 +10,8 @@
 from ScoutSuite.providers.aws.resources.elasticache.service import ElastiCache
 from ScoutSuite.providers.aws.resources.elb.service import ELB
 from ScoutSuite.providers.aws.resources.elbv2.service import ELBv2
+from ScoutSuite.providers.aws.resources.iam.service import IAM
 from ScoutSuite.providers.aws.resources.emr.service import EMR
-from ScoutSuite.providers.aws.services.iam import IAMConfig
 from ScoutSuite.providers.aws.resources.route53.service import Route53
 from ScoutSuite.providers.aws.resources.rds.service import RDS
 from ScoutSuite.providers.aws.resources.redshift.service import Redshift
@@ -61,10 +61,10 @@ def __init__(self, metadata=None, thread_config=4, **kwargs):
         self.ec2 = EC2()
         self.efs = EFS()
         self.elasticache = ElastiCache()
+        self.iam = IAM()
         self.elb = ELB()
         self.elbv2 = ELBv2()
         self.emr = EMR()
-        self.iam = IAMConfig(thread_config)
         self.awslambda = Lambdas()
         self.route53 = Route53()
         self.redshift = Redshift()

ScoutSuite/providers/aws/facade/awslambda.py

@@ -1,6 +1,7 @@
 from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils
 from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade
 
+
 class LambdaFacade(AWSBaseFacade):
     async def get_functions(self, region):
         return await AWSFacadeUtils.get_all_pages('lambda', region, self.session, 'list_functions', 'Functions')

ScoutSuite/providers/aws/facade/cloudformation.py

@@ -9,7 +9,7 @@ class CloudFormation(AWSBaseFacade):
     async def get_stacks(self, region: str):
         stacks = await AWSFacadeUtils.get_all_pages('cloudformation', region, self.session, 'list_stacks', 'StackSummaries')
         stacks = [stack for stack in stacks if not CloudFormation._is_stack_deleted(stack)]
-        client = AWSFacadeUtils.get_client('cloudformation', region, self.session)
+        client = AWSFacadeUtils.get_client('cloudformation', self.session, region)
         for stack in stacks:
             stack_name = stack['StackName']
 

ScoutSuite/providers/aws/facade/cloudtrail.py

@@ -5,7 +5,7 @@
 
 class CloudTrailFacade(AWSBaseFacade):
     async def get_trails(self, region):
-        client = AWSFacadeUtils.get_client('cloudtrail', region, self.session)
+        client = AWSFacadeUtils.get_client('cloudtrail', self.session, region)
         trails = await run_concurrently(
             lambda: client.describe_trails()['trailList']
         )

ScoutSuite/providers/aws/facade/directconnect.py

@@ -5,5 +5,5 @@
 
 class DirectConnectFacade(AWSBaseFacade):
     async def get_connections(self, region):
-        client = AWSFacadeUtils.get_client('directconnect', region, self.session)
+        client = AWSFacadeUtils.get_client('directconnect', self.session, region)
         return await run_concurrently(lambda: client.describe_connections()['connections'])

ScoutSuite/providers/aws/facade/ec2.py

@@ -11,7 +11,7 @@ class EC2Facade(AWSBaseFacade):
     flow_logs_cache = {}
 
     async def get_instance_user_data(self, region: str, instance_id: str):
-        ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session)
+        ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region)
         user_data_response = await run_concurrently(
             lambda: ec2_client.describe_instance_attribute(Attribute='userData', InstanceId=instance_id))
 
@@ -45,7 +45,7 @@ async def get_vpcs(self, region: str):
 
     async def get_images(self, region: str, owner_id: str):
         filters = [{'Name': 'owner-id', 'Values': [owner_id]}]
-        client = AWSFacadeUtils.get_client('ec2', region, self.session)
+        client = AWSFacadeUtils.get_client('ec2', self.session, region)
         response = await run_concurrently(lambda: client.describe_images(Filters=filters))
 
         return response['Images']
@@ -63,7 +63,7 @@ async def get_snapshots(self, region: str, owner_id: str):
         snapshots = await AWSFacadeUtils.get_all_pages(
             'ec2', region, self.session, 'describe_snapshots', 'Snapshots', Filters=filters)
 
-        ec2_client = AWSFacadeUtils.get_client('ec2', region, self.session)
+        ec2_client = AWSFacadeUtils.get_client('ec2', self.session, region)
         for snapshot in snapshots:
             snapshot['CreateVolumePermissions'] = await run_concurrently(lambda: ec2_client.describe_snapshot_attribute(
                 Attribute='createVolumePermission',

ScoutSuite/providers/aws/facade/efs.py

@@ -8,7 +8,7 @@ async def get_file_systems(self, region: str):
         file_systems = await AWSFacadeUtils.get_all_pages(
             'efs', region, self.session, 'describe_file_systems', 'FileSystems')
 
-        client = AWSFacadeUtils.get_client('efs', region, self.session)
+        client = AWSFacadeUtils.get_client('efs', self.session, region)
         for file_system in file_systems:
             file_system_id = file_system['FileSystemId']
             file_system['Tags'] = await run_concurrently(

ScoutSuite/providers/aws/facade/elasticache.py

@@ -30,7 +30,7 @@ async def cache_clusters(self, region):
                     cluster['VpcId'] = subnet_group['VpcId']
 
     async def get_security_groups(self, region):
-        client = AWSFacadeUtils.get_client('elasticache', region, self.session)
+        client = AWSFacadeUtils.get_client('elasticache', self.session, region)
 
         try:
             return await AWSFacadeUtils.get_all_pages('elasticache', region, self.session, 'describe_cache_security_groups', 'CacheSecurityGroups')

ScoutSuite/providers/aws/facade/emr.py

@@ -7,7 +7,7 @@
 class EMRFacade(AWSBaseFacade):
     async def get_clusters(self, region):
         clusters_list = await AWSFacadeUtils.get_all_pages('emr', region, self.session, 'list_clusters', 'Clusters')
-        client = AWSFacadeUtils.get_client('emr', region, self.session)
+        client = AWSFacadeUtils.get_client('emr', self.session, region)
         clusters_descriptions = []
         cluster_ids = [cluster['Id'] for cluster in clusters_list]
         tasks = {

ScoutSuite/providers/aws/facade/facade.py

@@ -16,6 +16,7 @@
 from ScoutSuite.providers.aws.facade.route53 import Route53Facade
 from ScoutSuite.providers.aws.facade.sqs import SQSFacade
 from ScoutSuite.providers.aws.facade.elbv2 import ELBv2Facade
+from ScoutSuite.providers.aws.facade.iam import IAMFacade
 from ScoutSuite.providers.aws.facade.rds import RDSFacade
 from ScoutSuite.providers.aws.facade.redshift import RedshiftFacade
 from ScoutSuite.providers.aws.facade.ses import SESFacade
@@ -29,7 +30,7 @@
 except ImportError:
     pass
 
-
+  
 class AWSFacade(AWSBaseFacade):
     def __init__(self, credentials: dict = None):
         self._set_session(credentials)
@@ -77,6 +78,7 @@ def _instantiate_facades(self):
         self.route53 = Route53Facade(self.session)
         self.elb = ELBFacade(self.session)
         self.elbv2 = ELBv2Facade(self.session)
+        self.iam = IAMFacade(self.session)
         self.rds = RDSFacade(self.session)
         self.redshift = RedshiftFacade(self.session)
         self.ses = SESFacade(self.session)

ScoutSuite/providers/aws/facade/iam.py

@@ -0,0 +1,203 @@
+from ScoutSuite.providers.aws.facade.utils import AWSFacadeUtils
+from ScoutSuite.providers.aws.facade.basefacade import AWSBaseFacade
+from ScoutSuite.core.console import print_error, print_exception
+from ScoutSuite.providers.utils import run_concurrently, get_non_provider_id
+from ScoutSuite.providers.aws.utils import is_throttled
+from ScoutSuite.providers.utils import run_concurrently
+
+
+class IAMFacade(AWSBaseFacade):
+    async def get_credential_reports(self):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        response = await run_concurrently(client.generate_credential_report)
+
+        if response['State'] != 'COMPLETE':
+            print_error('Failed to generate a credential report.')
+            return []
+
+        report = (await run_concurrently(client.get_credential_report))['Content']
+
+        # The report is a CSV string. The first row contains the name of each column. The next rows
+        # each represent an individual account. This algorithm provides a simple initial parsing.
+        lines = report.splitlines()
+        keys = lines[0].decode('utf-8').split(',')
+
+        credential_reports = []
+        for line in lines[1:]:
+            credential_report = {}
+            values = line.decode('utf-8').split(',')
+            for key, value in zip(keys, values):
+                credential_report[key] = value
+
+            credential_reports.append(credential_report)
+
+        return credential_reports
+
+    async def get_groups(self):
+        groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups', 'Groups')
+        for group in groups:
+            group['Users'] = await self._fetch_group_users(group['GroupName'])
+            policies = self._get_inline_policies(
+                'group', group['GroupId'], group['GroupName'])
+            if len(policies):
+                group['inline_policies'] = policies
+            group['inline_policies_count'] = len(policies)
+        return groups
+
+    async def get_policies(self):
+        policies = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_policies', 'Policies', OnlyAttached=True)
+        client = AWSFacadeUtils.get_client('iam', self.session)
+
+        # TODO: Parallelize this
+        for policy in policies:
+            policy_version = await run_concurrently(
+                lambda: client.get_policy_version(PolicyArn=policy['Arn'], VersionId=policy['DefaultVersionId']))
+            policy['PolicyDocument'] = policy_version['PolicyVersion']['Document']
+
+            policy['attached_to'] = {}
+            attached_entities = await AWSFacadeUtils.get_multiple_entities_from_all_pages('iam', None, self.session, 'list_entities_for_policy',  ['PolicyGroups', 'PolicyRoles', 'PolicyUsers'], PolicyArn=policy['Arn'])
+
+            for entity_type in attached_entities:
+                resource_type = entity_type.replace('Policy', '').lower()
+                if len(attached_entities[entity_type]):
+                    policy['attached_to'][resource_type] = []
+
+                for entity in attached_entities[entity_type]:
+                    name_field = entity_type.replace('Policy', '')[
+                        :-1] + 'Name'
+                    resource_name = entity[name_field]
+                    id_field = entity_type.replace('Policy', '')[:-1] + 'Id'
+                    resource_id = entity[id_field]
+                    policy['attached_to'][resource_type].append(
+                        {'name': resource_name, 'id': resource_id})
+
+        return policies
+
+    async def get_users(self):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        users = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_users', 'Users')
+
+        # TODO: Parallelize this
+        for user in users:
+            user_name = user['UserName']
+            user_id = user['UserId']
+
+            policies = self._get_inline_policies('user', user_id, user_name)
+            if len(policies):
+                user['inline_policies'] = policies
+            user['inline_policies_count'] = len(policies)
+            user['groups'] = []
+            groups = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_groups_for_user', 'Groups', UserName=user_name)
+            for group in groups:
+                user['groups'].append(group['GroupName'])
+            try:
+                user['LoginProfile'] = await run_concurrently(
+                    lambda: client.get_login_profile(UserName=user_name)['LoginProfile'])
+            except Exception:
+                pass
+            user['AccessKeys'] = await self._get_user_acces_keys(user_name)
+            user['MFADevices'] = await self._get_user_mfa_devices(user_name)
+
+        return users
+
+    async def get_roles(self):
+        roles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_roles', 'Roles')
+
+        # Handle throttling errors
+        for role in roles:
+            role['instances_count'] = 'N/A'
+
+            # Get role policies
+            policies = self._get_inline_policies(
+                'role', role['RoleId'], role['RoleName'])
+            if len(policies):
+                role['inline_policies'] = policies
+            role['inline_policies_count'] = len(policies)
+
+            # Get instance profiles
+            profiles = await AWSFacadeUtils.get_all_pages('iam', None, self.session, 'list_instance_profiles_for_role', 'InstanceProfiles', RoleName=role['RoleName'])
+            role.setdefault('instance_profiles', {})
+            for profile in profiles:
+                profile_id = profile['InstanceProfileId']
+                role['instance_profiles'].setdefault(profile_id, {})
+                role['instance_profiles'][profile_id].setdefault(
+                    'arn', profile['Arn'])
+                role['instance_profiles'][profile_id].setdefault(
+                    'name', profile['InstanceProfileName'])
+
+            # Get trust relationship
+            role['assume_role_policy'] = {}
+            role['assume_role_policy']['PolicyDocument'] = role.pop(
+                'AssumeRolePolicyDocument')
+
+        return roles
+
+    async def get_password_policy(self):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        return (await run_concurrently(client.get_account_password_policy))['PasswordPolicy']
+
+    async def _get_user_acces_keys(self, user_name):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        response = await run_concurrently(lambda: client.list_access_keys(UserName=user_name))
+        return response['AccessKeyMetadata']
+
+    async def _get_user_mfa_devices(self, user_name):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        response = await run_concurrently(lambda: client.list_mfa_devices(UserName=user_name))
+        return response['MFADevices']
+
+    async def _fetch_group_users(self, group_name):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        fetched_users = client.get_group(GroupName=group_name)['Users']
+
+        users = []
+        for user in fetched_users:
+            users.append(user['UserId'])
+        return users
+
+    # TODO: Make this async
+    def _get_inline_policies(self, iam_resource_type, resource_id, resource_name):
+        client = AWSFacadeUtils.get_client('iam', self.session)
+        get_policy_method = getattr(
+            client, 'get_' + iam_resource_type + '_policy')
+        fetched_policies = {}
+        list_policy_method = getattr(
+            client, 'list_' + iam_resource_type + '_policies')
+        args = {iam_resource_type.title() + 'Name': resource_name}
+        try:
+            policy_names = list_policy_method(**args)['PolicyNames']
+        except Exception as e:
+            if is_throttled(e):
+                raise e
+            else:
+                print_exception(e)
+                return fetched_policies
+        try:
+            for policy_name in policy_names:
+                args['PolicyName'] = policy_name
+                policy_document = get_policy_method(**args)['PolicyDocument']
+                policy_id = get_non_provider_id(policy_name)
+                fetched_policies[policy_id] = {}
+                fetched_policies[policy_id]['PolicyDocument'] = self._normalize_statements(
+                    policy_document)
+                fetched_policies[policy_id]['name'] = policy_name
+                fetched_policies[policy_id] = fetched_policies[policy_id]
+        except Exception as e:
+            if is_throttled(e):
+                raise e
+            else:
+                print_exception(e)
+        return fetched_policies
+
+    def _normalize_statements(self, policy_document):
+        for statement in policy_document['Statement']:
+            # Action or NotAction
+            action_string = 'Action' if 'Action' in statement else 'NotAction'
+            if type(statement[action_string]) != list:
+                statement[action_string] = [statement[action_string]]
+            # Resource or NotResource
+            resource_string = 'Resource' if 'Resource' in statement else 'NotResource'
+            if type(statement[resource_string]) != list:
+                statement[resource_string] = [statement[resource_string]]
+
+        return policy_document

ScoutSuite/providers/aws/facade/utils.py

@@ -8,25 +8,34 @@ class AWSFacadeUtils:
     _clients = {}
 
     @staticmethod
-    async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, response_key: str, **paginator_args):
-        client = AWSFacadeUtils.get_client(service, region, session)
+    async def get_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entity: str, **paginator_args):
+        results = await AWSFacadeUtils.get_multiple_entities_from_all_pages(service, region, session, paginator_name, [entity], **paginator_args)
+        return results[entity]
+
+    @staticmethod
+    async def get_multiple_entities_from_all_pages(service: str, region: str, session: boto3.session.Session, paginator_name: str, entities: list, **paginator_args):
+        client = AWSFacadeUtils.get_client(service, session, region)
+
         # Building a paginator doesn't require any API call so no need to do it concurrently:
-        paginator = client.get_paginator(paginator_name).paginate(**paginator_args)
+        paginator = client.get_paginator(
+            paginator_name).paginate(**paginator_args)
 
         # Getting all pages from a paginator requires API calls so we need to do it concurrently:
-        return await run_concurrently(lambda: AWSFacadeUtils._get_all_pages_from_paginator(paginator, response_key))
+        return await run_concurrently(lambda: AWSFacadeUtils._get_all_pages_from_paginator(paginator, entities))
 
     @staticmethod
-    def _get_all_pages_from_paginator(paginator, key):
-        resources = []
+    def _get_all_pages_from_paginator(paginator, entities: list):
+        resources = {entity: [] for entity in entities}
+
         # There's an API call hidden behind each iteration:
         for page in paginator:
-            resources.extend(page[key])
+            for entity in entities:
+                resources[entity].extend(page[entity])
 
         return resources
 
     @staticmethod
-    def get_client(service: str, region: str, session: boto3.session.Session):
+    def get_client(service: str, session: boto3.session.Session, region: str=None):
         """
         Instantiates an AWS API client
 
@@ -38,5 +47,5 @@ def get_client(service: str, region: str, session: boto3.session.Session):
         """
 
         # TODO: investigate the use of a mutex to avoid useless creation of a same type of client among threads:
-        client = session.client(service, region_name=region)
+        client = session.client(service, region_name=region) if region else session.client(service)
         return AWSFacadeUtils._clients.setdefault((service, region), client)

ScoutSuite/providers/aws/provider.py

@@ -75,7 +75,11 @@ def preprocessing(self, ip_ranges=None, ip_ranges_name_key=None):
         if 'elbv2' in self.service_list:
             self._add_security_group_data_to_elbv2()
 
+        if 's3' in self.service_list and 'iam' in self.service_list:
+            self._match_iam_policies_and_buckets()
+
         self._add_cidr_display_name(ip_ranges, ip_ranges_name_key)
+        self._merge_route53_and_route53domains()
         self._match_iam_policies_and_buckets()
 
         super(AWSProvider, self).preprocessing()