Merge pull request #5 from Signiant/save-aws-creds

Add option to save credentials

Author: haggaret

README.md

@@ -15,6 +15,9 @@ Keep file contents in sync with matching parameters in AWS Parameter Store
         - export PARAM_PREFIX=TESTING_
         - TESTING_param1.txt will be compared against param1.txt
         - TESTING_param2.conf will be compared against param2.conf
+- SAVE_AWS_CREDS - List of tuples (filename, profile name) to save AWS Credentials to (saved to credentials dir)
+    - value should be space separated list, e.g. aws_cli_credentials default aws_boto_credentials prod
+    - if a space is required in the profile name, e.g. profile prod, substitute # for a space, e.g. profile#prod
 
 The docker container exposes /credentials as a volume - this can be shared with other
 containers or mounted to the local file system
@@ -26,7 +29,9 @@ containers or mounted to the local file system
 This example checks AWS Parameter Store in the default us-east-1 region every 600 seconds (10 minutes)
 for parameters containing 'TESTING_'. The credentials in AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are
 used to access the AWS Parameter Store. The parameter values will be checked against files in the local
-folder 'credentials-dir' which is mounted into the container at '/credentials'.
+folder 'credentials-dir' which is mounted into the container at '/credentials'. The credentials used to access
+the AWS Parameter Store will be saved to /credentials/aws_cli_credentials file with a profile name of 'default'
+and no (default) region specified.
 
 
 ````
@@ -35,6 +40,7 @@ docker run -d -e "FREQUENCY=600" \
  -e "AWS_ACCESS_KEY_ID=MY_ACCESS_KEY_ID" \
  -e "AWS_SECRET_ACCESS_KEY=MY_SECRET_KEY" \
  -e "PARAM_PREFIX=TESTING_" \
+ -e "SAVE_AWS_CREDS="aws_cli_credentials default None" \
  -v credentials-dir:/credentials \
  signiant/aws-parameter-syncer
 ````

parameter_sync.py

@@ -16,8 +16,8 @@
 
 
 def get_sha256_hash(file_path):
-    logger.debug('Hashing "%s" using SHA256' % file_path)
-    BUF_SIZE = 65536  # lets read stuff in 64kb chunks!
+    logger.debug(f'Hashing "{file_path}" using SHA256')
+    BUF_SIZE = 65536  # let's read stuff in 64kb chunks!
     sha256 = hashlib.sha256()
     with open(file_path, 'rb') as f:
         while True:
@@ -31,7 +31,7 @@ def get_sha256_hash(file_path):
 
 
 def process_parameters_with_prefix(param_prefix, cred_path, aws_region, aws_access_key=None, aws_secret_key=None, dryrun=False):
-    logger.debug('Searching for parameters with a prefix of %s' % param_prefix)
+    logger.debug(f'Searching for parameters with a prefix of {param_prefix}')
 
     def get_parameters(parameter_names_list):
         parameter_list = []
@@ -53,29 +53,29 @@ def process_parameter(param_name, param_value):
         if os.path.exists(full_cred_path):
             existing_file_sha256_hash = get_sha256_hash(full_cred_path)
         new_file_full_path = temp_dir + os.sep + filename + '.new'
-        logger.debug('Storing retrieved value for parameter "%s" in "%s"' % (param_name, new_file_full_path))
+        logger.debug(f'Storing retrieved value for parameter "{param_name}" in "{new_file_full_path}"')
         with open(new_file_full_path, 'w') as f:
             f.write(param_value)
         new_file_sha256_hash = get_sha256_hash(new_file_full_path)
         logger.debug('Comparing file hashes')
         if existing_file_sha256_hash != new_file_sha256_hash:
             if not existing_file_sha256_hash:
-                logger.info('This is a new credentials file: "%s"' % filename)
+                logger.info(f'This is a new credentials file: "{filename}"')
             else:
-                logger.info("Contents don't match - replacing \"%s\" contents with value from parameter store" % full_cred_path)
+                logger.info(f"Contents don't match - replacing \"{full_cred_path}\" contents with value from parameter store")
             if not dryrun:
                 if os.path.exists(new_file_full_path) and os.stat(new_file_full_path).st_size > 0:
                     shutil.copyfile(new_file_full_path, full_cred_path)
                 else:
-                    logger.error('file %s is missing or zero length - NOT replacing' % new_file_full_path)
+                    logger.error(f'file {new_file_full_path} is missing or zero length - NOT replacing')
             else:
-                logger.info('*** Dryrun selected - will NOT update "%s"' % full_cred_path)
+                logger.info(f'*** Dryrun selected - will NOT update "{full_cred_path}"')
         else:
-            logger.info('Contents of existing "%s" MATCH with value for "%s" from parameter store' % (full_cred_path, param_name))
+            logger.info(f'Contents of existing "{full_cred_path}" MATCH with value for "{param_name}" from parameter store')
 
         # Cleanup
         if new_file_full_path:
-            logger.debug('Removing %s' % new_file_full_path)
+            logger.debug(f'Removing {new_file_full_path}')
             os.remove(new_file_full_path)
 
     def get_parameters_with_prefix(prefix, next_token=None):
@@ -84,7 +84,7 @@ def get_parameters_with_prefix(prefix, next_token=None):
             query_result = ssm.describe_parameters(Filters=[{'Key': 'Name', 'Values': [prefix]}], NextToken=next_token)
         else:
             query_result = ssm.describe_parameters(Filters=[{'Key': 'Name', 'Values': [prefix]}])
-        logger.debug("Query result %s" % str(query_result))
+        logger.debug(f"Query result {str(query_result)}")
         if 'ResponseMetadata' in query_result:
             if 'HTTPStatusCode' in query_result['ResponseMetadata']:
                 if query_result['ResponseMetadata']['HTTPStatusCode'] == 200:
@@ -99,7 +99,7 @@ def get_parameters_with_prefix(prefix, next_token=None):
                     else:
                         logger.debug("No next token, storing")
                         parameter_list.extend(query_result['Parameters'])
-        logger.debug("Parameter List %s" % parameter_list)
+        logger.debug(f"Parameter List {parameter_list}")
         return parameter_list
 
 
@@ -130,6 +130,78 @@ def get_parameters_with_prefix(prefix, next_token=None):
             process_parameter(parameter_name, parameter_value)
 
 
+def create_aws_cred_file(key_id, secret, file_location, cred_filename, profile_name, aws_region):
+    file_path = f'{file_location}{os.sep}{cred_filename}'
+    if os.path.exists(file_path):
+        # File already exists - append to it
+        with open(file_path, 'a') as cred_file:
+            cred_file.write('\n')
+            cred_file.write(f'[{profile_name}]\n')
+            if aws_region:
+                cred_file.write(f'region={aws_region}\n')
+            cred_file.write(f'aws_access_key_id={key_id}\n')
+            cred_file.write(f'aws_secret_access_key={secret}\n')
+    else:
+        with open(file_path, 'w') as cred_file:
+            cred_file.write(f'[{profile_name}]\n')
+            if aws_region:
+                cred_file.write(f'region={aws_region}\n')
+            cred_file.write(f'aws_access_key_id={key_id}\n')
+            cred_file.write(f'aws_secret_access_key={secret}\n')
+    return file_path
+
+
+def write_aws_cli_creds(key_id, secret, base_cred_path, aws_cred_list):
+    aws_creds_tuples = []
+    have_all_info = False
+    for i, val in enumerate(aws_cred_list):
+        if (i % 3) == 0:
+            logging.debug(f'save-aws-creds - filename: {val}')
+            filename = val
+        elif (i % 3) == 1:
+            logging.debug(f'save-aws-creds - profile name: {val}')
+            if '#' in val:
+                profile = val.replace('#', ' ')
+            else:
+                profile = val
+        else:
+            logging.debug(f'save-aws-creds - region: {val}')
+            if 'none' in val.lower():
+                region = None
+            else:
+                region = val
+            have_all_info = True
+
+        if have_all_info:
+            aws_creds_tuples.append((filename, profile, region))
+            have_all_info = False
+
+    if len(aws_creds_tuples) > 0:
+        logging.debug('Provided with the following tuples:')
+        logging.debug(f'{aws_creds_tuples}')
+
+        # write to a tmp file first
+        temp_dir = tempfile.gettempdir()
+        if not os.path.exists(temp_dir):
+            os.makedirs(temp_dir)
+
+        aws_cred_files = []
+        for f, p, r in aws_creds_tuples:
+            tmp_cred_filepath = create_aws_cred_file(key_id, secret, temp_dir, f, p, r)
+            if tmp_cred_filepath not in aws_cred_files:
+                aws_cred_files.append(tmp_cred_filepath)
+
+        # Now copy the tmp files to the base_cred_path
+        for cred_file in aws_cred_files:
+            filename = cred_file.split(os.sep)[-1]
+            new_file_path = f"{base_cred_path}{os.sep}{filename}"
+            logger.info(f'Saving AWS Credentials to {new_file_path}')
+            shutil.copyfile(cred_file, new_file_path)
+            # Cleanup
+            logger.debug(f'Removing {cred_file}')
+            os.remove(cred_file)
+
+
 if __name__ == "__main__":
 
     description =  "Script to get all parameters from AWS Parameter\n"
@@ -147,6 +219,7 @@ def get_parameters_with_prefix(prefix, next_token=None):
     parser.add_argument("--param-prefix", help="Parameter prefix", dest='param_prefix', required=True)
     parser.add_argument("--credentials-path", help="Where credentials are stored", dest='cred_path', default='/credentials/')
     parser.add_argument("--verbose", help="Turn on DEBUG logging", action='store_true', required=False)
+    parser.add_argument("--save-aws-creds", help="Save AWS Creds [filename, profile]", nargs='*')
     parser.add_argument("--dryrun", help="Do a dryrun - no changes will be performed", dest='dryrun',
                         action='store_true', default=False,
                         required=False)
@@ -163,8 +236,23 @@ def get_parameters_with_prefix(prefix, next_token=None):
         logger.critical('AWS Secret Access Key not set - cannot continue')
 
     logger.debug('INIT')
-    logger.info('Getting parameters with prefix %s from AWS Parameter Store' % args.param_prefix)
-    logger.info('Parameter values will be compared against file contents in "%s" and updated if necessary' % args.cred_path)
+    logger.info(f'Getting parameters with prefix {args.param_prefix} from AWS Parameter Store')
+    logger.info(f'Parameter values will be compared against file contents in "{args.cred_path}" and updated if necessary')
     process_parameters_with_prefix(args.param_prefix, args.cred_path, args.aws_region,
                                    args.aws_access_key, args.aws_secret_key, args.dryrun)
+
+    if args.save_aws_creds:
+        if args.aws_access_key and args.aws_secret_key:
+            aws_access_key = args.aws_access_key
+            aws_secret_key = args.aws_secret_key
+        else:
+            aws_access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+            aws_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+
+        if len(args.save_aws_creds) % 3 != 0:
+            logging.critical('Must provide filename, profile_name and region for aws creds')
+            sys.exit(1)
+
+        result = write_aws_cli_creds(aws_access_key, aws_secret_key, args.cred_path, args.save_aws_creds)
+
     logger.info('COMPLETE')

parameter_sync.sh

@@ -44,12 +44,17 @@ else
     echo "Frequency set to $FREQUENCY seconds"
 fi
 
+if [ "$SAVE_AWS_CREDS" ]; then
+    echo "Save AWS Creds enabled"
+    SAVE_AWS_CREDS="--save-aws-creds ${SAVE_AWS_CREDS}"
+fi
+
 # Loop forever, sleeping for our frequency
 while true
 do
     echo "Awoke to check for new credentials with prefix ${PARAM_PREFIX} in AWS Parameter Store"
 
-    python /parameter_sync.py --credentials-path ${CRED_FOLDER_PATH} --param-prefix ${PARAM_PREFIX} --aws-region ${AWS_REGION} ${VERBOSE}
+    python /parameter_sync.py --credentials-path ${CRED_FOLDER_PATH} --param-prefix ${PARAM_PREFIX} --aws-region ${AWS_REGION} ${SAVE_AWS_CREDS} ${VERBOSE}
     echo "Sleeping for $FREQUENCY seconds"
     sleep $FREQUENCY
     echo