Adding code for aws-parameter-syncer

haggaret · haggaret · commit 3a960a8fe565 · 2017-05-10T14:33:00.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+.idea/*
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,11 @@
+FROM jfloff/alpine-python:2.7-slim
+
+MAINTAINER Signiant DevOps <devops@signiant.com>
+
+ADD parameter_sync.py /parameter_sync.py
+ADD parameter_sync.sh /parameter_sync.sh
+
+RUN pip install boto3
+RUN chmod a+x /parameter_sync.py /parameter_sync.sh
+
+ENTRYPOINT ["/parameter_sync.sh"]
diff --git a/README.md b/README.md
@@ -0,0 +1,61 @@
+# aws-parameter-syncer
+Keep file contents in sync with matching parameters in AWS Parameter Store
+
+## Variables
+
+- VERBOSE - enable more logging if set to 1
+- FREQUENCY - How often to check for changes (in seconds). Default is 300 seconds (5 minutes).
+- AWS_ACCESS_KEY_ID - The AWS Access Key Id value
+- AWS_SECRET_ACCESS_KEY - The AWS Secret Access Key value
+- AWS_REGION - AWS Region to search (defaults to us-east-1)
+- PARAM_PREFIX - The prefix for the parameters to keep in sync
+    - resulting filenames will be the parameter name minus the PARAM_PREFIX
+    - eg.
+      Following parameters in parameter store: TESTING_param1.txt, TESTING_param2.conf
+      export PARAM_PREFIX=TESTING_
+      TESTING_param1.txt will be compared against param1.txt
+      TESTING_param2.conf will be compared against param2.conf
+- CRED_FOLDER_PATH - path to where files are stored (defaults to /credentials)
+    - in order to access the files outside of the container, make sure to mount this path into the container
+
+
+## Example Docker runs
+
+
+This example checks AWS Parameter Store in the default us-east-1 region every 600 seconds (10 minutes)
+for parameters containing 'TESTING_'. The credentials in AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are
+used to access the AWS Parameter Store. The parameter values will be checked against files in the local
+folder 'credentials-dir' which is mounted into the container at '/credentials'.
+
+
+````
+docker run -d -e "FREQUENCY=600" \
+        -e "VERBOSE=1" \
+        -e "AWS_ACCESS_KEY_ID=MY_ACCESS_KEY_ID \
+        -e "AWS_SECRET_ACCESS_KEY=MY_SECRET_KEY \
+        -e "PARAM_PREFIX=TESTING_" \
+        -v credentials-dir:/credentials \
+        signiant/aws-parameter-syncer
+````
+
+This example checks AWS Parameter Store in the us-west-2 region every 120 seconds (2 minutes)
+for parameters containing 'TESTING_'. The credentials in AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are
+used to access the AWS Parameter Store. The parameter values will be checked against files in the local
+folder '/my/local/credentials/path' which is mounted into the container at '/some/other/path'.
+
+
+````
+docker run -d -e "FREQUENCY=120" \
+        -e "AWS_ACCESS_KEY_ID=MY_ACCESS_KEY_ID \
+        -e "AWS_SECRET_ACCESS_KEY=MY_SECRET_KEY \
+        -e "AWS_REGION=us-west-2"
+        -e "PARAM_PREFIX=TESTING_" \
+        -e "CRED_FOLDER_PATH=/some/other/path"
+        -v /my/local/credentials/path:/some/other/path \
+        signiant/aws-parameter-syncer
+````
+
+
+
+
+
diff --git a/parameter_sync.py b/parameter_sync.py
@@ -0,0 +1,183 @@
+import logging, logging.handlers
+import argparse
+import os
+import boto3
+import hashlib
+import tempfile
+import shutil
+
+logging.getLogger("botocore").setLevel(logging.CRITICAL)
+
+def get_sha256_hash(file_path):
+    logging.debug('Hashing "%s" using SHA256' % file_path)
+    BUF_SIZE = 65536  # lets read stuff in 64kb chunks!
+    sha256 = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        while True:
+            data = f.read(BUF_SIZE)
+            if not data:
+                break
+            sha256.update(data)
+
+    logging.debug('   SHA256 hash: {0}'.format(sha256.hexdigest()))
+    return sha256.digest()
+
+
+def process_parameters_with_prefix(param_prefix, cred_path, aws_region, aws_access_key=None, aws_secret_key=None, dryrun=False):
+    logging.debug('Searching for parameters with a prefix of %s' % param_prefix)
+
+    def get_parameters(parameter_names_list):
+        parameter_list = []
+        if parameter_names_list:
+            result = ssm.get_parameters(Names=parameter_names_list, WithDecryption=True)
+            if result:
+                if 'ResponseMetadata' in result:
+                    if 'HTTPStatusCode' in result['ResponseMetadata']:
+                        if result['ResponseMetadata']['HTTPStatusCode'] == 200:
+                            if 'Parameters' in result:
+                                parameter_list = result['Parameters']
+        return parameter_list
+
+    def process_parameter(param_name, param_value):
+        filename = param_name.split(param_prefix)[1]
+        full_cred_path = cred_path + os.sep + filename
+        existing_file_sha256_hash = None
+        if os.path.exists(full_cred_path):
+            existing_file_sha256_hash = get_sha256_hash(full_cred_path)
+        new_file_full_path = temp_dir + os.sep + filename + '.new'
+        logging.debug('Storing retrieved value for parameter "%s" in "%s"' % (param_name, new_file_full_path))
+        with open(new_file_full_path, 'w') as f:
+            f.write(param_value.replace('\\n', '\n'))
+        new_file_sha256_hash = get_sha256_hash(new_file_full_path)
+        logging.debug('Comparing file hashes')
+        if existing_file_sha256_hash != new_file_sha256_hash:
+            if not existing_file_sha256_hash:
+                logging.info('This is a new credentials file: "%s"' % filename)
+            else:
+                logging.info("Contents don't match - replacing existing file contents with value from parameter store")
+            if not dryrun:
+                if os.path.exists(new_file_full_path) and os.stat(new_file_full_path).st_size > 0:
+                    shutil.copyfile(new_file_full_path, full_cred_path)
+                else:
+                    logging.error('file %s is missing or zero length - NOT replacing' % new_file_full_path)
+            else:
+                logging.info('*** Dryrun selected - will NOT update "%s"' % full_cred_path)
+        else:
+            logging.info('Contents of existing "%s" MATCH with value for "%s" from parameter store' % (full_cred_path, param_name))
+
+        # Cleanup
+        if new_file_full_path:
+            logging.debug('Removing %s' % new_file_full_path)
+            os.remove(new_file_full_path)
+
+    def get_parameters_with_prefix(prefix):
+        parameter_names_list = []
+        result = ssm.describe_parameters(Filters=[{'Key': 'Name', 'Values': [param_prefix]}], MaxResults=50)
+        if result:
+            if 'ResponseMetadata' in result:
+                if 'HTTPStatusCode' in result['ResponseMetadata']:
+                    if result['ResponseMetadata']['HTTPStatusCode'] == 200:
+                        if 'Parameters' in result:
+                            for param in result['Parameters']:
+                                logging.debug('Found parameter "%s" - adding to list' % param['Name'])
+                                parameter_names_list.append(param['Name'])
+        next_token = None
+        if 'NextToken' in result:
+            next_token = result['NextToken']
+        while next_token:
+            result = ssm.describe_parameters(Filters=[{'Key': 'Name', 'Values': [param_prefix]}], MaxResults=50)
+            if result:
+                if 'ResponseMetadata' in result:
+                    if 'HTTPStatusCode' in result['ResponseMetadata']:
+                        if result['ResponseMetadata']['HTTPStatusCode'] == 200:
+                            if 'Parameters' in result:
+                                for param in result['Parameters']:
+                                    logging.debug('Found parameter "%s" - adding to list' % param['Name'])
+                                    parameter_names_list.append(param['Name'])
+            next_token = None
+            if 'NextToken' in result:
+                next_token = result['NextToken']
+        return parameter_names_list
+
+    # If aws_access_key and aws_secret_key provided, use those
+    if aws_access_key and aws_secret_key:
+        session = boto3.session.Session(aws_access_key_id=aws_access_key,
+                                        aws_secret_access_key=aws_secret_key,
+                                        region_name=aws_region)
+    else:
+        session = boto3.session.Session(region_name=aws_region)
+
+    ssm = session.client('ssm')
+
+    parameter_names_list = get_parameters_with_prefix(param_prefix)
+
+    if parameter_names_list:
+        # Make sure we have a temp dir to work with
+        temp_dir = tempfile.gettempdir()
+        if not os.path.exists(temp_dir):
+            os.makedirs(temp_dir)
+
+        for param in get_parameters(parameter_names_list):
+            parameter_name = param['Name']
+            parameter_value = param['Value']
+            process_parameter(parameter_name, parameter_value)
+
+
+if __name__ == "__main__":
+
+    LOG_FILENAME = 'parameter-sync.log'
+
+    description =  "Script to get all parameters from AWS Parameter\n"
+    description += "Store with a prefix that matches the given prefix\n\n"
+    description += "Note: The following environment variables can be set prior to execution\n"
+    description += "      of the script (or alternatively, set them using script parameters)\n\n"
+    description += "      AWS_ACCESS_KEY_ID\n"
+    description += "      AWS_SECRET_ACCESS_KEY"
+
+    parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument("--aws-access-key-id", help="AWS Access Key ID", dest='aws_access_key', required=False)
+    parser.add_argument("--aws-secret-access-key", help="AWS Secret Access Key", dest='aws_secret_key', required=False)
+    parser.add_argument("--aws-region", help="AWS Region", dest='aws_region', required=True)
+    parser.add_argument("--param-prefix", help="Parameter prefix", dest='param_prefix', required=True)
+    parser.add_argument("--credentials-path", help="Where credentials are stored", dest='cred_path', default='/credentials/')
+    parser.add_argument("--verbose", help="Turn on DEBUG logging", action='store_true', required=False)
+    parser.add_argument("--dryrun", help="Do a dryrun - no changes will be performed", dest='dryrun',
+                        action='store_true', default=False,
+                        required=False)
+    args = parser.parse_args()
+
+    log_level = logging.INFO
+
+    if args.verbose:
+        print('Verbose logging selected')
+        log_level = logging.DEBUG
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    # # create file handler which logs even debug messages
+    # fh = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=5242880, backupCount=5)
+    # fh.setLevel(logging.DEBUG)
+    # create console handler using level set in log_level
+    ch = logging.StreamHandler()
+    ch.setLevel(log_level)
+    console_formatter = logging.Formatter('%(levelname)8s: %(message)s')
+    ch.setFormatter(console_formatter)
+    # file_formatter = logging.Formatter('%(asctime)s - %(levelname)8s: %(message)s')
+    # fh.setFormatter(file_formatter)
+    # Add the handlers to the logger
+    # logger.addHandler(fh)
+    logger.addHandler(ch)
+
+    if not os.environ.get('AWS_ACCESS_KEY_ID') and not args.aws_access_key:
+        logging.critical('AWS Access Key Id not set - cannot continue')
+
+    if not os.environ.get('AWS_SECRET_ACCESS_KEY') and not args.aws_secret_key:
+        logging.critical('AWS Secret Access Key not set - cannot continue')
+
+    logging.debug('INIT')
+    logging.info('Getting parameters with prefix %s from AWS Parameter Store' % args.param_prefix)
+    logging.info('Parameter values will be compared against file contents in "%s" and updated if necessary' % args.cred_path)
+    process_parameters_with_prefix(args.param_prefix, args.cred_path, args.aws_region,
+                                   args.aws_access_key, args.aws_secret_key, args.dryrun)
+    logging.info('COMPLETE')
diff --git a/parameter_sync.sh b/parameter_sync.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+if [ "$VERBOSE" ]; then
+    echo "Verbose logging enabled"
+    set -x
+    VERBOSE='--verbose'
+fi
+
+# Check for required ENV Variables
+if [ -z "$PARAM_PREFIX" ]; then
+    echo "Must supply a parameter prefix by setting the PARAM_PREFIX environment variable"
+    exit 1
+else
+    echo "Parameter Prefix set to $PARAM_PREFIX"
+fi
+
+if [ -z "$AWS_ACCESS_KEY_ID" ]; then
+    echo "Must supply an AWS Access Key ID by setting the AWS_ACCESS_KEY_ID environment variable"
+    exit 1
+fi
+
+if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+    echo "Must supply an AWS Secret Access Key by setting the AWS_SECRET_ACCESS_KEY environment variable"
+    exit 1
+fi
+
+if [ -z "$AWS_REGION" ]; then
+    echo "AWS_REGION not set - defaulting to us-east-1"
+    AWS_REGION='us-east-1'
+fi
+
+# Set a default frequency of 300 seconds if not set in the env
+if [ -z "$FREQUENCY" ]; then
+    echo "FREQUENCY not set - defaulting to 300 seconds"
+    FREQUENCY=300
+else
+    echo "Frequency set to $FREQUENCY seconds"
+fi
+
+if [ -z "$CRED_FOLDER_PATH" ]; then
+    CRED_FOLDER_PATH=/credentials
+    echo "CRED_FOLDER_PATH environment variable missing - assuming default of ${CRED_FOLDER_PATH}"
+fi
+
+if [ ! -e "$CRED_FOLDER_PATH" ]; then
+    echo "${CRED_FOLDER_PATH} doesn't exist, will attempt to create it"
+    echo "NOTE: If this was not expected - please mount credentials path at ${CRED_FOLDER_PATH}"
+    mkdir -p ${CRED_FOLDER_PATH}
+    if [ "$?" -ne 0 ]; then
+        echo "Unable to create ${CRED_FOLDER_PATH} - exiting..."
+        exit 1
+    fi
+fi
+
+# Loop forever, sleeping for our frequency
+while true
+do
+    echo "Awoke to check for new credentials with prefix ${PARAM_PREFIX} in AWS Parameter Store"
+
+    python /parameter_sync.py --credentials-path ${CRED_FOLDER_PATH} --param-prefix ${PARAM_PREFIX} --aws-region ${AWS_REGION} ${VERBOSE}
+    echo "Sleeping for $FREQUENCY seconds"
+    sleep $FREQUENCY
+    echo
+done
+
+exit 0
