refactor: cleanup and optimize

.gitignore

@@ -0,0 +1 @@
+docker-compose.override.yml

Dockerfile

@@ -1,21 +1,35 @@
-ARG pg_alpine_branch
-FROM alpine:${pg_alpine_branch}
+ARG PG_ALPINE_BRANCH
+FROM alpine:${PG_ALPINE_BRANCH}
 
-ARG pg_alpine_branch
-ARG pg_version
+ARG PG_ALPINE_BRANCH
+ARG PG_VERSION
 
-# python for aws-cli, for s3 downloading
-RUN apk --no-cache add python py-pip && \
+#--------------------------------------------------------------------------------
+# Install dependencies
+#--------------------------------------------------------------------------------
+# "postgresql" is required for "pg_restore"
+# "python" is required for "aws-cli"
+#--------------------------------------------------------------------------------
+RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${PG_ALPINE_BRANCH}/main" >> /etc/apk/repositories
+
+RUN apk --no-cache add dumb-init postgresql=${PG_VERSION} python py-pip && \
 	pip install awscli && \
 	apk --purge -v del py-pip
 
-# postgresql for pg_restore
-RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${pg_alpine_branch}/main" >> /etc/apk/repositories
-RUN apk --no-cache add postgresql=${pg_version}
-
-COPY action.sh /
-RUN chmod +x action.sh
+#--------------------------------------------------------------------------------
+# Set script permissions and create required directories
+#--------------------------------------------------------------------------------
+COPY aws-mfa.sh action.sh /
+RUN chmod +x action.sh && chmod +x aws-mfa.sh
+RUN mkdir -p /cache && mkdir -p /root/.aws
 
-RUN mkdir -p /cache
+#--------------------------------------------------------------------------------
+# Use the `dumb-init` init system (PID 1) for process handling
+#--------------------------------------------------------------------------------
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
-CMD echo "${CRON_MINUTE:-$(shuf -i 0-59 -n1)} ${CRON_HOUR:-*} * * * /action.sh" > /var/spool/cron/crontabs/root && crond -d 8 -f
+#--------------------------------------------------------------------------------
+# Configure and apply a cronjob
+#--------------------------------------------------------------------------------
+CMD echo "${CRON_MINUTE:-$(shuf -i 0-59 -n1)} ${CRON_HOUR:-*} * * * /action.sh" \
+> /var/spool/cron/crontabs/root && crond -d 8 -f

README.md

@@ -70,6 +70,22 @@ To execute arbitrary psql / SQL commands before or after the internal _pg_restor
     -e PRE_RESTORE_PSQL="____" \ # "CREATE EXTENSION postgis; CREATE EXTENSION pg_trgm;"
 ```
 
+### AWS multi-factor authentication
+
+If an AWS MFA code isn't passed to `action.sh` as an argument it will be requested.
+
+Authentication via AWS MFA can be enabled by setting the following environment variable.
+
+```
+-e AWS_MFA_DEVICE_ARN=<AWS_MFA_DEVICE_ARN>
+```
+
+In order for the temporary AWS MFA credentials to persist until they expire a volume must be mounted.
+
+```
+-v /tmp/.aws/:/root/.aws/
+```
+
 ***Note**: the usual cron tricks apply to the hour and minute env values. For instance setting `CRON_HOUR` to `*/4` and `CRON_MINUTE` to `0`, will trigger once every 4 hours.*
 
 Creating database dumps can be accomplished with the `bluedrop360/postgres-dump-to-s3` repo.

action.sh

@@ -1,72 +1,116 @@
-#!/bin/sh
+#!/usr/bin/env sh
 
-echo "postgres restore from s3 - looking for dump in cache and on s3 at s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}"
-if [ -n "${DUMP_OBJECT}" ]; then
-  object=${DUMP_OBJECT}
-  dumpFile=$(echo ${DUMP_OBJECT} | sed 's/.*\///')
+################################################################
+# Variable definitions
+################################################################
+# shellcheck disable=SC2001
+DB_NAME=$(echo "${DATABASE_URL}" | sed "s|.*/\([^/]*\)\$|\\1|")
+
+# shellcheck disable=SC2001
+DB_ROOT_URL=$(echo "${DATABASE_URL}" | sed "s|/[^/]*\$|/template1|")
+
+DROP_RESULT=$(echo "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '${DB_NAME}'; \
+DROP DATABASE ${DB_NAME};" | psql "${DB_ROOT_URL}" 2>&1)
+
+################################################################
+# Locate the dump file in the cache or from AWS S3
+################################################################
+printf '%b\n' "\n> Searching for a dump file in the local cache..."
+
+if [ -n "$DUMP_OBJECT" ]; then
+    OBJECT=${DUMP_OBJECT}
+    DUMP_FILE=$(echo "${DUMP_OBJECT}" | sed 's/.*\///')
 else
-  if [ -n "${DUMP_OBJECT_DATE}" ]; then
-    dateFilter=${DUMP_OBJECT_DATE}
-  else
-    dateFilter=$(date +"%Y-%m-%dT%H:%M")
-  fi
-  # broaden filter until a match is found that is also less than dateFilter
-  filter=$dateFilter
-  while true; do
-    echo "postgres restore from s3 - using filter $filter"
-    if [ -f "/cache/$filter.dump" ]; then
-      # file exists in the cache, stop looking remotely
-      object=$filter
-      dumpFile=$filter.dump
-      break;
-    fi
-    dumpFile=$(aws --region ${AWS_REGION} s3 ls s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}${filter} | sed "s/.* //" | grep '^[0-9:T\-]\{16\}\.dump$' | sort | tail -n 1)
-    if [ -n "$dumpFile" ]; then
-      object=${DUMP_OBJECT_PREFIX}$dumpFile
-      # found an object, success
-      break;
-    fi
-    if [ -z "$filter" ]; then
-      # got to an empty filter and still nothing found
-      object=""
-      break;
+    if [ -n "$DUMP_OBJECT_DATE" ]; then
+        FILTER=${DUMP_OBJECT_DATE}
+    else
+        FILTER=$(date +"%Y-%m-%dT%H:%M")
     fi
-    filter="${filter%?}"
-  done
+
+    # Broaden filter until a match is found that is also less than FILTER
+    while true; do
+        printf '%b\n' "    Trying filter: ${FILTER}"
+
+        # File exists in the cache, stop looking remotely
+        if [ -f "/cache/$FILTER.dump" ]; then
+            OBJECT=${FILTER}
+            DUMP_FILE=${FILTER}.dump
+            break;
+        fi
+
+        DUMP_FILE=$(aws --region "${AWS_REGION}" s3 ls "s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}${FILTER}" | sed "s/.* //" | grep '^[0-9:T\-]\{16\}\.dump$' | sort | tail -n 1)
+
+        # Found an object, success
+        if [ -n "${DUMP_FILE}" ]; then
+            OBJECT=${DUMP_OBJECT_PREFIX}${DUMP_FILE}
+            break;
+        fi
+
+        # Got to an empty filter and still nothing found
+        if [ -z "$FILTER" ]; then
+            OBJECT=""
+            break;
+        fi
+
+        FILTER="${FILTER%?}"
+    done
 fi
-if [ -z "$object" ]; then
-  echo "postgres restore from s3 - dump file not found on s3"
-  exit 1
+
+if [ -z "$OBJECT" ]; then
+    printf '%b\n' "> Dump file not found in AWS S3 bucket"
+    exit 1
 fi
-if [ -f "/cache/$dumpFile" ]; then
-  echo "postgres restore from s3 - using cached $dumpFile"
+
+if [ -f "/cache/${DUMP_FILE}" ]; then
+    printf '%b\n' "    Using cached dump: \"${DUMP_FILE}\""
 else
-  echo "postgres restore from s3 - downloading dump from s3 - $object"
-  aws --region ${AWS_REGION} s3 cp s3://${AWS_BUCKET}/$object /cache/$dumpFile
+    printf '%b\n' "    Not found: Attempting to download the dump from an AWS S3 bucket"
+
+    # Download the dump
+    printf '%b\n' "\n> Downloading the latest dump from: \"s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}\""
+    aws --region "${AWS_REGION}" s3 cp "s3://${AWS_BUCKET}/${OBJECT}" "/cache/${DUMP_FILE}" || exit 1
 fi
-echo "postgres restore from s3 - dropping old database"
-dbName=$(echo $DATABASE_URL | sed "s|.*/\([^/]*\)\$|\\1|")
-dbRootUrl=$(echo $DATABASE_URL | sed "s|/[^/]*\$|/template1|")
-dropResult=$(echo "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$dbName'; \
-  DROP DATABASE $dbName;" | psql $dbRootUrl 2>&1)
-if echo $dropResult | grep "other session using the database" -> /dev/null; then
-  echo "RESTORE FAILED - another database session is preventing drop of database $dbName"
-  exit 1
+
+################################################################
+# Drop the target database
+################################################################
+printf '%b\n' '\n> Dropping the target database...'
+printf '%b\n' "    DROP DATABASE ${DB_NAME};"
+
+if echo "${DROP_RESULT}" | grep "other session using the database" >/dev/null 2>&1; then
+    echo "RESTORE FAILED - another database session is preventing drop of database ${DB_NAME}"
+    exit 1
 fi
-createResult=$(echo "CREATE DATABASE $dbName;" | psql $dbRootUrl 2>&1)
-echo "postgres restore from s3 - filling target database with dump"
+
+################################################################
+# Restore the target database
+################################################################
+printf '%b\n' '\n> Restoring the target database...'
+printf '%b\n' "    CREATE DATABASE ${DB_NAME};\n    REVOKE connect ON DATABASE ${DB_NAME} FROM PUBLIC;\n    ALTER DATABASE ${DB_NAME} OWNER TO ${DB_NAME};"
+
+printf '%s' \
+"CREATE DATABASE ${DB_NAME}; REVOKE connect ON DATABASE ${DB_NAME} FROM PUBLIC; ALTER DATABASE ${DB_NAME} OWNER TO ${DB_NAME};" | \
+psql "${DB_ROOT_URL}" >/dev/null 2>&1
+
+printf '%b\n' "\n> Rebuilding the target database..."
+
 if [ -n "$PRE_RESTORE_PSQL" ]; then
-  echo "postgres restore from s3 - executing pre restore psql"
-  printf %s "$PRE_RESTORE_PSQL" | psql $DATABASE_URL
+    printf '%b\n' "> Executing pre-restore psql"
+    printf '%b\n' "${PRE_RESTORE_PSQL}" | psql "${DATABASE_URL}"
 fi
+
 if [ -n "$SCHEMA" ]; then
-  echo "postgres restore from s3 - schema - $SCHEMA"
-  pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d $DATABASE_URL /cache/$dumpFile
+    printf '%s' "    pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d <DATABASE_URL> /cache/${DUMP_FILE}"
+    pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --schema "$SCHEMA" --no-owner -d "${DATABASE_URL}" "/cache/${DUMP_FILE}"
 else
-  pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d $DATABASE_URL /cache/$dumpFile
+    printf '%s' "    pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d <DATABASE_URL> /cache/${DUMP_FILE}"
+    pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --no-owner -d "${DATABASE_URL}" "/cache/${DUMP_FILE}"
 fi
+
 if [ -n "$POST_RESTORE_PSQL" ]; then
-  echo "postgres restore from s3 - executing post restore psql"
-  printf %s "$POST_RESTORE_PSQL" | psql $DATABASE_URL
+    printf '%b\n' "> Executing post-restore psql"
+    printf '%s' "${POST_RESTORE_PSQL}" | psql "${DATABASE_URL}"
 fi
-echo "postgres restore from s3 - complete - $object"
+
+echo ""
+echo "COMPLETE: ${OBJECT}"

build_push.sh

@@ -1,20 +1,21 @@
-#! /bin/sh
+#!/usr/bin/env sh
 
-builds=$(echo '
+BUILDS=$(echo '
 9.6.10    9.6.10-r0   3.6
 9.6       9.6.10-r0   3.6
 9         9.6.10-r0   3.6
 latest    9.6.10-r0   3.6
 ' | grep -v '^#' | tr -s ' ')
 
+# shellcheck disable=SC2039
 IFS=$'\n'
-for build in $builds; do
-  tag=$(echo $build | cut -d ' ' -f 1 )
-  pgVersion=$(echo $build | cut -d ' ' -f 2)
-  pgAlpineBranch=$(echo $build | cut -d ' ' -f 3)
-  
-  echo docker build --tag bluedrop360/postgres-restore-from-s3:$tag --build-arg pg_version=$pgVersion --build-arg pg_alpine_branch=$pgAlpineBranch .
-  eval docker build --tag bluedrop360/postgres-restore-from-s3:$tag --build-arg pg_version=$pgVersion --build-arg pg_alpine_branch=$pgAlpineBranch .
-  echo docker push bluedrop360/postgres-restore-from-s3:$tag
-  eval docker push bluedrop360/postgres-restore-from-s3:$tag
+for BUILD in $BUILDS; do
+  TAG=$(echo "${BUILD}" | cut -d ' ' -f 1 )
+  PG_VERSION=$(echo "${BUILD}" | cut -d ' ' -f 2)
+  PG_ALPINE_VERSION=$(echo "${BUILD}" | cut -d ' ' -f 3)
+
+  echo docker build --tag bluedrop360/postgres-restore-from-s3:"${TAG}" --build-arg pg_version="${PG_VERSION}" --build-arg pg_alpine_branch="${PG_ALPINE_VERSION}" .
+  eval docker build --tag bluedrop360/postgres-restore-from-s3:"${TAG}" --build-arg pg_version="${PG_VERSION}" --build-arg pg_alpine_branch="${PG_ALPINE_VERSION}" .
+  echo docker push bluedrop360/postgres-restore-from-s3:"${TAG}"
+  eval docker push bluedrop360/postgres-restore-from-s3:"${TAG}"
 done

docker-compose.yml

@@ -0,0 +1,28 @@
+########################################################
+# Docker Compose: https://docs.docker.com/compose
+########################################################
+# This file is for local testing
+# - Copy contents to `docker-compose.override.yml`
+# - Update the volume and environment variable values
+# - BUILD: docker-compose build
+# - BUILD AND RUN: docker-compose up --build
+########################################################
+version: '3'
+
+services:
+  postgres-restore-from-s3:
+    image: postgres-restore-from-s3:9.6.10
+    network_mode: 'host'
+    build:
+      context: ./
+      dockerfile: ./Dockerfile
+      args:
+        PG_ALPINE_BRANCH: '3.6'
+        PG_VERSION: '9.6.10-r0'
+    environment:
+      AWS_BUCKET: <AWS_BUCKET_NAME>
+      AWS_REGION: <AWS_REGION_NAME>
+      DATABASE_URL: postgres://<DB_NAME>:<PASSPHRASE>@<HOST>:<PORT>/<DB_NAME>
+      DUMP_OBJECT_PREFIX: <DB_NAME>/postgres/
+    volumes:
+      - <PROJECT_DIRECTORY>/build/postgres/dump:/cache:rw