Portability additions

.circleci/aws-local.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # A wrapper script for the AWS CLI that redirects all calls to localhost:5000 so that they go to moto instead of the
 # real AWS servers. This script should be installed in the PATH so it gets called instead of the real AWS CLI, and this
 # script will, in turn, call the real AWS CLI.

.circleci/config.yml

@@ -1,11 +1,20 @@
 version: 2
 jobs:
-  build:
+  shellcheck:
+    machine: true
+    steps:
+      - checkout
+      - run: docker-compose up shellcheck
+  bats:
     # We need to run Docker Compose with privileged settings, which isn't supported by CircleCI's Docker executor, so
     # we have to use the machine executor instead.
     machine: true
     steps:
       - checkout
-      - run: |
-          cd test
-          docker-compose up
+      - run: docker-compose up bats
+workflows:
+  version: 2
+  checks:
+    jobs:
+      - shellcheck
+      - bats

.circleci/shellcheck.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+# shellcheck source=./modules/bash-commons/src/os.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../modules/bash-commons/src/bootstrap.sh"
+
+function run_shellcheck {
+  local -r format="${1}"
+  shift
+  local -i exit_code
+  local -a results
+
+  shellcheck --version
+  echo
+
+  printf "Shellcheck is scanning some files (%s):\n" "$#"
+  printf "  %s\n" "$@"
+  echo
+
+  set +e
+  IFS=$'\n' \
+    read -ra results <<< \
+      "$(shellcheck \
+        --exclude=SC1117 \
+        --external-sources \
+        --format="$format" \
+        "$@" 2>&1
+      )"
+  exit_code=$?
+  set -e
+
+  case "$exit_code" in
+    0)
+      echo "All files successfully scanned with no issues"
+      ;;
+
+    1)
+      printf "All files successfully scanned with some issues (%s):\n" ${#results[@]}
+      printf "  %s\n" "${results[@]}"
+      exit $exit_code
+      ;;
+
+    2)
+      printf "Some files could not be processed (%s):\n" ${#results[@]}
+      printf "  %s\n" "${results[@]}"
+      exit $exit_code
+      ;;
+
+    3)
+      echo "ShellCheck was invoked with bad syntax:"
+      printf "  %s\n" "${results[@]}"
+      exit $exit_code
+      ;;
+
+    4)
+      echo "ShellCheck was invoked with bad options:"
+      printf "  %s\n" "${results[@]}"
+      exit $exit_code
+      ;;
+
+    *)
+      echo "Unrecognized exit code '$exit_code' returned from shellcheck"
+      set -x
+      exit 1
+
+  esac
+}
+
+# Runs shellcheck against shell scripts and displays results.
+# 
+# format {1}: Shellcheck format options, either one of "tty", "gcc", "checkstyle"
+#             or "json". Default is "gcc".
+# filename {2}: If not given, it will search for all files whose first line
+#               begin with a shebang like "#!/usr/bin/env bash".
+#               If given a string of space separated files, only those files
+#               will be scanned.
+function main {
+  local -a check_files
+  local -r format="${1:-gcc}"
+  local filename="${2:-}"
+  local line
+
+  # If `CIRCLE_WORKING_DIRECTORY` is not set, assume the project root dir.
+  if [[ -z "${CIRCLE_WORKING_DIRECTORY:-}" ]]; then
+    CIRCLE_WORKING_DIRECTORY="$(
+      readlink -f \
+        "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
+    )"
+  fi
+
+  if [[ -z "$filename" ]]; then
+    # Since no filenames are provided, look for files based on shebang.
+    while read -r filename; do
+      set +e
+      IFS= read -rd '' line < <(head -n 1 "$filename")
+      set -e
+
+      if [[ "$line" =~ ^#!/usr/bin/env\ +bash ]]; then
+        check_files+=( "$filename" )
+      fi
+    done < <(find "$CIRCLE_WORKING_DIRECTORY" -path ./.git -prune -o -type f -print)
+  else
+    check_files=( "$filename" )
+  fi
+
+  run_shellcheck "$format" "${check_files[@]}"
+}
+
+main "$@"
+exit 0

.circleci/Dockerfile → Dockerfile.bats

@@ -21,4 +21,4 @@ RUN pip install flask moto moto[server]
 RUN apt-get install -y net-tools iptables
 
 # Copy mock AWS CLI into the PATH
-COPY ./aws-local.sh /usr/local/bin/aws
+COPY ./.circleci/aws-local.sh /usr/local/bin/aws

Dockerfile.shellcheck

@@ -0,0 +1,30 @@
+FROM bash:3.2
+
+# TODO: labels here. See: http://label-schema.org/rc1/
+
+ARG SHELLCHECK_VERSION=stable
+ARG SHELLCHECK_FORMAT=gcc
+
+# Install dependencies.
+RUN set -e; \
+    apk --update add \
+        git \
+        outils-sha512 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm /var/cache/apk/*
+
+# Install shellcheck.
+RUN set -e; \
+    mkdir -p ~/stage \
+    && wget "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" \
+    && wget "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz.sha512sum" \
+    && sha512 -c shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz.sha512sum \
+    && tar --xz -xvf shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz \
+    && cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/ \
+    && shellcheck --version \
+    && rm -rf ~/stage
+
+WORKDIR /usr/local/src/bash-commons
+COPY ./.circleci/ /usr/local/src/bash-commons/.circleci/
+
+CMD ["bash"]

README.md

@@ -11,9 +11,11 @@ into your bash scripts using `source`.
 ## Examples
 
 Once you have `bash-commons` installed (see the [install instructions](#install)), you use `source` to import the
-modules and start calling the functions within them:
+modules and start calling the functions within them. Before you import any modules, make sure you `source` the
+`bootstrap.sh` file which sets some important defaults to encourage good code:
 
 ```bash
+source /opt/gruntwork/bash-commons/bootstrap.sh
 source /opt/gruntwork/bash-commons/log.sh
 source /opt/gruntwork/bash-commons/assert.sh
 source /opt/gruntwork/bash-commons/os.sh
@@ -150,17 +152,17 @@ The code in this repo aims to be compatible with:
 All the code should mainly follow the [Google Shell Style Guide](https://google.github.io/styleguide/shell.xml).
 In particular:
 
-* The first line of every script should be `#!/bin/bash`.
+* The first line of every script should be `#!/usr/bin/env bash`.
 * All code should be defined in functions.
 * Functions should exit or return 0 on success and non-zero on error.
 * Functions should return output by writing it to `stdout`.
 * Functions should log to `stderr`.
 * All variables should be `local`. No global variables are allowed at all.
 * Make as many variables `readonly` as possible.
-* If calling to a subshell and storing the output in a variable (foo=`$( ... )`), do NOT use `local` and `readonly`
-  in the same statement or the [exit code will be
-  lost](https://blog.gruntwork.io/yak-shaving-series-1-all-i-need-is-a-little-bit-of-disk-space-6e5ef1644f67). Instead,
-  declare the variable as `local` on one line and then call the subshell on the next line.
+* If a variable is both local and readonly, use `local -r`. 
+* If calling to a subshell and storing the output in a variable (foo=`$( ... )`), do NOT use `local -r`  in the same
+  statement or the [exit code will be lost](https://blog.gruntwork.io/yak-shaving-series-1-all-i-need-is-a-little-bit-of-disk-space-6e5ef1644f67).
+  Instead, declare the variable as `local` on one line and then call the subshell on the next line.
 * Quote all strings.
 * Use `[[ ... ]]` instead of `[ ... ]`.
 * Use snake_case for function and variable names. Use UPPER_SNAKE_CASE for constants.

docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+services:
+  shellcheck:
+    build:
+      context: ./
+      dockerfile: Dockerfile.shellcheck
+    volumes:
+      - ./:/usr/local/src/bash-commons
+    working_dir: /usr/local/src/bash-commons/.circleci
+    command: ./shellcheck.sh
+  bats:
+    build:
+      context: ./
+      dockerfile: Dockerfile.bats
+    volumes:
+      # Mount all the files so you have "hot reload" of all changes from the host
+      - ./:/usr/local/src/bash-commons
+    working_dir: /usr/local/src/bash-commons
+    command: bats test
+    # Necessary so we can run a mock EC2 metadata service on port 80 on a special IP
+    privileged: true
+    ports:
+      # For moto
+      - "5000:5000"
+      # For ec2-metadata-mock
+      - "8111:8111"

modules/bash-commons/install.sh

@@ -1,13 +1,16 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # This script is used by the Gruntwork Installer to install the bash-commons library.
 
 set -e
 
 readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly BASH_COMMONS_SRC_DIR="$SCRIPT_DIR/src"
 
+# shellcheck source=./modules/bash-commons/src/log.sh
 source "$BASH_COMMONS_SRC_DIR/log.sh"
+# shellcheck source=./modules/bash-commons/src/assert.sh
 source "$BASH_COMMONS_SRC_DIR/assert.sh"
+# shellcheck source=./modules/bash-commons/src/os.sh
 source "$BASH_COMMONS_SRC_DIR/os.sh"
 
 readonly DEFAULT_INSTALL_DIR="/opt/gruntwork/bash-commons"
@@ -37,7 +40,7 @@ function install {
   local install_dir_owner="$DEFAULT_USER_NAME"
   local install_dir_group="$DEFAULT_USER_GROUP_NAME"
 
-  while [[ $# > 0 ]]; do
+  while [[ $# -gt 0 ]]; do
     local key="$1"
 
     case "$key" in

modules/bash-commons/src/array.sh

@@ -1,12 +1,10 @@
-#!/bin/bash
-
-set -e
+#!/usr/bin/env bash
 
 # Returns 0 if the given item (needle) is in the given array (haystack); returns 1 otherwise.
 function array_contains {
-  local readonly needle="$1"
+  local -r needle="$1"
   shift
-  local readonly haystack=("$@")
+  local -ra haystack=("$@")
 
   local item
   for item in "${haystack[@]}"; do
@@ -26,9 +24,9 @@ function array_contains {
 #   Returns: "A,B,C"
 #
 function array_join {
-  local readonly separator="$1"
+  local -r separator="$1"
   shift
-  local readonly values=("$@")
+  local -ra values=("$@")
 
   local out=""
   for (( i=0; i<"${#values[@]}"; i++ )); do

modules/bash-commons/src/assert.sh

@@ -1,18 +1,19 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # A collection of useful assertions. Each one checks a condition and if the condition is not satisfied, exits the
 # program. This is useful for defensive programming.
 
-
-set -e
-
+# shellcheck source=./modules/bash-commons/src/log.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/log.sh"
+# shellcheck source=./modules/bash-commons/src/array.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/array.sh"
+# shellcheck source=./modules/bash-commons/src/string.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/string.sh"
+# shellcheck source=./modules/bash-commons/src/os.sh
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/os.sh"
 
 # Check that the given binary is available on the PATH. If it's not, exit with an error.
 function assert_is_installed {
-  local readonly name="$1"
+  local -r name="$1"
 
   if ! os_command_is_installed "$name"; then
     log_error "The command '$name' is required by this script but is not installed or in the system's PATH."
@@ -22,9 +23,9 @@ function assert_is_installed {
 
 # Check that the value of the given arg is not empty. If it is, exit with an error.
 function assert_not_empty {
-  local readonly arg_name="$1"
-  local readonly arg_value="$2"
-  local readonly reason="$3"
+  local -r arg_name="$1"
+  local -r arg_value="$2"
+  local -r reason="$3"
 
   if [[ -z "$arg_value" ]]; then
     log_error "The value for '$arg_name' cannot be empty. $reason"
@@ -34,9 +35,9 @@ function assert_not_empty {
 
 # Check that the value of the given arg is empty. If it isn't, exit with an error.
 function assert_empty {
-  local readonly arg_name="$1"
-  local readonly arg_value="$2"
-  local readonly reason="$3"
+  local -r arg_name="$1"
+  local -r arg_value="$2"
+  local -r reason="$3"
 
   if [[ ! -z "$arg_value" ]]; then
     log_error "The value for '$arg_name' must be empty. $reason"
@@ -47,8 +48,8 @@ function assert_empty {
 # Check that the given response from AWS is not empty or null (the null often comes from trying to parse AWS responses
 # with jq). If it is, exit with an error.
 function assert_not_empty_or_null {
-  local readonly response="$1"
-  local readonly description="$2"
+  local -r response="$1"
+  local -r description="$2"
 
   if string_is_empty_or_null "$response"; then
     log_error "Got empty response for $description"
@@ -58,10 +59,10 @@ function assert_not_empty_or_null {
 
 # Check that the given value is one of the values from the given list. If not, exit with an error.
 function assert_value_in_list {
-  local readonly arg_name="$1"
-  local readonly arg_value="$2"
+  local -r arg_name="$1"
+  local -r arg_value="$2"
   shift 2
-  local readonly list=("$@")
+  local -ar list=("$@")
 
   if ! array_contains "$arg_value" "${list[@]}"; then
     log_error "'$arg_value' is not a valid value for $arg_name. Must be one of: [${list[@]}]."