feat: allow containers to connect to host machine/dockest inside cont…

…ainer support (#91)

* patch: allow to resolve the container host based on the containerId

* feat: add internal config option for making dockest aware of whether it is executed inside a container or not.

This is a prerequisite for using dockest inside docker containers. see #79

* feat: create bridge network if executed inside docker container.

Detect whether dockest is run inside a docker container. Create network and link all runner containers to the dockest container.

* feat: use target port and alias host in inside docker container mode

Revert option of using a function for setting the host. Automatically set the host to the service name in dockest inside docker container mode.

* chore: add aws-codebuild example.

Example that showcases dockest in docker. Works inside docker container and outside docker container.

* chore: fix eslint errors

* test: update snapshots

* chore: run aws-code-build-example on ci.

* chore: disable tests outside container

* feat: add helper methods that easily allow writing tests for "dockest inside docker" and "dockest on docker host" environments.

* refactor: change approach of adding hostnames

* test: pin docker image to hash

* Update src/index.ts

Co-Authored-By: Erik Engervall <erik.engervall@gmail.com>

examples/aws-code-build-example/.gitignore

@@ -0,0 +1,2 @@
+dockest.tgz
+.artifacts

examples/aws-code-build-example/README.md

@@ -0,0 +1,23 @@
+# aws-code-build-example
+
+Exampel that showcases usage with AWS Codebuild
+
+It can be reused for any CI System that runs your build inside a docker container with an injected docker socket.
+
+## Running the build
+
+```bash
+./run_tests.sh
+```
+
+This test should also pass when not being run inside a container.
+
+## Differences to running dockest on the host
+
+- Dockest creates a network that connects the container that runs dockest to the other containers
+- Dockest uses the target ports on the containers (instead of the published on the host)
+- Services can be accessed via their service name as the hostname
+
+# Development
+
+Dockest must be bundeled as a .tgz and put inside this folder, because the codebuild container cannot resolve the parent directories (check `run_tests.sh`).

examples/aws-code-build-example/app/Dockerfile

@@ -0,0 +1,9 @@
+FROM node:12-alpine
+
+COPY package.json /app/package.json
+RUN sh -c "cd /app && yarn install" 
+COPY index.js /app/index.js
+
+EXPOSE 9000
+
+CMD ["node", "/app/index.js"]

examples/aws-code-build-example/app/index.js

@@ -0,0 +1,19 @@
+'use strict'
+
+const bodyParser = require('body-parser')
+const app = require('express')()
+const fetch = require('node-fetch')
+
+app.use(bodyParser.text())
+
+app.post('/', (req, res) => {
+  const url = req.body
+
+  res.status(200).send('OK.')
+
+  setTimeout(() => {
+    fetch(url)
+  }, 2000)
+})
+
+app.listen(9000)

examples/aws-code-build-example/app/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "app",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "private": true,
+  "dependencies": {
+    "body-parser": "1.19.0",
+    "express": "4.17.1",
+    "node-fetch": "2.6.0"
+  }
+}

examples/aws-code-build-example/buildspec.yml

@@ -0,0 +1,13 @@
+version: 0.2
+
+phases:
+  install:
+    commands:
+      # docker in docker integration
+      - nohup /usr/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2&
+      - timeout -t 15 sh -c "until docker info; do echo .; sleep 1; done"
+      - yarn
+
+  build:
+    commands:
+      - yarn test

examples/aws-code-build-example/codebuild_build.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+
+function allOSRealPath() {
+    if isOSWindows
+    then
+        path=""
+        case $1 in
+            .* ) path="$PWD/${1#./}" ;;
+            /* ) path="$1" ;;
+            *  ) path="/$1" ;;
+        esac
+
+        echo "/$path" | sed -e 's/\\/\//g' -e 's/://' -e 's/./\U&/3'
+    else
+        case $1 in
+            /* ) echo "$1"; exit;;
+            *  ) echo "$PWD/${1#./}"; exit;;
+        esac
+    fi
+}
+
+function isOSWindows() {
+    if [ $OSTYPE == "msys" ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function usage {
+    echo "usage: codebuild_build.sh [-i image_name] [-a artifact_output_directory] [options]"
+    echo "Required:"
+    echo "  -i        Used to specify the customer build container image."
+    echo "  -a        Used to specify an artifact output directory."
+    echo "Options:"
+    echo "  -l IMAGE  Used to override the default local agent image."
+    echo "  -s        Used to specify source information. Defaults to the current working directory for primary source."
+    echo "               * First (-s) is for primary source"
+    echo "               * Use additional (-s) in <sourceIdentifier>:<sourceLocation> format for secondary source"
+    echo "               * For sourceIdentifier, use a value that is fewer than 128 characters and contains only alphanumeric characters and underscores"
+    echo "  -c        Use the AWS configuration and credentials from your local host. This includes ~/.aws and any AWS_* environment variables."
+    echo "  -p        Used to specify the AWS CLI Profile."
+    echo "  -b FILE   Used to specify a buildspec override file. Defaults to buildspec.yml in the source directory."
+    echo "  -m        Used to mount the source directory to the customer build container directly."
+    echo "  -e FILE   Used to specify a file containing environment variables."
+    echo "            (-e) File format expectations:"
+    echo "               * Each line is in VAR=VAL format"
+    echo "               * Lines beginning with # are processed as comments and ignored"
+    echo "               * Blank lines are ignored"
+    echo "               * File can be of type .env or .txt"
+    echo "               * There is no special handling of quotation marks, meaning they will be part of the VAL"
+    exit 1
+}
+
+image_flag=false
+artifact_flag=false
+awsconfig_flag=false
+mount_src_dir_flag=false
+
+while getopts "cmi:a:s:b:e:l:p:h" opt; do
+    case $opt in
+        i  ) image_flag=true; image_name=$OPTARG;;
+        a  ) artifact_flag=true; artifact_dir=$OPTARG;;
+        b  ) buildspec=$OPTARG;;
+        c  ) awsconfig_flag=true;;
+        m  ) mount_src_dir_flag=true;;
+        s  ) source_dirs+=("$OPTARG");;
+        e  ) environment_variable_file=$OPTARG;;
+        l  ) local_agent_image=$OPTARG;;
+        p  ) aws_profile=$OPTARG;;
+        h  ) usage; exit;;
+        \? ) echo "Unknown option: -$OPTARG" >&2; exit 1;;
+        :  ) echo "Missing option argument for -$OPTARG" >&2; exit 1;;
+        *  ) echo "Invalid option: -$OPTARG" >&2; exit 1;;
+    esac
+done
+
+if  ! $image_flag
+then
+    echo "The image name flag (-i) must be included for a build to run" >&2
+fi
+
+if  ! $artifact_flag
+then
+    echo "The artifact directory (-a) must be included for a build to run" >&2
+fi
+
+if  ! $image_flag ||  ! $artifact_flag
+then
+    exit 1
+fi
+
+docker_command="docker run "
+if isOSWindows
+then
+    docker_command+="-v //var/run/docker.sock:/var/run/docker.sock -e "
+else
+    docker_command+="-v /var/run/docker.sock:/var/run/docker.sock -e "
+fi
+
+docker_command+="\"IMAGE_NAME=$image_name\" -e \
+    \"ARTIFACTS=$(allOSRealPath $artifact_dir)\""
+
+if [ -z "$source_dirs" ]
+then
+    docker_command+=" -e \"SOURCE=$(allOSRealPath $PWD)\""
+else
+    for index in "${!source_dirs[@]}"; do
+        if [ $index -eq 0 ]
+        then
+            docker_command+=" -e \"SOURCE=$(allOSRealPath ${source_dirs[$index]})\""
+        else
+            identifier=${source_dirs[$index]%%:*}
+            src_dir=$(allOSRealPath ${source_dirs[$index]#*:})
+
+            docker_command+=" -e \"SECONDARY_SOURCE_$index=$identifier:$src_dir\""
+        fi
+    done
+fi
+
+if [ -n "$buildspec" ]
+then
+    docker_command+=" -e \"BUILDSPEC=$(allOSRealPath $buildspec)\""
+fi
+
+if [ -n "$environment_variable_file" ]
+then
+    environment_variable_file_path=$(allOSRealPath "$environment_variable_file")
+    environment_variable_file_dir=$(dirname "$environment_variable_file_path")
+    environment_variable_file_basename=$(basename "$environment_variable_file")
+    docker_command+=" -v \"$environment_variable_file_dir:/LocalBuild/envFile/\" -e \"ENV_VAR_FILE=$environment_variable_file_basename\""
+fi
+
+if [ -n "$local_agent_image" ]
+then
+    docker_command+=" -e \"LOCAL_AGENT_IMAGE_NAME=$local_agent_image\""
+fi
+
+if $awsconfig_flag
+then
+    if [ -d "$HOME/.aws" ]
+    then
+        configuration_file_path=$(allOSRealPath "$HOME/.aws")
+        docker_command+=" -e \"AWS_CONFIGURATION=$configuration_file_path\""
+    else
+        docker_command+=" -e \"AWS_CONFIGURATION=NONE\""
+    fi
+
+    if [ -n "$aws_profile" ]
+    then
+        docker_command+=" -e \"AWS_PROFILE=$aws_profile\""
+    fi
+
+    docker_command+="$(env | grep ^AWS_ | while read -r line; do echo " -e \"$line\""; done )"
+fi
+
+if $mount_src_dir_flag
+then
+    docker_command+=" -e \"MOUNT_SOURCE_DIRECTORY=TRUE\""
+fi
+
+if isOSWindows
+then
+    docker_command+=" -e \"INITIATOR=$USERNAME\""
+else
+    docker_command+=" -e \"INITIATOR=$USER\""
+fi
+
+docker_command+=" amazon/aws-codebuild-local:latest"
+
+# Note we do not expose the AWS_SECRET_ACCESS_KEY or the AWS_SESSION_TOKEN
+exposed_command=$docker_command
+secure_variables=( "AWS_SECRET_ACCESS_KEY=" "AWS_SESSION_TOKEN=")
+for variable in "${secure_variables[@]}"
+do
+    exposed_command="$(echo $exposed_command | sed "s/\($variable\)[^ ]*/\1********\"/")"
+done
+
+echo "Build Command:"
+echo ""
+echo $exposed_command
+echo ""
+
+eval $docker_command

examples/aws-code-build-example/dockest.ts

@@ -0,0 +1,24 @@
+import Dockest, { runners, logLevel } from 'dockest'
+
+const runner = new runners.GeneralPurposeRunner({
+  service: 'website',
+  build: './app',
+  ports: [
+    {
+      target: 9000,
+      published: 9000,
+    },
+  ],
+  responsivenessTimeout: 5,
+  connectionTimeout: 5,
+})
+
+const dockest = new Dockest({
+  opts: {
+    logLevel: logLevel.DEBUG,
+  },
+})
+
+dockest.attachRunners([runner])
+
+dockest.run()

examples/aws-code-build-example/integration-test/hello-world.spec.ts

@@ -0,0 +1,46 @@
+import http from 'http'
+import fetch from 'node-fetch'
+import { getHostAddress, getServiceAddress } from 'dockest/dist/test-helper'
+
+const TARGET_HOST = getServiceAddress('website', 9000)
+
+// hostname is either our docker container hostname or if not run inside a docker container the docker host
+const HOSTNAME = getHostAddress()
+const PORT = 8080
+
+let server: http.Server
+
+afterEach(async () => {
+  if (server) {
+    await new Promise((resolve, reject) => {
+      server.close(err => {
+        if (err) {
+          reject(err)
+          return
+        }
+        resolve()
+      })
+    })
+  }
+})
+
+test('can send a request to the container and it can send a request to us', async done => {
+  await new Promise(resolve => {
+    server = http
+      .createServer((_req, res) => {
+        res.write('Hello World!')
+        res.end()
+        done()
+      })
+      .listen(PORT, () => {
+        console.log(`Serving on http://${HOSTNAME}:${PORT}`)
+        resolve()
+      })
+  })
+
+  const res = await fetch(`http://${TARGET_HOST}`, {
+    method: 'post',
+    body: `http://${HOSTNAME}:${PORT}`,
+  }).then(res => res.text())
+  expect(res).toEqual('OK.')
+})

examples/aws-code-build-example/jest.config.js

@@ -0,0 +1,8 @@
+'use strict'
+
+module.exports = {
+  roots: ['<rootDir>/integration-test'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+}

examples/aws-code-build-example/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "aws--codebuild-example",
+  "version": "1.0.0",
+  "main": "index.js",
+  "author": "n1ru4l <laurinquast@googlemail.com>",
+  "license": "MIT",
+  "private": true,
+  "devDependencies": {
+    "@types/jest": "24.0.18",
+    "@types/node": "12.7.5",
+    "@types/node-fetch": "2.5.2",
+    "is-docker": "2.0.0",
+    "jest": "24.9.0",
+    "node-fetch": "2.6.0",
+    "ts-jest": "24.1.0",
+    "ts-node": "8.4.1",
+    "typescript": "3.6.3"
+  },
+  "scripts": {
+    "test": "ts-node dockest.ts"
+  },
+  "dependencies": {
+    "dockest": "file:./dockest.tgz"
+  }
+}

examples/aws-code-build-example/run_tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euxo pipefail
+
+cd ../..
+yarn build
+yarn pack --filename examples/aws-code-build-example/dockest.tgz
+cd examples/aws-code-build-example
+
+# build dockest
+yarn cache clean
+yarn install --no-lockfile
+yarn test
+
+# build with dockest inside docker container
+rm -rf node_modules
+./codebuild_build.sh -i n1ru4l/aws-codebuild-node:7712cfae8d65fd3b704f74e84f688739de5bd357 -a .artifacts