[ML] Add a dev-tools/ci script to automate the Linux/macOS portion of the CI (#29)

To facilitate migration from ML team CI to Infra CI we need to make the
steps triggered by Jenkins as simple as possible.

This change adds a ci script that can be used for Linux/macOS.

Author: droberts195

dev-tools/ci

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# The non-Windows part of ML C++ CI:
+#
+# 1. Build and unit test the Linux version of the C++
+# 2. Build the macOS version of the C++
+# 3. Upload the builds to the artifacts directory on S3 that
+#    subsequent Java builds will download the C++ components from
+#
+# The builds run in Docker containers that ensure OS dependencies
+# are appropriate given the support matrix.
+#
+# The macOS build cannot be unit tested as it is cross-compiled.
+
+set -e
+
+# Change directory to the directory containing this script
+cd `dirname $0`
+
+# Default to a snapshot build
+if [ -z "$SNAPSHOT" ] ; then
+    SNAPSHOT=yes
+fi
+
+# Remove any old builds
+rm -rf ../builds
+
+# Build and test Linux
+./docker_test.sh linux
+
+# Build macOS
+./docker_build.sh macosx
+
+# Upload
+cd ..
+./gradlew --info -b upload.gradle upload
+

dev-tools/docker/build_linux_test_image.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# Builds the Docker image that can be used to unit test the machine learning
+# C++ code for Linux.
+#
+# This script is not intended to be run regularly.  When changing the tools
+# or 3rd party components required to build the machine learning C++ code
+# increment the version, change the Dockerfile and build a new image to be
+# used for subsequent builds on this branch.  Then update the version to be
+# used for builds in docker/linux_builder/Dockerfile.
+
+ACCOUNT=droberts195
+REPOSITORY=ml-linux-test
+VERSION=2
+
+set -e
+
+cd `dirname $0`
+
+docker build --no-cache -t $ACCOUNT/$REPOSITORY:$VERSION linux_test_image
+docker login
+docker push $ACCOUNT/$REPOSITORY:$VERSION
+

dev-tools/docker/docker_entrypoint.sh

@@ -45,4 +45,9 @@ mkdir ../distributions
 zip -9 ../distributions/$ARTIFACT_NAME-$PRODUCT_VERSION-$BUNDLE_PLATFORM.zip `find * | egrep -v '\.lib$|libMlTest|\.dSYM|-debug$|\.pdb$|/core'`
 # Include only debug files
 zip -9 ../distributions/$ARTIFACT_NAME-debug-$PRODUCT_VERSION-$BUNDLE_PLATFORM.zip `find * | egrep '\.dSYM|-debug$|\.pdb$'`
+cd ../..
+
+if [ "x$1" = "x--test" ] ; then
+    make -j`grep -c '^processor' /proc/cpuinfo` ML_KEEP_GOING=1 test
+fi
 

dev-tools/docker/linux_test_image/Dockerfile

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# Increment the version here when a new tools/3rd party components image is built
+FROM droberts195/ml-linux-build:3
+
+MAINTAINER David Roberts <dave.roberts@elastic.co>
+
+# Build cppunit
+RUN \
+  wget --quiet -O - http://dev-www.libreoffice.org/src/cppunit-1.13.2.tar.gz | tar zxf - && \
+  cd cppunit-1.13.2 && \
+  ./configure --prefix=/usr/local/gcc73 && \
+  make -j`grep -c '^processor' /proc/cpuinfo` && \
+  make install && \
+  cd .. && \
+  rm -rf cppunit-1.13.2

dev-tools/docker/linux_tester/Dockerfile

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# Increment the version here when a new unit test image is built
+FROM droberts195/ml-linux-test:2
+
+MAINTAINER David Roberts <dave.roberts@elastic.co>
+
+# Copy the current Git repository into the container
+COPY . /ml-cpp/
+
+# Pass through whether this is a snapshot build (default yes if not specified)
+ARG SNAPSHOT=yes
+
+# Run the build and unit tests
+RUN \
+  /ml-cpp/dev-tools/docker/docker_entrypoint.sh --test
+

dev-tools/docker_build.sh

@@ -60,6 +60,10 @@ cd "$TOOLS_DIR/.."
 for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u`
 do
 
+    # This Dockerfile is for the temporary image that is used to do the build.
+    # It is based on a pre-built build image stored on Docker Hub, but will have
+    # the local repository contents copied into it before the entrypoint script
+    # is run.  This temporary image is discarded after the build is complete.
     DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_builder/Dockerfile"
     TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$
 

dev-tools/docker_test.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# Builds the machine learning C++ code for Linux in a Docker container,
+# then runs the unit tests.
+#
+# The output .zip files are then copied out of the container to the
+# location in the current repository that they'd be in had they been
+# built outside of Docker.
+#
+# Finally, the Docker container used for the build/test is deleted.
+
+usage()
+{
+    echo "Usage: $0 linux ..."
+    exit 1
+}
+
+PLATFORMS=
+
+while [ -n "$1" ]
+do
+
+    case "$1" in
+        linux)
+            PLATFORMS="$1 $PLATFORMS"
+            ;;
+        *)
+            usage
+            ;;
+    esac
+
+    shift
+
+done
+
+if [ -z "$PLATFORMS" ] ; then
+    usage
+fi
+
+# Default to a snapshot build
+if [ -z "$SNAPSHOT" ] ; then
+    SNAPSHOT=yes
+fi
+
+set -e
+
+# The build needs to be done with the Docker context set to the root of the
+# repository so that we can copy it into the container.
+MY_DIR=`dirname "$BASH_SOURCE"`
+TOOLS_DIR=`cd "$MY_DIR" && pwd`
+
+# The Docker context here is the root directory of the outer repository.
+cd "$TOOLS_DIR/.."
+
+for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u`
+do
+
+    # This Dockerfile is for the temporary image that is used to do the build
+    # and unit tests.  It is based on a pre-built test image stored on Docker
+    # Hub, but will have the local repository contents copied into it before
+    # the entrypoint script is run.  This temporary image is discarded after
+    # the build and unit tests are complete.
+    DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_tester/Dockerfile"
+    TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$
+
+    docker build -t $TEMP_TAG --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" .
+    # Using tar to copy the build and test artifacts out of the container seems
+    # more reliable than docker cp, and also means the files end up with the
+    # correct uid/gid
+    docker run --workdir=/ml-cpp $TEMP_TAG bash -c 'find . -name cppunit_results.xml | xargs tar cf - build/distributions' | tar xvf -
+    docker rmi --force $TEMP_TAG
+
+done
+