Skip to content

Commit b4154d4

Browse files
droberts195droberts195
authored
[ML] Add a dev-tools/ci script to automate the Linux/macOS portion of the CI (#29)
To facilitate migration from ML team CI to Infra CI we need to make the steps triggered by Jenkins as simple as possible. This change adds a ci script that can be used for Linux/macOS.
1 parent efa2fb5 commit b4154d4

File tree

7 files changed

+244
-0
lines changed

7 files changed

+244
-0
lines changed

‎dev-tools/ci

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
#
3+
# ELASTICSEARCH CONFIDENTIAL
4+
#
5+
# Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
6+
#
7+
# Notice: this software, and all information contained
8+
# therein, is the exclusive property of Elasticsearch BV
9+
# and its licensors, if any, and is protected under applicable
10+
# domestic and foreign law, and international treaties.
11+
#
12+
# Reproduction, republication or distribution without the
13+
# express written consent of Elasticsearch BV is
14+
# strictly prohibited.
15+
#
16+
17+
# The non-Windows part of ML C++ CI:
18+
#
19+
# 1. Build and unit test the Linux version of the C++
20+
# 2. Build the macOS version of the C++
21+
# 3. Upload the builds to the artifacts directory on S3 that
22+
# subsequent Java builds will download the C++ components from
23+
#
24+
# The builds run in Docker containers that ensure OS dependencies
25+
# are appropriate given the support matrix.
26+
#
27+
# The macOS build cannot be unit tested as it is cross-compiled.
28+
29+
set -e
30+
31+
# Change directory to the directory containing this script
32+
cd `dirname $0`
33+
34+
# Default to a snapshot build
35+
if [ -z "$SNAPSHOT" ] ; then
36+
SNAPSHOT=yes
37+
fi
38+
39+
# Remove any old builds
40+
rm -rf ../builds
41+
42+
# Build and test Linux
43+
./docker_test.sh linux
44+
45+
# Build macOS
46+
./docker_build.sh macosx
47+
48+
# Upload
49+
cd ..
50+
./gradlew --info -b upload.gradle upload
51+

‎dev-tools/docker/build_linux_test_image.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
#
3+
# ELASTICSEARCH CONFIDENTIAL
4+
#
5+
# Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
6+
#
7+
# Notice: this software, and all information contained
8+
# therein, is the exclusive property of Elasticsearch BV
9+
# and its licensors, if any, and is protected under applicable
10+
# domestic and foreign law, and international treaties.
11+
#
12+
# Reproduction, republication or distribution without the
13+
# express written consent of Elasticsearch BV is
14+
# strictly prohibited.
15+
#
16+
17+
# Builds the Docker image that can be used to unit test the machine learning
18+
# C++ code for Linux.
19+
#
20+
# This script is not intended to be run regularly. When changing the tools
21+
# or 3rd party components required to build the machine learning C++ code
22+
# increment the version, change the Dockerfile and build a new image to be
23+
# used for subsequent builds on this branch. Then update the version to be
24+
# used for builds in docker/linux_builder/Dockerfile.
25+
26+
ACCOUNT=droberts195
27+
REPOSITORY=ml-linux-test
28+
VERSION=2
29+
30+
set -e
31+
32+
cd `dirname $0`
33+
34+
docker build --no-cache -t $ACCOUNT/$REPOSITORY:$VERSION linux_test_image
35+
docker login
36+
docker push $ACCOUNT/$REPOSITORY:$VERSION
37+

‎dev-tools/docker/docker_entrypoint.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,9 @@ mkdir ../distributions
5454
zip -9 ../distributions/$ARTIFACT_NAME-$PRODUCT_VERSION-$BUNDLE_PLATFORM.zip `find * | egrep -v '\.lib$|libMlTest|\.dSYM|-debug$|\.pdb$|/core'`
5555
# Include only debug files
5656
zip -9 ../distributions/$ARTIFACT_NAME-debug-$PRODUCT_VERSION-$BUNDLE_PLATFORM.zip `find * | egrep '\.dSYM|-debug$|\.pdb$'`
57+
cd ../..
58+
59+
if [ "x$1" = "x--test" ] ; then
60+
make -j`grep -c '^processor' /proc/cpuinfo` ML_KEEP_GOING=1 test
61+
fi
5762

‎dev-tools/docker/linux_test_image/Dockerfile

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#
2+
# ELASTICSEARCH CONFIDENTIAL
3+
#
4+
# Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
5+
#
6+
# Notice: this software, and all information contained
7+
# therein, is the exclusive property of Elasticsearch BV
8+
# and its licensors, if any, and is protected under applicable
9+
# domestic and foreign law, and international treaties.
10+
#
11+
# Reproduction, republication or distribution without the
12+
# express written consent of Elasticsearch BV is
13+
# strictly prohibited.
14+
#
15+
16+
# Increment the version here when a new tools/3rd party components image is built
17+
FROM droberts195/ml-linux-build:3
18+
19+
MAINTAINER David Roberts <dave.roberts@elastic.co>
20+
21+
# Build cppunit
22+
RUN \
23+
wget --quiet -O - http://dev-www.libreoffice.org/src/cppunit-1.13.2.tar.gz | tar zxf - && \
24+
cd cppunit-1.13.2 && \
25+
./configure --prefix=/usr/local/gcc73 && \
26+
make -j`grep -c '^processor' /proc/cpuinfo` && \
27+
make install && \
28+
cd .. && \
29+
rm -rf cppunit-1.13.2

‎dev-tools/docker/linux_tester/Dockerfile

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#
2+
# ELASTICSEARCH CONFIDENTIAL
3+
#
4+
# Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
5+
#
6+
# Notice: this software, and all information contained
7+
# therein, is the exclusive property of Elasticsearch BV
8+
# and its licensors, if any, and is protected under applicable
9+
# domestic and foreign law, and international treaties.
10+
#
11+
# Reproduction, republication or distribution without the
12+
# express written consent of Elasticsearch BV is
13+
# strictly prohibited.
14+
#
15+
16+
# Increment the version here when a new unit test image is built
17+
FROM droberts195/ml-linux-test:2
18+
19+
MAINTAINER David Roberts <dave.roberts@elastic.co>
20+
21+
# Copy the current Git repository into the container
22+
COPY . /ml-cpp/
23+
24+
# Pass through whether this is a snapshot build (default yes if not specified)
25+
ARG SNAPSHOT=yes
26+
27+
# Run the build and unit tests
28+
RUN \
29+
/ml-cpp/dev-tools/docker/docker_entrypoint.sh --test
30+

‎dev-tools/docker_build.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ cd "$TOOLS_DIR/.."
6969
for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u`
7070
do
7171

72+
# This Dockerfile is for the temporary image that is used to do the build.
73+
# It is based on a pre-built build image stored on Docker Hub, but will have
74+
# the local repository contents copied into it before the entrypoint script
75+
# is run. This temporary image is discarded after the build is complete.
7276
DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_builder/Dockerfile"
7377
TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$
7478

‎dev-tools/docker_test.sh

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
#
3+
# ELASTICSEARCH CONFIDENTIAL
4+
#
5+
# Copyright (c) 2018 Elasticsearch BV. All Rights Reserved.
6+
#
7+
# Notice: this software, and all information contained
8+
# therein, is the exclusive property of Elasticsearch BV
9+
# and its licensors, if any, and is protected under applicable
10+
# domestic and foreign law, and international treaties.
11+
#
12+
# Reproduction, republication or distribution without the
13+
# express written consent of Elasticsearch BV is
14+
# strictly prohibited.
15+
#
16+
17+
# Builds the machine learning C++ code for Linux in a Docker container,
18+
# then runs the unit tests.
19+
#
20+
# The output .zip files are then copied out of the container to the
21+
# location in the current repository that they'd be in had they been
22+
# built outside of Docker.
23+
#
24+
# Finally, the Docker container used for the build/test is deleted.
25+
26+
usage()
27+
{
28+
echo "Usage: $0 linux ..."
29+
exit 1
30+
}
31+
32+
PLATFORMS=
33+
34+
while [ -n "$1" ]
35+
do
36+
37+
case "$1" in
38+
linux)
39+
PLATFORMS="$1 $PLATFORMS"
40+
;;
41+
*)
42+
usage
43+
;;
44+
esac
45+
46+
shift
47+
48+
done
49+
50+
if [ -z "$PLATFORMS" ] ; then
51+
usage
52+
fi
53+
54+
# Default to a snapshot build
55+
if [ -z "$SNAPSHOT" ] ; then
56+
SNAPSHOT=yes
57+
fi
58+
59+
set -e
60+
61+
# The build needs to be done with the Docker context set to the root of the
62+
# repository so that we can copy it into the container.
63+
MY_DIR=`dirname "$BASH_SOURCE"`
64+
TOOLS_DIR=`cd "$MY_DIR" && pwd`
65+
66+
# The Docker context here is the root directory of the outer repository.
67+
cd "$TOOLS_DIR/.."
68+
69+
for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u`
70+
do
71+
72+
# This Dockerfile is for the temporary image that is used to do the build
73+
# and unit tests. It is based on a pre-built test image stored on Docker
74+
# Hub, but will have the local repository contents copied into it before
75+
# the entrypoint script is run. This temporary image is discarded after
76+
# the build and unit tests are complete.
77+
DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_tester/Dockerfile"
78+
TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$
79+
80+
docker build -t $TEMP_TAG --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" .
81+
# Using tar to copy the build and test artifacts out of the container seems
82+
# more reliable than docker cp, and also means the files end up with the
83+
# correct uid/gid
84+
docker run --workdir=/ml-cpp $TEMP_TAG bash -c 'find . -name cppunit_results.xml | xargs tar cf - build/distributions' | tar xvf -
85+
docker rmi --force $TEMP_TAG
86+
87+
done
88+

0 commit comments

Comments
 (0)