Skip to content

Commit

Permalink
Merge pull request #7666 from EOSIO/jerstern/kiss
Browse files Browse the repository at this point in the history
K.I.S.S Project
  • Loading branch information
NorseGaud authored Jul 18, 2019
2 parents 1bd6901 + 075aed7 commit 3596f16
Show file tree
Hide file tree
Showing 11 changed files with 101 additions and 102 deletions.
47 changes: 47 additions & 0 deletions .cicd/.helpers
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
DRYRUN=${DRYRUN:-false}
VERBOSE=${VERBOSE:-true}
export PROJECT_NAME="eosio"

# Set IMAGE_TAG using arg (generate-base-images.sh) or env (travis.yml)
( [[ $(uname) != 'Darwin' ]] && [[ -z $IMAGE_TAG ]] && [[ -z $1 ]] ) && echo "You must provide the distro IMAGE_TAG name (example: ubuntu-18.04) as argument \$1 or set it within your ENV" && exit 1
export IMAGE_TAG=${IMAGE_TAG:-$1}

function execute() {
$VERBOSE && echo "--- Executing: $@"
$DRYRUN || "$@"
}

function determine-hash() {
# Determine the sha1 hash of all dockerfiles in the .cicd directory.
[[ -z $1 ]] && echo "Please provide the files to be hashed (wildcards supported)" && exit 1
echo "Obtaining Hash of files from $1..."
# Collect all files, hash them, then hash those.
HASHES=()
for FILE in $(find $1 -type f); do
HASH=$(sha1sum $FILE | sha1sum | awk '{ print $1 }')
HASHES=($HASH "${HASHES[*]}")
echo "$FILE - $HASH"
done
export DETERMINED_HASH=$(echo ${HASHES[*]} | sha1sum | awk '{ print $1 }')
export HASHED_IMAGE_TAG="${IMAGE_TAG}-${DETERMINED_HASH}"
}

function generate_docker_image() {
# If we cannot pull the image, we build and push it first.
docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
cd ./.cicd
docker build -t $FULL_TAG -f ./${IMAGE_TAG}.dockerfile .
docker push $FULL_TAG
cd -
}

function docker_tag_exists() {
ORG_REPO=$(echo $1 | cut -d: -f1)
TAG=$(echo $1 | cut -d: -f2)
EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG")
( [[ $EXISTS =~ '404 page not found' ]] || [[ $EXISTS =~ 'manifest unknown' ]] ) && return 1 || return 0
}

determine-hash ".cicd/${IMAGE_TAG}.dockerfile"
[[ -z $DETERMINED_HASH ]] && echo "DETERMINED_HASH empty! (check determine-hash function)" && exit 1
export FULL_TAG="eosio/producer:${PROJECT_NAME}-$HASHED_IMAGE_TAG"
11 changes: 10 additions & 1 deletion .cicd/amazonlinux-2.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,13 @@ ENV PATH=${PATH}:/mongodb-linux-x86_64-amazon-3.6.3/bin

# CCACHE
RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c/ccache-3.3.4-1.el7.x86_64.rpm \
&& yum install -y ccache-3.3.4-1.el7.x86_64.rpm
&& yum install -y ccache-3.3.4-1.el7.x86_64.rpm

# PRE_COMMANDS: Executed pre-cmake
# CMAKE_EXTRAS: Executed right before the cmake path (on the end)
ENV PRE_COMMANDS="export PATH=/usr/lib64/ccache:$PATH &&"
ENV CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'"

CMD bash -c "$PRE_COMMANDS ccache -s && \
mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \
ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
10 changes: 9 additions & 1 deletion .cicd/centos-7.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -86,4 +86,12 @@ RUN curl -LO http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/c
## Needed as devtoolset uses c++ and it's not in ccache by default
RUN cd /usr/lib64/ccache && ln -s ../../bin/ccache c++
## We need to tell ccache to actually use devtoolset-8 instead of the default system one (ccache resets anything set in PATH when it launches)
ENV CCACHE_PATH="/opt/rh/devtoolset-8/root/usr/bin"
ENV CCACHE_PATH="/opt/rh/devtoolset-8/root/usr/bin"

# PRE_COMMANDS: Executed pre-cmake
# CMAKE_EXTRAS: Executed right before the cmake path (on the end)
ENV PRE_COMMANDS="source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:$PATH &&"

CMD bash -c "$PRE_COMMANDS ccache -s && \
mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \
ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
47 changes: 4 additions & 43 deletions .cicd/generate-base-images.sh
Original file line number Diff line number Diff line change
@@ -1,45 +1,6 @@
#!/usr/bin/env bash
set -eo pipefail

[[ -z $1 ]] && echo "Must provide the distro IMAGE_TAG name (example: ubuntu-18.04)" && exit 1
export IMAGE_TAG=$1

function determine-hash() {
# Determine the sha1 hash of all dockerfiles in the .cicd directory.
[[ -z $1 ]] && echo "Please provide the files to be hashed (wildcards supported)" && exit 1
echo "Obtaining Hash of files from $1..."
# Collect all files, hash them, then hash those.
HASHES=()
for FILE in $(find $1 -type f); do
HASH=$(sha1sum $FILE | sha1sum | awk '{ print $1 }')
HASHES=($HASH "${HASHES[*]}")
echo "$FILE - $HASH"
done
export DETERMINED_HASH=$(echo ${HASHES[*]} | sha1sum | awk '{ print $1 }')
export HASHED_IMAGE_TAG="${IMAGE_TAG}-${DETERMINED_HASH}"
}

function generate_docker_image() {
# If we cannot pull the image, we build and push it first.
docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
cd ./.cicd
docker build -t eosio/producer:ci-${HASHED_IMAGE_TAG} -f ./${IMAGE_TAG}.dockerfile .
docker push eosio/producer:ci-${HASHED_IMAGE_TAG}
cd -
}

function docker_tag_exists() {
ORG_REPO=$(echo $1 | cut -d: -f1)
TAG=$(echo $1 | cut -d: -f2)
EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG")
( [[ $EXISTS =~ '404 page not found' ]] || [[ $EXISTS =~ 'manifest unknown' ]] ) && return 1 || return 0
}

determine-hash ".cicd/${IMAGE_TAG}.dockerfile"
[[ -z $DETERMINED_HASH ]] && echo "DETERMINED_HASH empty! (check determine-hash function)" && exit 1
echo "Looking for $IMAGE_TAG-$DETERMINED_HASH"
if docker_tag_exists eosio/producer:ci-${HASHED_IMAGE_TAG}; then
echo "eosio/producer:ci-${HASHED_IMAGE_TAG} already exists"
else
generate_docker_image
fi
cd $( dirname "${BASH_SOURCE[0]}" )/.. # Ensure we're in the repo root and not inside of scripts
. ./.cicd/.helpers
echo "Looking for $FULL_TAG"
docker_tag_exists $FULL_TAG && echo "$FULL_TAG already exists" || generate_docker_image
24 changes: 5 additions & 19 deletions .cicd/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,58 +15,44 @@ steps:
- wait

- label: ":aws: [Amazon] 2 Build"
command:
- "mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang' /workdir && make -j $(getconf _NPROCESSORS_ONLN)"
- "cd /workdir/build && ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
agents:
queue: "automation-eos-builder-fleet"
plugins:
- docker#v3.2.0:
shell: ["/bin/bash", "-i", "-e", "-c"]
debug: $DEBUG
image: "eosio/producer:ci-amazonlinux-2-266a1fe17771b1c1e9aeb3087516dbe8570e0e25"
image: "eosio/producer:eosio-amazonlinux-2-1c266e25276ad1f1147a4df5c12921968079c49b"
timeout: $BUILD_TIMEOUT
skip: $SKIP_AMAZON_LINUX_2

- label: ":centos: [CentOS] 7 Build"
command:
- "mkdir /workdir/build && cd /workdir/build && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true /workdir && make -j $(getconf _NPROCESSORS_ONLN)"
- "cd /workdir/build && ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
- ""
agents:
queue: "automation-eos-builder-fleet"
plugins:
- docker#v3.2.0:
shell: ["/bin/bash", "-i", "-e", "-c"]
debug: $DEBUG
image: "eosio/producer:ci-centos-7-9b1b0f1e15eba74234860434ebd82b93b23dd1de"
image: "eosio/producer:eosio-centos-7-d45789dfb1a5b830a62381afd205b34d3d7c2d63"
timeout: $BUILD_TIMEOUT
skip: $SKIP_CENTOS_7

- label: ":ubuntu: [Ubuntu] 16.04 Build"
command:
- "mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' /workdir && make -j $(getconf _NPROCESSORS_ONLN)"
- "cd /workdir/build && ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
agents:
queue: "automation-eos-builder-fleet"
plugins:
- docker#v3.2.0:
shell: ["/bin/bash", "-i", "-e", "-c"]
debug: $DEBUG
image: "eosio/producer:ci-ubuntu-16.04-0b3d8a452a4dd471d073d12d7e5971e2416d0536"
image: "eosio/producer:eosio-ubuntu-16.04-4f430ef21e359ef2581c25255c99e5411adfaed9"
timeout: $BUILD_TIMEOUT
skip: $SKIP_UBUNTU_16

- label: ":ubuntu: [Ubuntu] 18.04 Build"
command:
- "mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true /workdir && make -j $(getconf _NPROCESSORS_ONLN)"
- "cd /workdir/build && ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
agents:
queue: "automation-eos-builder-fleet"
plugins:
- docker#v3.2.0:
shell: ["/bin/bash", "-i", "-e", "-c"]
debug: $DEBUG
image: "eosio/producer:ci-ubuntu-18.04-254c25845ae6f00c8c4c63a90a5f49e7e61b5662"
image: "eosio/producer:eosio-ubuntu-18.04-d3c8fb71bf95415b85617bd29dae56590faf5267"
timeout: $BUILD_TIMEOUT
skip: $SKIP_UBUNTU_18

Expand Down
5 changes: 0 additions & 5 deletions .cicd/travis-before-cache.sh

This file was deleted.

8 changes: 0 additions & 8 deletions .cicd/travis-before-install.sh

This file was deleted.

23 changes: 6 additions & 17 deletions .cicd/travis-build.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
#!/bin/bash
set -e
#!/usr/bin/env bash
set -eo pipefail
cd $( dirname "${BASH_SOURCE[0]}" )/.. # Ensure we're in the repo root and not inside of scripts
. ./.cicd/.helpers

CPU_CORES=$(getconf _NPROCESSORS_ONLN)
if [[ "$(uname)" == Darwin ]]; then
ccache -s
export PATH="/usr/local/opt/ccache/libexec:$PATH"
echo 'Detected Darwin, building natively.'
[[ -d eos ]] && cd eos
[[ ! -d build ]] && mkdir build
Expand All @@ -16,17 +17,5 @@ if [[ "$(uname)" == Darwin ]]; then
echo "$ ctest -j $CPU_CORES -LE _tests --output-on-failure -T Test"
ctest -j $CPU_CORES -LE _tests --output-on-failure -T Test # run unit tests
else # linux
echo 'Detected Linux, building in Docker.'
echo "$ docker pull eosio/producer:ci-${IMAGE_TAG}"
docker pull eosio/producer:ci-${IMAGE_TAG}
# per-distro additions to docker commands
[[ $IMAGE_TAG == centos-7 ]] && PRE_COMMANDS="source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable &&"
[[ $IMAGE_TAG == amazonlinux-2 ]] && CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" # explicitly set to clang as gcc is default
[[ $IMAGE_TAG == ubuntu-16.04 ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" # pinned only
[[ $IMAGE_TAG == amazonlinux-2 || $IMAGE_TAG == centos-7 ]] && EXPORTS="export PATH=/usr/lib64/ccache:$PATH &&" || EXPORTS="$EXPORTS export PATH=/usr/lib/ccache:$PATH &&" # ccache needs to come first in the list (devtoolset-8 overrides that if we include this in the Dockerfile)
# docker
docker run --rm -v $(pwd):/eos -v /usr/lib/ccache -v $HOME/.ccache:/opt/.ccache -e CCACHE_DIR=/opt/.ccache eosio/producer:ci-${IMAGE_TAG} bash -c " \
$PRE_COMMANDS ccache -s && \
mkdir /eos/build && cd /eos/build && $EXPORTS cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /eos && make -j $(getconf _NPROCESSORS_ONLN)
ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
execute docker run --rm -v $(pwd):/workdir -v /usr/lib/ccache -v $HOME/.ccache:/opt/.ccache -e CCACHE_DIR=/opt/.ccache $FULL_TAG
fi
9 changes: 9 additions & 0 deletions .cicd/ubuntu-16.04.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,12 @@ RUN curl -LO https://github.com/ccache/ccache/releases/download/v3.4.1/ccache-3.
&& make \
&& make install \
&& cd / && rm -rf ccache-3.4.1/

# PRE_COMMANDS: Executed pre-cmake
# CMAKE_EXTRAS: Executed right before the cmake path (on the end)
ENV PRE_COMMANDS="export PATH=/usr/lib/ccache:$PATH &&"
ENV CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE='/tmp/pinned_toolchain.cmake' -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"

CMD bash -c "$PRE_COMMANDS ccache -s && \
mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \
ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
13 changes: 9 additions & 4 deletions .cicd/ubuntu-18.04.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ RUN apt-get update && apt-get upgrade -y \
bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \
autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \
autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \
libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang
libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache

# Build appropriate version of CMake.
RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz \
Expand Down Expand Up @@ -57,7 +57,12 @@ RUN curl -L https://github.com/mongodb/mongo-cxx-driver/archive/r3.4.0.tar.gz -o
&& cd / \
&& rm -f mongo-cxx-driver-r3.4.0.tar.gz

# CCACHE
RUN apt-get install -y ccache
ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin

ENV PATH=${PATH}:/mongodb-linux-x86_64-ubuntu1804-4.1.1/bin
# PRE_COMMANDS: Executed pre-cmake
# CMAKE_EXTRAS: Executed right before the cmake path (on the end)
ENV PRE_COMMANDS="export PATH=/usr/lib/ccache:$PATH &&"

CMD bash -c "$PRE_COMMANDS ccache -s && \
mkdir /workdir/build && cd /workdir/build && cmake -DCMAKE_BUILD_TYPE='Release' -DCORE_SYMBOL_NAME='SYS' -DOPENSSL_ROOT_DIR='/usr/include/openssl' -DBUILD_MONGO_DB_PLUGIN=true $CMAKE_EXTRAS /workdir && make -j $(getconf _NPROCESSORS_ONLN) && \
ctest -j$(getconf _NPROCESSORS_ONLN) -LE _tests --output-on-failure -T Test"
6 changes: 2 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,8 @@ matrix:
- openssl
- boost@1.70
- ccache
before_cache: |
./.cicd/travis-before-cache.sh
before_install: |
./.cicd/travis-before-install.sh
env:
- PATH="/usr/local/opt/ccache/libexec:$PATH"
script: |
./.cicd/travis-build.sh
notifications:
Expand Down

0 comments on commit 3596f16

Please sign in to comment.