Jenkins integration (#80)

Updates conda scripts to uses sccache (ccache work-alike with an S3 backend) Adds 4 Jenkins stages: 1. check 1. build 1. test 1. docs build Results are publised into S3 with a 30 day retention police and are available via: `https://downloads.rapids.ai/ci/morpheus/pull-request/<pull-req id>/<git commit>/<arch>/` ex: https://downloads.rapids.ai/ci/morpheus/pull-request/80/5161d30673f4d5fb34452df5a8ac6f68b8462685/x86_64/ Intermediate Artifacts pushed to S3: * conda_env.tar.gz - Morpheus conda environment used by test & documentation stages * workspace.tar.bz - Morpheus workspace used by test & documentation stages Results pushed to S3: * docs.tar.bz - Sphinx HTML documentation * report_pytest.xml - Pytest junit report * report_pytest_coverage.xml - Pytest coverage report * conda_build.tar.bz - Morpheus conda build (not currently working) Conda builds for cudf are cached seperately (even with sscache this step was still took 17 minutes). S3 cache location at: `https://downloads.rapids.ai/ci/morpheus/cudf/<cuda ver>/<python ver>/<rapids ver>/<latest git commit to ci/conda>/<arch>cudf_conda.tar.bz` ex: https://downloads.rapids.ai/ci/morpheus/cudf/11.4/3.8/21.10/b7f728e6db4fc6124970935f461a6791818bfb1d/x86_64/cudf_conda.tar.bz TODO List: - [x] Need a test stage - [x] Need to run style checks - [x] Need to add sccache - [x] Need to cache cudf conda packages - [x] Need to be able to build without using a gpu - [x] Output junit style xml - [x] Output docs build - [ ] Add conda build step - [ ] Need to figure out a way to cache lfs Closes #102 Closes #104 Authors: - David Gardner (https://github.com/dagardner-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: #80
nv-morpheus · May 23, 2022 · c52cb24 · c52cb24
1 parent 91225b9
commit c52cb24
Show file tree

Hide file tree

Showing 12 changed files with 526 additions and 11 deletions.
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -0,0 +1,147 @@
+pipeline {
+    agent any
+    options {
+        disableConcurrentBuilds(abortPrevious: true)
+    }
+    environment {
+        PYTHON_VER = '3.8'
+        RAPIDS_VER = '21.10'
+        CUDA_VER = '11.4'
+        HOME = "${WORKSPACE}"
+    }
+    stages {
+        stage('Checks & Builds') {
+            failFast true
+            parallel {
+                stage('Check') {
+                    options {
+                        timeout(time: 1, unit: 'HOURS')
+                    }
+                    agent {
+                        docker {
+                            image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                            label 'cpu'
+                        }
+                    }
+                    steps {
+                        cleanWs(
+                            deleteDirs: true,
+                            externalDelete: 'sudo rm -rf %s'
+                        )
+                        checkout scm
+                        sh "${WORKSPACE}/ci/scripts/jenkins/checks.sh"
+                    }
+                }
+                stage('Build:linux:x86_64:gcc') {
+                    options {
+                        timeout(time: 4, unit: 'HOURS')
+                    }
+                    environment {
+                        PARALLEL_LEVEL = '10'
+                        USE_SCCACHE = '1'
+                        HOME = "${WORKSPACE}"
+                    }
+                    agent {
+                        docker {
+                            image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                            label 'driver-495'
+                            args '--runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"'
+                        }
+                    }
+                    steps {
+                        cleanWs(
+                            deleteDirs: true,
+                            externalDelete: 'sudo rm -rf %s'
+                        )
+                        checkout scm
+                        withCredentials([[
+                            $class: 'AmazonWebServicesCredentialsBinding',
+                            credentialsId: "aws-s3-gpuci",
+                            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                        ]])
+                        {
+                            sh "${WORKSPACE}/ci/scripts/jenkins/build.sh"
+                        }
+                    }
+                }
+            }
+        }
+        stage('Post Build Stages') {
+            failFast true
+            parallel {
+                stage('Test') {
+                    options {
+                        timeout(time: 1, unit: 'HOURS')
+                    }
+                    agent {
+                        docker {
+                            image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                            label 'driver-495'
+                            args '--runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"'
+                        }
+                    }
+                    environment {
+                        HOME = "${WORKSPACE}"
+                    }
+                    steps {
+                        cleanWs(
+                            deleteDirs: true,
+                            externalDelete: 'sudo rm -rf %s'
+                        )
+                        checkout scm
+                        withCredentials([[
+                            $class: 'AmazonWebServicesCredentialsBinding',
+                            credentialsId: "aws-s3-gpuci",
+                            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                        ]])
+                        {
+                            sh "${WORKSPACE}/ci/scripts/jenkins/test.sh"
+                        }
+                    }
+                }
+                stage('Documentation') {
+                    options {
+                        timeout(time: 1, unit: 'HOURS')
+                    }
+                    agent {
+                        docker {
+                            image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                            label 'driver-495'
+                            args '--runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"'
+                        }
+                    }
+                    environment {
+                        HOME = "${WORKSPACE}"
+                        MORPHEUS_NO_CPP = '1'
+                    }
+                    steps {
+                        cleanWs(
+                            deleteDirs: true,
+                            externalDelete: 'sudo rm -rf %s'
+                        )
+                        checkout scm
+                        withCredentials([[
+                            $class: 'AmazonWebServicesCredentialsBinding',
+                            credentialsId: "aws-s3-gpuci",
+                            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                        ]])
+                        {
+                            sh "${WORKSPACE}/ci/scripts/jenkins/docs.sh"
+                        }
+                    }
+                }
+            }
+        }
+    }
+    post {
+        always {
+                cleanWs(
+                    deleteDirs: true,
+                    externalDelete: 'sudo rm -rf %s'
+                )
+        }
+    }
+}
diff --git a/ci/conda/recipes/cudf/meta.yaml b/ci/conda/recipes/cudf/meta.yaml
@@ -53,7 +53,7 @@ requirements:
     - {{ compiler("cxx") }}
     - ccache
   host:
-    - protobuf
+    - protobuf=3.19
     - python {{ python }}
     - cython >=0.29,<0.30
     - setuptools
@@ -65,7 +65,7 @@ requirements:
     - cudatoolkit {{ cuda_version }}
     - rapidjson=1.1
   run:
-    - protobuf
+    - protobuf=3.19
     - python
     - typing_extensions
     - pandas >=1.0,<1.4.0dev0

diff --git a/ci/conda/recipes/morpheus/morpheus_build.sh b/ci/conda/recipes/morpheus/morpheus_build.sh
@@ -21,6 +21,7 @@
 CMAKE_ARGS=${CMAKE_ARGS:-""}
 
 export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
+export USE_SCCACHE=${USE_SCCACHE:-""}
 
 # Check for some neo environment variables. Append to front of args to allow users to overwrite them
 if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
@@ -36,7 +37,13 @@ CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
 CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
+
+if [[ "${USE_SCCACHE}" == "" ]]; then
+   CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
+else
+   CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=OFF ${CMAKE_ARGS}"
+fi
+
 CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}"

diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh
@@ -39,6 +39,7 @@ export NEO_GIT_TAG=${NEO_GIT_TAG:-"5b55e37c6320c1a5747311a1e29e7ebb049d12bc"}
 
 # Set CONDA_CHANNEL_ALIAS to mimic the conda config channel_alias property during the build
 CONDA_CHANNEL_ALIAS=${CONDA_CHANNEL_ALIAS:-""}
+export USE_SCCACHE=${USE_SCCACHE:-""}
 
 export CUDA="$(conda list | grep cudatoolkit | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+")"
 export PYTHON_VER="$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")"
@@ -48,21 +49,31 @@ echo "PYTHON_VER  : ${PYTHON_VER}"
 echo "NEO_GIT_TAG : ${NEO_GIT_TAG}"
 echo ""
 
+export CMAKE_GENERATOR="Ninja"
+
 # Export variables for the cache
 export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"}
-
-# Export CCACHE variables
 export CCACHE_DIR="${MORPHEUS_CACHE_DIR}/ccache"
 export CCACHE_NOHASHDIR=1
-export CMAKE_GENERATOR="Ninja"
-export CMAKE_C_COMPILER_LAUNCHER="ccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="ccache"
-export CMAKE_CUDA_COMPILER_LAUNCHER="ccache"
 
 # Ensure the necessary folders exist before continuing
 mkdir -p ${MORPHEUS_CACHE_DIR}
 mkdir -p ${CCACHE_DIR}
 
+# Local builds use ccache
+# ci builds will use sccache which is a ccache work-alike but uses an S3 backend
+# (https://github.com/mozilla/sccache)
+if [[ "${USE_SCCACHE}" == "" ]]; then
+   # Export CCACHE variables
+   export CMAKE_C_COMPILER_LAUNCHER="ccache"
+   export CMAKE_CXX_COMPILER_LAUNCHER="ccache"
+   export CMAKE_CUDA_COMPILER_LAUNCHER="ccache"
+else
+   export CMAKE_C_COMPILER_LAUNCHER="sccache"
+   export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
+   export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
+fi
+
 # Holds the arguments in an array to allow for complex json objects
 CONDA_ARGS_ARRAY=()
 

diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -25,8 +25,8 @@ export PY_DIRS="${PY_ROOT} ci/scripts"
 export YAPF_EXCLUDE_FLAGS="-e ${PY_ROOT}/versioneer.py -e ${PY_ROOT}/morpheus/_version.py"
 
 # Determine the commits to compare against. If running in CI, these will be set. Otherwise, diff with main
-export BASE_SHA=${CI_MERGE_REQUEST_DIFF_BASE_SHA:-${BASE_SHA:-main}}
-export COMMIT_SHA=${CI_COMMIT_SHA:-${COMMIT_SHA:-HEAD}}
+export BASE_SHA=${CHANGE_TARGET:-${BASE_SHA:-main}}
+export COMMIT_SHA=${GIT_COMMIT:-${COMMIT_SHA:-HEAD}}
 
 export CPP_FILE_REGEX='^(\.\/)?(src|include|tests|benchmarks|python)\/.*\.(cc|cpp|h|hpp)$'
 export PYTHON_FILE_REGEX='^(\.\/)?(?!\.|build).*\.(py|pyx|pxd)$'

diff --git a/ci/scripts/jenkins/build.sh b/ci/scripts/jenkins/build.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+source ${WORKSPACE}/ci/scripts/jenkins/common.sh
+
+gpuci_logger "Creating conda env"
+conda config --add pkgs_dirs /opt/conda/pkgs
+conda config --env --add channels conda-forge
+conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS:-"https://conda.anaconda.org"}
+mamba install -q -y -n base -c conda-forge "boa >=0.10"
+mamba create -q -y -n morpheus python=${PYTHON_VER}
+conda activate morpheus
+
+gpuci_logger "Installing CI dependencies"
+mamba env update -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_ci.yml
+
+gpuci_logger "Check versions"
+python3 --version
+gcc --version
+g++ --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+gpuci_logger "Checking S3 cuDF cache"
+CUDF_CONDA_BLD_DIR=/opt/conda/conda-bld
+CUDF_CONDA_COMMIT=$(git log -n 1 --pretty=format:%H -- ci/conda)
+CUDF_CONDA_CACHE_PATH="/cudf/${CUDA_VER}/${PYTHON_VER}/${RAPIDS_VER}/${CUDF_CONDA_COMMIT}/${NVARCH}/cudf_conda.tar.bz"
+CUDF_CONDA_CACHE_URL="${S3_URL}${CUDF_CONDA_CACHE_PATH}"
+CUDF_CONDA_TAR="${WORKSPACE_TMP}/cudf_conda.tar.bz"
+
+gpuci_logger "Checking ${DISPLAY_URL}${CUDF_CONDA_CACHE_PATH}"
+set +e
+aws s3 cp --no-progress ${CUDF_CONDA_CACHE_URL} ${CUDF_CONDA_TAR}
+CUDF_CACHE_CHECK=$?
+set -e
+
+if [[ "${CUDF_CACHE_CHECK}" != "0" ]]; then
+      gpuci_logger "Cache miss, Building cuDF"
+      mkdir -p ${CUDF_CONDA_BLD_DIR}
+      # The --no-build-id bit is needed for sccache
+      CONDA_ARGS="--no-build-id --output-folder ${CUDF_CONDA_BLD_DIR} --skip-existing --no-test" ${MORPHEUS_ROOT}/ci/conda/recipes/run_conda_build.sh libcudf cudf
+
+      gpuci_logger "sccache usage for cudf build:"
+      sccache --show-stats
+      sccache --zero-stats 2>&1 > /dev/null
+
+      gpuci_logger "Archiving cuDF build"
+      cd $(dirname ${CUDF_CONDA_BLD_DIR})
+      tar cfj ${CUDF_CONDA_TAR} $(basename ${CUDF_CONDA_BLD_DIR})
+      cd -
+      aws s3 cp --no-progress ${CUDF_CONDA_TAR} ${CUDF_CONDA_CACHE_URL}
+else
+      gpuci_logger "Cache hit, using cached cuDF"
+      cd $(dirname ${CUDF_CONDA_BLD_DIR})
+      tar xf ${CUDF_CONDA_TAR}
+      cd -
+fi
+
+gpuci_logger "Installing cuDF"
+mamba install -q -y -c local -c nvidia -c rapidsai -c conda-forge libcudf cudf
+
+gpuci_logger "Installing other dependencies"
+mamba env update -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml
+
+gpuci_logger "Check cmake & ninja"
+cmake --version
+ninja --version
+
+gpuci_logger "Configuring cmake for Morpheus"
+cmake -B build -G Ninja \
+      -DCMAKE_MESSAGE_CONTEXT_SHOW=ON \
+      -DMORPHEUS_BUILD_BENCHMARKS=ON \
+      -DMORPHEUS_BUILD_EXAMPLES=ON \
+      -DMORPHEUS_BUILD_TESTS=ON \
+      -DMORPHEUS_USE_CONDA=ON \
+      -DMORPHEUS_PYTHON_INPLACE_BUILD=ON \
+      -DMORPHEUS_USE_CCACHE=OFF \
+      -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+      -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+      -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache \
+      .
+
+gpuci_logger "Building Morpheus"
+cmake --build build -j --parallel ${PARALLEL_LEVEL}
+
+gpuci_logger "sccache usage for morpheus build:"
+sccache --show-stats
+
+gpuci_logger "Installing Morpheus"
+pip install -e ${MORPHEUS_ROOT}
+
+gpuci_logger "Archiving results"
+mamba pack --quiet --force --ignore-editable-packages --ignore-missing-files --n-threads ${PARALLEL_LEVEL} -n morpheus -o ${WORKSPACE_TMP}/conda_env.tar.gz
+tar cfj ${WORKSPACE_TMP}/workspace.tar.bz --exclude=".git" --exclude="models" --exclude=".cache" ./
+ls -lh ${WORKSPACE_TMP}/
+
+gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
+aws s3 cp --no-progress "${WORKSPACE_TMP}/conda_env.tar.gz" "${ARTIFACT_URL}/conda_env.tar.gz"
+aws s3 cp --no-progress "${WORKSPACE_TMP}/workspace.tar.bz" "${ARTIFACT_URL}/workspace.tar.bz"
+
+gpuci_logger "Success"
+exit 0
diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+source ${WORKSPACE}/ci/scripts/jenkins/common.sh
+
+conda activate rapids
+
+gpuci_logger "Installing CI dependencies"
+mamba install -q -y -c conda-forge "yapf=0.32"
+
+gpuci_logger "Runing Python style checks"
+${MORPHEUS_ROOT}/ci/scripts/python_checks.sh
+
+gpuci_logger "Checking copyright headers"
+python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}