Updating build_conda_packages to specify which packages to build

docker/build_conda_packages.sh

@@ -16,6 +16,13 @@
 
 set -e
 
+NUMARGS=$#
+ARGS=$*
+
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Need to build the development docker container to setup git safe.directory
@@ -30,15 +37,54 @@ echo "Building container"
 ${SCRIPT_DIR}/build_container_dev.sh
 
 # Now run the container with the volume mount to build the packages
-export DOCKER_EXTRA_ARGS=""
+CONDA_ARGS=()
+CONDA_ARGS+=("--output-folder" "/workspace/.conda-bld")
+CONDA_ARGS+=("--skip-existing")
 
-echo "Running conda build"
+DOCKER_EXTRA_ARGS=()
+
+if hasArg libneo; then
+   # If libneo is specified, you must set NEO_GIT_URL
+   DOCKER_EXTRA_ARGS+=("--env" "NEO_GIT_URL=${NEO_GIT_URL:?"Cannot build libneo. Must set NEO_GIT_URL to git repo location to allow checkout of neo repository"}")
+
+   url=${NEO_GIT_URL}
+
+   # Remove the http/https/ssh
+   url="${url#http://}"
+   url="${url#https://}"
+   url="${url#ssh://}"
+
+   # Remove git@
+   url="${url#git@}"
 
-CONDA_ARGS="--output-folder=/workspace/.conda-bld"
-export DOCKER_EXTRA_ARGS="--env CONDA_ARGS=${CONDA_ARGS} --env NEO_GIT_URL=${NEO_GIT_URL:?"Cannot build libneo. Must set NEO_GIT_URL to git repo location to allow checkout of neo repository"}"
+   # Remove username/password
+   url="${url#*:*@}"
+   url="${url#*@}"
+
+   # Remove remaining
+   url=${url%%/*}
+
+   port=${url##*:}
+   url=${url%%:*}
+
+   # Add the command to auto accept this url/port combo
+   if [[ -n "${port}" ]]; then
+      BUILD_SCRIPT="${BUILD_SCRIPT:+${BUILD_SCRIPT}\n}mkdir -p \$HOME/.ssh && ssh-keyscan -t rsa -p ${port} ${url} > ~/.ssh/known_hosts"
+   else
+      BUILD_SCRIPT="${BUILD_SCRIPT:+${BUILD_SCRIPT}\n}mkdir -p \$HOME/.ssh && ssh-keyscan -t rsa ${url} > ~/.ssh/known_hosts"
+   fi
+fi
+
+# Build the script to execute inside of the container (needed to set multiple statements in CONDA_ARGS)
+BUILD_SCRIPT="${BUILD_SCRIPT}
+export CONDA_ARGS=\"${CONDA_ARGS[@]}\"
+./ci/conda/recipes/run_conda_build.sh "$@"
+"
+
+echo "Running conda build"
 
 # Run with an output folder that is mounted and skip existing to avoid repeated builds
-${SCRIPT_DIR}/run_container_dev.sh ./ci/conda/recipes/run_conda_build.sh libneo libcudf cudf
+DOCKER_EXTRA_ARGS="${DOCKER_EXTRA_ARGS[@]}" ${SCRIPT_DIR}/run_container_dev.sh bash -c "${BUILD_SCRIPT}"
 
 echo "Conda packages have been built. Use the following to install into an environment:"
-echo "    mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge neo cudf"
+echo "    mamba install -c file://$(realpath ${MORPHEUS_ROOT}/.conda-bld) -c nvidia -c rapidsai -c conda-forge $@"

docker/build_container.sh

@@ -29,7 +29,6 @@ LINUX_VER=${LINUX_VER:-20.04}
 RAPIDS_VER=${RAPIDS_VER:-21.10}
 PYTHON_VER=${PYTHON_VER:-3.8}
 TENSORRT_VERSION=${TENSORRT_VERSION:-8.2.1.3}
-NEO_GIT_URL=${NEO_GIT_URL:-UNKNOWN}
 
 DOCKER_ARGS="-t ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
 DOCKER_ARGS="${DOCKER_ARGS} --target ${DOCKER_TARGET}"
@@ -40,7 +39,6 @@ DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg TENSORRT_VERSION=${TENSORRT_VERSION}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg NEO_GIT_URL=${NEO_GIT_URL}"
 DOCKER_ARGS="${DOCKER_ARGS} --network=host"
 
 if [[ "${DOCKER_BUILDKIT}" = "1" ]]; then
@@ -62,7 +60,6 @@ echo "   LINUX_VER       : ${LINUX_VER}"
 echo "   RAPIDS_VER      : ${RAPIDS_VER}"
 echo "   PYTHON_VER      : ${PYTHON_VER}"
 echo "   TENSORRT_VERSION: ${TENSORRT_VERSION}"
-echo "   NEO_GIT_URL     : ${NEO_GIT_URL}"
 echo ""
 echo "   COMMAND: docker buildx build ${DOCKER_ARGS} -f docker/Dockerfile ."
 echo "   Note: add '--progress plain' to DOCKER_ARGS to show all container build output"

docker/build_container_release.sh

@@ -18,7 +18,7 @@
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 export DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"}
-export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"}
+export DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"runtime-$(git describe --tags --abbrev=0)"}
 export DOCKER_TARGET=${DOCKER_TARGET:-"runtime"}
 
 # Call the general build script

docker/run_container_dev.sh

@@ -39,4 +39,6 @@ fi
 
 echo -e "${g}Launching ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}...${x}"
 
+set -x
 docker run --rm -ti ${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS} ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} "${@:-bash}"
+set +x