Updating tests to force .pyi files to be committed into the repo (nv-…

…morpheus#1111)

This PR changes the stub generation to be committed into the repo instead of as part of the build. This will allow CI steps that need the PYI files to not require a GPU as well.

This works by always copying the PYI files back to the source tree so they will show up as diffs. Any diffs during the test phase will be reported as an error.

Closes nv-morpheus#766

Authors:
  - Michael Demoret (https://github.com/mdemoret-nv)
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Christopher Harris (https://github.com/cwharris)
  - David Gardner (https://github.com/dagardner-nv)

URL: nv-morpheus#1111

.gitignore

@@ -1,7 +1,4 @@
-./build
-*.engine
-.Dockerfile
-
+# Put new items at the bottom of this list!!!
 
 # Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology
 
@@ -181,26 +178,27 @@ tags
 
 
 # End of https://www.gitignore.io/api/vim,c++,cmake,python,synology
+
+./build
+.conda-bld
+.Dockerfile
 .tmp
-examples/data/dfp
-examples/digital_fingerprinting/**/dfp_detections_*.csv
-viz_frames*
-notebooks/output
-dask-worker-space
 *.dirlock
+*.engine
 *.lock
-triton_models/**/*.onnx
-triton_models/**/*.engine
+/build*/
+/conda-build/
+dask-worker-space
+data/*
 docs/source/_lib
 docs/source/_modules
-data/*
+examples/data/dfp
+examples/digital_fingerprinting/**/dfp_detections_*.csv
 mlruns/*
-/build*/
-/conda-build/
-.conda-bld
-
-# Ignore generated pyi files for pybind and cython modules
-morpheus/_lib/**/*.pyi
+notebooks/output
+triton_models/**/*.engine
+triton_models/**/*.onnx
+viz_frames*
 
 # Explicitly ignore .vscode/. Shared settings should go in morpheus.code-workspace
 # and user settings will go in .vscode/

ci/release/update-version.sh

@@ -67,7 +67,12 @@ function sed_runner() {
 # .gitmodules
 git submodule set-branch -b branch-${NEXT_SHORT_TAG} external/morpheus-visualizations
 git submodule set-branch -b branch-${NEXT_SHORT_TAG} morpheus_utils
-git submodule update --remote --recursive
+
+if [[ "$(git diff --name-only | grep .gitmodules)" != "" ]]; then
+   # Only update the submodules if setting the branch changed .gitmodules. Otherwise this will undo the current commit
+   # for any submodules resulting in differences always appearing in CI
+   git submodule update --remote --recursive
+fi
 
 # Root CMakeLists.txt
 sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt

ci/scripts/common.sh

@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# set -x
-
 export SCRIPT_DIR=${SCRIPT_DIR:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"}
-export REPO_DIR=$(realpath ${REPO_DIR:-"${SCRIPT_DIR}/../.."})
-export PY_ROOT="${REPO_DIR}"
+
+# The root to the Morpheus repo
+export MORPHEUS_ROOT=${MORPHEUS_ROOT:-"$(realpath ${SCRIPT_DIR}/../..)"}
+
+export PY_ROOT="${MORPHEUS_ROOT}"
 export PY_CFG="${PY_ROOT}/setup.cfg"
 export PY_DIRS="${PY_ROOT} ci/scripts"
 
@@ -40,7 +41,7 @@ export SKIP_ISORT=${SKIP_ISORT:-""}
 export SKIP_YAPF=${SKIP_YAPF:-""}
 
 # Set BUILD_DIR to use a different build folder
-export BUILD_DIR=${BUILD_DIR:-"${REPO_DIR}/build"}
+export BUILD_DIR=${BUILD_DIR:-"${MORPHEUS_ROOT}/build"}
 
 # Speficy the clang-tools version to use. Default 14
 export CLANG_TOOLS_VERSION=${CLANG_TOOLS_VERSION:-14}
@@ -194,4 +195,4 @@ function cleanup {
 trap cleanup EXIT
 
 # Change directory to the repo root
-pushd "${REPO_DIR}" &> /dev/null
+pushd "${MORPHEUS_ROOT}" &> /dev/null

ci/scripts/github/test.sh

@@ -21,17 +21,46 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-download_artifact "wheel.tar.bz"
-download_artifact "cpp_tests.tar.bz"
-download_artifact "morhpeus_libs.tar.bz"
-
-tar xf "${WORKSPACE_TMP}/wheel.tar.bz"
-tar xf "${WORKSPACE_TMP}/morhpeus_libs.tar.bz"
-tar xf "${WORKSPACE_TMP}/cpp_tests.tar.bz"
-
-# Install the built Morpheus python package
-cd ${MORPHEUS_ROOT}
-pip install ${MORPHEUS_ROOT}/build/dist/*.whl
+rapids-logger "Check versions"
+python3 --version
+x86_64-conda-linux-gnu-cc --version
+x86_64-conda-linux-gnu-c++ --version
+cmake --version
+ninja --version
+sccache --version
+
+git submodule update --init --recursive
+
+CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=ON"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=OFF"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON"
+if [[ "${LOCAL_CI}" == "" ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
+fi
+
+rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}"
+cmake -B build -G Ninja ${CMAKE_FLAGS} .
+
+rapids-logger "Building Morpheus"
+cmake --build build --parallel ${PARALLEL_LEVEL}
+
+if [[ "${LOCAL_CI}" == "" ]]; then
+    rapids-logger "sccache usage for morpheus build:"
+    sccache --show-stats
+fi
+
+rapids-logger "Checking Python stub files"
+
+# Check for git diffs which would mean the build is out of sync with the repo
+if [[ $(git status --short --untracked | grep .pyi) != "" ]]; then
+
+    echo "The Python stubs (*.pyi) are out of sync with repo. Please rerun the build locally with "
+    echo "'-DMORPHEUS_PYTHON_BUILD_STUBS=ON' and commit the stub files into the repo"
+    git status
+    exit 1
+fi
 
 CPP_TESTS=($(find ${MORPHEUS_ROOT}/build -name "*.x"))
 

ci/scripts/run_iwyu_for_ci.sh

@@ -23,7 +23,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Call iwyu_tool.py and append IWYU arguments onto the end
 ${IWYU_TOOL_PY:-iwyu_tool.py} "$@" -- \
-   -Xiwyu --mapping_file=${REPO_DIR:-${SCRIPT_DIR}/../..}/ci/iwyu/mappings.imp \
+   -Xiwyu --mapping_file=${MORPHEUS_ROOT:-${SCRIPT_DIR}/../..}/ci/iwyu/mappings.imp \
    -Xiwyu --verbose=${IWYU_VERBOSITY:-1} \
    -Xiwyu --no_fwd_decls \
    -Xiwyu --quoted_includes_first \