Improve the python CMake functionality to speed up the configure step (…

…nv-morpheus#754)

This is the corresponding update to incorporate changes from nv-morpheus/utilities#17.

Summary of changes:

1. This reorganizes the python CMake checks so they only get executed once (handled in utilities). Significantly speeding up the configure step.
1. Makes the CMake variable names consistent (breaking change)
   1. For example, `MORPHEUS_BUILD_PYTHON_STUBS` -> `MORPHEUS_PYTHON_BUILD_STUBS`
1. Removes extra cmake code that is now handled by utilities

Authors:
  - Michael Demoret (https://github.com/mdemoret-nv)

Approvers:
  - Christopher Harris (https://github.com/cwharris)

URL: nv-morpheus#754

CMakeLists.txt

@@ -19,20 +19,22 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 list(APPEND CMAKE_MESSAGE_CONTEXT "morpheus")
 
-# Global options
+# Global options (Keep sorted!)
 option(BUILD_SHARED_LIBS "Default value for whether or not to build shared or static libraries" ON)
 option(MORPHEUS_BUILD_BENCHMARKS "Whether or not to build benchmarks" OFF)
+option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF)
 option(MORPHEUS_BUILD_EXAMPLES "Whether or not to build examples" OFF)
 option(MORPHEUS_BUILD_TESTS "Whether or not to build tests" OFF)
-option(MORPHEUS_BUILD_PYTHON_STUBS "Whether or not to generated .pyi stub files for C++ Python modules. Disable to avoid requiring loading the NVIDIA GPU Driver during build" ON)
-option(MORPHEUS_BUILD_PYTHON_WHEEL "Whether or not to build the morpheus .whl file" OFF)
 option(MORPHEUS_ENABLE_DEBUG_INFO "Enable printing debug information" OFF)
+option(MORPHEUS_PYTHON_BUILD_STUBS "Whether or not to generated .pyi stub files for C++ Python modules. Disable to avoid requiring loading the NVIDIA GPU Driver during build" ON)
+option(MORPHEUS_PYTHON_BUILD_WHEEL "Whether or not to build the morpheus .whl file" OFF)
+option(MORPHEUS_PYTHON_INPLACE_BUILD "Whether or not to copy built python modules back to the source tree for debug purposes." OFF)
+option(MORPHEUS_PYTHON_PERFORM_INSTALL "Whether or not to automatically `pip install` any built python library. WARNING: This may overwrite any existing installation of the same name." OFF)
 option(MORPHEUS_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(MORPHEUS_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
-option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg.
-  Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" OFF)
+option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg. Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" OFF)
 option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
-option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF)
+
 set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory")
 
 set(MORPHEUS_RAPIDS_VERSION "22.10" CACHE STRING "Sets default versions for RAPIDS libraries.")
@@ -116,15 +118,17 @@ if("${CMAKE_EXECUTABLE_SUFFIX}" STREQUAL "")
   set(CMAKE_EXECUTABLE_SUFFIX ".x")
 endif()
 
+# Create a custom target to allow preparing for style checks
+add_custom_target(${PROJECT_NAME}_style_checks
+  COMMENT "Building dependencies for style checks"
+)
+
 
 ##################################
 ##### Morpheus Python Setup ######
 ##################################
 
-# Create a custom target to allow preparing for style checks
-add_custom_target(${PROJECT_NAME}_style_checks
-  COMMENT "Building dependencies for style checks"
-)
+morpheus_utils_python_configure()
 
 # Include the main morpheus code
 morpheus_utils_create_python_package(morpheus
@@ -139,7 +143,7 @@ if(MORPHEUS_PYTHON_INPLACE_BUILD)
   list(APPEND extra_args "IS_INPLACE")
 endif()
 
-if(MORPHEUS_BUILD_PYTHON_WHEEL)
+if(MORPHEUS_PYTHON_BUILD_WHEEL)
   list(APPEND extra_args "BUILD_WHEEL")
 endif()
 

ci/conda/recipes/morpheus/meta.yaml

@@ -37,7 +37,7 @@ outputs:
         # Note: Setting a default value here will overwrite the value from the
         # shell! Defaults should be set in the run_conda_build.sh script
         - CMAKE_CUDA_ARCHITECTURES
-        - MORPHEUS_BUILD_PYTHON_STUBS
+        - MORPHEUS_PYTHON_BUILD_STUBS
         - MORPHEUS_CACHE_DIR
         - PARALLEL_LEVEL
       run_exports:

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -38,9 +38,9 @@ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
 CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_STUBS=${MORPHEUS_BUILD_PYTHON_STUBS=-"ON"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_WHEEL=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"all"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"

ci/conda/recipes/run_conda_build.sh

@@ -45,7 +45,7 @@ export MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
 
 # Export script_env variables that must be set for conda build
 export CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:-"ALL"}
-export MORPHEUS_BUILD_PYTHON_STUBS=${MORPHEUS_BUILD_PYTHON_STUBS:-"ON"}
+export MORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS:-"ON"}
 export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"}
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
 

ci/scripts/github/build.sh

@@ -32,7 +32,7 @@ rapids-logger "Configuring cmake for Morpheus"
 git submodule update --init --recursive
 cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} \
     -DCCACHE_PROGRAM_PATH=$(which sccache) \
-    -DMORPHEUS_BUILD_PYTHON_WHEEL=ON \
+    -DMORPHEUS_PYTHON_BUILD_WHEEL=ON \
     -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON .
 
 rapids-logger "Building Morpheus"

ci/scripts/github/common.sh

@@ -60,7 +60,7 @@ export SCCACHE_REGION="us-east-2"
 export SCCACHE_IDLE_TIMEOUT=32768
 #export SCCACHE_LOG=debug
 
-export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_BUILD_PYTHON_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
+export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
 
 export FETCH_STATUS=0
 

docker/Dockerfile

@@ -161,7 +161,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke
     source activate base &&\
     # Need to get around recent versions of git locking paths until they are deemed safe
     git config --global --add safe.directory "*" &&\
-    MORPHEUS_BUILD_PYTHON_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus
+    MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus
 
 # sid_visualization is a submodule we need to init
 RUN git submodule update --init --recursive

morpheus/CMakeLists.txt

@@ -15,20 +15,9 @@
 
 list(APPEND CMAKE_MESSAGE_CONTEXT "morpheus")
 
-option(MORPHEUS_PYTHON_INPLACE_BUILD
-        "Whether or not to copy built python modules back to the source tree for debug purposes." OFF)
-option(MORPHEUS_PYTHON_PERFORM_INSTALL
-        "Whether or not to automatically `pip install` any built python library. WARNING: This may overwrite any existing installation of the same name." OFF)
-option(MORPHEUS_BUILD_PYTHON_STUBS
-        "Whether or not to generate stubs for python files." ON)
-
-morpheus_utils_python_modules_ensure_loaded()
-
 # Save the root of the python for relative paths
 set(MORPHEUS_PY_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..)
 
-# Determine the install directory for built modules
-
 add_subdirectory(_lib)
 
 

morpheus/_lib/CMakeLists.txt

@@ -13,13 +13,6 @@
 # =============================================================================
 list(APPEND CMAKE_MESSAGE_CONTEXT "_lib")
 
-find_package(pybind11 REQUIRED)
-
-if(MORPHEUS_PYTHON_INPLACE_BUILD)
-  message(STATUS "Performing inplace build for Python modules and library dependencies. Use `pip install -e .`\
-   from the root folder to install Morpheus.")
-endif()
-
 ########### morpheus ###########
 set(MORPHEUS_LIB_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -33,6 +26,9 @@ include(cmake/libraries/cudf_helpers.cmake)
 include(cmake/libraries/morpheus.cmake)
 
 ########### py_morpheus ########
+# Set the default link targets to avoid repeating this
+morpheus_utils_python_package_set_default_link_targets(morpheus mrc::pymrc)
+
 #----------morpheus._lib.stages---------
 include(cmake/python_modules/stages.cmake)
 

morpheus/_lib/cmake/libraries/cuda_utils.cmake

@@ -35,9 +35,6 @@ set_target_properties(
 target_include_directories(cuda_utils_objs
     PUBLIC
       "${MORPHEUS_LIB_ROOT}/include"
-      cudf::cudf
-      matx::matx
-      mrc::pymrc
 )
 
 target_link_libraries(cuda_utils_objs
@@ -83,14 +80,11 @@ set_target_properties(cuda_utils
     PROPERTIES OUTPUT_NAME ${PROJECT_NAME}_utils
 )
 
-message(STATUS " Install dest: (cuda_utils) ${MORPHEUS_LIB_INSTALL_DIR}")
 install(
     TARGETS
       cuda_utils
     EXPORT
       ${PROJECT_NAME}-exports
-    LIBRARY DESTINATION
-      "${MORPHEUS_LIB_INSTALL_DIR}"
     COMPONENT Wheel
 )
 

morpheus/_lib/cmake/libraries/cudf_helpers.cmake

@@ -12,20 +12,14 @@
 # the License.
 # =============================================================================
 
-morpheus_utils_add_cython_library(
+morpheus_add_cython_library(
     cudf_helpers
-    MODULE_ROOT
-      "${MORPHEUS_LIB_ROOT}"
     PYX_FILE
       "${MORPHEUS_LIB_ROOT}/cudf_helpers.pyx"
-    INCLUDE_DIRS
-      "${MORPHEUS_LIB_ROOT}/include"
     LINK_TARGETS
       cuda_utils
     OUTPUT_TARGET
       cudf_helpers_target
-    INSTALL_DEST
-      ${MORPHEUS_LIB_INSTALL_DIR}
 )
 
 # This target generates headers used by other parts of the code base.
@@ -42,7 +36,3 @@ set_target_properties(
       CXX_INCLUDE_WHAT_YOU_USE ""
       EXPORT_COMPILE_COMMANDS OFF
 )
-
-if (MORPHEUS_PYTHON_INPLACE_BUILD)
-  morpheus_utils_inplace_build_copy(${cudf_helpers_target} ${MORPHEUS_LIB_ROOT})
-endif()

morpheus/_lib/cmake/libraries/morpheus.cmake

@@ -70,18 +70,16 @@ target_link_libraries(morpheus
 target_include_directories(morpheus
     PUBLIC
       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+      $<INSTALL_INTERFACE:include>
 )
 
 set_target_properties(morpheus PROPERTIES CXX_VISIBILITY_PRESET hidden)
 
-message(STATUS " Install dest: (morpheus) ${MORPHEUS_LIB_INSTALL_DIR}")
 install(
     TARGETS
       morpheus
     EXPORT
       ${PROJECT_NAME}-exports
-    LIBRARY DESTINATION
-      "${MORPHEUS_LIB_INSTALL_DIR}"
     COMPONENT Wheel
 )
 

morpheus/_lib/cmake/python_modules/common.cmake

@@ -12,23 +12,8 @@
 # the License.
 # =============================================================================
 
-morpheus_utils_add_pybind11_module(
+morpheus_add_pybind11_module(
     common
-    MODULE_ROOT
-      "${MORPHEUS_LIB_ROOT}"
     SOURCE_FILES
       "${MORPHEUS_LIB_ROOT}/src/python_modules/common.cpp"
-    INCLUDE_DIRS
-      "${MORPHEUS_LIB_ROOT}/include"
-    LINK_TARGETS
-      morpheus
-      mrc::pymrc
-    OUTPUT_TARGET
-      common_target
-    INSTALL_DEST
-      ${MORPHEUS_LIB_INSTALL_DIR}
 )
-
-if(MORPHEUS_PYTHON_INPLACE_BUILD)
-  morpheus_utils_inplace_build_copy(${common_target} ${MORPHEUS_LIB_ROOT})
-endif()

morpheus/_lib/cmake/python_modules/messages.cmake

@@ -14,21 +14,6 @@
 
 morpheus_utils_add_pybind11_module(
     messages
-    MODULE_ROOT
-      "${MORPHEUS_LIB_ROOT}"
     SOURCE_FILES
       "${MORPHEUS_LIB_ROOT}/src/python_modules/messages.cpp"
-    INCLUDE_DIRS
-      "${MORPHEUS_LIB_ROOT}/include"
-    LINK_TARGETS
-      morpheus
-      mrc::pymrc
-    OUTPUT_TARGET
-      messages_target
-    INSTALL_DEST
-      ${MORPHEUS_LIB_INSTALL_DIR}
 )
-
-if(MORPHEUS_PYTHON_INPLACE_BUILD)
-  morpheus_utils_inplace_build_copy(${messages_target} ${MORPHEUS_LIB_ROOT})
-endif()

morpheus/_lib/cmake/python_modules/stages.cmake

@@ -14,21 +14,6 @@
 
 morpheus_utils_add_pybind11_module(
     stages
-    MODULE_ROOT
-      "${MORPHEUS_LIB_ROOT}"
     SOURCE_FILES
       "${MORPHEUS_LIB_ROOT}/src/python_modules/stages.cpp"
-    INCLUDE_DIRS
-      "${MORPHEUS_LIB_ROOT}/include"
-    LINK_TARGETS
-      morpheus
-      mrc::pymrc
-    OUTPUT_TARGET
-      stages_target
-    INSTALL_DEST
-      ${MORPHEUS_LIB_INSTALL_DIR}
 )
-
-if(MORPHEUS_PYTHON_INPLACE_BUILD)
-  morpheus_utils_inplace_build_copy(${stages_target} ${MORPHEUS_LIB_ROOT})
-endif()