[morpheus-refactor] Move morpheus source to python/morpheus (nv-morph…

…eus#1836)

This is the first PR to update the source layout. It moves the morpheus source code to a new sub-tree that will allow breaking out morpheus functional blocks from the morpheus-base -
```
.
|___ CMakeLists.txt
|___ python
|    |___ CMakeLists.txt
|    |___ morpheus
|    |    |___ CMakeLists.txt
|    |    |___ setup.py
|    |    |___ setup.cfg
|    |    |___ MANIFEST.in
|    |    |___ morpheus
|    |    |    |___ __init__.py
|    |    |    |___ _lib
|    |    |    |___ etc.
|    |___ morpheus_N
```

## By Submitting this PR I confirm:
- I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md).
- When the PR is ready for review, new or existing tests cover these changes.
- When the PR is ready for review, the documentation is up to date with these changes.

Authors:
  - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah)

Approvers:
  - David Gardner (https://github.com/dagardner-nv)
  - Christopher Harris (https://github.com/cwharris)

URL: nv-morpheus#1836

CMakeLists.txt

@@ -143,36 +143,7 @@ morpheus_utils_compiler_set_defaults(MORPHEUS_USE_CLANG_TIDY)
 # Setup IWYU if enabled
 include(environment/init_iwyu)
 
-# #################################
-# #### Morpheus Python Setup ######
-# #################################
-morpheus_utils_python_configure()
-
-# Include the main morpheus code
-morpheus_utils_create_python_package(morpheus
-  PROJECT_DIRECTORY "${CMAKE_SOURCE_DIR}"
-  SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/morpheus"
-)
-
-add_subdirectory(morpheus)
-
-# Complete the python package
-if(MORPHEUS_PYTHON_INPLACE_BUILD)
-  list(APPEND extra_args "IS_INPLACE")
-endif()
-
-if(MORPHEUS_PYTHON_BUILD_WHEEL)
-  list(APPEND extra_args "BUILD_WHEEL")
-endif()
-
-if(MORPHEUS_PYTHON_PERFORM_INSTALL)
-  list(APPEND extra_args "INSTALL_WHEEL")
-endif()
-
-file(GLOB morpheus_data_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus/data/*")
-morpheus_utils_add_python_sources(${morpheus_data_files})
-
-morpheus_utils_build_python_package(morpheus ${extra_args})
+add_subdirectory(python)
 
 if(MORPHEUS_BUILD_EXAMPLES)
   add_subdirectory(examples)

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -93,4 +93,4 @@ cmake -B ${BUILD_DIR} \
 cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
 
 # Install just the python wheel components
-${PYTHON} -m pip install -vv ${BUILD_DIR}/dist/*.whl
+${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl

ci/scripts/common.sh

@@ -19,7 +19,7 @@ export SCRIPT_DIR=${SCRIPT_DIR:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /de
 export MORPHEUS_ROOT=${MORPHEUS_ROOT:-"$(realpath ${SCRIPT_DIR}/../..)"}
 
 export PY_ROOT="${MORPHEUS_ROOT}"
-export PY_CFG="${PY_ROOT}/setup.cfg"
+export PY_CFG="${PY_ROOT}/python/morpheus/setup.cfg"
 export PY_DIRS="${PY_ROOT} ci/scripts"
 
 # Determine the commits to compare against. If running in CI, these will be set. Otherwise, diff with main

ci/scripts/github/build.sh

@@ -41,13 +41,13 @@ cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 log_sccache_stats
 
 rapids-logger "Archiving results"
-tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/dist
+tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" ${BUILD_DIR}/python/morpheus/dist
 
-MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
+MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;) \
                 $(find ${MORPHEUS_ROOT}/examples -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
 tar cfj "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${MORPHEUS_LIBS[@]}"
 
-CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
+CPP_TESTS=($(find ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
 tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}"
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"

ci/scripts/github/checks.sh

@@ -47,7 +47,7 @@ cmake --build ${BUILD_DIR} --parallel ${PARALLEL_LEVEL}
 log_sccache_stats
 
 rapids-logger "Installing Morpheus"
-pip install ./
+pip install ./python/morpheus
 
 rapids-logger "Checking copyright headers"
 python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}

ci/scripts/github/docs.sh

@@ -29,7 +29,7 @@ download_artifact "wheel.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/wheel.tar.bz"
 
-pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/dist/*.whl
+pip install ${MORPHEUS_ROOT}/${BUILD_DIR}/python/morpheus/dist/*.whl
 
 rapids-logger "Pulling LFS assets"
 cd ${MORPHEUS_ROOT}

docs/source/conf.py

@@ -64,7 +64,7 @@
 # built documents.
 
 # Load the _version file according to https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
-spec = importlib.util.spec_from_file_location("_version", "../../morpheus/_version.py")
+spec = importlib.util.spec_from_file_location("_version", "../../python/morpheus/morpheus/_version.py")
 module = importlib.util.module_from_spec(spec)
 sys.modules["_version"] = module
 spec.loader.exec_module(module)
@@ -135,7 +135,7 @@
         FILE_PATTERNS = *.c *.cc *.cpp *.h *.hpp *.cu *.cuh *.md
         HAVE_DOT = YES
         HIDE_UNDOC_MEMBERS = NO
-        INPUT = ../../morpheus/_lib
+        INPUT = ../../python/morpheus/morpheus/_lib
         INTERACTIVE_SVG = YES
         SOURCE_BROWSER = YES
         ENABLE_PREPROCESSING = YES

docs/source/developer_guide/contributing.md

@@ -197,7 +197,7 @@ git submodule update --init --recursive
    This script will run both CMake Configure with default options and CMake build.
 1. Install Morpheus
    ```bash
-   pip install -e ${MORPHEUS_ROOT}
+   pip install -e ${MORPHEUS_ROOT}/python/morpheus
    ```
    Once Morpheus has been built, it can be installed into the current virtual environment.
 1. Test the build (Note: some tests will be skipped)\

pyproject.toml

@@ -134,7 +134,7 @@ ignore = ["_version.py", "conf.py", "cudf_helpers.pyx", "github_link.py"]
 # ignore-list. The regex matches against paths and can be in Posix or Windows
 # format. Because '\\' represents the directory delimiter on Windows systems, it
 # can't be used as an escape character.
-ignore-paths = ["morpheus/models/dfencoder/*"]
+ignore-paths = ["python/morpheus/morpheus/models/dfencoder/*"]
 
 # Files or directories matching the regular expression patterns are skipped. The
 # regex matches against base names, not paths. The default value ignores Emacs

python/CMakeLists.txt

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+list(APPEND CMAKE_MESSAGE_CONTEXT "python")
+
+add_subdirectory(morpheus)
+
+list(POP_BACK CMAKE_MESSAGE_CONTEXT)

python/morpheus/CMakeLists.txt

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+list(APPEND CMAKE_MESSAGE_CONTEXT "morpheus")
+
+# Save the root of the python for relative paths
+# Where is this used, fixme
+set(MORPHEUS_PY_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..)
+
+
+# #################################
+# #### Morpheus Python Setup ######
+# #################################
+morpheus_utils_python_configure()
+
+# Include the main morpheus code
+# fixme: add another subdirectory src and change SOURCE_DIRECTORY to ${CMAKE_CURRENT_SOURCE_DIR}/src
+morpheus_utils_create_python_package(morpheus
+  PROJECT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+  SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/morpheus"
+)
+
+add_subdirectory(morpheus)
+
+# Complete the python package
+if(MORPHEUS_PYTHON_INPLACE_BUILD)
+  list(APPEND extra_args "IS_INPLACE")
+endif()
+
+if(MORPHEUS_PYTHON_BUILD_WHEEL)
+  list(APPEND extra_args "BUILD_WHEEL")
+endif()
+
+if(MORPHEUS_PYTHON_PERFORM_INSTALL)
+  list(APPEND extra_args "INSTALL_WHEEL")
+endif()
+
+file(GLOB morpheus_data_files "${CMAKE_CURRENT_SOURCE_DIR}/morpheus/data/*")
+morpheus_utils_add_python_sources(${morpheus_data_files})
+
+morpheus_utils_build_python_package(morpheus ${extra_args})
+
+list(POP_BACK CMAKE_MESSAGE_CONTEXT)

morpheus/_lib/common/module.cpp → python/morpheus/morpheus/_lib/common/module.cpp

@@ -1,4 +1,4 @@
-/*
+    /*
  * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *

morpheus/_lib/tests/stages/test_preprocess_nlp.cpp → python/morpheus/morpheus/_lib/tests/stages/test_preprocess_nlp.cpp

@@ -48,7 +48,7 @@ TEST_F(TestPreprocessNLP, TestProcessControlMessageAndMultiMessage)
     auto test_data_dir               = test::get_morpheus_root() / "tests/tests_data";
     std::filesystem::path input_file = test_data_dir / "countries_sample.csv";
 
-    auto test_vocab_hash_file_dir         = test::get_morpheus_root() / "morpheus/data";
+    auto test_vocab_hash_file_dir         = test::get_morpheus_root() / "python/morpheus/morpheus/data";
     std::filesystem::path vocab_hash_file = test_vocab_hash_file_dir / "bert-base-cased-hash.txt";
 
     // Create a dataframe from a file

python/morpheus/morpheus/data

@@ -0,0 +1 @@
+../../../models/data