Merge remote-tracking branch 'devin/devin_integrated_training_595' in…

…to bhargav_integrated_training_595

.devcontainer/initialize-command.sh

@@ -18,7 +18,8 @@
 docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus
 
 # create the parent conda folder so it's found when mounting
-mkdir -p ../.conda
+mkdir -p .cache/conda/envs
+mkdir -p ../.conda/pkgs
 
 # create a config directory if it does not exist so it's found when mounting
 mkdir -p ../.config

CMakeLists.txt

@@ -169,8 +169,8 @@ if(MORPHEUS_ENABLE_DEBUG_INFO)
 
   morpheus_utils_print_target_properties(
       TARGETS
-        cuda_utils
         morpheus
+        morpheus_utils
       WRITE_TO_FILE
   )
 

ci/conda/recipes/morpheus/meta.yaml

@@ -69,7 +69,7 @@ outputs:
         - scikit-build >=0.12
         - versioneer-518
       run:
-        # Runtime only requirements. This + setup.y is the definitive runtime requirement list
+        # Runtime only requirements. This + setup.py is the definitive runtime requirement list
         - click >=8
         - configargparse 1.5.*
         - cuda-python <=11.7.0 # Needed to ensure cudatoolkit and not cuda-toolkit is installed
@@ -91,30 +91,29 @@ outputs:
         - tqdm 4.*
         - typing_utils 0.1.*
         - watchdog 2.1.*
-
+      run_constrained:
+        # If cuml is installed, we need to constrain these two libraries. See:
+        # https://docs.rapids.ai/notices/rsn0026/ for libcusolver & libcusparse
+        - libcusolver<=11.4.1.48
+        - libcusparse<12
     test:
       requires:
         - cudatoolkit {{ cuda_version }}.*
         - gputil
         - pytest
         - pytest-cov
         - pytest-benchmark
+        # test that cuml can be installed in the env
+        - cuml {{ rapids_version }}
       source_files:
-        # - .git/*
         - docker/conda/environments/*
         - pyproject.toml
         - scripts/fetch_data.py
         - tests/*
       imports:
         - morpheus
-        - morpheus._lib
       commands:
-        # Install dependencies that are pip-only
-        - mamba env update --prefix ${CONDA_PREFIX} --file docker/conda/environments/cuda{{ cuda_version }}_runtime.yml
-        # Ensure the test data is downloaded
-        - ./scripts/fetch_data.py fetch tests
-        # Run pytest
-        - pytest
+          - echo # make sure test requirements get installed
 
 about:
   home: https://github.com/nv-morpheus/Morpheus

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -44,6 +44,7 @@ CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"all"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
 
 if [[ "${USE_SCCACHE}" == "1" ]]; then
    CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"

ci/conda/recipes/run_conda_build.sh

@@ -101,6 +101,7 @@ CONDA_ARGS_ARRAY+=("--variants" "{python: 3.8}")
 # And default channels (with optional channel alias)
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapidsai")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/cuda-11.8.0")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev")
 CONDA_ARGS_ARRAY+=("-c" "conda-forge")
 

ci/scripts/github/build.sh

@@ -30,6 +30,7 @@ sccache --version
 
 rapids-logger "Configuring cmake for Morpheus"
 git submodule update --init --recursive
+
 cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} \
     -DCCACHE_PROGRAM_PATH=$(which sccache) \
     -DMORPHEUS_PYTHON_BUILD_WHEEL=ON \
@@ -45,8 +46,16 @@ sccache --show-stats
 rapids-logger "Archiving results"
 tar cfj "${WORKSPACE_TMP}/wheel.tar.bz" build/dist
 
+MORPHEUS_LIBS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib -name "*.so" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
+tar cfj "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${MORPHEUS_LIBS[@]}"
+
+CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib/tests -name "*.x" -exec realpath --relative-to ${MORPHEUS_ROOT} {} \;))
+tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}"
+
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
 aws s3 cp --no-progress "${WORKSPACE_TMP}/wheel.tar.bz" "${ARTIFACT_URL}/wheel.tar.bz"
+aws s3 cp --no-progress "${WORKSPACE_TMP}/morhpeus_libs.tar.bz" "${ARTIFACT_URL}/morhpeus_libs.tar.bz"
+aws s3 cp --no-progress "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${ARTIFACT_URL}/cpp_tests.tar.bz"
 
 rapids-logger "Success"
 exit 0

ci/scripts/github/test.sh

@@ -22,16 +22,20 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 update_conda_env
 
 aws s3 cp --no-progress "${ARTIFACT_URL}/wheel.tar.bz" "${WORKSPACE_TMP}/wheel.tar.bz"
+aws s3 cp --no-progress "${ARTIFACT_URL}/cpp_tests.tar.bz" "${WORKSPACE_TMP}/cpp_tests.tar.bz"
+aws s3 cp --no-progress "${ARTIFACT_URL}/morhpeus_libs.tar.bz" "${WORKSPACE_TMP}/morhpeus_libs.tar.bz"
 
 tar xf "${WORKSPACE_TMP}/wheel.tar.bz"
+tar xf "${WORKSPACE_TMP}/morhpeus_libs.tar.bz"
+tar xf "${WORKSPACE_TMP}/cpp_tests.tar.bz"
 
 # Install the built Morpheus python package
+cd ${MORPHEUS_ROOT}
 pip install ${MORPHEUS_ROOT}/build/dist/*.whl
 
 CPP_TESTS=($(find ${MORPHEUS_ROOT}/build -name "*.x"))
 
 rapids-logger "Pulling LFS assets"
-cd ${MORPHEUS_ROOT}
 
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation
@@ -43,6 +47,12 @@ git lfs ls-files
 REPORTS_DIR="${WORKSPACE_TMP}/reports"
 mkdir -p ${WORKSPACE_TMP}/reports
 
+rapids-logger "Running C++ tests"
+# Running the tests from the tests dir. Normally this isn't nescesary, however since
+# we are testing the installed version of morpheus in site-packages and not the one
+# in the repo dir, the pytest coverage module reports incorrect coverage stats.
+pushd ${MORPHEUS_ROOT}/tests
+
 TEST_RESULTS=0
 for cpp_test in "${CPP_TESTS[@]}"; do
        test_name=$(basename ${cpp_test})
@@ -57,11 +67,6 @@ for cpp_test in "${CPP_TESTS[@]}"; do
 done
 
 rapids-logger "Running Python tests"
-# Running the tests from the tests dir. Normally this isn't nescesary, however since
-# we are testing the installed version of morpheus in site-packages and not the one
-# in the repo dir, the pytest coverage module reports incorrect coverage stats.
-cd ${MORPHEUS_ROOT}/tests
-
 set +e
 
 python -I -m pytest --run_slow --run_kafka \
@@ -74,6 +79,7 @@ PYTEST_RESULTS=$?
 TEST_RESULTS=$(($TEST_RESULTS+$PYTEST_RESULTS))
 
 set -e
+popd
 
 rapids-logger "Archiving test reports"
 cd $(dirname ${REPORTS_DIR})

cmake/dependencies.cmake

@@ -21,34 +21,39 @@ if (VERBOSE)
   morpheus_utils_print_config()
 endif()
 
-# Load direct physical package dependencies first, so we fail early.
-find_package(Protobuf REQUIRED)
-find_package(CUDAToolkit REQUIRED) # Required by Morpheus. Fail early if we don't have it.
+# Load direct physical package dependencies first, so we fail early. Add all dependencies to our export set
+rapids_find_package(Protobuf
+  REQUIRED
+  BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports
+  INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports
+)
+
+find_package(CUDAToolkit REQUIRED)
 
 if(MORPHEUS_BUILD_BENCHMARKS)
   # google benchmark
   # - Expects package to pre-exist in the build environment
   # ================
   rapids_find_package(benchmark REQUIRED
-          GLOBAL_TARGETS      benchmark::benchmark
-          BUILD_EXPORT_SET    ${PROJECT_NAME}-exports
-          INSTALL_EXPORT_SET  ${PROJECT_NAME}-exports
-          FIND_ARGS
-          CONFIG
-          )
+    GLOBAL_TARGETS      benchmark::benchmark
+    BUILD_EXPORT_SET    ${PROJECT_NAME}-exports
+    INSTALL_EXPORT_SET  ${PROJECT_NAME}-exports
+    FIND_ARGS
+    CONFIG
+  )
 endif()
 
 if(MORPHEUS_BUILD_TESTS)
   # google test
   # - Expects package to pre-exist in the build environment
   # ===========
   rapids_find_package(GTest REQUIRED
-          GLOBAL_TARGETS      GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main
-          BUILD_EXPORT_SET    ${PROJECT_NAME}-exports
-          INSTALL_EXPORT_SET  ${PROJECT_NAME}-exports
-          FIND_ARGS
-          CONFIG
-          )
+    GLOBAL_TARGETS      GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main
+    BUILD_EXPORT_SET    ${PROJECT_NAME}-exports
+    INSTALL_EXPORT_SET  ${PROJECT_NAME}-exports
+    FIND_ARGS
+    CONFIG
+  )
 endif()
 
 # libcudacxx -- get an explicit lubcudacxx build, matx tries to pull a tag that doesn't exist.

docker/Dockerfile

@@ -50,9 +50,10 @@ RUN apt-get update &&\
     apt-get install --no-install-recommends -y \
         bc \
         build-essential \
-        cuda-gdb-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-cudart-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-cupti-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        cuda-gdb-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         curl \
         git-lfs \
         jq \
@@ -79,9 +80,8 @@ WORKDIR ${MORPHEUS_ROOT}
 # Install mamba to speed the solve up
 RUN conda config --set ssl_verify false &&\
     conda config --add pkgs_dirs /opt/conda/pkgs &&\
-    conda config --env --add channels conda-forge &&\
     # Install mamba, boa and git here. Conda build breaks with other git installs
-    /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" "tini=0.19" &&\
+    /opt/conda/bin/mamba install -y -n base -c conda-forge "boa" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" "tini=0.19" &&\
     source activate base &&\
     git lfs install
     # conda clean -afy
@@ -142,14 +142,15 @@ ARG CUDA_MINOR_VER
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
     apt install --no-install-recommends -y \
+        cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
-        libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
-        libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \
+        libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -161,7 +162,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke
     source activate base &&\
     # Need to get around recent versions of git locking paths until they are deemed safe
     git config --global --add safe.directory "*" &&\
-    MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus
+    MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld ./ci/conda/recipes/run_conda_build.sh morpheus
 
 # sid_visualization is a submodule we need to init
 RUN git submodule update --init --recursive
@@ -178,7 +179,7 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target
     --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
     source activate morpheus &&\
     # Install morpheus
-    CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus -c local -c rapidsai -c nvidia -c nvidia/label/dev -c conda-forge morpheus &&\
+    CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus -c local -c rapidsai -c nvidia -c nvidia/label/cuda-11.8.0 -c nvidia/label/dev -c conda-forge morpheus &&\
     # Install runtime dependencies that are pip-only
     /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml &&\
     # Clean and activate
@@ -210,13 +211,14 @@ ARG CUDA_MINOR_VER
 RUN apt update && \
     DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
     apt install --no-install-recommends -y \
+        cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
+        libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
-        libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \
         nodejs \
         npm && \
     apt clean && \
@@ -245,7 +247,6 @@ ARG MORPHEUS_USER="root"
 RUN source activate morpheus \
     && MORPHEUS_ROOT=/workspace \
        CONDA_BLD_PATH=/opt/conda/conda-bld \
-       CONDA_ARGS="--no-test" \
        ./ci/conda/recipes/run_conda_build.sh pydebug \
     && ./ci/conda/recipes/python_dbg_install.sh \
        -s $( ls /opt/conda/conda-bld/src_cache/Python-${PYTHON_VER}*.tar.xz ) \

docker/conda/environments/cuda11.8_dev.yml

@@ -17,6 +17,7 @@ name: morpheus
 channels:
     - rapidsai
     - nvidia
+    - nvidia/label/cuda-11.8.0
     - nvidia/label/dev # For pre-releases of MRC. Should still default to full releases if available
     - conda-forge
 dependencies:
@@ -37,6 +38,7 @@ dependencies:
     - cupy=9.5.0
     - cython=0.29.24
     - datacompy=0.8
+    - dill
     - docker-compose=1.29.2
     - docker-py=5.0
     - faker=12.3.0
@@ -80,6 +82,7 @@ dependencies:
     - python=3.8
     - rapidjson=1.1.0
     - scikit-build=0.13
+    - scikit-learn=0.23.1
     - sphinx
     - sphinx_rtd_theme
     - sysroot_linux-64=2.17

docker/conda/environments/requirements.txt

@@ -4,7 +4,6 @@
 
 ####### Pip-only runtime dependencies (keep sorted!) #######
 # Packages listed here should also be listed in setup.py
-git+https://github.com/nv-morpheus/dfencoder.git@branch-23.01#egg=dfencoder
 ipywidgets
 jupyter-core>=4.11.2,<5.0
 jupyterlab

docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md

@@ -1,3 +1,20 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
 # Introduction to Digital Fingerprinting Pipeline in Morpheus
 
 ## Table of Contents

docs/source/developer_guide/guides/2_real_world_phishing.md

@@ -109,7 +109,7 @@ import typing
 
 import mrc
 
-from morpheus._lib.common import TypeId
+from morpheus.common import TypeId
 from morpheus.cli.register_stage import register_stage
 from morpheus.config import Config
 from morpheus.config import PipelineModes
@@ -435,7 +435,7 @@ pipeline.add_stage(WriteToFileStage(config, filename=results_file, overwrite=Tru
 
 Note that we didn't specify the output format. In our example, the result file contains the extension `.jsonlines`. Morpheus will infer the output format based on the extension. At time of writing the extensions that Morpheus will infer are: `.csv`, `.json` & `.jsonlines`
 
-To explicitly set the output format we could specify the `file_type` argument to the `WriteToFileStage` which is an enumeration defined in `morpheus._lib.common.FileTypes`. Current values defined are:
+To explicitly set the output format we could specify the `file_type` argument to the `WriteToFileStage` which is an enumeration defined in `morpheus.common.FileTypes`. Current values defined are:
 * `FileTypes.Auto`
 * `FileTypes.JSON`
 * `FileTypes.CSV`

docs/source/developer_guide/guides/6_digital_fingerprinting_reference.md

@@ -225,8 +225,8 @@ The `DFPFileToDataFrameStage` ([examples/digital_fingerprinting/production/morph
 | `c` | `morpheus.config.Config` | Morpheus config object |
 | `schema` | `DataFrameInputSchema` | Schema specifying columns to load, along with any necessary renames and data type conversions  |
 | `filter_null` | `bool` | Optional: Whether to filter null rows after loading, by default True. |
-| `file_type` | `morpheus._lib.common.FileTypes` (enum) | Optional: Indicates file type to be loaded. Currently supported values at time of writing are: `FileTypes.Auto`, `FileTypes.CSV`, and `FileTypes.JSON`. Default value is `FileTypes.Auto` which will infer the type based on the file extension, set this value if using a custom extension |
-| `parser_kwargs` | `dict` or `None` | Optional: additional keyword arguments to be passed into the `DataFrame` parser, currently this is going to be either [`pandas.read_csv`](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html) or [`pandas.read_json`](https://pandas.pydata.org/docs/reference/api/pandas.read_json.html) |
+| `file_type` | `morpheus.common.FileTypes` (enum) | Optional: Indicates file type to be loaded. Currently supported values at time of writing are: `FileTypes.Auto`, `FileTypes.CSV`, `FileTypes.JSON` and `FileTypes.PARQUET`. Default value is `FileTypes.Auto` which will infer the type based on the file extension, set this value if using a custom extension |
+| `parser_kwargs` | `dict` or `None` | Optional: additional keyword arguments to be passed into the `DataFrame` parser, currently this is going to be either [`pandas.read_csv`](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html), [`pandas.read_json`](https://pandas.pydata.org/docs/reference/api/pandas.read_json.html) or [`pandas.read_parquet`](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html) |
 | `cache_dir` | `str` | Optional: path to cache location, defaults to `./.cache/dfp` |
 
 This stage is able to download and load data files concurrently by multiple methods.  Currently supported methods are: `single_thread`, `multiprocess`, `dask`, and `dask_thread`.  The method used is chosen by setting the `FILE_DOWNLOAD_TYPE` environment variable, and `dask_thread` is used by default, and `single_thread` effectively disables concurrent loading.