Update Morpheus to Use SRF 22.06 (#152)

This PR updates all references from Neo->SRF. It also updates the CMake build to allow building SRF from source again.

Authors:
  - Michael Demoret (https://github.com/mdemoret-nv)
  - Pete MacKinnon (https://github.com/pdmack)
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Christopher Harris (https://github.com/cwharris)
  - David Gardner (https://github.com/dagardner-nv)
  - Bartley Richardson (https://github.com/BartleyR)
  - Devin Robison (https://github.com/drobison00)
  - https://github.com/raykallen

URL: #152

.clang-tidy

@@ -8,7 +8,7 @@
 Checks: -*,readability-identifier-naming,modernize-*,-modernize-use-trailing-return-type,-modernize-use-nodiscard,google-readability-todo
 
 #WarningsAsErrors: '*'
-HeaderFilterRegex: '.*\/include\/trtlab\/neo\/.*'
+HeaderFilterRegex: '.*\/include\/morpheus\/.*'
 AnalyzeTemporaryDtors: false
 FormatStyle:     file
 CheckOptions:

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-allow-list=neo,morpheus._lib
+extension-pkg-allow-list=srf,morpheus._lib
 
 # Specify a score threshold to be exceeded before program exits with error.
 fail-under=10

CONTRIBUTING.md

@@ -301,4 +301,4 @@ Ex:
 
 ## Attribution
 Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md \
-Portions adopted from https://github.com/dask/dask/blob/master/docs/source/develop.rst
+Portions adopted from https://github.com/dask/dask/blob/master/docs/source/develop.rst

ci/conda/recipes/morpheus/meta.yaml

@@ -59,7 +59,7 @@ outputs:
         - cython >=0.29,<0.30
         - libcudf {{ rapids_version }}
         - librdkafka 1.7
-        - neo 22.04.*
+        - srf {{ minor_version }}
         - pip
         - pybind11-stubgen
         - python {{ python }}

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -23,7 +23,7 @@ CMAKE_ARGS=${CMAKE_ARGS:-""}
 export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
 export USE_SCCACHE=${USE_SCCACHE:-""}
 
-# Check for some neo environment variables. Append to front of args to allow users to overwrite them
+# Check for some srf environment variables. Append to front of args to allow users to overwrite them
 if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
    # Set the cache variable, then set the Staging prefix to allow for host searching
    CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"

ci/conda/recipes/run_conda_build.sh

@@ -38,9 +38,6 @@ export MORPHEUS_BUILD_PYTHON_STUBS=${MORPHEUS_BUILD_PYTHON_STUBS:-"ON"}
 export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"}
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
 
-# Set the tag for the neo commit to use
-export NEO_GIT_TAG=${NEO_GIT_TAG:-"5b55e37c6320c1a5747311a1e29e7ebb049d12bc"}
-
 # Set CONDA_CHANNEL_ALIAS to mimic the conda config channel_alias property during the build
 CONDA_CHANNEL_ALIAS=${CONDA_CHANNEL_ALIAS:-""}
 export USE_SCCACHE=${USE_SCCACHE:-""}
@@ -50,7 +47,6 @@ export PYTHON_VER="$(python -c "import sys; print('.'.join(map(str, sys.version_
 export CUDA=11.5
 echo "CUDA        : ${CUDA}"
 echo "PYTHON_VER  : ${PYTHON_VER}"
-echo "NEO_GIT_TAG : ${NEO_GIT_TAG}"
 echo ""
 
 export CMAKE_GENERATOR="Ninja"
@@ -98,42 +94,8 @@ CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvid
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev")
 CONDA_ARGS_ARRAY+=("-c" "conda-forge")
 
-if hasArg libneo; then
-
-   export NEO_ROOT="${MORPHEUS_CACHE_DIR}/src_cache/libneo"
-   export NEO_CACHE_DIR=${MORPHEUS_CACHE_DIR}
-
-   # First need to download the repo into the cache
-   if [[ ! -d "${NEO_ROOT}" ]]; then
-      git clone ${NEO_GIT_URL:?"Cannot build libneo. Must set NEO_GIT_URL to git repo location to allow checkout of neo repository"} ${NEO_ROOT}
-   fi
-
-   pushd ${NEO_ROOT}
-
-   # Ensure we have the latest checkout
-   git fetch
-   git checkout ${NEO_GIT_TAG}
-
-   if [[ "$(git branch --show-current | wc -l)" == "1" ]]; then
-      git pull
-   fi
-
-   # Set GIT_VERSION to set the project version inside of meta.yaml
-   export GIT_VERSION="$(get_version)"
-
-   echo "Running conda-build for libneo..."
-   set -x
-   conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/libneo
-   set +x
-
-   unset GIT_DESCRIBE_TAG
-
-   popd
-fi
-
 if hasArg morpheus; then
    # Set GIT_VERSION to set the project version inside of meta.yaml
-   # Do this after neo in case they are different
    export GIT_VERSION="$(get_version)"
 
    echo "Running conda-build for morpheus..."

ci/iwyu/mappings.imp

@@ -1,11 +1,11 @@
 [
 
 ## Include mappings
-# neo protos
-{ "include": [ "\"neo/protos/architect.pb.h\"", private, "<neo/protos/architect.pb.h>", "public" ] },
-{ "include": [ "\"neo/protos/codable.pb.h\"", private, "<neo/protos/codable.pb.h>", "public" ] },
-{ "include": [ "\"neo/protos/remote_descriptor.pb.h\"", private, "<neo/protos/remote_descriptor.pb.h>", "public" ] },
-{ "include": [ "\"neo/protos/tensor_meta_data.pb.h\"", private, "<neo/protos/tensor_meta_data.pb.h>", "public" ] },
+# srf protos
+{ "include": [ "\"srf/protos/architect.pb.h\"", private, "<srf/protos/architect.pb.h>", "public" ] },
+{ "include": [ "\"srf/protos/codable.pb.h\"", private, "<srf/protos/codable.pb.h>", "public" ] },
+{ "include": [ "\"srf/protos/remote_descriptor.pb.h\"", private, "<srf/protos/remote_descriptor.pb.h>", "public" ] },
+{ "include": [ "\"srf/protos/tensor_meta_data.pb.h\"", private, "<srf/protos/tensor_meta_data.pb.h>", "public" ] },
 
 # stdlib
 { "include": [ "<bits/cxxabi_forced.h>", private, "<mutex>", "public" ] },
@@ -87,7 +87,7 @@
 # xtensor
 { "symbol": ["xt::no_ownership", "private", "<xtensor/xadapt.hpp>", "public"] },
 
-# neo
-{ "symbol": ["std::__decay_and_strip<std::shared_ptr<neo::TraceStatistics> &>::__type" , "private", "<neo/benchmarking/trace_statistics.hpp>", "public"] },
+# srf
+{ "symbol": ["std::__decay_and_strip<std::shared_ptr<srf::TraceStatistics> &>::__type" , "private", "<srf/benchmarking/trace_statistics.hpp>", "public"] },
 
 ]

cmake/dependencies.cmake

@@ -102,10 +102,10 @@ include(deps/Configure_pybind11)
 set(RDKAFKA_VERSION 1.6.2)
 include(deps/Configure_rdkafka)
 
-# NEO (Should come after all third party but before NVIDIA repos)
+# SRF (Should come after all third party but before NVIDIA repos)
 # =====
-set(NEO_VERSION 22.04 CACHE STRING "Which version of Neo to use")
-include(deps/Configure_neo)
+set(SRF_VERSION 22.06 CACHE STRING "Which version of SRF to use")
+include(deps/Configure_srf)
 
 # CuDF
 # =====

cmake/deps/Configure_srf.cmake

@@ -0,0 +1,75 @@
+#=============================================================================
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+function(find_and_configure_srf version)
+
+  list(APPEND CMAKE_MESSAGE_CONTEXT "srf")
+
+  rapids_cpm_find(srf ${version}
+    GLOBAL_TARGETS
+      srf::srf srf::pysrf
+    BUILD_EXPORT_SET
+      ${PROJECT_NAME}-exports
+    INSTALL_EXPORT_SET
+      ${PROJECT_NAME}-exports
+    CPM_ARGS
+      GIT_REPOSITORY  https://github.com/nv-morpheus/SRF.git
+      GIT_TAG         branch-${version}
+      GIT_SHALLOW     TRUE
+      OPTIONS         "SRF_BUILD_EXAMPLES OFF"
+                      "SRF_BUILD_TESTS OFF"
+                      "SRF_BUILD_BENCHMARKS OFF"
+                      "SRF_BUILD_PYTHON ON"
+                      "SRF_ENABLE_XTENSOR ON"
+                      "SRF_ENABLE_MATX ON"
+                      "SRF_USE_CONDA ${MORPHEUS_USE_CONDA}"
+                      "SRF_USE_CCACHE ${MORPHEUS_USE_CCACHE}"
+                      "SRF_USE_CLANG_TIDY ${MORPHEUS_USE_CLANG_TIDY}"
+                      "SRF_PYTHON_INPLACE_BUILD ${MORPHEUS_PYTHON_INPLACE_BUILD}"
+                      "RMM_VERSION ${RAPIDS_VERSION}"
+  )
+
+  if(srf_ADDED)
+
+    # Now ensure its installed
+    find_package(Python3 COMPONENTS Interpreter REQUIRED)
+
+    # detect virtualenv and set Pip args accordingly
+    if(DEFINED ENV{VIRTUAL_ENV} OR DEFINED ENV{CONDA_PREFIX})
+      set(_pip_args)
+    else()
+      set(_pip_args "--user")
+    endif()
+
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+      list(APPEND _pip_args "-e")
+    endif()
+
+    add_custom_command(
+      OUTPUT ${srf_BINARY_DIR}/python/srf.egg-info/PKG-INFO
+      COMMAND ${Python3_EXECUTABLE} -m pip install ${_pip_args} ${srf_BINARY_DIR}/python
+      DEPENDS srf_python_rebuild
+      COMMENT "Installing srf python package"
+    )
+
+    add_custom_target(
+      install_srf_python ALL
+      DEPENDS ${srf_BINARY_DIR}/python/srf.egg-info/PKG-INFO
+    )
+  endif()
+endfunction()
+
+find_and_configure_srf(${SRF_VERSION})

docker/build_conda_packages.sh

@@ -48,38 +48,6 @@ CONDA_ARGS+=("--skip-existing")
 
 DOCKER_EXTRA_ARGS=()
 
-if hasArg libneo; then
-   # If libneo is specified, you must set NEO_GIT_URL
-   DOCKER_EXTRA_ARGS+=("--env" "NEO_GIT_URL=${NEO_GIT_URL:?"Cannot build libneo. Must set NEO_GIT_URL to git repo location to allow checkout of neo repository"}")
-
-   url=${NEO_GIT_URL}
-
-   # Remove the http/https/ssh
-   url="${url#http://}"
-   url="${url#https://}"
-   url="${url#ssh://}"
-
-   # Remove git@
-   url="${url#git@}"
-
-   # Remove username/password
-   url="${url#*:*@}"
-   url="${url#*@}"
-
-   # Remove remaining
-   url=${url%%/*}
-
-   port=${url##*:}
-   url=${url%%:*}
-
-   # Add the command to auto accept this url/port combo
-   if [[ -n "${port}" ]]; then
-      BUILD_SCRIPT="${BUILD_SCRIPT:+${BUILD_SCRIPT}\n}mkdir -p \$HOME/.ssh && ssh-keyscan -t rsa -p ${port} ${url} > ~/.ssh/known_hosts"
-   else
-      BUILD_SCRIPT="${BUILD_SCRIPT:+${BUILD_SCRIPT}\n}mkdir -p \$HOME/.ssh && ssh-keyscan -t rsa ${url} > ~/.ssh/known_hosts"
-   fi
-fi
-
 # Build the script to execute inside of the container (needed to set multiple statements in CONDA_ARGS)
 BUILD_SCRIPT="${BUILD_SCRIPT}
 export CONDA_ARGS=\"${CONDA_ARGS[@]}\"

docker/conda/environments/cuda11.5_dev.yml

@@ -17,7 +17,7 @@ name: morpheus
 channels:
     - rapidsai
     - nvidia
-    - nvidia/label/dev # For pre-releases of Neo. Should still default to full releases if available
+    - nvidia/label/dev # For pre-releases of SRF. Should still default to full releases if available
     - nvidia/label/cuda-11.5.2 # For cuda-nvml-dev=11.5, which is not published under nvidia channel yet.
     - conda-forge
 dependencies:
@@ -42,21 +42,21 @@ dependencies:
     - gcc_linux-64=9.4
     - gflags=2.2
     - git>=2.35.3 # Needed for wildcards on safe.directory
-    - glog=0.5
+    - glog=0.6
     - gmock=1.10
     - grpc-cpp>=1.43
     - gtest=1.10
     - gxx_linux-64=9.4
     - isort
     - mlflow>=1.23
     - myst-parser==0.17
-    - neo 22.04.*
+    - srf 22.06.*
     - ninja=1.10
     - nodejs=17.4.0
     - pandas=1.3
     - pip
-    - pkg-config # for neo cmake
-    - protobuf=3.19
+    - pkg-config # for srf cmake
+    - protobuf=3.20
     - pybind11-stubgen
     - pydot
     - pytest