Morpheus refactor to MRC (#530)

Resolves #496

Authors:
  - Devin Robison (https://github.com/drobison00)
  - Michael Demoret (https://github.com/mdemoret-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #530

.clang-format

@@ -58,7 +58,6 @@ ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 2
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
-DerivePointerAlignment: true
 DisableFormat: false
 ExperimentalAutoDetectBinPacking: false
 FixNamespaceComments: true
@@ -158,7 +157,7 @@ SpacesInContainerLiterals: true
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
-Standard: c++17
+Standard: c++20
 StatementMacros:
   - Q_UNUSED
   - QT_REQUIRE_VERSION

.clang-tidy

@@ -181,7 +181,7 @@ CheckOptions:
   - key:             readability-identifier-naming.TypeTemplateParameterCase
     value:           CamelCase
   - key:             readability-identifier-naming.TypeTemplateParameterIgnoredRegexp
-    value:           ^[A-Z]$
+    value:           '^([A-Z]|expr-type)$'
   - key:             readability-identifier-naming.TypeTemplateParameterPrefix
     value:           ''
   - key:             readability-identifier-naming.TypeTemplateParameterSuffix

CMakeLists.txt

@@ -36,7 +36,7 @@ set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Lo
 set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MORPHEUS_CACHE_DIR)
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -23,7 +23,7 @@ CMAKE_ARGS=${CMAKE_ARGS:-""}
 export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
 export USE_SCCACHE=${USE_SCCACHE:-""}
 
-# Check for some srf environment variables. Append to front of args to allow users to overwrite them
+# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
 if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
    # Set the cache variable, then set the Staging prefix to allow for host searching
    CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"

ci/iwyu/mappings.imp

@@ -1,11 +1,11 @@
 [
 
 ## Include mappings
-# srf protos
-{ "include": [ "\"srf/protos/architect.pb.h\"", private, "<srf/protos/architect.pb.h>", "public" ] },
-{ "include": [ "\"srf/protos/codable.pb.h\"", private, "<srf/protos/codable.pb.h>", "public" ] },
-{ "include": [ "\"srf/protos/remote_descriptor.pb.h\"", private, "<srf/protos/remote_descriptor.pb.h>", "public" ] },
-{ "include": [ "\"srf/protos/tensor_meta_data.pb.h\"", private, "<srf/protos/tensor_meta_data.pb.h>", "public" ] },
+# mrc protos
+{ "include": [ "\"mrc/protos/architect.pb.h\"", private, "<mrc/protos/architect.pb.h>", "public" ] },
+{ "include": [ "\"mrc/protos/codable.pb.h\"", private, "<mrc/protos/codable.pb.h>", "public" ] },
+{ "include": [ "\"mrc/protos/remote_descriptor.pb.h\"", private, "<mrc/protos/remote_descriptor.pb.h>", "public" ] },
+{ "include": [ "\"mrc/protos/tensor_meta_data.pb.h\"", private, "<mrc/protos/tensor_meta_data.pb.h>", "public" ] },
 
 # stdlib
 { "include": [ "<bits/cxxabi_forced.h>", private, "<mutex>", "public" ] },
@@ -138,7 +138,7 @@
 # xtensor
 { "symbol": ["xt::no_ownership", "private", "<xtensor/xadapt.hpp>", "public"] },
 
-# srf
-{ "symbol": ["std::__decay_and_strip<std::shared_ptr<srf::TraceStatistics> &>::__type" , "private", "<srf/benchmarking/trace_statistics.hpp>", "public"] },
+# mrc
+{ "symbol": ["std::__decay_and_strip<std::shared_ptr<mrc::TraceStatistics> &>::__type" , "private", "<mrc/benchmarking/trace_statistics.hpp>", "public"] },
 
 ]

ci/scripts/common.sh

@@ -45,11 +45,18 @@ export BUILD_DIR=${BUILD_DIR:-"${REPO_DIR}/build"}
 # Speficy the clang-tools version to use. Default 14
 export CLANG_TOOLS_VERSION=${CLANG_TOOLS_VERSION:-14}
 
+# Returns the `branch-YY.MM` that is used as the base for merging
+function get_base_branch() {
+   local major_minor_version=$(git describe --tags | grep -o -E '[0-9][0-9]\.[0-9][0-9]')
+
+   echo "branch-${major_minor_version}"
+}
+
 # Determine the merge base as the root to compare against. Optionally pass in a
 # result variable otherwise the output is printed to stdout
 function get_merge_base() {
    local __resultvar=$1
-   local result=$(git merge-base ${BASE_SHA:-main} ${COMMIT_SHA:-HEAD})
+   local result=$(git merge-base ${BASE_SHA:-$(get_merge_base)} ${COMMIT_SHA:-HEAD})
 
    if [[ "$__resultvar" ]]; then
       eval $__resultvar="'${result}'"

ci/scripts/gitutils.py

@@ -1,4 +1,4 @@
-#! /bin/env python3
+#!/usr/bin/env python3
 # SPDX-FileCopyrightText: Copyright (c) 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #

cmake/dependencies.cmake

@@ -101,10 +101,10 @@ include(deps/Configure_pybind11)
 set(RDKAFKA_VERSION 1.6.2)
 include(deps/Configure_rdkafka)
 
-# SRF (Should come after all third party but before NVIDIA repos)
+# MRC (Should come after all third party but before NVIDIA repos)
 # =====
-set(SRF_VERSION 22.11 CACHE STRING "Which version of SRF to use")
-include(deps/Configure_srf)
+set(MRC_VERSION 23.01 CACHE STRING "Which version of MRC to use")
+include(deps/Configure_mrc)
 
 # CuDF
 # =====

cmake/deps/Configure_srf.cmake → cmake/deps/Configure_mrc.cmake

@@ -14,36 +14,36 @@
 # limitations under the License.
 #=============================================================================
 
-function(find_and_configure_srf version)
+function(find_and_configure_mrc version)
 
-  list(APPEND CMAKE_MESSAGE_CONTEXT "srf")
+  list(APPEND CMAKE_MESSAGE_CONTEXT "mrc")
 
-  rapids_cpm_find(srf ${version}
+  rapids_cpm_find(mrc ${version} CONFIG REQUIRED
     GLOBAL_TARGETS
-      srf::srf srf::pysrf
+      mrc::mrc mrc::pymrc
     BUILD_EXPORT_SET
       ${PROJECT_NAME}-exports
     INSTALL_EXPORT_SET
       ${PROJECT_NAME}-exports
     CPM_ARGS
-      GIT_REPOSITORY  https://github.com/nv-morpheus/SRF.git
+      GIT_REPOSITORY  https://github.com/nv-morpheus/MRC.git
       GIT_TAG         branch-${version}
       GIT_SHALLOW     TRUE
-      OPTIONS         "SRF_BUILD_EXAMPLES OFF"
-                      "SRF_BUILD_TESTS OFF"
-                      "SRF_BUILD_BENCHMARKS OFF"
-                      "SRF_BUILD_PYTHON ON"
-                      "SRF_ENABLE_XTENSOR ON"
-                      "SRF_ENABLE_MATX ON"
-                      "SRF_USE_CONDA ${MORPHEUS_USE_CONDA}"
-                      "SRF_USE_CCACHE ${MORPHEUS_USE_CCACHE}"
-                      "SRF_USE_CLANG_TIDY ${MORPHEUS_USE_CLANG_TIDY}"
-                      "SRF_PYTHON_INPLACE_BUILD OFF"
-                      "SRF_PYTHON_PERFORM_INSTALL ON"
-                      "SRF_PYTHON_BUILD_STUBS ${MORPHEUS_BUILD_PYTHON_STUBS}"
+      OPTIONS         "MRC_BUILD_EXAMPLES OFF"
+                      "MRC_BUILD_TESTS OFF"
+                      "MRC_BUILD_BENCHMARKS OFF"
+                      "MRC_BUILD_PYTHON ON"
+                      "MRC_ENABLE_XTENSOR ON"
+                      "MRC_ENABLE_MATX ON"
+                      "MRC_USE_CONDA ${MORPHEUS_USE_CONDA}"
+                      "MRC_USE_CCACHE ${MORPHEUS_USE_CCACHE}"
+                      "MRC_USE_CLANG_TIDY ${MORPHEUS_USE_CLANG_TIDY}"
+                      "MRC_PYTHON_INPLACE_BUILD OFF"
+                      "MRC_PYTHON_PERFORM_INSTALL ON"
+                      "MRC_PYTHON_BUILD_STUBS ${MORPHEUS_BUILD_PYTHON_STUBS}"
                       "RMM_VERSION ${MORPHEUS_RAPIDS_VERSION}"
   )
 
 endfunction()
 
-find_and_configure_srf(${SRF_VERSION})
+find_and_configure_MRC(${MRC_VERSION})

docker/conda/environments/cuda11.5_dev.yml

@@ -29,10 +29,10 @@ dependencies:
     - ccache>=3.7
     - clangdev=14
     - click >=8
-    - cmake=3.22
+    - cmake=3.24
     - configargparse=1.5
     - cuda-nvml-dev=11.5
-    - cuda-python<=11.7.0 # Remove when Issue #251 is closed
+    - cuda-python<=11.5.0 # Remove when Issue #251 is closed
     - cudatoolkit=11.5
     - cudf 22.08
     - cupy=9.5.0
@@ -43,30 +43,32 @@ dependencies:
     - faker=12.3.0
     - flake8
     - flatbuffers=2.0
-    - gcc_linux-64=9.4
+    - gcc_linux-64=11.2
     - gflags=2.2
-    - git>=2.35.3 # Needed for wildcards on safe.directory
     - git-lfs=3.2
+    - git>=2.35.3 # Needed for wildcards on safe.directory
     - glog=0.6
     - gmock=1.10
     - gputil
     - grpc-cpp>=1.43
     - grpcio
     - gtest=1.10
-    - gxx_linux-64=9.4
+    - gxx_linux-64=11.2
     - include-what-you-use=0.18
     - isort
     - librdkafka=1.7.0
     - mlflow>1.29,<2
+    - mrc=23.01
     - myst-parser==0.17
     - networkx=2.8
     - ninja=1.10
     - nodejs=17.4.0
     - numba==0.55
     - numpydoc=1.4
+    - nvcc_linux-64=11.5
     - pandas=1.3
     - pip
-    - pkg-config # for srf cmake
+    - pkg-config # for mrc cmake
     - pluggy=1.0
     - protobuf=3.20
     - pybind11-stubgen=0.10.5
@@ -81,7 +83,6 @@ dependencies:
     - scikit-build=0.13
     - sphinx
     - sphinx_rtd_theme
-    - srf 22.11.*
     - sysroot_linux-64=2.17
     - tqdm=4
     - typing_utils=0.1

docs/source/developer_guide/guides/1_simple_python_stage.md

@@ -34,7 +34,7 @@ We start our class definition with a few basic imports:
 ```python
 import typing
 
-import srf
+import mrc
 
 from morpheus.cli.register_stage import register_stage
 from morpheus.pipeline.single_port_stage import SinglePortStage
@@ -75,7 +75,7 @@ Our `on_data` method accepts the incoming message and returns a message. The ret
 
 Finally, the `_build_single` method will be used at build time to wire our stage into the pipeline. `_build_single` receives an instance of the SRF pipeline segment along with a `StreamPair` instance. We will be using the segment instance to build a node from our stage and add it to the pipeline segment. The `StreamPair` argument is a tuple; the first element is our parent node, and the second is our parent node's output type. The return type of this method is also a `StreamPair`. Typically, we will be returning our newly constructed node along with our output type.
 ```python
-    def _build_single(self, builder: srf.Builder, input_stream: StreamPair) -> StreamPair:
+    def _build_single(self, builder: mrc.Builder, input_stream: StreamPair) -> StreamPair:
         node = builder.make_node(self.unique_name, self.on_data)
         builder.make_edge(input_stream[0], node)
 
@@ -101,7 +101,7 @@ return node, input_stream[1]
 ```python
 import typing
 
-import srf
+import mrc
 
 from morpheus.cli.register_stage import register_stage
 from morpheus.pipeline.single_port_stage import SinglePortStage
@@ -128,7 +128,7 @@ class PassThruStage(SinglePortStage):
         # Return the message for the next stage
         return message
 
-    def _build_single(self, builder: srf.Builder, input_stream: StreamPair) -> StreamPair:
+    def _build_single(self, builder: mrc.Builder, input_stream: StreamPair) -> StreamPair:
         node = builder.make_node(self.unique_name, self.on_data)
         builder.make_edge(input_stream[0], node)
 

docs/source/developer_guide/guides/2_real_world_phishing.md

@@ -84,7 +84,7 @@ Our `_build_single` method remains unchanged; even though we are modifying the i
 ```python
 import typing
 
-import srf
+import mrc
 
 from morpheus.cli.register_stage import register_stage
 from morpheus.messages.message_meta import MessageMeta
@@ -125,7 +125,7 @@ class RecipientFeaturesStage(SinglePortStage):
         # Return the message for the next stage
         return message
 
-    def _build_single(self, builder: srf.Builder, input_stream: StreamPair) -> StreamPair:
+    def _build_single(self, builder: mrc.Builder, input_stream: StreamPair) -> StreamPair:
         node = builder.make_node(self.unique_name, self.on_data)
         builder.make_edge(input_stream[0], node)
 
@@ -455,7 +455,7 @@ In this example, we will create a source that reads messages from a [RabbitMQ](h
 The `_build_source` method is similar to the `_build_single` method; it receives an instance of the pipeline segment and returns a `StreamPair`. However, unlike in the previous examples, source stages do not have parent stages and therefore do not receive a `StreamPair` as input. We also will no longer build our node by calling `make_node`. Instead, we will call `make_source` with the parameter `self.source_generator`, which is a method that we will define next.
 
 ```python
-def _build_source(self, builder: srf.Builder) -> StreamPair:
+def _build_source(self, builder: mrc.Builder) -> StreamPair:
     node = builder.make_source(self.unique_name, self.source_generator)
     return node, MessageMeta
 ```
@@ -498,7 +498,7 @@ from io import StringIO
 import cudf
 import pandas as pd
 import pika
-import srf
+import mrc
 
 from morpheus.cli.register_stage import register_stage
 from morpheus.config import Config
@@ -568,7 +568,7 @@ class RabbitMQSourceStage(SingleOutputSource):
 
         return super().stop()
 
-    def _build_source(self, builder: srf.Builder) -> StreamPair:
+    def _build_source(self, builder: mrc.Builder) -> StreamPair:
         node = builder.make_source(self.unique_name, self.source_generator)
         return node, MessageMeta
 
@@ -605,7 +605,7 @@ class WriteToRabbitMQStage(SinglePortStage):
 
 In our `_build_single` we will be making use of the `make_sink` method rather than `make_node` or `make_source`
 ```python
-def _build_single(self, builder: srf.Builder, input_stream: StreamPair) -> StreamPair:
+def _build_single(self, builder: mrc.Builder, input_stream: StreamPair) -> StreamPair:
     node = builder.make_sink(self.unique_name, self.on_data, self.on_error, self.on_complete)
     builder.make_edge(input_stream[0], node)
     return input_stream
@@ -647,7 +647,7 @@ import typing
 from io import StringIO
 
 import pika
-import srf
+import mrc
 
 from morpheus.cli.register_stage import register_stage
 from morpheus.config import Config
@@ -697,7 +697,7 @@ class WriteToRabbitMQStage(SinglePortStage):
     def supports_cpp_node(self) -> bool:
         return False
 
-    def _build_single(self, builder: srf.Builder, input_stream: StreamPair) -> StreamPair:
+    def _build_single(self, builder: mrc.Builder, input_stream: StreamPair) -> StreamPair:
         node = builder.make_sink(self.unique_name, self.on_data, self.on_error, self.on_complete)
         builder.make_edge(input_stream[0], node)
         return input_stream