Updated branch changes to reflect 23.11

.clang-format

@@ -1,6 +1,7 @@
----
 # Refer to the following link for the explanation of each params:
 #   https://releases.llvm.org/14.0.0/tools/clang/docs/ClangFormatStyleOptions.html
+
+---
 Language: Cpp
 # BasedOnStyle: Google
 AccessModifierOffset: -2
@@ -164,3 +165,7 @@ StatementMacros:
 # Be consistent with indent-width, even for people who use tab for indentation!
 TabWidth: 4
 UseTab: Never
+
+---
+Language: Json
+TabWidth: 2

.clang-tidy

@@ -91,7 +91,7 @@ CheckOptions:
   - key:             readability-identifier-naming.GetConfigPerFile
     value:           'true'
   - key:             readability-identifier-naming.GlobalConstantCase
-    value:           CamelCase
+    value:           UPPER_CASE
   - key:             readability-identifier-naming.GlobalConstantIgnoredRegexp
     value:           ''
   - key:             readability-identifier-naming.GlobalConstantPrefix

.devcontainer/Dockerfile

@@ -13,20 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM nvidia/cuda:11.5.1-devel-ubuntu20.04 AS base
+FROM rapidsai/devcontainers:23.10-cpp-cuda11.8-mambaforge-ubuntu22.04 AS base
 
-RUN apt-get update && \
-    apt-get install --no-install-recommends --yes \
-        wget git-lfs gdb iputils-ping
-
-ENV CONDA_DIR=/opt/conda
-
-RUN wget "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" && \
-    bash Mambaforge-$(uname)-$(uname -m).sh -b -p ${CONDA_DIR} && \
-    echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
-    echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
-
-ENV PATH="${CONDA_DIR}/bin:${PATH}"
-
-ENV MAMBA_NO_BANNER=1
 ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin"

.devcontainer/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

.devcontainer/bin/dev-kafka-logs

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml"
 
 docker compose -f $COMPOSE_FILE logs zookeeper kafka

.devcontainer/bin/dev-kafka-produce

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml"
 
 if [[ $# -eq 2 ]]; then

.devcontainer/bin/dev-kafka-start

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml"
 
 docker compose -f $COMPOSE_FILE up -d zookeeper kafka

.devcontainer/bin/dev-kafka-stop

@@ -14,4 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 docker compose -f ${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml stop kafka zookeeper

.devcontainer/bin/dev-pytest-kafka-install

@@ -0,0 +1,28 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Replicates the install instructions from the CI runner image
+sudo apt update
+
+# Install openjdk
+sudo apt install -y openjdk-11-jre-headless
+
+# Download kafka
+python ${MORPHEUS_ROOT}/ci/scripts/download_kafka.py
+
+pip install pytest-kafka==0.6.0

.devcontainer/bin/dev-triton-start

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 args=""
 
 if [ $# -gt 0 ] ; then

.devcontainer/bin/dev-triton-stop

@@ -14,4 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 docker compose -f ${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml stop triton

.devcontainer/devcontainer.json

@@ -38,27 +38,68 @@
 
     "containerEnv": {
         "HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}",
-        "MORPHEUS_ROOT": "/workspaces/morpheus"
+        "MORPHEUS_ROOT": "${containerWorkspaceFolder}",
+        "DEFAULT_CONDA_ENV": "morpheus",
+        "MAMBA_NO_BANNER": "1",
+        "CONDA_ALWAYS_YES": "true",
+        "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+        "DISPLAY": "${localEnv:DISPLAY:-}",
+        "XAUTHORITY": "${localEnv:XAUTHORITY:-}",
+        "XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE:-}",
+        "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR:-}",
+        "DBUS_SESSION_BUS_ADDRESS": "${localEnv:DBUS_SESSION_BUS_ADDRESS:-}"
     },
 
-    "postStartCommand": "${MORPHEUS_ROOT}/.devcontainer/bin/setup-morpheus-env",
-
-    "initializeCommand": [ "./.devcontainer/init.sh" ],
+    "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ],
 
     "remoteUser": "coder",
-
     "mounts": [
-        {"type":"bind", "source": "/var/run/docker.sock", "target": "/var/run/docker.sock"},
-        {"type":"bind", "source": "${localWorkspaceFolder}/.cache/conda", "target": "/home/coder/.conda"}
+        {
+            "type": "bind",
+            "source": "/tmp/.X11-unix",
+            "target": "/tmp/.X11-unix"
+        },
+        {
+            "type": "bind",
+            "source": "${localEnv:XDG_RUNTIME_DIR}",
+            "target": "${localEnv:XDG_RUNTIME_DIR}"
+        },
+        {
+            "type": "bind",
+            "source": "/run/dbus/system_bus_socket",
+            "target": "/run/dbus/system_bus_socket"
+        },
+        {
+            "type":"bind",
+            "source": "/var/run/docker.sock",
+            "target": "/var/run/docker.sock"
+        },
+        {
+            "type": "bind",
+            "source": "${localWorkspaceFolder}/.cache/conda/envs",
+            "target": "/home/coder/.conda/envs"
+        },
+        {
+            "type": "bind",
+            "source": "${localWorkspaceFolder}/../.conda/pkgs",
+            "target": "/home/coder/.conda/pkgs"
+        },
+        {
+            "type":"bind",
+            "source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs
+            "target": "/home/coder/.config"
+        },
+        {
+            "type": "bind",
+            "source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus",
+            "target": "/opt/morpheus"
+        },
+
     ],
 
     "features": {
-        "ghcr.io/devcontainers/features/common-utils:1": {
-            "uid": "1000",
-            "gid": "1000",
-            "username": "coder"
-        },
-        "ghcr.io/devcontainers/features/docker-from-docker": {}
+        "ghcr.io/devcontainers/features/docker-from-docker": {},
+        "ghcr.io/devcontainers/features/dotnet:1": {}
     },
 
     "customizations": {
@@ -72,11 +113,13 @@
 				"nvidia.nsight-vscode-edition",
 				"twxs.cmake",
 				"xaver.clang-format"
-			]
+			],
+            "settings": {
+                "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake",
+                "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet",
+                "C_Cpp.intelliSenseEngine": "disabled",
+                "python.terminal.activateEnvironment": false
+            }
         }
     },
-    "settings": {
-        "C_Cpp.intelliSenseEngine": "disabled",
-        "python.terminal.activateEnvironment": false
-    }
 }

.devcontainer/docker-compose.yml

@@ -20,15 +20,15 @@ services:
   triton:
     container_name: morpheus-triton
     runtime: nvidia
-    image: nvcr.io/nvidia/tritonserver:22.08-py3
-    command: tritonserver --model-repository=/models --exit-on-error=false ${TRITON_MODEL_ARGS}
+    image: nvcr.io/nvidia/tritonserver:23.06-py3
+    command: tritonserver --model-repository=/models/triton-model-repo --exit-on-error=false ${TRITON_MODEL_ARGS}
     ports:
       - 8000:8000
       - 8001:8001
       - 8002:8002
     volumes:
-      - ${HOST_MORPHEUS_ROOT}/models/triton-model-repo:/models
-      - ${HOST_MORPHEUS_ROOT}/examples/abp_pcap_detection/abp-pcap-xgb:/models/abp-pcap-xgb
+      - ${HOST_MORPHEUS_ROOT}/models:/models
+      - ${HOST_MORPHEUS_ROOT}/examples/abp_pcap_detection/abp-pcap-xgb:/models/triton-model-repo/abp-pcap-xgb
 
   zookeeper:
     image: bitnami/zookeeper:latest

.devcontainer/init.sh → .devcontainer/initialize-command.sh

@@ -14,8 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 # create a docker network for morpheus
 docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus
 
 # create the parent conda folder so it's found when mounting
-mkdir -p ./.cache/conda
+mkdir -p .cache/conda/envs
+mkdir -p ../.conda/pkgs
+
+# create a config directory if it does not exist so it's found when mounting
+mkdir -p ../.config

.devcontainer/bin/setup-morpheus-env → .devcontainer/opt/morpheus/bin/post-attach-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Ensure our ~/.config directory has the correct permissions. If ~/.config did
+# not exist, and you mount ~/.config/gh from the host, then ~/.config will be
+# created with root permissions which can break things
+
 conda_env_find(){
     conda env list | grep "${@}" >/dev/null 2>/dev/null
 }
@@ -23,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus}
 sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
-then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \
-else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \
 fi

.devcontainer/opt/morpheus/bin/update-content-command.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ensure our ~/.config directory has the correct permissions. If ~/.config did
+# not exist, and you mount ~/.config/gh from the host, then ~/.config will be
+# created with root permissions which can break things
+
+if [[ ! -f ~/.config/.gdbinit ]]; then
+    cp /opt/morpheus/etc/.gdbinit ~/.config/.gdbinit
+fi

.devcontainer/opt/morpheus/etc/.gdbinit

@@ -0,0 +1,3 @@
+add-auto-load-safe-path /opt/conda
+
+source /opt/morpheus/etc/enable_conda_libstd_pretty_print.py

.devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -*- python -*-
+import os
+import subprocess
+import sys
+
+import gdb
+
+conda_env_path = os.environ.get("CONDA_PREFIX", None)
+
+if (conda_env_path is not None):
+
+    gcc_path = os.environ.get("GCC", None)
+
+    if (gcc_path is None):
+        print(
+            "Could not find gcc from $GCC: '{}'. Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment"
+            .format(gcc_path))
+    else:
+        # Get the GCC version
+        result = subprocess.run([gcc_path, '-dumpversion'], stdout=subprocess.PIPE)
+        gcc_version = result.stdout.decode("utf-8").strip()
+
+        # Build the gcc python path
+        gcc_python_path = os.path.join(conda_env_path, "share", "gcc-{}".format(gcc_version), "python")
+
+        if (os.path.exists(gcc_python_path)):
+
+            # Add to the path for the pretty printer
+            sys.path.insert(0, gcc_python_path)
+
+            # Now register the pretty printers
+            from libstdcxx.v6 import register_libstdcxx_printers
+            register_libstdcxx_printers(gdb.current_objfile())
+
+            print("Loaded stdlibc++ pretty printers")
+        else:
+            print("Could not find gcc python files at: {}".format(gcc_python_path))
+            print(
+                "Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment"
+            )

.gitattributes

@@ -1,8 +1,11 @@
 docs/source/developer_guide/guides/img/** filter=lfs diff=lfs merge=lfs -text
 examples/data/** filter=lfs diff=lfs merge=lfs -text
+examples/digital_fingerprinting/production/grafana/img/** filter=lfs diff=lfs merge=lfs -text
 examples/digital_fingerprinting/visualization/img/** filter=lfs diff=lfs merge=lfs -text
 morpheus/_version.py export-subst
 tests/mock_triton_server/payloads/** filter=lfs diff=lfs merge=lfs -text
 tests/tests_data/** filter=lfs diff=lfs merge=lfs -text
 examples/basic_usage/img/** filter=lfs diff=lfs merge=lfs -text
 docs/source/img/* filter=lfs diff=lfs merge=lfs -text
+git filter=lfs diff=lfs merge=lfs -text
+status filter=lfs diff=lfs merge=lfs -text