Merge branch 'branch-23.11' of github.com:nv-morpheus/Morpheus into d…

…avid-pipe-build-refactor-stage-schema

CHANGELOG.md

@@ -15,6 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 -->
 
+# Morpheus 23.07.03 (11 Oct 2023)
+
+## 🐛 Bug Fixes
+- Add pinned libwebp to resolve CVE ([#1236](https://github.com/nv-morpheus/Morpheus/pull/1236)) [@drobison00](https://github.com/drobison00)
+- Add libwebp to meta.yaml for CVE 2307 ([#1242](https://github.com/nv-morpheus/Morpheus/pull/1242)) [@drobison00](https://github.com/drobison00)
+- [BUG] Fix Control Message Utils & SQL Max Connections Exhaust ([#1243](https://github.com/nv-morpheus/Morpheus/pull/1243)) [@bsuryadevara](https://github.com/bsuryadevara)
+
 # Morpheus 23.07.02 (25 Jul 2023)
 
 ## 🐛 Bug Fixes

docker/conda/environments/cuda11.8_dev.yml

@@ -70,7 +70,7 @@ dependencies:
     - mrc=23.11
     - networkx=3.1
     - ninja=1.10
-    - nodejs=18.15.0
+    - nodejs=18.*
     - numba>=0.56.2
     - numpydoc=1.4
     - nvtabular=23.06
@@ -110,3 +110,5 @@ dependencies:
         # Add additional dev dependencies here
         - databricks-connect
         - pytest-kafka==0.6.0
+        - pymilvus==2.3.1
+        - milvus

docs/source/conf.py

@@ -169,6 +169,7 @@
     "morpheus.cli.commands",  # Dont document the CLI in Sphinx
     "nvtabular",
     "pandas",
+    "pymilvus",
     "tensorrt",
     "torch",
     "tqdm",

models/training-tuning-scripts/abp-models/requirements.yml

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - rapidsai
+  - nvidia
+  - conda-forge
+dependencies:
+  - cuml=23.06
+  - jupyterlab
+  - python=3.10
+  - scikit-learn=1.2.2
+  - xgboost

models/training-tuning-scripts/dfp-models/hammah-20211017-script.py

@@ -102,7 +102,7 @@ def main():
         min_cats=1  # cut off for minority categories
     )
 
-    model.fit(x_train, epochs=25, validation_data=x_val)
+    model.fit(x_train, epochs=25, validation_data=x_val, run_validation=True)
 
     torch.save(model.state_dict(), args.trainingdata[:-4] + ".pkl")
     with open(args.trainingdata[:-4] + 'dill' + '.pkl', 'wb') as f:

models/training-tuning-scripts/dfp-models/requirements.yml

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - nvidia
+  - pytorch
+  - conda-forge
+dependencies:
+  - dill
+  - jupyterlab
+  - matplotlib
+  - pandas=1.3
+  - python=3.10
+  - pytorch-cuda=11.8
+  - pytorch=2.0.1
+  - scikit-learn=1.2.2
+  - seaborn

models/training-tuning-scripts/fraud-detection-models/requirements.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +14,20 @@
 # limitations under the License.
 
 channels:
-    - rapidsai
-    - nvidia
-    - conda-forge
+  - rapidsai
+  - nvidia
+  - dglteam/label/cu118
+  - pytorch
+  - conda-forge
 dependencies:
-    - click==8.1.3
-    - cuml=23.06
-    - dgl==1.0.2+cu118
-    - numpy==1.23.5
-    - pandas==1.5.3
-    - scikit_learn==1.2.2
-    - torch==2.0.0+cu118
-    - torchmetrics==0.11.4
-    - tqdm==4.65.0
-    - xgboost==1.7.1
+  - click>=8
+  - cuml=23.06
+  - dgl
+  - jupyterlab
+  - matplotlib
+  - pytorch-cuda=11.8
+  - pytorch=2.0.1
+  - scikit-learn=1.2.2
+  - tqdm=4
+  - torchmetrics
+  - xgboost

models/training-tuning-scripts/log-parsing-models/requirements.yml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - rapidsai
+  - nvidia
+  - pytorch
+  - conda-forge
+dependencies:
+  - cudf=23.06
+  - jupyterlab
+  - onnx
+  - python=3.10
+  - pytorch-cuda=11.8
+  - pytorch=2.0.1
+  - seqeval=1.2.2
+  - tqdm=4
+  - transformers

models/training-tuning-scripts/phishing-models/requirements.yml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - rapidsai
+  - nvidia
+  - pytorch
+  - conda-forge
+dependencies:
+  - cudf=23.06
+  - jupyterlab
+  - onnx
+  - python=3.10
+  - pytorch-cuda=11.8
+  - pytorch=2.0.1
+  - scikit-learn=1.2.2
+  - tqdm=4
+  - transformers

models/training-tuning-scripts/ransomware-models/requirements.yml

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - conda-forge
+dependencies:
+  - jupyterlab
+  - matplotlib
+  - pandas=1.3
+  - python=3.10
+  - scikit-learn=1.2.2
+  - seaborn

models/training-tuning-scripts/root-cause-models/requirements.yml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - rapidsai
+  - nvidia
+  - pytorch
+  - conda-forge
+dependencies:
+  - cudf=23.06
+  - jupyterlab
+  - onnx
+  - python=3.10
+  - pytorch-cuda=11.8
+  - pytorch=2.0.1
+  - scikit-learn=1.2.2
+  - tqdm=4
+  - transformers

models/training-tuning-scripts/sid-models/requirements.yml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+channels:
+  - rapidsai
+  - nvidia
+  - pytorch
+  - conda-forge
+dependencies:
+  - cudf=23.06
+  - jupyterlab
+  - onnx
+  - python=3.10
+  - pytorch-cuda=11.8
+  - pytorch=2.0.1
+  - scikit-learn=1.2.2
+  - tqdm=4
+  - transformers

models/training-tuning-scripts/sid-models/sid-minibert-20230424.ipynb

@@ -31,7 +31,6 @@
    "outputs": [],
    "source": [
     "from os import path\n",
-    "import s3fs\n",
     "import torch\n",
     "from torch.nn import BCEWithLogitsLoss\n",
     "from transformers import AutoModelForSequenceClassification, AdamW\n",

morpheus/_lib/modules/module.cpp

@@ -21,8 +21,11 @@
 
 #include <mrc/modules/module_registry_util.hpp>
 #include <nlohmann/json.hpp>
+#include <pybind11/cast.h>      // for object_api::operator(), object::cast
 #include <pybind11/pybind11.h>  // for arg, init, class_, module_, str_attr_accessor, PYBIND11_MODULE, pybind11
+#include <pybind11/pytypes.h>
 
+#include <array>  // for array
 #include <sstream>
 #include <vector>
 
@@ -39,11 +42,19 @@ PYBIND11_MODULE(modules, _module)
 
         )pbdoc";
 
-    const std::vector<unsigned int> MorpheusModuleVersion{
-        morpheus_VERSION_MAJOR, morpheus_VERSION_MINOR, morpheus_VERSION_PATCH};
+    // Get the MRC version that we are registering these modules for. Ideally, this would be able to get it directly
+    // from <mrc/version.hpp> but that file isnt exported
+    std::vector<unsigned int> mrc_version;
 
-    mrc::modules::ModelRegistryUtil::create_registered_module<DataLoaderModule>(
-        "DataLoader", "morpheus", MorpheusModuleVersion);
+    auto mrc_version_list = pybind11::module_::import("mrc").attr("__version__").attr("split")(".").cast<py::list>();
+
+    for (const auto& l : mrc_version_list)
+    {
+        auto i = py::int_(py::reinterpret_borrow<py::object>(l));
+        mrc_version.push_back(i.cast<unsigned int>());
+    }
+
+    mrc::modules::ModelRegistryUtil::create_registered_module<DataLoaderModule>("DataLoader", "morpheus", mrc_version);
 
     _module.attr("__version__") =
         MORPHEUS_CONCAT_STR(morpheus_VERSION_MAJOR << "." << morpheus_VERSION_MINOR << "." << morpheus_VERSION_PATCH);

morpheus/service/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.