Final cleanup for 23.01 release (nv-morpheus#654)

Includes various fixes for the following:

1. Fixing copyright headers
2. Fixing python style checks which were missing file exclusions
3. Fixing python style
4. Fixing cpp style
5. Fixing release container build

Authors:
  - Michael Demoret (https://github.com/mdemoret-nv)

Approvers:
  - Devin Robison (https://github.com/drobison00)
  - David Gardner (https://github.com/dagardner-nv)
  - Bartley Richardson (https://github.com/BartleyR)
  - Tad ZeMicheal (https://github.com/tzemicheal)
  - https://github.com/raykallen
  - Bhargav Suryadevara (https://github.com/bsuryadevara)

URL: nv-morpheus#654

.devcontainer/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

CHANGELOG.md

@@ -1,3 +1,20 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
 # Morpheus 22.11.00 (18 Nov 2022)
 
 ## 🐛 Bug Fixes

CONTRIBUTING.md

@@ -1 +1,18 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
 Refer to the [Contributing to Morpheus](./docs/source/developer_guide/contributing.md) guide.

ci/conda/channel/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

ci/conda/recipes/morpheus/meta.yaml

@@ -55,37 +55,42 @@ outputs:
         - ccache
         - ninja
       host:
+        - cuda-python <=11.7.0
+        - cudatoolkit {{ cuda_version }}.*
         - cudf {{ rapids_version }}
         - cython >=0.29,<0.30
         - libcudf {{ rapids_version }}
         - librdkafka 1.7
+        - mrc {{ minor_version }}
         - pip
         - pybind11-stubgen 0.10.5
         - python {{ python }}
         - rapidjson 1.1
         - scikit-build >=0.12
-        - mrc {{ minor_version }}
         - versioneer-518
       run:
         # Runtime only requirements. This + setup.y is the definitive runtime requirement list
         - click >=8
-        - configargparse 1.5
-        - cudf {{ rapids_version }}
-        - cudf_kafka {{ rapids_version }}
+        - configargparse 1.5.*
+        - cuda-python <=11.7.0 # Needed to ensure cudatoolkit and not cuda-toolkit is installed
+        - cudf
+        - cudf_kafka {{ rapids_version }}.*
         - cupy # Version determined from cudf
-        - datacompy 0.8
+        - datacompy 0.8.*
         - distributed
-        - docker-py 5.0
+        - docker-py 5.0.*
         - grpcio # Version determined from cudf
+        - libmrc
         - mlflow >1.29,<2
-        - networkx 2.8
-        - numpydoc 1.4
-        - pandas 1.3
-        - pluggy 1.0
+        - mrc
+        - networkx 2.8.*
+        - numpydoc 1.4.*
+        - pandas 1.3.*
+        - pluggy 1.0.*
         - python
-        - tqdm 4
-        - typing_utils 0.1
-        - watchdog 2.1
+        - tqdm 4.*
+        - typing_utils 0.1.*
+        - watchdog 2.1.*
 
     test:
       requires:

ci/conda/recipes/morpheus/morpheus_build.sh

@@ -40,6 +40,8 @@ CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_STUBS=${MORPHEUS_BUILD_PYTHON_STUBS=-"ON"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_WHEEL=ON ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"all"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
 
@@ -77,10 +79,4 @@ cmake -B ${BUILD_DIR} \
 cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)}
 
 # Install just the python wheel components
-cmake -DCOMPONENT=Wheel -P ${BUILD_DIR}/cmake_install.cmake
-
-# Change to the wheel install dir
-cd ${BUILD_DIR}/wheel
-
-# Install the python library
-${PYTHON} -m pip install -vv --no-deps .
+${PYTHON} -m pip install -vv --no-deps ${BUILD_DIR}/dist/*.whl

ci/conda/recipes/run_conda_build.sh

@@ -34,6 +34,9 @@ export e="\033[0;90m"
 export y="\033[0;33m"
 export x="\033[0m"
 
+# Ensure yes is always selected otherwise it can stop halfway through on a prelink message
+export CONDA_ALWAYS_YES=true
+
 # Change this to switch between build/mambabuild/debug
 export CONDA_COMMAND=${CONDA_COMMAND:-"mambabuild"}
 
@@ -97,8 +100,8 @@ CONDA_ARGS_ARRAY+=("--variants" "{python: 3.8}")
 
 # And default channels (with optional channel alias)
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapidsai")
+CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/cuda-11.5.2") # Prefer labeled versions over the latest
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia")
-CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/cuda-11.5.2")
 CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev")
 CONDA_ARGS_ARRAY+=("-c" "conda-forge")
 

ci/runner/README.md

@@ -1,5 +1,5 @@
 <!--
- SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  SPDX-License-Identifier: Apache-2.0
 
  Licensed under the Apache License, Version 2.0 (the "License");

ci/scripts/common.sh

@@ -22,7 +22,7 @@ export PY_CFG="${PY_ROOT}/setup.cfg"
 export PY_DIRS="${PY_ROOT} ci/scripts"
 
 # work-around for known yapf issue https://github.com/google/yapf/issues/984
-export YAPF_EXCLUDE_FLAGS="-e ${PY_ROOT}/versioneer.py -e ${PY_ROOT}/morpheus/_version.py"
+export YAPF_EXCLUDE_FLAGS="-e versioneer.py -e morpheus/_version.py"
 
 # Determine the commits to compare against. If running in CI, these will be set. Otherwise, diff with main
 export BASE_SHA=${CHANGE_TARGET:-${BASE_SHA:-$(${SCRIPT_DIR}/gitutils.py get_merge_target)}}

ci/scripts/copyright.py

@@ -18,6 +18,7 @@
 import argparse
 import datetime
 import io
+import logging
 import os
 import re
 import sys
@@ -231,6 +232,8 @@ def checkCopyright_main():
     retVal = 0
     global ExemptFiles
 
+    logging.basicConfig(level=logging.DEBUG)
+
     argparser = argparse.ArgumentParser("Checks for a consistent copyright header in git's modified files")
     argparser.add_argument("--update-start-year",
                            dest='update_start_year',

ci/scripts/python_checks.sh

@@ -54,7 +54,7 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
 
    if [[ "${SKIP_YAPF}" == "" ]]; then
       # Run yapf. Will return 1 if there are any diffs
-      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
+      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} ${YAPF_EXCLUDE_FLAGS} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
       YAPF_RETVAL=$?
    fi
 

docker/conda/environments/cuda11.5_dev.yml

@@ -16,9 +16,9 @@
 name: morpheus
 channels:
     - rapidsai
+    - nvidia/label/cuda-11.5.2 # Prefer packages with a specific cuda label over ones without
     - nvidia
     - nvidia/label/dev # For pre-releases of MRC. Should still default to full releases if available
-    - nvidia/label/cuda-11.5.2 # For cuda-nvml-dev=11.5, which is not published under nvidia channel yet.
     - conda-forge
 dependencies:
     ####### Morpheus Dependencies (keep sorted!) #######
@@ -32,6 +32,7 @@ dependencies:
     - cmake=3.24
     - configargparse=1.5
     - cuda-nvml-dev=11.5
+    - cuda-python<=11.7.0 # Remove when Issue #251 is closed
     - cudatoolkit=11.5
     - cudf 22.10
     - cupy=9.5.0

docs/source/developer_guide/guides/index.rst

@@ -1,5 +1,5 @@
 ..
-   SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+   SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
    SPDX-License-Identifier: Apache-2.0
 
    Licensed under the Apache License, Version 2.0 (the "License");

docs/source/examples/index.rst

@@ -1,5 +1,5 @@
 ..
-   SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+   SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
    SPDX-License-Identifier: Apache-2.0
 
    Licensed under the Apache License, Version 2.0 (the "License");

docs/source/sphinxext/github_link.py

@@ -47,8 +47,7 @@ def isfunction(obj):
 
 REVISION_CMD = 'git rev-parse --short HEAD'
 
-source_regex = re.compile(r"^File: (.*?) \(starting at line ([0-9]*?)\)$",
-                          re.MULTILINE)
+source_regex = re.compile(r"^File: (.*?) \(starting at line ([0-9]*?)\)$", re.MULTILINE)
 
 
 def _get_git_revision():
@@ -118,9 +117,7 @@ def _linkcode_resolve(domain, info, package, url_fmt, revision):
 
             # fn is expected to be the absolute path.
             fn = os.path.relpath(source_file, start=package)
-            print("{}:{}".format(
-                os.path.abspath(os.path.join("..", "python", "cuml", fn)),
-                lineno))
+            print("{}:{}".format(os.path.abspath(os.path.join("..", "python", "cuml", fn)), lineno))
         else:
             return
     else:
@@ -130,9 +127,7 @@ def _linkcode_resolve(domain, info, package, url_fmt, revision):
             fn = os.path.abspath(os.path.join("..", "python", fn))
 
         # Convert to relative from module root
-        fn = os.path.relpath(fn,
-                             start=os.path.dirname(
-                                 __import__(package).__file__))
+        fn = os.path.relpath(fn, start=os.path.dirname(__import__(package).__file__))
 
     # Get the line number if we need it. (Can work without it)
     if (lineno is None):
@@ -145,10 +140,7 @@ def _linkcode_resolve(domain, info, package, url_fmt, revision):
                 lineno = obj.__code__.co_firstlineno
             else:
                 lineno = ''
-    return url_fmt.format(revision=revision,
-                          package=package,
-                          path=fn,
-                          lineno=lineno)
+    return url_fmt.format(revision=revision, package=package, path=fn, lineno=lineno)
 
 
 def make_linkcode_resolve(package, url_fmt):
@@ -163,7 +155,4 @@ def make_linkcode_resolve(package, url_fmt):
                                    '{path}#L{lineno}')
     """
     revision = _get_git_revision()
-    return partial(_linkcode_resolve,
-                   revision=revision,
-                   package=package,
-                   url_fmt=url_fmt)
+    return partial(_linkcode_resolve, revision=revision, package=package, url_fmt=url_fmt)

examples/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

examples/digital_fingerprinting/production/morpheus/dfp/utils/module_ids.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

models/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

models/datasets/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

models/mlflow/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

models/training-tuning-scripts/fraud-detection-models/README.md

@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");

models/training-tuning-scripts/phishing-models/dataset.py

@@ -14,6 +14,7 @@
 
 
 class Dataset(object):
+
     def __init__(self, df):
         self._df = df.reset_index(drop=True)
         self._dataset_len = self._df.shape[0]

models/training-tuning-scripts/root-cause-models/dataset.py

@@ -16,6 +16,7 @@
 
 
 class Dataset(object):
+
     def __init__(self, df):
         self._df = df.reset_index(drop=True)
         self._dataset_len = self._df.shape[0]

models/training-tuning-scripts/root-cause-models/root-cause-bert.py

@@ -39,15 +39,11 @@ def train(trainingdata, unseenerrors):
 
     random_seed = 42
 
-    dflogs = pd.read_csv(trainingdata, header=None, names=['label',
-                         'log'])
+    dflogs = pd.read_csv(trainingdata, header=None, names=['label', 'log'])
 
-    dfnewerror = pd.read_csv(unseenerrors, header=None, names=['label',
-                             'log'])
+    dfnewerror = pd.read_csv(unseenerrors, header=None, names=['label', 'log'])
 
-    (X_train, X_test, y_train, y_test) = train_test_split(
-        dflogs, dflogs.label, random_state=random_seed
-    )
+    (X_train, X_test, y_train, y_test) = train_test_split(dflogs, dflogs.label, random_state=random_seed)
 
     X_train.reset_index(drop=True, inplace=True)
 
@@ -73,17 +69,15 @@ def train(trainingdata, unseenerrors):
     torch.cuda.manual_seed(random_seed)
     np.random.seed(random_seed)
     torch.backends.cudnn.deterministic = True
-    seq_classifier.train_model(X_train['log'], y_train, batch_size=128,
-                               epochs=1, learning_rate=3.6e-04)
+    seq_classifier.train_model(X_train['log'], y_train, batch_size=128, epochs=1, learning_rate=3.6e-04)
 
     timestr = time.strftime('%Y%m%d-%H%M%S')
 
     seq_classifier.save_model(timestr)
 
     print(seq_classifier.evaluate_model(X_test['log'], y_test))
 
-    test_preds = seq_classifier.predict(X_test['log'], batch_size=128,
-                                        threshold=0.5)
+    test_preds = seq_classifier.predict(X_test['log'], batch_size=128, threshold=0.5)
 
     tests = test_preds[0].to_numpy()
 
@@ -99,12 +93,11 @@ def main():
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument('--trainingdata', required=True,
-                        help='Labelled data in CSV format')
-    parser.add_argument('--unseenerrors', required=True,
+    parser.add_argument('--trainingdata', required=True, help='Labelled data in CSV format')
+    parser.add_argument('--unseenerrors',
+                        required=True,
                         help="""Labelled data to be added to test set for
-                        evaluation after training"""
-                        )
+                        evaluation after training""")
     args = parser.parse_args()
 
 main()