Slimmed down CI runners and published artifact urls (nv-morpheus#1112)

* Publish artifact urls in job summaries fixes nv-morpheus#1105
* Remove redundant apt installs for packages that exist in the conda env fixes nv-morpheus#1106
* Checks now performs a build first. This allows pylint to accurately warn for undefined members. fixes nv-morpheus#956
* Custom pylint checker to validate correct importing of `morpheus._lib` fixes nv-morpheus#727

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: nv-morpheus#1112

.github/workflows/pull_request.yml

@@ -46,7 +46,7 @@ jobs:
     uses: ./.github/workflows/ci_pipe.yml
     with:
       run_check: ${{ startsWith(github.ref_name, 'pull-request/') }}
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230725
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230725
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230801
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230801
     secrets:
       NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

ci/pylint/morpheus_lib_import_checker.py

@@ -0,0 +1,78 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from pylint.checkers import BaseChecker
+
+
+class MorpheusLibImportChecker(BaseChecker):
+    """
+    Custom pylint checker to detect incorrect imports of morpheus._lib modules.
+
+    refer to https://pylint.readthedocs.io/en/stable/development_guide/how_tos/custom_checkers.html
+    """
+    name = 'morpheus_lib_import_checker'
+    msgs = {
+        'W9901': ('Incorrect import of %(name)s as %(alias)s. Imports from morpheus._lib should be in the form of: '
+                  '"import morpheus._lib.%(name)s as %(expected_alias)s"',
+                  'morpheus-incorrect-lib-import',
+                  'Used when a forbidden library is imported.'),
+        'W9902':
+            ('Incorrect import of %(name)s from %(modname)s as %(alias)s. Importing symbols from morpheus._lib should '
+             'only occur from an __init__.py file: '
+             '"from morpheus.%(short_mod)s import %(name)s"',
+             'morpheus-incorrect-lib-from-import',
+             'Used when a forbidden library is imported.'),
+    }
+
+    _LIB_MODULES = ['morpheus._lib']
+
+    def visit_import(self, node) -> None:
+        """
+        Bans the following:
+        * import morpheus._lib.XXX
+        * import morpheus._lib.XXX as YYY
+        """
+        for name, alias in node.names:
+            for lib_module in self._LIB_MODULES:
+                if name.startswith(lib_module):
+                    expected_alias = f"_{name.split('.')[-1]}"
+                    if alias != expected_alias:
+                        self.add_message('morpheus-incorrect-lib-import',
+                                         node=node,
+                                         args={
+                                             'name': name, 'alias': alias, 'expected_alias': expected_alias
+                                         })
+
+    def visit_importfrom(self, node) -> None:
+        """
+        Bans: from morpheus._lib.XXX import YYY
+        """
+        for name, alias in node.names:
+            for lib_module in self._LIB_MODULES:
+                if (node.modname.startswith(lib_module) and os.path.basename(node.root().file) != '__init__.py'):
+                    self.add_message('morpheus-incorrect-lib-from-import',
+                                     node=node,
+                                     args={
+                                         'name': name,
+                                         'alias': alias,
+                                         'modname': node.modname,
+                                         'short_mod': node.modname.split('.')[-1]
+                                     })
+
+
+def register(linter):
+    linter.register_checker(MorpheusLibImportChecker(linter))

ci/runner/Dockerfile

@@ -33,14 +33,6 @@ ARG CUDA_SHORT_VER
 
 SHELL ["/bin/bash",  "-c"]
 
-# OS deps
-RUN apt update && \
-    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
-    apt install --no-install-recommends -y \
-        libnuma1 && \
-    apt clean && \
-    rm -rf /var/lib/apt/lists/*
-
 # Create conda environment
 COPY ./docker/conda/environments/* /tmp/conda/
 
@@ -56,19 +48,6 @@ FROM base as build
 
 # Add any build only dependencies here.
 
-ARG CUDA_PKG_VER
-
-RUN apt update && \
-    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
-    apt install --no-install-recommends -y \
-        cuda-nvtx-${CUDA_PKG_VER} \
-        libcublas-dev-${CUDA_PKG_VER} \
-        libcufft-dev-${CUDA_PKG_VER} \
-        libcurand-dev-${CUDA_PKG_VER} \
-        libcusolver-dev-${CUDA_PKG_VER} && \
-    apt clean && \
-    rm -rf /var/lib/apt/lists/*
-
 # ============ test ==================
 FROM base as test
 

ci/scripts/github/build.sh

@@ -20,13 +20,7 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-rapids-logger "Check versions"
-python3 --version
-x86_64-conda-linux-gnu-cc --version
-x86_64-conda-linux-gnu-c++ --version
-cmake --version
-ninja --version
-sccache --version
+log_toolchain
 
 git submodule update --init --recursive
 
@@ -60,6 +54,7 @@ CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/morpheus/_lib/tests -name "*.x" -exec r
 tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${CPP_TESTS[@]}"
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
+set_job_summary_preamble
 upload_artifact "${WORKSPACE_TMP}/wheel.tar.bz"
 upload_artifact "${WORKSPACE_TMP}/morhpeus_libs.tar.bz"
 upload_artifact "${WORKSPACE_TMP}/cpp_tests.tar.bz"

ci/scripts/github/checks.sh

@@ -20,31 +20,44 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-fetch_base_branch
+log_toolchain
 
-rapids-logger "Checking copyright headers"
-python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}
+cd ${MORPHEUS_ROOT}
 
-rapids-logger "Runing Python style checks"
-${MORPHEUS_ROOT}/ci/scripts/python_checks.sh
+fetch_base_branch
 
 git submodule update --init --recursive
 
 rapids-logger "Configuring cmake for Morpheus"
 CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF"
+export CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_INPLACE_BUILD=ON"
 if [[ "${LOCAL_CI}" == "" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
 fi
+
 cmake -B build -G Ninja ${CMAKE_FLAGS} .
 
-rapids-logger "Building targets that generate source code"
-cmake --build build --target morpheus_style_checks --parallel ${PARALLEL_LEVEL}
+rapids-logger "Building Morpheus"
+cmake --build build --parallel ${PARALLEL_LEVEL}
 
 if [[ "${LOCAL_CI}" == "" ]]; then
     rapids-logger "sccache usage for source build:"
     sccache --show-stats
 fi
 
+rapids-logger "Installing Morpheus"
+pip install ./
+
+# Setting this prevents loading of cudf since we don't have a GPU
+export MORPHEUS_IN_SPHINX_BUILD=1
+
+rapids-logger "Checking copyright headers"
+python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}
+
+rapids-logger "Running Python style checks"
+${MORPHEUS_ROOT}/ci/scripts/python_checks.sh
+
 rapids-logger "Checking versions"
 ${MORPHEUS_ROOT}/ci/scripts/version_checks.sh
 

ci/scripts/github/common.sh

@@ -51,7 +51,7 @@ export ARTIFACT_URL="${S3_URL}${ARTIFACT_ENDPOINT}"
 if [[ "${LOCAL_CI}" == "1" ]]; then
     export DISPLAY_ARTIFACT_URL="${LOCAL_CI_TMP}"
 else
-    export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}/"
+    export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}"
 fi
 
 # Set sccache env vars
@@ -95,10 +95,11 @@ function update_conda_env() {
         conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML}
     fi
 
-    rapids-logger "Checking for updates to conda env"
-
-    # Update the packages
-    rapids-mamba-retry env update -n morpheus --prune -q --file ${ENV_YAML}
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        rapids-logger "Checking for updates to conda env"
+        # Update the packages
+        rapids-mamba-retry env update -n morpheus --prune -q --file ${ENV_YAML}
+    fi
 
     # Finally, reactivate
     conda activate morpheus
@@ -156,6 +157,16 @@ function show_conda_info() {
     conda list --show-channel-urls
 }
 
+function log_toolchain() {
+    rapids-logger "Check versions"
+    python3 --version
+    x86_64-conda-linux-gnu-cc --version
+    x86_64-conda-linux-gnu-c++ --version
+    cmake --version
+    ninja --version
+    sccache --version
+}
+
 function upload_artifact() {
     FILE_NAME=$1
     BASE_NAME=$(basename "${FILE_NAME}")
@@ -164,6 +175,7 @@ function upload_artifact() {
         cp ${FILE_NAME} "${LOCAL_CI_TMP}/${BASE_NAME}"
     else
         aws s3 cp --only-show-errors "${FILE_NAME}" "${ARTIFACT_URL}/${BASE_NAME}"
+        echo "- ${DISPLAY_ARTIFACT_URL}/${BASE_NAME}" >> ${GITHUB_STEP_SUMMARY}
     fi
 }
 
@@ -176,3 +188,9 @@ function download_artifact() {
         aws s3 cp --only-show-errors "${ARTIFACT_URL}/${ARTIFACT}" "${WORKSPACE_TMP}/${ARTIFACT}"
     fi
 }
+
+function set_job_summary_preamble() {
+    msg="Note: NVIDIA VPN access is required to view these URLs."
+    echo $msg >> ${GITHUB_STEP_SUMMARY}
+    rapids-logger $msg
+}

ci/scripts/github/docs.sh

@@ -44,6 +44,7 @@ rapids-logger "Archiving the docs"
 tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
+set_job_summary_preamble
 upload_artifact "${WORKSPACE_TMP}/docs.tar.bz"
 
 rapids-logger "Success"

ci/scripts/github/test.sh

@@ -115,6 +115,7 @@ cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR})
 
 rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}"
+set_job_summary_preamble
 upload_artifact ${WORKSPACE_TMP}/test_reports.tar.bz
 
 exit ${TEST_RESULTS}

ci/scripts/run_ci_local.sh

@@ -49,7 +49,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-230725}
+CONTAINER_VER=${CONTAINER_VER:-230801}
 CUDA_VER=${CUDA_VER:-11.8}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
@@ -63,6 +63,7 @@ ENV_LIST="${ENV_LIST} --env GIT_BRANCH=${GIT_BRANCH}"
 ENV_LIST="${ENV_LIST} --env GIT_COMMIT=${GIT_COMMIT}"
 ENV_LIST="${ENV_LIST} --env PARALLEL_LEVEL=$(nproc)"
 ENV_LIST="${ENV_LIST} --env CUDA_VER=${CUDA_VER}"
+ENV_LIST="${ENV_LIST} --env SKIP_CONDA_ENV_UPDATE=${SKIP_CONDA_ENV_UPDATE}"
 
 mkdir -p ${LOCAL_CI_TMP}
 cp ${MORPHEUS_ROOT}/ci/scripts/bootstrap_local_ci.sh ${LOCAL_CI_TMP}

docker/conda/environments/cuda11.8_dev.yml

@@ -95,7 +95,7 @@ dependencies:
     - tqdm=4
     - typing_utils=0.1
     - watchdog=2.1
-    - yapf=0.32.0
+    - yapf=0.40.1
     ####### Morpheus Pip Dependencies (keep sorted!) #######
     - pip:
         # Add additional dev dependencies here

morpheus/loaders/file_to_df_loader.py

@@ -28,8 +28,8 @@
 
 import cudf
 
-from morpheus._lib.common import FileTypes
 from morpheus.cli.utils import str_to_file_type
+from morpheus.common import FileTypes
 from morpheus.io.deserializers import read_file_to_df
 from morpheus.messages import ControlMessage
 from morpheus.messages.message_meta import MessageMeta

morpheus/modules/payload_batcher.py

@@ -21,8 +21,8 @@
 
 import cudf
 
-from morpheus._lib.messages import ControlMessage
-from morpheus._lib.messages import MessageMeta
+from morpheus.messages import ControlMessage
+from morpheus.messages import MessageMeta
 from morpheus.utils.control_message_utils import cm_default_failure_context_manager
 from morpheus.utils.control_message_utils import cm_skip_processing_if_failed
 from morpheus.utils.module_ids import MORPHEUS_MODULE_NAMESPACE

morpheus/stages/doca/doca_source_stage.py

@@ -54,9 +54,9 @@ def __init__(
         # Attempt to import the C++ stage on creation
         try:
             # pylint: disable=C0415
-            from morpheus._lib.doca import DocaSourceStage as _DocaSourceStage
+            import morpheus._lib.doca as _doca
 
-            self._doca_source_class = _DocaSourceStage
+            self._doca_source_class = _doca.DocaSourceStage
         except ImportError as ex:
             raise NotImplementedError(("The Morpheus DOCA components could not be imported. "
                                        "Ensure the DOCA components have been built and installed. Error message: ") +

morpheus/utils/loader_utils.py

@@ -14,7 +14,7 @@
 
 import logging
 
-from morpheus._lib.messages import DataLoaderRegistry as registry
+from morpheus.messages import DataLoaderRegistry as registry
 
 logger = logging.getLogger(__name__)
 
@@ -38,9 +38,9 @@ def inner_func(func):
         # Register a loader if not exists in the registry.
         if not registry.contains(loader_id):
             registry.register_loader(loader_id, func)
-            logger.debug("Loader '{}' was successfully registered.".format(loader_id))
+            logger.debug("Loader '%s' was successfully registered.", loader_id)
         else:
-            logger.debug("Loader: '{}' already exists.".format(loader_id))
+            logger.debug("Loader: '%s' already exists.", loader_id)
 
         return func
 

pyproject.toml

@@ -122,7 +122,13 @@ ignore-patterns = ["^\\.#"]
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-# init-hook =
+init-hook = """
+import sys
+from pylint.config import find_default_config_files
+
+config_file = next(find_default_config_files())
+sys.path.append(str((config_file / "../ci/pylint").resolve()))
+"""
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use, and will cap the count on Windows to
@@ -136,7 +142,7 @@ limit-inference-results = 100
 
 # List of plugins (as comma separated values of python module names) to load,
 # usually to register additional checkers.
-load-plugins = ["pylint.extensions.docparams"]
+load-plugins = ["pylint.extensions.docparams", "morpheus_lib_import_checker"]
 
 # Pickle collected data for later comparisons.
 persistent = true
@@ -441,7 +447,6 @@ disable = [
   "missing-class-docstring",
   "missing-function-docstring",
   "missing-module-docstring",
-  "no-member", # pylint isn't aware of some of our attrs issue #956
   "protected-access",
   "raw-checker-failed",
   "superfluous-parens",