Misc Conda Improvements (#1462)

* Update examples to use package relative paths where possible allowing users of the Morpheus conda package to run examples. * Install `pyarrow-hotfix` via conda rather than pip * Ensure `pyarrow-hotfix` is installed during conda build enabling stubs to be generated. * Fix version restriction for sqlalchemy in `meta.yaml` * Fix type-o in `dependencies.yaml` causing duplicate entries for the `click` package. * Add examples target to `dependencies.yaml`. * Specify a version for sysroot_linux-64, prevents older kernel headers from being installed causing glibc link errors. * Update `docker/build_conda_packages.sh` to create `.conda-bld` dir if it doesn't already exist Closes #1461 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - David Gardner (https://github.com/dagardner-nv) - Michael Demoret (https://github.com/mdemoret-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: #1462
nv-morpheus · Jan 31, 2024 · 949279f · 949279f
1 parent 659e735
commit 949279f
Show file tree

Hide file tree

Showing 11 changed files with 38 additions and 28 deletions.
diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml
@@ -63,6 +63,7 @@ outputs:
         - mrc {{ minor_version }}
         - pip
         - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+        - pyarrow-hotfix # CVE-2023-47248
         - pybind11-stubgen 0.10.5
         - python {{ python }}
         - rapidjson 1.1
@@ -92,14 +93,15 @@ outputs:
         - pandas 1.3.*
         - pluggy 1.0.*
         - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+        - pyarrow-hotfix # CVE-2023-47248
         - python
         - python-confluent-kafka 1.9.2
         - pytorch 2.0.1
         - pytorch-cuda
         - requests=2.31
         - requests-cache=1.1
         - scikit-learn 1.2.2.*
-        - sqlalchemy <=2.0 # 2.0 is incompatible with pandas=1.3
+        - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3
         - tqdm 4.*
         - tritonclient 2.26.*
         - typing_utils 0.1.*

diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -48,6 +48,7 @@ export CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES:-"RAPIDS"}
 export MORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS:-"ON"}
 export MORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR:-"${MORPHEUS_ROOT}/.cache"}
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
+export MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF}
 
 # Set CONDA_CHANNEL_ALIAS to mimic the conda config channel_alias property during the build
 CONDA_CHANNEL_ALIAS=${CONDA_CHANNEL_ALIAS:-""}

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -19,7 +19,6 @@ dependencies:
 - breathe=4.34.0
 - ccache>=3.7
 - clangdev=16
-- click >=8
 - click>=8
 - cmake=3.25
 - cuda-cudart-dev=11.8
@@ -76,13 +75,14 @@ dependencies:
 - pluggy=1.0
 - pre-commit
 - protobuf=4.21.*
+- pyarrow-hotfix
 - pybind11-stubgen=0.10
 - pylint>=2.17.4,<2.18
 - pypdf=3.16
-- pytest
 - pytest-asyncio
 - pytest-benchmark>=4.0
 - pytest-cov
+- pytest=7.4
 - python-confluent-kafka=1.9.2
 - python-dotenv=1.0
 - python-graphviz
@@ -97,6 +97,7 @@ dependencies:
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
+- sysroot_linux-64=2.17
 - tqdm=4
 - transformers=4.30.2
 - tritonclient=2.26
@@ -113,7 +114,6 @@ dependencies:
   - grpcio-status==1.58
   - milvus==2.3.2
   - nemollm
-  - pyarrow_hotfix
   - pymilvus==2.3.2
   - pytest-kafka==0.6.0
 name: all_cuda-118_arch-x86_64_py-310

diff --git a/conda/environments/dev_cuda-118_arch-x86_64.yaml b/conda/environments/dev_cuda-118_arch-x86_64.yaml
@@ -17,7 +17,6 @@ dependencies:
 - breathe=4.34.0
 - ccache>=3.7
 - clangdev=16
-- click >=8
 - click>=8
 - cmake=3.25
 - cuda-cudart-dev=11.8
@@ -61,12 +60,13 @@ dependencies:
 - pluggy=1.0
 - pre-commit
 - protobuf=4.21.*
+- pyarrow-hotfix
 - pybind11-stubgen=0.10
 - pylint>=2.17.4,<2.18
-- pytest
 - pytest-asyncio
 - pytest-benchmark>=4.0
 - pytest-cov
+- pytest=7.4
 - python-confluent-kafka=1.9.2
 - python-graphviz
 - pytorch-cuda
@@ -78,6 +78,7 @@ dependencies:
 - sphinx
 - sphinx_rtd_theme
 - sqlalchemy<2.0
+- sysroot_linux-64=2.17
 - tqdm=4
 - tritonclient=2.26
 - typing_utils=0.1
@@ -89,7 +90,6 @@ dependencies:
 - pip:
   - databricks-connect
   - milvus==2.3.2
-  - pyarrow_hotfix
   - pymilvus==2.3.2
   - pytest-kafka==0.6.0
 name: dev_cuda-118_arch-x86_64_py-310

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -30,7 +30,6 @@ files:
       - test_python_morpheus
       - docs
       - cudatoolkit
-      - runtime
       - checks
 
   dev:
@@ -47,9 +46,18 @@ files:
       - test_python_morpheus
       - docs
       - cudatoolkit
-      - runtime
       - checks
 
+  examples:
+    output: conda
+    matrix:
+      cuda: ["11.8"]
+      arch: [x86_64]
+    includes:
+      - examples
+      - runtime
+      - cudatoolkit
+
   build:
     output: none
     matrix:
@@ -143,6 +151,7 @@ dependencies:
           - pybind11-stubgen=0.10
           - rapidjson=1.1.0
           - scikit-build=0.17.1
+          - sysroot_linux-64=2.17
           - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21
           - ucx=1.14
 
@@ -185,7 +194,7 @@ dependencies:
           - sphinx_rtd_theme
           - numpydoc=1.4
           - pluggy=1.0
-          - &click click >=8
+          - &click click>=8
 
   benchmark_cpp:
     common:
@@ -245,6 +254,7 @@ dependencies:
           - nb_conda_kernels
           - numpydoc=1.4
           - nvtabular=23.06
+          - pyarrow-hotfix
           - python-confluent-kafka=1.9.2
           - python-graphviz
           - pytorch-cuda
@@ -260,15 +270,14 @@ dependencies:
           - pip:
             - databricks-connect
             - milvus==2.3.2
-            - pyarrow_hotfix
             - pymilvus==2.3.2
 
   test_python_morpheus:
     common:
       - output_types: [conda]
         packages:
           - nodejs=18.*
-          - pytest
+          - pytest=7.4 # 8.0 appears to be incompatible with pytest-asyncio
           - pytest-asyncio
           - pytest-benchmark>=4.0
           - pytest-cov

diff --git a/docker/build_conda_packages.sh b/docker/build_conda_packages.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -36,6 +36,7 @@ CUR_GID=$(id -g ${LOGNAME})
 MORPHEUS_ROOT=${MORPHEUS_ROOT:-$(git rev-parse --show-toplevel)}
 mkdir -p ${MORPHEUS_ROOT}/.cache/ccache
 mkdir -p ${MORPHEUS_ROOT}/.cache/cpm
+mkdir -p ${MORPHEUS_ROOT}/.conda-bld
 
 echo "Building container"
 # Call the build script to get a container ready to build conda packages

diff --git a/docker/conda/environments/cuda11.8_dev.yml b/docker/conda/environments/cuda11.8_dev.yml
@@ -83,10 +83,11 @@ dependencies:
     - pluggy=1.0
     - protobuf=4.21.*
     - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
+    - pyarrow-hotfix # CVE-2023-47248. See morpheus/__init__.py for more details
     - pybind11-stubgen=0.10.5
     - pydot
     - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support
-    - pytest
+    - pytest=7.4
     - pytest-asyncio
     - pytest-benchmark>=4.0
     - pytest-cov
@@ -124,6 +125,5 @@ dependencies:
         # Add additional dev dependencies here
         - databricks-connect
         - milvus==2.3.2
-        - pyarrow_hotfix # CVE-2023-47248. See morpheus/__init__.py for more details
         - pymilvus==2.3.2
         - pytest-kafka==0.6.0
diff --git a/examples/abp_nvsmi_detection/README.md b/examples/abp_nvsmi_detection/README.md
@@ -1,5 +1,5 @@
 <!--
-SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 SPDX-License-Identifier: Apache-2.0
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -107,13 +107,12 @@ The following command line is the entire command to build and launch the pipelin
 
 From the  Morpheus repo root directory, run:
 ```bash
-export MORPHEUS_ROOT=$(pwd)
 # Launch Morpheus printing debug messages
 morpheus --log_level=DEBUG \
    `# Run a pipeline with 8 threads and a model batch size of 1024 (Must be equal or less than Triton config)` \
    run --num_threads=8 --pipeline_batch_size=1024 --model_max_batch_size=1024 \
    `# Specify a NLP pipeline with 256 sequence length (Must match Triton config)` \
-   pipeline-fil --columns_file=${MORPHEUS_ROOT}/morpheus/data/columns_fil.txt \
+   pipeline-fil --columns_file=data/columns_fil.txt \
    `# 1st Stage: Read from file` \
    from-file --filename=examples/data/nvsmi.jsonlines \
    `# 2nd Stage: Deserialize from JSON strings to objects` \

diff --git a/examples/log_parsing/README.md b/examples/log_parsing/README.md
@@ -1,5 +1,5 @@
 <!--
-# Copyright (c) 2021-2023, NVIDIA CORPORATION.
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -63,7 +63,7 @@ python run.py \
     --num_threads 1 \
     --input_file ${MORPHEUS_ROOT}/models/datasets/validation-data/log-parsing-validation-data-input.csv \
     --output_file ./log-parsing-output.jsonlines \
-    --model_vocab_hash_file=${MORPHEUS_ROOT}/morpheus/data/bert-base-cased-hash.txt \
+    --model_vocab_hash_file=data/bert-base-cased-hash.txt \
     --model_vocab_file=${MORPHEUS_ROOT}/models/training-tuning-scripts/sid-models/resources/bert-base-cased-vocab.txt \
     --model_seq_length=256 \
     --model_name log-parsing-onnx \
@@ -114,7 +114,7 @@ morpheus --log_level INFO \
 	pipeline-nlp \
 	from-file --filename ./models/datasets/validation-data/log-parsing-validation-data-input.csv  \
 	deserialize \
-	preprocess --vocab_hash_file ${MORPHEUS_ROOT}/morpheus/data/bert-base-cased-hash.txt --stride 64 --column=raw \
+	preprocess --vocab_hash_file data/bert-base-cased-hash.txt --stride 64 --column=raw \
 	monitor --description "Preprocessing rate" \
 	inf-logparsing --model_name log-parsing-onnx --server_url localhost:8001 --force_convert_inputs=True \
 	monitor --description "Inference rate" --unit inf \

diff --git a/examples/log_parsing/run.py b/examples/log_parsing/run.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2023, NVIDIA CORPORATION.
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@
 from inference import LogParsingInferenceStage
 from postprocessing import LogParsingPostProcessingStage
 
+from morpheus.cli.utils import MorpheusRelativePath
 from morpheus.config import Config
 from morpheus.config import PipelineModes
 from morpheus.pipeline import LinearPipeline
@@ -61,7 +62,7 @@
 )
 @click.option('--model_vocab_hash_file',
               required=True,
-              type=click.Path(exists=True, dir_okay=False),
+              type=MorpheusRelativePath(exists=True, dir_okay=False),
               help="Model vocab hash file to use for pre-processing.")
 @click.option('--model_vocab_file',
               required=True,

diff --git a/setup.py b/setup.py
@@ -38,10 +38,7 @@
     author="NVIDIA Corporation",
     include_package_data=True,
     packages=find_packages(include=["morpheus*"], exclude=['tests']),
-    install_requires=[
-        # Only list the packages which cannot be installed via conda here.
-        "pyarrow_hotfix",  # CVE-2023-47248. See morpheus/__init__.py for more details
-    ],
+    install_requires=[],
     license="Apache",
     python_requires='>=3.10, <4',
     cmdclass=versioneer.get_cmdclass(),  # pylint: disable=no-member