Auditing the dependencies and syncing dependencies.yaml with `meta.…

…yaml` (#1743) In Progress ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Michael Demoret (https://github.com/mdemoret-nv) - David Gardner (https://github.com/dagardner-nv) Approvers: - David Gardner (https://github.com/dagardner-nv) URL: #1743
nv-morpheus · Jun 26, 2024 · a09261c · a09261c
1 parent 7f24aef
commit a09261c
Show file tree

Hide file tree

Showing 9 changed files with 227 additions and 118 deletions.
diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml
@@ -58,53 +58,60 @@ outputs:
         - pkg-config =0.29 # for mrc cmake
         - sysroot_linux-64 =2.17
       host:
+        # CUDA dependencies
         - cuda-cudart-dev {{ cuda_compiler_version }}.*
         - cuda-libraries-dev {{ cuda_compiler_version }}.* # required by matx for libcublas-dev
         - cuda-nvml-dev {{ cuda_compiler_version }}.*
         - cuda-nvrtc-dev {{ cuda_compiler_version }}.*
         - cuda-nvtx-dev {{ cuda_compiler_version }}.*
         - cuda-version {{ cuda_compiler_version }}.*
+        # Non-CUDA dependencies
         - cudf {{ rapids_version }}
-        - cython =3.0
+        - cython 3.0.*
+        - glog 0.6.*
         - libcudf {{ rapids_version }}
         - librdkafka >=1.9.2,<1.10.0a0
         - mrc {{ minor_version }}
+        - nlohmann_json 3.11.*
         - pip
-        - pybind11-stubgen =0.10.5
+        - pybind11-stubgen 0.10.5
         - python {{ python }}
-        - rapidjson =1.1.0
+        - rapidjson 1.1.0
         - rdma-core >=48 # Needed for DOCA.
-        - scikit-build =0.17.6
+        - scikit-build 0.17.6
         - versioneer-518
-        - zlib # required to build triton client
+        - zlib 1.2.13 # required to build triton client
       run:
         # Runtime only requirements. This + setup.py is the definitive runtime requirement list
+        # This should be synced with `runtime` in dependencies.yaml
+        - appdirs
+        - beautifulsoup4
         - click >=8
-        - configargparse =1.5.*
         - cuda-version {{ cuda_compiler_version }}.*
         - cudf
         - cupy # Version determined from cudf
         - datacompy =0.10.*
-        - dill
-        - distributed
+        - dill =0.3.7
         - docker-py =5.0.*
-        - grpcio # Version determined from cudf
-        - libmrc
+        - elasticsearch ==8.9.0
+        - feedparser =6.0.*
+        - grpcio =1.59.*
         - libwebp>=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863
         - mlflow>=2.10.0,<3
         - mrc
-        - networkx>=2.8
+        - networkx=2.8.8
         - numpydoc =1.5.*
         - nvtabular =23.08.00
-        - pandas
+        - pydantic
         - pluggy =1.3.*
-        - pyarrow * *_cuda # Ensure we get a CUDA build. Version determined by cuDF
         - python
         - python-confluent-kafka >=1.9.2,<1.10.0a0
-        - pytorch
+        - python-graphviz
         - pytorch-cuda
+        - pytorch * *cuda*
+        - rapids-dask-dependency {{ rapids_version }} # provides dask and distributed
         - requests
-        - requests-cache =1.1
+        - requests-cache =1.1.*
         - scikit-learn =1.3.2.*
         - sqlalchemy <2.0 # 2.0 is incompatible with pandas=1.3
         - tqdm =4.*

diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh
@@ -21,7 +21,7 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 rapids-dependency-file-generator \
   --output conda \
-  --file-key test \
+  --file-key all \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${WORKSPACE_TMP}/env.yaml"
 
 update_conda_env "${WORKSPACE_TMP}/env.yaml"

diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml
@@ -12,40 +12,42 @@ dependencies:
 - anyio>=3.7
 - appdirs
 - arxiv=1.4
-- automake
+- automake=1.16.5
 - beautifulsoup4=4.12
 - benchmark=1.8.3
-- boost-cpp=1.84
 - boto3
 - breathe=4.35.0
 - ccache
 - clangdev=16
-- click >=8
+- click>=8
 - cmake=3.27
 - cuda-cudart-dev=12.1
-- cuda-nvcc
+- cuda-cudart=12.1
+- cuda-nvcc=12.1
 - cuda-nvml-dev=12.1
 - cuda-nvrtc-dev=12.1
+- cuda-nvrtc=12.1
 - cuda-nvtx-dev=12.1
+- cuda-nvtx=12.1
 - cuda-tools=12.1
 - cuda-version=12.1
 - cudf=24.02
 - cuml=24.02.*
+- cupy
 - cxx-compiler
 - cython=3.0
-- dask=2023.12.1
 - datacompy=0.10
 - dill=0.3.7
-- distributed=2023.12.1
+- docker-py=5.0
 - doxygen=1.9.2
 - elasticsearch==8.9.0
 - exhale=0.3.6
-- feedparser=6.0.10
+- feedparser=6.0
 - flake8
 - gcc_linux-64=11.2
 - git-lfs
 - glog=0.6
-- grpcio-status==1.59
+- grpcio-status=1.59
 - grpcio=1.59
 - gxx_linux-64=11.2
 - huggingface_hub=0.20.2
@@ -54,6 +56,7 @@ dependencies:
 - isort
 - jsonpatch>=1.33
 - kfp
+- libcudf=24.02
 - librdkafka>=1.9.2,<1.10.0a0
 - libtool
 - libwebp=1.3.2
@@ -76,7 +79,6 @@ dependencies:
 - pkg-config=0.29
 - pluggy=1.3
 - pre-commit
-- protobuf=4.24
 - pybind11-stubgen=0.10.5
 - pydantic
 - pylint=3.0.3
@@ -92,6 +94,7 @@ dependencies:
 - pytorch-cuda
 - pytorch=*=*cuda*
 - rapidjson=1.1.0
+- rapids-dask-dependency=24.02
 - rdma-core>=48
 - requests
 - requests-cache=1.1
@@ -102,14 +105,14 @@ dependencies:
 - sentence-transformers=2.7
 - sphinx
 - sphinx_rtd_theme
-- sqlalchemy
+- sqlalchemy<2.0
 - sysroot_linux-64=2.17
 - tqdm=4
 - transformers=4.36.2
 - tritonclient=2.34
 - typing_utils=0.1
-- ucx=1.15
 - versioneer
+- versioneer-518
 - watchdog=3.0
 - websockets
 - yapf=0.40.1
@@ -123,7 +126,7 @@ dependencies:
   - dgl==2.0.0
   - dglgo
   - faiss-gpu==1.7.*
-  - google-search-results==2.4.*
+  - google-search-results==2.4
   - langchain-nvidia-ai-endpoints==0.0.11
   - langchain==0.1.16
   - milvus==2.3.5

diff --git a/conda/environments/dev_cuda-121_arch-x86_64.yaml b/conda/environments/dev_cuda-121_arch-x86_64.yaml
@@ -10,41 +10,46 @@ channels:
 - pytorch
 dependencies:
 - appdirs
-- automake
+- automake=1.16.5
+- beautifulsoup4=4.12
 - benchmark=1.8.3
-- boost-cpp=1.84
 - breathe=4.35.0
 - ccache
 - clangdev=16
-- click >=8
+- click>=8
 - cmake=3.27
 - cuda-cudart-dev=12.1
-- cuda-nvcc
+- cuda-nvcc=12.1
 - cuda-nvml-dev=12.1
 - cuda-nvrtc-dev=12.1
 - cuda-nvtx-dev=12.1
 - cuda-tools=12.1
 - cuda-version=12.1
 - cudf=24.02
+- cupy
 - cxx-compiler
 - cython=3.0
 - datacompy=0.10
 - dill=0.3.7
+- docker-py=5.0
 - doxygen=1.9.2
 - elasticsearch==8.9.0
 - exhale=0.3.6
-- feedparser=6.0.10
+- feedparser=6.0
 - flake8
 - gcc_linux-64=11.2
 - git-lfs
 - glog=0.6
+- grpcio-status=1.59
 - grpcio=1.59
 - gxx_linux-64=11.2
 - include-what-you-use=0.20
 - ipython
 - isort
+- libcudf=24.02
 - librdkafka>=1.9.2,<1.10.0a0
 - libtool
+- libwebp=1.3.2
 - mlflow>=2.10.0,<3
 - mrc=24.06
 - myst-parser=0.18.1
@@ -59,7 +64,6 @@ dependencies:
 - pkg-config=0.29
 - pluggy=1.3
 - pre-commit
-- protobuf=4.24
 - pybind11-stubgen=0.10.5
 - pydantic
 - pylint=3.0.3
@@ -74,20 +78,21 @@ dependencies:
 - pytorch-cuda
 - pytorch=*=*cuda*
 - rapidjson=1.1.0
+- rapids-dask-dependency=24.02
 - rdma-core>=48
 - requests
 - requests-cache=1.1
 - scikit-build=0.17.6
 - scikit-learn=1.3.2
 - sphinx
 - sphinx_rtd_theme
-- sqlalchemy
+- sqlalchemy<2.0
 - sysroot_linux-64=2.17
 - tqdm=4
 - tritonclient=2.34
 - typing_utils=0.1
-- ucx=1.15
 - versioneer
+- versioneer-518
 - watchdog=3.0
 - websockets
 - yapf=0.40.1

diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-121_arch-x86_64.yaml
@@ -14,21 +14,23 @@ dependencies:
 - arxiv=1.4
 - beautifulsoup4=4.12
 - boto3
-- click >=8
+- click>=8
+- cudf=24.02
 - cuml=24.02.*
-- dask=2023.12.1
+- cupy
 - datacompy=0.10
 - dill=0.3.7
-- distributed=2023.12.1
+- docker-py=5.0
 - elasticsearch==8.9.0
-- feedparser=6.0.10
-- grpcio-status==1.59
+- feedparser=6.0
+- grpcio-status=1.59
 - grpcio=1.59
 - huggingface_hub=0.20.2
 - jsonpatch>=1.33
 - kfp
 - libwebp=1.3.2
 - mlflow>=2.10.0,<3
+- mrc=24.06
 - networkx=2.8.8
 - newspaper3k=0.2
 - nodejs=18.*
@@ -39,22 +41,25 @@ dependencies:
 - openai=1.13
 - papermill=2.4.0
 - pip
+- pluggy=1.3
 - pydantic
 - pypdf=3.17.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
 - pytorch=*=*cuda*
+- rapids-dask-dependency=24.02
 - requests
 - requests-cache=1.1
 - requests-toolbelt=1.0
 - s3fs=2023.12.2
 - scikit-learn=1.3.2
 - sentence-transformers=2.7
-- sqlalchemy
+- sqlalchemy<2.0
 - tqdm=4
 - transformers=4.36.2
+- tritonclient=2.34
 - typing_utils=0.1
 - watchdog=3.0
 - websockets
@@ -67,7 +72,7 @@ dependencies:
   - dgl==2.0.0
   - dglgo
   - faiss-gpu==1.7.*
-  - google-search-results==2.4.*
+  - google-search-results==2.4
   - langchain-nvidia-ai-endpoints==0.0.11
   - langchain==0.1.16
   - milvus==2.3.5

diff --git a/conda/environments/runtime_cuda-121_arch-x86_64.yaml b/conda/environments/runtime_cuda-121_arch-x86_64.yaml
@@ -10,28 +10,42 @@ channels:
 - pytorch
 dependencies:
 - appdirs
-- click >=8
+- beautifulsoup4=4.12
+- click>=8
+- cuda-cudart=12.1
+- cuda-nvrtc=12.1
+- cuda-nvtx=12.1
+- cuda-version=12.1
+- cudf=24.02
+- cupy
 - datacompy=0.10
 - dill=0.3.7
+- docker-py=5.0
 - elasticsearch==8.9.0
-- feedparser=6.0.10
+- feedparser=6.0
+- grpcio-status=1.59
 - grpcio=1.59
+- libwebp=1.3.2
 - mlflow>=2.10.0,<3
+- mrc=24.06
 - networkx=2.8.8
 - numpydoc=1.5
 - nvtabular=23.08.00
 - pip
+- pluggy=1.3
 - pydantic
 - python-confluent-kafka>=1.9.2,<1.10.0a0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
 - pytorch=*=*cuda*
+- rapids-dask-dependency=24.02
 - requests
 - requests-cache=1.1
 - scikit-learn=1.3.2
-- sqlalchemy
+- sqlalchemy<2.0
 - tqdm=4
+- tritonclient=2.34
 - typing_utils=0.1
 - watchdog=3.0
 - websockets