Prefer conda package over pip dependencies (nv-morpheus#1135)

Prefer conda package over pip dependencies

Todo:
- [x] build and test inside devcontainer
- [x] build and test inside development container
- [ ] build and test inside release container

Closes nv-morpheus#1129

Authors:
  - Christopher Harris (https://github.com/cwharris)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: nv-morpheus#1135

.devcontainer/bin/dev-pytest-kafka-install

@@ -24,3 +24,5 @@ sudo apt install -y openjdk-11-jre-headless
 
 # Download kafka
 python ${MORPHEUS_ROOT}/ci/scripts/download_kafka.py
+
+pip install pytest-kafka==0.6.0

ci/scripts/github/common.sh

@@ -64,7 +64,6 @@ export SCCACHE_IDLE_TIMEOUT=32768
 export CONDA_ENV_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml
 export CONDA_EXAMPLES_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_examples.yml
 export CONDA_DOCS_YML=${MORPHEUS_ROOT}/docs/conda_docs.yml
-export PIP_REQUIREMENTS=${MORPHEUS_ROOT}/docker/conda/environments/requirements.txt
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
 
@@ -88,9 +87,6 @@ function update_conda_env() {
             YAMLS="${YAMLS} ${CONDA_DOCS_YML}"
         fi
 
-        # Conda is going to expect a requirements.txt file to be in the same directory as the env yaml
-        cp ${PIP_REQUIREMENTS} ${WORKSPACE_TMP}/requirements.txt
-
         rapids-logger "Merging conda envs: ${YAMLS}"
         conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML}
     fi

docker/Dockerfile

@@ -179,7 +179,6 @@ ARG CONDA_CHANNEL_ALIAS
 
 # Copy the development dependencies file
 COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml ./docker/conda/environments/
-COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/requirements.txt ./docker/conda/environments/
 
 # Update the morpheus environment
 RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
@@ -237,8 +236,6 @@ ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
 ARG CUDA_MINOR_VER
 
-# Manually need to install some pip-only dependencies. Once these can get moved to conda, they can be removed.
-COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/requirements.txt ./docker/conda/environments/
 COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml ./docker/conda/environments/
 
 # Mount Morpheus conda package build in `conda_bld_morpheus`
@@ -257,7 +254,6 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target
         -c nvidia \
         -c pytorch \
         -c conda-forge morpheus &&\
-    # Install runtime dependencies that are pip-only
     /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml
 
 # Only copy specific files/folders over that are necessary for runtime

docker/conda/environments/cuda11.8_dev.yml

@@ -42,6 +42,7 @@ dependencies:
     - datacompy=0.8
     - dill
     - docker-py=5.0
+    - docutils
     - faker=12.3.0
     - flake8
     - flatbuffers=2.0
@@ -56,7 +57,10 @@ dependencies:
     - gtest>=1.13.0
     - gxx_linux-64=11.2
     - include-what-you-use=0.18
+    - ipywidgets
     - isort
+    - jupyter_core>=4.11.2,<5.0
+    - jupyterlab
     - libgrpc>=1.49
     - librdkafka=1.9.2
     - mlflow>=2.2.1,<3
@@ -95,11 +99,9 @@ dependencies:
     - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21
     - typing_utils=0.1
     - watchdog=2.1
+    - websockets
     - yapf=0.40.1
     ####### Morpheus Pip Dependencies (keep sorted!) #######
     - pip:
         # Add additional dev dependencies here
-        - docutils
         - pytest-kafka==0.6.0
-        # Ensure all runtime requirements are installed using the requirements file
-        - --requirement requirements.txt

docker/conda/environments/cuda11.8_runtime.yml

@@ -22,7 +22,3 @@ channels:
 dependencies:
     - nb_conda_kernels
     - pip
-    ####### Morpheus Pip Dependencies (keep sorted!) #######
-    - pip:
-        # Ensure all runtime requirements are installed using the requirements file
-        - --requirement requirements.txt

setup.py

@@ -51,8 +51,7 @@
     include_package_data=True,
     packages=find_packages(include=["morpheus*"], exclude=['tests']),
     install_requires=[
-        # Only list the packages which cannot be installed via conda here. Should mach the requirements in
-        # docker/conda/environments/requirements.txt
+        # Only list the packages which cannot be installed via conda here.
     ],
     license="Apache",
     python_requires='>=3.10, <4',