diff --git a/.devcontainer/bin/dev-pytest-kafka-install b/.devcontainer/bin/dev-pytest-kafka-install index fa8e803cd9..f5bef68935 100755 --- a/.devcontainer/bin/dev-pytest-kafka-install +++ b/.devcontainer/bin/dev-pytest-kafka-install @@ -24,3 +24,5 @@ sudo apt install -y openjdk-11-jre-headless # Download kafka python ${MORPHEUS_ROOT}/ci/scripts/download_kafka.py + +pip install pytest-kafka==0.6.0 diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 5af9ed17cf..e598e64482 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -64,7 +64,6 @@ export SCCACHE_IDLE_TIMEOUT=32768 export CONDA_ENV_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml export CONDA_EXAMPLES_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_examples.yml export CONDA_DOCS_YML=${MORPHEUS_ROOT}/docs/conda_docs.yml -export PIP_REQUIREMENTS=${MORPHEUS_ROOT}/docker/conda/environments/requirements.txt export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON" @@ -88,9 +87,6 @@ function update_conda_env() { YAMLS="${YAMLS} ${CONDA_DOCS_YML}" fi - # Conda is going to expect a requirements.txt file to be in the same directory as the env yaml - cp ${PIP_REQUIREMENTS} ${WORKSPACE_TMP}/requirements.txt - rapids-logger "Merging conda envs: ${YAMLS}" conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML} fi diff --git a/docker/Dockerfile b/docker/Dockerfile index a087ead7e1..e8c2f8a7fc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -179,7 +179,6 @@ ARG CONDA_CHANNEL_ALIAS # Copy the development dependencies file COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml ./docker/conda/environments/ -COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/requirements.txt ./docker/conda/environments/ # Update the morpheus environment RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ @@ -237,8 +236,6 @@ ARG MORPHEUS_ROOT_HOST ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER -# Manually need to install some pip-only dependencies. Once these can get moved to conda, they can be removed. -COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/requirements.txt ./docker/conda/environments/ COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml ./docker/conda/environments/ # Mount Morpheus conda package build in `conda_bld_morpheus` @@ -257,7 +254,6 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target -c nvidia \ -c pytorch \ -c conda-forge morpheus &&\ - # Install runtime dependencies that are pip-only /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml # Only copy specific files/folders over that are necessary for runtime diff --git a/docker/conda/environments/cuda11.8_dev.yml b/docker/conda/environments/cuda11.8_dev.yml index cea252c9a4..6dd491a71f 100644 --- a/docker/conda/environments/cuda11.8_dev.yml +++ b/docker/conda/environments/cuda11.8_dev.yml @@ -42,6 +42,7 @@ dependencies: - datacompy=0.8 - dill - docker-py=5.0 + - docutils - faker=12.3.0 - flake8 - flatbuffers=2.0 @@ -56,7 +57,10 @@ dependencies: - gtest>=1.13.0 - gxx_linux-64=11.2 - include-what-you-use=0.18 + - ipywidgets - isort + - jupyter_core>=4.11.2,<5.0 + - jupyterlab - libgrpc>=1.49 - librdkafka=1.9.2 - mlflow>=2.2.1,<3 @@ -95,11 +99,9 @@ dependencies: - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21 - typing_utils=0.1 - watchdog=2.1 + - websockets - yapf=0.40.1 ####### Morpheus Pip Dependencies (keep sorted!) ####### - pip: # Add additional dev dependencies here - - docutils - pytest-kafka==0.6.0 - # Ensure all runtime requirements are installed using the requirements file - - --requirement requirements.txt diff --git a/docker/conda/environments/cuda11.8_runtime.yml b/docker/conda/environments/cuda11.8_runtime.yml index 414598feb6..4ff10a6837 100644 --- a/docker/conda/environments/cuda11.8_runtime.yml +++ b/docker/conda/environments/cuda11.8_runtime.yml @@ -22,7 +22,3 @@ channels: dependencies: - nb_conda_kernels - pip - ####### Morpheus Pip Dependencies (keep sorted!) ####### - - pip: - # Ensure all runtime requirements are installed using the requirements file - - --requirement requirements.txt diff --git a/docker/conda/environments/requirements.txt b/docker/conda/environments/requirements.txt deleted file mode 100644 index 394407192e..0000000000 --- a/docker/conda/environments/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -####### Pip-only runtime dependencies (keep sorted!) ####### -# Packages listed here should also be listed in setup.py -ipywidgets -jupyter-core>=4.11.2,<5.0 -jupyterlab -nvidia-pyindex -# Duplicated in conda dev to ensure parity with libprotobuf -protobuf==4.21.* -websockets diff --git a/setup.py b/setup.py index 381c102574..f91c299eaa 100644 --- a/setup.py +++ b/setup.py @@ -51,8 +51,7 @@ include_package_data=True, packages=find_packages(include=["morpheus*"], exclude=['tests']), install_requires=[ - # Only list the packages which cannot be installed via conda here. Should mach the requirements in - # docker/conda/environments/requirements.txt + # Only list the packages which cannot be installed via conda here. ], license="Apache", python_requires='>=3.10, <4',