Move Morpheus Data Files Out of LFS (nv-morpheus#242)

* Testing why LFS isnt working

* Removing the morpheus/data files from git LFS

* Removing tensorrt

* Removing duplicate conda requriements

* Adding git=2.35.2 and git-lfs to the ci dependencies to fix an issue with the build

* Trying to use fetchinclude still

* Trying to pin cuda-python to <=11.7.0

* Go back to exclude

* Override exclude

Co-authored-by: David Gardner <dagardner@nvidia.com>

.gitattributes

@@ -1,5 +1,4 @@
 examples/data/** filter=lfs diff=lfs merge=lfs -text
 morpheus/_version.py export-subst
-morpheus/data/* filter=lfs diff=lfs merge=lfs -text
 tests/mock_triton_server/payloads/** filter=lfs diff=lfs merge=lfs -text
 tests/tests_data/** filter=lfs diff=lfs merge=lfs -text

.lfsconfig

@@ -1,2 +1,2 @@
 [lfs]
-	fetchinclude = morpheus/data/*
+	fetchexclude = *

ci/conda/recipes/morpheus/meta.yaml

@@ -86,15 +86,19 @@ outputs:
         - pytest-cov
         - pytest-benchmark
       source_files:
+        # - .git/*
+        - docker/conda/environments/*
         - pyproject.toml
+        - scripts/fetch_data.py
         - tests/*
-        - docker/conda/environments/*
       imports:
         - morpheus
         - morpheus._lib
       commands:
         # Install dependencies that are pip-only
         - mamba env update --prefix ${CONDA_PREFIX} --file docker/conda/environments/cuda{{ cuda_version }}_runtime.yml
+        # Ensure the test data is downloaded
+        - ./scripts/fetch_data.py fetch tests
         # Run pytest
         - pytest
 

ci/scripts/jenkins/test.sh

@@ -19,21 +19,33 @@ set -e
 source ${WORKSPACE}/ci/scripts/jenkins/common.sh
 /usr/bin/nvidia-smi
 
+# Restore the environment and then ensure we have the CI dependencies
 restore_conda_env
+
+gpuci_logger "Installing CI dependencies"
+mamba env update -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_ci.yml
+
+# Install the built Morpheus python package
 pip install ${MORPHEUS_ROOT}/build/wheel
 
 CPP_TESTS=($(find ${MORPHEUS_ROOT}/build/wheel -name "*.x"))
 
 gpuci_logger "Installing test dependencies"
 npm install --silent -g camouflage-server
-mamba install -q -y -c conda-forge "git-lfs=3.1.4"
+
+# Before running any .git commands, set this as a safe directory
+git config --global --add safe.directory ${MORPHEUS_ROOT}
 
 gpuci_logger "Pulling LFS assets"
 cd ${MORPHEUS_ROOT}
 
 git lfs install
 ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation
 
+# List missing files
+gpuci_logger "Listing missing files"
+git lfs ls-files
+
 REPORTS_DIR="${WORKSPACE_TMP}/reports"
 mkdir -p ${WORKSPACE_TMP}/reports
 

docker/Dockerfile

@@ -61,7 +61,9 @@ RUN conda config --set ssl_verify false &&\
     conda config --add pkgs_dirs /opt/conda/pkgs &&\
     conda config --env --add channels conda-forge &&\
     # Install mamba, boa and git here. Conda build breaks with other git installs
-    /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" "git >=2.35.3" python=${PYTHON_VER}
+    /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" &&\
+    source activate base &&\
+    git lfs install
     # conda clean -afy
 
 # ============ Stage: conda_env ============

docker/conda/environments/cuda11.5_ci.yml

@@ -20,5 +20,7 @@ channels:
 dependencies:
     - bash=5.1.16 # bash >=4.4 is necessary for morpheus cli auto-complete tests to pass.
     - conda-pack=0.7
+    - git-lfs=3.2
+    - git>=2.35.3 # Needed for wildcards on safe.directory
     - pkg-config=0.29
     - sccache=0.3

docker/conda/environments/cuda11.5_dev.yml

@@ -51,7 +51,6 @@ dependencies:
     - isort
     - mlflow>=1.23
     - myst-parser==0.17
-    - srf 22.06.*
     - ninja=1.10
     - nodejs=17.4.0
     - pandas=1.3
@@ -69,6 +68,7 @@ dependencies:
     - scikit-build=0.13
     - sphinx
     - sphinx_rtd_theme
+    - srf 22.06.*
     - sysroot_linux-64=2.17
     - tqdm
     - yapf=0.32.0

docker/run_container_dev.sh

@@ -28,7 +28,7 @@ DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"morpheus"}
 DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"dev-$(date +'%y%m%d')"}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
-DOCKER_ARGS="--env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace --net=host --gpus=all"
+DOCKER_ARGS="--env WORKSPACE_VOLUME=${PWD} -v $PWD:/workspace --net=host --gpus=all --cap-add=sys_nice"
 
 if [[ -z "${SSH_AUTH_SOCK}" ]]; then
    echo -e "${y}No ssh-agent auth socket found. Dependencies in private git repos may fail during build.${x}"

docker/run_container_release.sh

@@ -27,7 +27,7 @@ DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/morpheus"}
 DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"$(git describe --tags --abbrev=0)-runtime"}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
-DOCKER_ARGS="--env WORKSPACE_VOLUME=${PWD} -v $PWD/models:/workspace/models --net=host --gpus=all ${DOCKER_EXTRA_ARGS}"
+DOCKER_ARGS="--env WORKSPACE_VOLUME=${PWD} -v $PWD/models:/workspace/models --net=host --gpus=all --cap-add=sys_nice ${DOCKER_EXTRA_ARGS}"
 
 if [[ -z "${SSH_AUTH_SOCK}" ]]; then
    echo -e "${y}No ssh-agent auth socket found. Dependencies in private git repos may fail during build.${x}"