Merge branch 'master' into fix/dali_batch_input

Author: agunapal

.github/workflows/docker-nightly-build.yml

@@ -1,9 +1,9 @@
 name: Push Docker Nightly
 
 on:
-  # run every day at 11:15am
+  # run every day at 1:15pm
   schedule:
-    - cron:  '15 11 * * *'
+    - cron:  '15 13 * * *'
 jobs:
   nightly:
     runs-on: ubuntu-20.04

.github/workflows/regression_tests_docker.yml

@@ -0,0 +1,55 @@
+name: Run Regression Tests on Docker
+
+on:
+  # run every day at 5:15am
+  schedule:
+    - cron:  '15 5 * * *'
+
+concurrency:
+  group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker-regression:
+    strategy:
+      fail-fast: false
+      matrix:
+        hardware: [ubuntu-20.04, [self-hosted, regression-test-gpu]]
+    runs-on:
+      - ${{ matrix.hardware }}
+    steps:
+      - name: Clean up previous run
+        run: |
+          echo "Cleaning up previous run"
+          ls -la ./
+          sudo rm -rf ./* || true
+          sudo rm -rf ./.??* || true
+          ls -la ./
+          docker system prune -f
+      - name: Checkout TorchServe
+        uses: actions/checkout@v3
+      - name: Branch name
+        run: |
+          echo $GITHUB_REF_NAME
+      - name: Build CPU Docker Image
+        if: contains(matrix.hardware, 'ubuntu')
+        run: |
+          cd docker
+          ./build_image.sh -bt ci -n -b $GITHUB_REF_NAME -t pytorch/torchserve:ci
+      - name: Build GPU Docker Image
+        if: false == contains(matrix.hardware, 'ubuntu')
+        run: |
+          cd docker
+          ./build_image.sh -g -cv cu117 -bt ci -n -b $GITHUB_REF_NAME -t pytorch/torchserve:ci
+      - name: Torchserve GPU Regression Tests
+        if: false == contains(matrix.hardware, 'ubuntu')
+        run: |
+          docker run --gpus all -v $GITHUB_WORKSPACE:/home/serve pytorch/torchserve:ci
+      - name: Torchserve CPU Regression Tests
+        if: contains(matrix.hardware, 'ubuntu')
+        run: |
+          docker run -v $GITHUB_WORKSPACE:/home/serve pytorch/torchserve:ci
+      - name: Cleanup Docker Images
+        if: success()
+        run: |
+          docker system prune -f && docker rmi pytorch/torchserve:ci

.github/workflows/regression_tests_gpu.yml

@@ -20,11 +20,10 @@ jobs:
       - name: Clean up previous run
         run: |
           echo "Cleaning up previous run"
-          cd $RUNNER_WORKSPACE
-          pwd
-          cd ..
-          pwd
-          rm -rf _tool
+          ls -la ./
+          sudo rm -rf ./* || true
+          sudo rm -rf ./.??* || true
+          ls -la ./
       - name: Update git
         run: sudo add-apt-repository ppa:git-core/ppa -y && sudo apt-get update && sudo apt-get install git -y
       - name: Check git version

CONTRIBUTING.md

@@ -20,9 +20,9 @@ Your contributions will fall into two categories:
 
          For GPU
            ```bash
-           python ts_scripts/install_dependencies.py --environment=dev --cuda=cu102
+           python ts_scripts/install_dependencies.py --environment=dev --cuda=cu118
            ```
-            > Supported cuda versions as cu117, cu116, cu113, cu111, cu102, cu101, cu92
+            > Supported cuda versions as cu118, cu117, cu116, cu113, cu111, cu102, cu101, cu92
         - Install `pre-commit` to your Git flow:
             ```bash
             pre-commit install

README.md

@@ -72,7 +72,7 @@ Refer to [torchserve docker](docker/README.md) for details.
 
 
 ## 🏆 Highlighted Examples
-* [🤗 HuggingFace Transformers](examples/Huggingface_Transformers) with a [Better Transformer Integration](examples/Huggingface_Transformers#Speed-up-inference-with-Better-Transformer)
+* [🤗 HuggingFace Transformers](examples/Huggingface_Transformers) with a [Better Transformer Integration/ Flash Attention & Xformer Memory Efficient ](examples/Huggingface_Transformers#Speed-up-inference-with-Better-Transformer)
 * [Model parallel inference](examples/Huggingface_Transformers#model-parallelism)
 * [MultiModal models with MMF](https://github.com/pytorch/serve/tree/master/examples/MMF-activity-recognition) combining text, audio and video
 * [Dual Neural Machine Translation](examples/Workflows/nmt_transformers_pipeline) for a complex workflow DAG
@@ -101,7 +101,7 @@ To learn more about how to contribute, see the contributor guide [here](https://
 * [Grokking Intel CPU PyTorch performance from first principles( Part 2): a TorchServe case study](https://pytorch.org/tutorials/intermediate/torchserve_with_ipex_2.html)
 * [Case Study: Amazon Ads Uses PyTorch and AWS Inferentia to Scale Models for Ads Processing](https://pytorch.org/blog/amazon-ads-case-study/)
 * [Optimize your inference jobs using dynamic batch inference with TorchServe on Amazon SageMaker](https://aws.amazon.com/blogs/machine-learning/optimize-your-inference-jobs-using-dynamic-batch-inference-with-torchserve-on-amazon-sagemaker/)
-* [Using AI to bring children's drawings to life](https://ai.facebook.com/blog/using-ai-to-bring-childrens-drawings-to-life/)
+* [Using AI to bring children's drawings to life](https://ai.meta.com/blog/using-ai-to-bring-childrens-drawings-to-life/)
 * [🎥 Model Serving in PyTorch](https://www.youtube.com/watch?v=2A17ZtycsPw)
 * [Evolution of Cresta's machine learning architecture: Migration to AWS and PyTorch](https://aws.amazon.com/blogs/machine-learning/evolution-of-crestas-machine-learning-architecture-migration-to-aws-and-pytorch/)
 * [🎥 Explain Like I’m 5: TorchServe](https://www.youtube.com/watch?v=NEdZbkfHQCk)

benchmarks/auto_benchmark.py

@@ -149,7 +149,7 @@ def install_torchserve(skip_ts_install, hw, ts_version):
 
     # install_dependencies.py
     if hw == "gpu":
-        cmd = "python ts_scripts/install_dependencies.py --environment dev --cuda cu117"
+        cmd = "python ts_scripts/install_dependencies.py --environment dev --cuda cu118"
     elif hw == "neuronx":
         cmd = "python ts_scripts/install_dependencies.py --environment dev --neuronx"
     else:

benchmarks/benchmark-ab.py

@@ -659,7 +659,10 @@ def plot_line(fig, data, color="blue", title=None):
         title="Combined Graph",
     )
     fig5.grid()
-    plt.savefig("api-profile1.png", bbox_inches="tight")
+    plt.savefig(
+        f"{execution_params['report_location']}/benchmark/api-profile1.png",
+        bbox_inches="tight",
+    )
 
 
 def stop_torchserve():

docker/Dockerfile

@@ -30,6 +30,7 @@ ARG PYTHON_VERSION=3.9
 FROM ${BASE_IMAGE} AS compile-image
 ARG BASE_IMAGE=ubuntu:rolling
 ARG PYTHON_VERSION
+ARG BUILD_NIGHTLY
 ENV PYTHONUNBUFFERED TRUE
 
 RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
@@ -82,10 +83,15 @@ RUN \
     fi
 
 # Make sure latest version of torchserve is uploaded before running this
-RUN python -m pip install --no-cache-dir torchserve torch-model-archiver torch-workflow-archiver
+RUN \
+    if echo "$BUILD_NIGHTLY" | grep -q "false"; then \
+        python -m pip install --no-cache-dir torchserve torch-model-archiver torch-workflow-archiver;\
+    else \
+        python -m pip install --no-cache-dir torchserve-nightly torch-model-archiver-nightly torch-workflow-archiver-nightly;\
+    fi
 
 # Final image for production
-FROM ${BASE_IMAGE} AS runtime-image
+FROM ${BASE_IMAGE} AS production-image
 # Re-state ARG PYTHON_VERSION to make it active in this build-stage (uses default define at the top)
 ARG PYTHON_VERSION
 ENV PYTHONUNBUFFERED TRUE
@@ -130,3 +136,48 @@ WORKDIR /home/model-server
 ENV TEMP=/home/model-server/tmp
 ENTRYPOINT ["/usr/local/bin/dockerd-entrypoint.sh"]
 CMD ["serve"]
+
+# Final image for docker regression
+FROM ${BASE_IMAGE} AS ci-image
+# Re-state ARG PYTHON_VERSION to make it active in this build-stage (uses default define at the top)
+ARG PYTHON_VERSION
+ARG BRANCH_NAME
+ENV PYTHONUNBUFFERED TRUE
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install software-properties-common -y && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt remove python-pip  python3-pip && \
+    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+    python$PYTHON_VERSION \
+    python3-distutils \
+    python$PYTHON_VERSION-dev \
+    python$PYTHON_VERSION-venv \
+    # using openjdk-17-jdk due to circular dependency(ca-certificates) bug in openjdk-17-jre-headless debian package
+    # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1009905
+    openjdk-17-jdk \
+    build-essential \
+    wget \
+    numactl \
+    nodejs \
+    npm \
+    zip \
+    unzip \
+    && npm install -g newman newman-reporter-htmlextra markdown-link-check \
+    && rm -rf /var/lib/apt/lists/* \
+    && cd /tmp
+
+
+COPY --from=compile-image /home/venv /home/venv
+
+ENV PATH="/home/venv/bin:$PATH"
+
+RUN python -m pip install --no-cache-dir -r https://raw.githubusercontent.com/pytorch/serve/$BRANCH_NAME/requirements/developer.txt
+
+RUN mkdir /home/serve
+ENV TS_RUN_IN_DOCKER True
+
+WORKDIR /home/serve
+CMD ["python", "test/regression_tests.py"]

docker/README.md

@@ -28,16 +28,17 @@ cd serve/docker
 
 # Create TorchServe docker image
 
-Use `build_image.sh` script to build the docker images. The script builds the `production`, `dev` and `codebuild` docker images.
+Use `build_image.sh` script to build the docker images. The script builds the `production`, `dev` , `ci` and `codebuild` docker images.
 | Parameter | Description |
 |------|------|
 |-h, --help|Show script help|
 |-b, --branch_name|Specify a branch name to use. Default: master |
 |-g, --gpu|Build image with GPU based ubuntu base image|
-|-bt, --buildtype|Which type of docker image to build. Can be one of : production, dev, codebuild|
+|-bt, --buildtype|Which type of docker image to build. Can be one of : production, dev, ci, codebuild|
 |-t, --tag|Tag name for image. If not specified, script uses torchserve default tag names.|
 |-cv, --cudaversion| Specify to cuda version to use. Supported values `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`. Default `cu117`|
 |-ipex, --build-with-ipex| Specify to build with intel_extension_for_pytorch. If not specified, script builds without intel_extension_for_pytorch.|
+|-n, --nightly| Specify to build with TorchServe nightly.|
 |--codebuild| Set if you need [AWS CodeBuild](https://aws.amazon.com/codebuild/)|
 |-py, --pythonversion| Specify the python version to use. Supported values `3.8`, `3.9`, `3.10`. Default `3.9`|
 
@@ -52,7 +53,7 @@ Creates a docker image with publicly available `torchserve` and `torch-model-arc
 ./build_image.sh
 ```
 
- - To create a GPU based image with cuda 10.2. Options are `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`
+ - To create a GPU based image with cuda 10.2. Options are `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`
 
   ```bash
   ./build_image.sh -g -cv cu102

docker/build_image.sh

@@ -11,6 +11,7 @@ USE_CUSTOM_TAG=false
 CUDA_VERSION=""
 USE_LOCAL_SERVE_FOLDER=false
 BUILD_WITH_IPEX=false
+BUILD_NIGHTLY=false
 PYTHON_VERSION=3.9
 
 for arg in "$@"
@@ -27,6 +28,7 @@ do
           echo "-lf, --use-local-serve-folder specify this option for the benchmark image if the current 'serve' folder should be used during automated benchmarks"
           echo "-ipex, --build-with-ipex specify to build with intel_extension_for_pytorch"
           echo "-py, --pythonversion specify to python version to use: Possible values: 3.8 3.9 3.10"
+          echo "-n, --nightly specify to build with TorchServe nightly"
           exit 0
           ;;
         -b|--branch_name)
@@ -43,7 +45,7 @@ do
         -g|--gpu)
           MACHINE=gpu
           DOCKER_TAG="pytorch/torchserve:latest-gpu"
-          BASE_IMAGE="nvidia/cuda:11.7.1-base-ubuntu20.04"
+          BASE_IMAGE="nvidia/cuda:11.8.0-base-ubuntu20.04"
           CUDA_VERSION="cu117"
           shift
           ;;
@@ -66,6 +68,10 @@ do
           BUILD_WITH_IPEX=true
           shift
           ;;
+        -n|--nightly)
+          BUILD_NIGHTLY=true
+          shift
+          ;;
         -py|--pythonversion)
           PYTHON_VERSION="$2"
           if [[ $PYTHON_VERSION = 3.8 || $PYTHON_VERSION = 3.9 || $PYTHON_VERSION = 3.10 ]]; then
@@ -137,7 +143,10 @@ fi
 
 if [ "${BUILD_TYPE}" == "production" ]
 then
-  DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t "${DOCKER_TAG}" .
+  DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" -t "${DOCKER_TAG}" --target production-image  .
+elif [ "${BUILD_TYPE}" == "ci" ]
+then
+  DOCKER_BUILDKIT=1 docker build --file Dockerfile --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg CUDA_VERSION="${CUDA_VERSION}"  --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg BUILD_NIGHTLY="${BUILD_NIGHTLY}" --build-arg BRANCH_NAME="${BRANCH_NAME}"  -t "${DOCKER_TAG}" --target ci-image  .
 elif [ "${BUILD_TYPE}" == "benchmark" ]
 then
   DOCKER_BUILDKIT=1 docker build --pull --no-cache --file Dockerfile.benchmark --build-arg USE_LOCAL_SERVE_FOLDER=$USE_LOCAL_SERVE_FOLDER --build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg BRANCH_NAME="${BRANCH_NAME}" --build-arg CUDA_VERSION="${CUDA_VERSION}" --build-arg MACHINE_TYPE="${MACHINE}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t "${DOCKER_TAG}" .