From a0e34806999bad50ed5d54ef6281d42e5946a71f Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Wed, 23 Feb 2022 15:48:19 -0500 Subject: [PATCH] [Test refactor 5/5] Build docker images (#15729) --- .github/workflows/build-docker-images.yml | 70 +++++++++++++++++++ docker/transformers-all-latest-gpu/Dockerfile | 22 ++++++ .../Dockerfile | 21 ++++++ docker/transformers-pytorch-gpu/Dockerfile | 43 +++++------- docker/transformers-tensorflow-gpu/Dockerfile | 37 +++++----- 5 files changed, 149 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/build-docker-images.yml create mode 100644 docker/transformers-all-latest-gpu/Dockerfile create mode 100644 docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml new file mode 100644 index 00000000000000..e2c3bb97709e3d --- /dev/null +++ b/.github/workflows/build-docker-images.yml @@ -0,0 +1,70 @@ +name: Build docker images (scheduled) + +on: + push: + branches: + - master + - updating-dockerfiles + paths: + - "src/**" + - "tests/**" + - ".github/**" + - "templates/**" + - "utils/**" + - "docker/**" + repository_dispatch: + schedule: + - cron: "0 1 * * *" + +jobs: + latest-docker: + name: "Latest PyTorch + TensorFlow [dev]" + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Check out code + uses: actions/checkout@v2 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./docker/transformers-all-latest-gpu + build-args: | + REF=master + push: true + tags: huggingface/transformers-all-latest-gpu + + latest-torch-deepspeed-docker: + name: "Latest PyTorch + DeepSpeed" + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Check out code + uses: actions/checkout@v2 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./docker/transformers-pytorch-deepspeed-latest-gpu + build-args: | + REF=master + push: true + tags: huggingface/transformers-pytorch-deepspeed-latest-gpu diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile new file mode 100644 index 00000000000000..3a61866f13f616 --- /dev/null +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -0,0 +1,22 @@ +FROM nvidia/cuda:11.2.2-cudnn8-runtime-ubuntu20.04 +LABEL maintainer="Hugging Face" + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg +RUN python3 -m pip install --no-cache-dir --upgrade pip + +ARG REF=master +RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF +RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] + +RUN python3 -m pip install --no-cache-dir -U torch tensorflow +RUN python3 -m pip uninstall -y flax jax +RUN python3 -m pip install --no-cache-dir torch-scatter -f https://data.pyg.org/whl/torch-$(python3 -c "from torch import version; print(version.__version__.split('+')[0])")+cpu.html +RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract https://github.com/kpu/kenlm/archive/master.zip +RUN python3 -m pip install -U "itsdangerous<2.1.0" + +# When installing in editable mode, `transformers` is not recognized as a package. +# this line must be added in order for python to be aware of transformers. +RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile new file mode 100644 index 00000000000000..f20763e7c829d7 --- /dev/null +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -0,0 +1,21 @@ +FROM nvcr.io/nvidia/pytorch:21.03-py3 +LABEL maintainer="Hugging Face" + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt -y update +RUN apt install -y libaio-dev +RUN python3 -m pip install --no-cache-dir --upgrade pip + +ARG REF=master +RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF +RUN python3 -m pip install --no-cache-dir -e ./transformers[testing,deepspeed] + +RUN git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build && \ + DS_BUILD_CPU_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 python3 -m pip install -e . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1 + +# When installing in editable mode, `transformers` is not recognized as a package. +# this line must be added in order for python to be aware of transformers. +RUN cd transformers && python3 setup.py develop + +RUN python3 -c "from deepspeed.launcher.runner import main" diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index 5ed2bd70fd2faa..8b5594c77b079d 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -1,30 +1,25 @@ -FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04 +FROM nvidia/cuda:11.2.2-cudnn8-runtime-ubuntu20.04 LABEL maintainer="Hugging Face" -LABEL repository="transformers" -RUN apt update && \ - apt install -y bash \ - build-essential \ - git \ - curl \ - ca-certificates \ - python3 \ - python3-pip && \ - rm -rf /var/lib/apt/lists +ARG DEBIAN_FRONTEND=noninteractive -RUN python3 -m pip install --no-cache-dir --upgrade pip && \ - python3 -m pip install --no-cache-dir \ - mkl \ - torch +RUN apt update +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg +RUN python3 -m pip install --no-cache-dir --upgrade pip -RUN git clone https://github.com/NVIDIA/apex -RUN cd apex && \ - python3 setup.py install && \ - pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./ +ARG REF=master +RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF +RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] -WORKDIR /workspace -COPY . transformers/ -RUN cd transformers/ && \ - python3 -m pip install --no-cache-dir . +# If set to nothing, will install the latest version +ARG PYTORCH='' -CMD ["/bin/bash"] +RUN [ ${#PYTORCH} -gt 0 ] && VERSION='torch=='$PYTORCH'.*' || VERSION='torch'; python3 -m pip install --no-cache-dir -U $VERSION +RUN python3 -m pip uninstall -y tensorflow flax + +RUN python3 -m pip install --no-cache-dir torch-scatter -f https://data.pyg.org/whl/torch-$(python3 -c "from torch import version; print(version.__version__.split('+')[0])")+cpu.html +RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract https://github.com/kpu/kenlm/archive/master.zip + +# When installing in editable mode, `transformers` is not recognized as a package. +# this line must be added in order for python to be aware of transformers. +RUN cd transformers && python3 setup.py develop diff --git a/docker/transformers-tensorflow-gpu/Dockerfile b/docker/transformers-tensorflow-gpu/Dockerfile index 3277434c9f0a60..ed2016386730a6 100644 --- a/docker/transformers-tensorflow-gpu/Dockerfile +++ b/docker/transformers-tensorflow-gpu/Dockerfile @@ -1,25 +1,22 @@ -FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 +FROM nvidia/cuda:11.2.2-cudnn8-runtime-ubuntu20.04 LABEL maintainer="Hugging Face" -LABEL repository="transformers" -RUN apt update && \ - apt install -y bash \ - build-essential \ - git \ - curl \ - ca-certificates \ - python3 \ - python3-pip && \ - rm -rf /var/lib/apt/lists +ARG DEBIAN_FRONTEND=noninteractive -RUN python3 -m pip install --no-cache-dir --upgrade pip && \ - python3 -m pip install --no-cache-dir \ - mkl \ - tensorflow +RUN apt update +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg +RUN python3 -m pip install --no-cache-dir --upgrade pip -WORKDIR /workspace -COPY . transformers/ -RUN cd transformers/ && \ - python3 -m pip install --no-cache-dir . +ARG REF=master +RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF +RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] -CMD ["/bin/bash"] \ No newline at end of file +# If set to nothing, will install the latest version +ARG TENSORFLOW='' + +RUN [ ${#TENSORFLOW} -gt 0 ] && VERSION='tensorflow=='$TENSORFLOW'.*' || VERSION='tensorflow'; python3 -m pip install --no-cache-dir -U $VERSION +RUN python3 -m pip uninstall -y torch flax + +# When installing in editable mode, `transformers` is not recognized as a package. +# this line must be added in order for python to be aware of transformers. +RUN cd transformers && python3 setup.py develop