Use buildx for Docker builds

RasaHQ · Jan 15, 2021 · 374f2a6 · 374f2a6
1 parent 771cc4b
commit 374f2a6
Show file tree

Hide file tree

Showing 15 changed files with 575 additions and 380 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -4,3 +4,5 @@ docs
 **/*.pyc
 **/__pycache__
 !docker/configs
+rasa/tests
+rasa/scripts
diff --git a/.gitattributes b/.gitattributes
@@ -1 +1,3 @@
-* -text
+* -text
+# Reclassifies `Dockerfile*` files as Dockerfile:
+Dockerfile.* linguist-language=Dockerfile
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -28,6 +28,8 @@ env:
   # needed to fix issues with boto during testing:
   # https://github.com/travis-ci/travis-ci/issues/7940
   BOTO_CONFIG: /dev/null
+  IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
+  DOCKERHUB_USERNAME: tmbo
 
 jobs:
   changes:
@@ -279,27 +281,170 @@ jobs:
         COVERALLS_SERVICE_NAME: github
       run: poetry run coveralls
 
+  build_docker_base_images_and_set_env:
+    name: Build Docker base images and setup environment
+    runs-on: ubuntu-20.04
+    outputs:
+      base_image_hash: ${{ steps.check_image.outputs.base_image_hash }}
+      base_mitie_image_hash: ${{ steps.check_image.outputs.base_mitie_image_hash }}
+      # Tag name used for images created during Docker image builds, e.g. 3886 - a PR number
+      image_tag: ${{ steps.set_output.outputs.image_tag }}
+      # Return 'true' if tag version is equal or higher than the latest tagged Rasa version
+      is_newest_version: ${{ steps.rasa_get_version.outputs.is_newest_version }}
+    steps:
+      # Due to an issue with checking out a wrong commit, we make sure
+      # to checkout HEAD commit for a pull request.
+      # More details: https://github.com/actions/checkout/issues/299
+    - name: Checkout pull request HEAD commit instead of merge commit 🕝
+      uses: actions/checkout@v2
+      if: github.event_name == 'pull_request'
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@v2
+      if: github.event_name != 'pull_request'
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+        driver: docker
+
+    - name: Login to DockerHub Registry 🔢
+      run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ env.DOCKERHUB_USERNAME }} --password-stdin || true
+
+    - name: Check if tag version is equal or higher than the latest tagged Rasa version
+      id: rasa_get_version
+      if: env.IS_TAG_BUILD == 'true'
+      run: |
+        # Get latest tagged Rasa version
+        git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
+        # Fetch branch history
+        git fetch --prune --unshallow
+        LATEST_TAGGED_NON_ALPHA_RASA_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1)
+        CURRENT_TAG=${GITHUB_TAG/refs\/tags\//}
+        # Return 'true' if tag version is equal or higher than the latest tagged Rasa version
+        IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_VERSION}" "$CURRENT_TAG" \
+          | sort -V -C && echo true || echo false) || true)
+        # Avoid that the script gets released for alphas or release candidates
+        if [[ "${IS_NEWEST_VERSION}" == "true" && "$CURRENT_TAG" =~ ^[0-9.]+$ ]]; then
+          echo "::set-output name=is_newest_version::true"
+        else
+          echo "::set-output name=is_newest_version::false"
+        fi
+
+    - name: Check if a base image exists
+      id: check_image
+      run: |
+        DOCKERHUB_TAGS_URL="https://registry.hub.docker.com/v2/repositories/rasa/rasa/tags?page_size=10000"
+
+        # Base image
+        BASE_IMAGE_HASH=${{ hashFiles('docker/Dockerfile.base') }}
+        BASE_IMAGE_TAG=$(curl -s ${DOCKERHUB_TAGS_URL} | jq -r '.results[] | select(.name == "base-'${BASE_IMAGE_HASH}'") | .name')
+
+        echo "::set-output name=base_image_hash::${BASE_IMAGE_HASH}"
+
+        echo $BASE_IMAGE_TAG $BASE_IMAGE_HASH
+        if [[ "${BASE_IMAGE_TAG}" == "base-${BASE_IMAGE_HASH}" ]]; then
+          echo "::set-output name=base_exists::true"
+        else
+          echo "::set-output name=base_exists::false"
+        fi
+
+        # Base MITIE image
+        BASE_MITIE_IMAGE_HASH=${{ hashFiles('docker/Dockerfile.base-mitie') }}
+        BASE_MITIE_IMAGE_TAG=$(curl -s ${DOCKERHUB_TAGS_URL} | jq -r '.results[] | select(.name == "base-mitie-'${BASE_MITIE_IMAGE_HASH}'") | .name')
+
+        echo "::set-output name=base_mitie_image_hash::${BASE_MITIE_IMAGE_HASH}"
+
+        echo $BASE_MITIE_IMAGE_TAG $BASE_MITIE_IMAGE_HASH
+        if [[ "${BASE_MITIE_IMAGE_TAG}" == "base-mitie-${BASE_MITIE_IMAGE_HASH}" ]]; then
+          echo "::set-output name=base_mitie_exists::true"
+        else
+          echo "::set-output name=base_mitie_exists::false"
+        fi
+
+    - name: Build Docker base image and push 🛠 ⬆
+      if: steps.check_image.outputs.base_exists == 'false' || env.IS_TAG_BUILD == 'true'
+      run: |
+        export IMAGE_TAG=${{ hashFiles('docker/Dockerfile.base') }}
+        docker buildx bake -f docker/docker-bake.hcl base --push
+
+    - name: Build Docker mitie base image and push 🛠 ⬆
+      if: steps.check_image.outputs.base_mitie_exists == 'false'
+      run: |
+        export IMAGE_TAG=${{ hashFiles('docker/Dockerfile.base-mitie') }}
+        docker buildx bake -f docker/docker-bake.hcl base-mitie --push
+
+    # Set environment variables for a pull request
+    #
+    # In this scenario, we've created a PR #1234
+    #
+    # Example output:
+    # IMAGE_TAG=1234
+    - name: Set environment variables - pull_request
+      if: github.event_name == 'pull_request' && env.IS_TAG_BUILD == 'false'
+      run: |
+        echo "IMAGE_TAG=${{ github.event.number }}" >> $GITHUB_ENV
+
+    # Set environment variables for a tag
+    #
+    # In this scenario, we've pushed the '2.0.6' tag
+    #
+    # Example output:
+    # TAG_NAME=2.0.6
+    # IMAGE_TAG=2.0.6
+    - name: Set environment variables - push - tag
+      if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true'
+      run: |
+        TAG_NAME=${GITHUB_REF#refs/tags/}
+        echo "IMAGE_TAG=${TAG_NAME}" >> $GITHUB_ENV
+
+    # Set environment variables for a branch
+    #
+    # In this scenario, we've pushed changes into the master branch
+    #
+    # Example output:
+    # IMAGE_TAG=master
+    - name: Set environment variables - push - branch
+      if: github.event_name == 'push' && env.IS_TAG_BUILD == 'false'
+      run: |
+        BRANCH_NAME=${GITHUB_REF#refs/heads/}
+        SAFE_BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\./-/g')
+        echo "IMAGE_TAG=${BRANCH_NAME}" >> $GITHUB_ENV
+
+    - name: Set output
+      id: set_output
+      run: |
+        echo "::set-output name=image_tag::${{ env.IMAGE_TAG }}"
+
   docker:
     name: Build Docker
     runs-on: ubuntu-latest
-    needs: [changes]
+    needs: [changes, build_docker_base_images_and_set_env]
+    env:
+      IMAGE_TAG: ${{ needs.build_docker_base_images_and_set_env.outputs.image_tag }}
+      BASE_IMAGE_HASH: ${{ needs.build_docker_base_images_and_set_env.outputs.base_image_hash }}
+      BASE_MITIE_IMAGE_HASH: ${{ needs.build_docker_base_images_and_set_env.outputs.base_mitie_image_hash }}
 
     strategy:
       matrix:
-        image:
-          - {"file": "Dockerfile", "tag_ext": ""}
-          - {"file": "docker/Dockerfile_full", "tag_ext": "-full"}
-          - {"file": "docker/Dockerfile_pretrained_embeddings_mitie_en", "tag_ext": "-mitie-en"}
-          - {"file": "docker/Dockerfile_pretrained_embeddings_spacy_de", "tag_ext": "-spacy-de"}
-          - {"file": "docker/Dockerfile_pretrained_embeddings_spacy_en", "tag_ext": "-spacy-en"}
-
-    env:
-      DOCKERHUB_USERNAME: tmbo
+        image: [default, full, mitie-en, spacy-de, spacy-en]
 
     steps:
+      # Due to an issue with checking out a wrong commit, we make sure
+      # to checkout HEAD commit for a pull request.
+      # More details: https://github.com/actions/checkout/issues/299
+    - name: Checkout pull request HEAD commit instead of merge commit 🕝
+      uses: actions/checkout@v2
+      if: github.event_name == 'pull_request'
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
     - name: Checkout git repository 🕝
-      if: needs.changes.outputs.docker == 'true'
       uses: actions/checkout@v2
+      if: github.event_name != 'pull_request'
 
     - name: Free disk space
       if: needs.changes.outputs.docker == 'true'
@@ -312,14 +457,16 @@ jobs:
         docker rmi $(docker image ls -aq)
         df -h
 
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+        driver: docker
+
     - name: Login to DockerHub Registry 🔢
       if: needs.changes.outputs.docker == 'true'
       run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ env.DOCKERHUB_USERNAME }} --password-stdin || true
 
-    - name: Pull latest${{ matrix.image.tag_ext }} Docker image for caching
-      if: needs.changes.outputs.docker == 'true'
-      run: docker pull rasa/rasa:latest${{ matrix.image.tag_ext }} || true
-
     - name: Copy Segment write key to the package
       if: needs.changes.outputs.docker == 'true' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && github.repository == 'RasaHQ/rasa'
       env:
@@ -328,22 +475,38 @@ jobs:
       run: |
         ./scripts/write_keys_file.sh
 
-    - name: Build latest${{ matrix.image.tag_ext }} Docker image
+    - name: Build Docker image
       if: needs.changes.outputs.docker == 'true'
-      run: docker build . --file ${{ matrix.image.file }} --tag rasa/rasa:latest${{ matrix.image.tag_ext }} --cache-from rasa/rasa:latest${{ matrix.image.tag_ext }}
+      run: |
+        docker buildx bake -f docker/docker-bake.hcl ${{ matrix.image }}
 
-    - name: Push image with latest tag 📦
+    - name: Push image with master tag 📦
       if: needs.changes.outputs.docker == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'RasaHQ/rasa'
-      run: docker push rasa/rasa:latest${{ matrix.image.tag_ext }}
+      run: |
+        docker buildx bake -f docker/docker-bake.hcl ${{ matrix.image }} --push
 
     - name: Push image with ${{github.ref}} tag 📦
-      env:
-        GITHUB_TAG: ${{ github.ref }}
-      if: needs.changes.outputs.docker == 'true' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && github.repository == 'RasaHQ/rasa'
+      if: needs.changes.outputs.docker == 'true' && github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa'
       run: |
-        GITHUB_TAG=${GITHUB_TAG/refs\/tags\//}
-        docker tag rasa/rasa:latest${{ matrix.image.tag_ext }} rasa/rasa:${GITHUB_TAG}${{ matrix.image.tag_ext }}
-        docker push rasa/rasa:${GITHUB_TAG}${{ matrix.image.tag_ext }}
+        IS_NEWEST_VERSION=${{ needs.build_docker_base_images_and_set_env.outputs.is_newest_version }}
+
+        docker buildx bake -f docker/docker-bake.hcl ${{ matrix.image }} --push
+
+        # Tag the image as latest
+        if [[ "${IS_NEWEST_VERSION}" == "true" ]]; then
+          if [[ "${{ matrix.image }}" == "default" ]]; then
+            RELEASE_TAG="${IMAGE_TAG}"
+          else
+            RELEASE_TAG="${IMAGE_TAG}-${{ matrix.image }}"
+          fi
+
+          LATEST_TAG=$(echo $RELEASE_TAG | sed 's/'$GITHUB_TAG'/latest/g')
+
+          docker tag rasa/rasa:${RELEASE_TAG} rasa/rasa:${LATEST_TAG}
+          docker push rasa/rasa:${LATEST_TAG}
+        fi
+
+
 
   deploy:
     name: Deploy to PyPI

diff --git a/Dockerfile b/Dockerfile
@@ -1,34 +1,8 @@
-FROM python:3.7-slim as base
+# The default Docker image
+ARG IMAGE_BASE_NAME
+ARG BASE_IMAGE_HASH
 
-RUN apt-get update -qq \
- && apt-get install -y --no-install-recommends \
-    # required by psycopg2 at build and runtime
-    libpq-dev \
-     # required for health check
-    curl \
- && apt-get autoremove -y
-
-FROM base as builder
-
-RUN apt-get update -qq && \
-  apt-get install -y --no-install-recommends \
-  build-essential \
-  wget \
-  openssh-client \
-  graphviz-dev \
-  pkg-config \
-  git-core \
-  openssl \
-  libssl-dev \
-  libffi6 \
-  libffi-dev \
-  libpng-dev
-
-# install poetry
-# keep this in sync with the version in pyproject.toml and Dockerfile
-ENV POETRY_VERSION 1.1.4
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
-ENV PATH "/root/.poetry/bin:/opt/venv/bin:${PATH}"
+FROM ${IMAGE_BASE_NAME}:base-${BASE_IMAGE_HASH} as builder
 
 # copy files
 COPY . /build/
@@ -46,7 +20,7 @@ RUN python -m venv /opt/venv && \
   rm -rf dist *.egg-info
 
 # start a new build stage
-FROM base as runner
+FROM ${IMAGE_BASE_NAME}:base-${BASE_IMAGE_HASH} as runner
 
 # copy everything from /opt
 COPY --from=builder /opt/venv /opt/venv

diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
@@ -0,0 +1,39 @@
+# The base image used for all images
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND="noninteractive"
+
+RUN apt-get update -qq && \
+  apt-get install -y --no-install-recommends \
+  python3 \
+  python3-venv \
+  python3-pip \
+  python3-dev \
+  build-essential \
+  wget \
+  openssh-client \
+  graphviz-dev \
+  pkg-config \
+  git-core \
+  openssl \
+  libssl-dev \
+  libffi7 \
+  libffi-dev \
+  libpng-dev \
+  # required by psycopg2 at build and runtime
+  libpq-dev \
+  # required for health check
+  curl \
+  && apt-get autoremove -y
+
+# Make sure that all security updates are installed
+RUN apt-get update && apt-get dist-upgrade -y --no-install-recommends
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 100 \
+   && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 100
+
+# install poetry
+# keep this in sync with the version in pyproject.toml and Dockerfile
+ENV POETRY_VERSION 1.1.4
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+ENV PATH "/root/.poetry/bin:/opt/venv/bin:${PATH}"
diff --git a/docker/Dockerfile.base-mitie b/docker/Dockerfile.base-mitie
@@ -0,0 +1,7 @@
+# The base image used for all images that require a MITIE model
+FROM alpine:latest
+
+# download mitie model
+RUN mkdir -p /build/data/ && wget -P /build/data/ https://github.com/mit-nlp/MITIE/releases/download/v0.4/MITIE-models-v0.2.tar.bz2 && \
+  tar -xvjf /build/data/MITIE-models-v0.2.tar.bz2 --strip-components 2 -C /build/data/ MITIE-models/english/total_word_feature_extractor.dat && \
+  rm /build/data/MITIE*.bz2