Update CircleCI config to use public docker images (#1171)

kedro-org · Jan 21, 2022 · 70736d6 · 70736d6
1 parent 483cfff
commit 70736d6
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 30 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,7 +16,7 @@ executors:
       python_version:
         type: string
     docker:
-      - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:<<parameters.python_version>>
+      - image: public.ecr.aws/g0x0s3o2/kedro-builder:<<parameters.python_version>>
     resource_class: medium+
 
 commands:
@@ -344,11 +344,11 @@ jobs:
           command: ./tools/circleci/github_scripts/attempt_merge_pr.sh "merge-main-to-develop" "develop" "${GITHUB_TAGGING_TOKEN}"
 
   build_docker_image:
+    parameters:
+      python_version:
+        type: string
     docker:
       - image: cimg/python:3.8
-    environment:
-      ECR_IMAGE_URL: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder
-      AWS_REGION: eu-west-2
     steps:
       - setup_remote_docker:
           docker_layer_caching: true
@@ -358,16 +358,14 @@ jobs:
           command: pip install -U awscli
       - run:
           name: Login to AWS ECR
-          command: |
-            aws ecr get-login-password --region ${AWS_REGION} | \
-            docker login --username AWS --password-stdin "${ECR_IMAGE_URL}"
+          command: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
       - run:
           name: Build docker images
-          command: ./tools/circleci/docker_build_img/build.sh "." "${ECR_IMAGE_URL}"
+          command: ./tools/circleci/docker_build_img/build.sh "." "public.ecr.aws/g0x0s3o2/kedro-builder" "<<parameters.python_version>>"
           no_output_timeout: 20m
       - run:
           name: Logout from AWS ECR
-          command: docker logout "${ECR_IMAGE_URL}"
+          command: docker logout public.ecr.aws
           when: always  # logout even if the previous step has failed
 
   # This is effectively just a combination of the lint, unit_tests and e2e_tests jobs.
@@ -520,21 +518,45 @@ workflows:
     jobs:
       - merge_pr_to_develop
 
-  nightly_build:
+  # Python versions that are supported on `main`.
+  nightly_build_main:
     triggers:
       - schedule:
           cron: 30 2 * * *
           filters:
             branches:
               only: main
     jobs:
-      - build_docker_image
+      - build_docker_image:
+          matrix:
+            parameters:
+              python_version: ["3.6", "3.7", "3.8"]
       - build_kedro:
           matrix:
             parameters:
               python_version: ["3.6", "3.7", "3.8"]
           requires:
-            - build_docker_image
+            - build_docker_image-<<matrix.python_version>>
+
+  # Python version that are *only* supported on `develop`.
+  nightly_build_develop:
+    triggers:
+      - schedule:
+          cron: 30 2 * * *
+          filters:
+            branches:
+              only: develop
+    jobs:
+      - build_docker_image:
+          matrix:
+            parameters:
+              python_version: ["3.9"]
+      - build_kedro:
+          matrix:
+            parameters:
+              python_version: ["3.9"]
+          requires:
+            - build_docker_image-<<matrix.python_version>>
 
   kedro_release:
     when: <<pipeline.parameters.release_kedro>>  # only run if 'release_kedro' flag is set

diff --git a/tools/circleci/docker_build_img/build.sh b/tools/circleci/docker_build_img/build.sh
@@ -4,6 +4,7 @@ cd "$(dirname "$0")"
 
 KEDRO_REPO=$1
 ECR_IMAGE_URL=$2
+PY_VERSION=$3
 
 get_pip_reqs() {
     local project_path=$1
@@ -12,37 +13,24 @@ get_pip_reqs() {
 
 docker_build() {
     local pip_reqs="$1"
-    local py_version=$2
-    local image=$ECR_IMAGE_URL:$py_version
+    local image=$ECR_IMAGE_URL:$PY_VERSION
     echo "Building docker image: $image"
     docker build -t $image \
         --build-arg PIP_REQS="$pip_reqs" \
-        --build-arg PY_VERSION=$py_version \
+        --build-arg PY_VERSION=$PY_VERSION \
         .
 }
 
 docker_push() {
-    local py_version=$1
-    local image=$ECR_IMAGE_URL:$py_version
+    local image=$ECR_IMAGE_URL:$PY_VERSION
     echo "Pushing docker image: $image"
     docker push $image
 }
 
 main() {
     local pip_reqs="$(get_pip_reqs $KEDRO_REPO)"
-
-    # Image for python 3.6
-    docker_build "$pip_reqs" 3.6
-    docker_push 3.6
-
-    # Image for python 3.7
-    docker_build "$pip_reqs" 3.7
-    docker_push 3.7
-
-    # Image for python 3.8
-    docker_build "$pip_reqs" 3.8
-    docker_push 3.8
-
+    docker_build "$pip_reqs"
+    docker_push
 }
 
 main