brefphp · mnapoli · Jan 28, 2023 · Jan 22, 2023 · Jan 22, 2023 · Jan 23, 2023
diff --git a/.env.example b/.env.example
@@ -7,3 +7,8 @@
 # Limit the parallelization of layer publication.
 # Default is 7, we recommend a lower number when publishing from a laptop.
 MAX_PARALLEL_PUBLISH=3
+
+# In the CI and on local machines in the core team we build using https://depot.dev
+# as it is much faster to build cross-platform images.
+# Do not uncomment this line, unless you have a depot.dev account.
+#USE_DEPOT=1
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,57 +15,64 @@ permissions:
     contents: read  # This is required for actions/checkout
 
 jobs:
-    release-x86:
-        name: Publish x86 layers
-        runs-on: ubuntu-latest
-        steps:
-            -   uses: actions/checkout@v3
-            -   name: Configure AWS credentials
-                uses: aws-actions/configure-aws-credentials@v1
-                with:
-                    role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions
-                    role-session-name: bref-layer-publisher-github-actions
-                    aws-region: us-east-1
-            -   name: Configure Docker Hub credentials
-                uses: docker/login-action@v2
-                with:
-                    username: ${{ secrets.DOCKER_USERNAME }}
-                    password: ${{ secrets.DOCKER_PASSWORD }}
-            -   run: make -f cpu-x86.Makefile layers
-            -   run: make -f cpu-x86.Makefile test
-            -   run: make -f cpu-x86.Makefile upload-layers
-            -   run: make -f cpu-x86.Makefile upload-to-docker-hub
 
-    release-arm:
-        name: Publish ARM layers
+    release:
+        name: Publish ${{ matrix.cpu }} layers
         runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                cpu:
+                    - x86
+                    - arm
         steps:
             -   uses: actions/checkout@v3
+
+            # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
+            -   name: Set up QEMU to run ARM images (that were built with Depot)
+                uses: docker/setup-qemu-action@v2
+
+            -   uses: depot/setup-action@v1
+
             -   name: Configure AWS credentials
                 uses: aws-actions/configure-aws-credentials@v1
                 with:
                     role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions
                     role-session-name: bref-layer-publisher-github-actions
                     aws-region: us-east-1
+
             -   name: Configure Docker Hub credentials
                 uses: docker/login-action@v2
                 with:
                     username: ${{ secrets.DOCKER_USERNAME }}
                     password: ${{ secrets.DOCKER_PASSWORD }}
-            # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
-            -   name: Set up QEMU to build ARM images
-                uses: docker/setup-qemu-action@v2
-            -   name: Set up Docker buildx to build ARM images
-                uses: docker/setup-buildx-action@v2
-            -   run: make -f cpu-arm.Makefile layers
-            -   run: make -f cpu-arm.Makefile test
-            -   run: make -f cpu-arm.Makefile upload-layers
-            -   run: make -f cpu-arm.Makefile upload-to-docker-hub
+
+            -   name: Build Docker images
+                run: make docker-images
+                env:
+                    CPU: ${{ matrix.cpu }}
+                    USE_DEPOT: 1
+                    DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
+
+            -   run: make layers
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   run: make test
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   run: make upload-layers
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   run: make upload-to-docker-hub
+                env:
+                    CPU: ${{ matrix.cpu }}
 
     update-layer-versions:
         name: Update layer versions in brefphp/bref
         runs-on: ubuntu-latest
-        needs: [ release-x86, release-arm ]
+        needs: [ release ]
         steps:
             -   name: Trigger layer update in brefphp/bref
                 uses: actions/github-script@v6
@@ -82,7 +89,7 @@ jobs:
     update-layer-js-versions:
         name: Update layer versions in brefphp/layers.js
         runs-on: ubuntu-latest
-        needs: [ release-x86, release-arm ]
+        needs: [ release ]
         steps:
             -   name: Trigger release in brefphp/layers.js
                 uses: actions/github-script@v6

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
     tests:
-        name: Build and tests layers
+        name: Build and tests PHP ${{ matrix.php_version }}, ${{ matrix.cpu }}
         runs-on: ubuntu-latest
         strategy:
             fail-fast: false
@@ -21,51 +21,29 @@ jobs:
                     - 81
                     - 82
         steps:
-
             -   uses: actions/checkout@v3
 
             # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
-            -   name: Set up QEMU to build ARM images
+            -   name: Set up QEMU to run ARM images (that were built with Depot)
                 uses: docker/setup-qemu-action@v2
 
-            -   name: Set up Docker buildx to use BuildKit features
-                uses: docker/setup-buildx-action@v2
-                with:
-                    # Sets up `docker build` command as an alias to `docker buildx`
-                    install: true
+            -   uses: depot/setup-action@v1
 
             -   name: Build Docker images
-                uses: docker/bake-action@v2.3.0
+                run: make docker-images-php-${{ matrix.php_version }}
                 env:
-                    PHP_VERSION: ${{ matrix.php_version }}
                     CPU: ${{ matrix.cpu }}
-                    CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }}
-                    IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }}
-                with:
-                    # This is needed to make the built images available in later steps
-                    # https://docs.docker.com/engine/reference/commandline/buildx_build/#load
-                    load: true
-                    # Cache Docker layers in GitHub Actions cache, scoped per image
-                    # https://github.com/docker/bake-action/issues/87#issuecomment-1184659151
-                    # We unfortunately don't use `mode=max` (which caches ALL layers instead of just tags)
-                    # nor do we cache all images because it creates a huge number of cache requests
-                    # and we get GitHub Actions cache timeouts:
-                    # https://github.com/moby/buildkit/issues/2804
-                    set: |
-                        base-devel.cache-from=type=gha,scope=base-devel-${{ matrix.cpu }}
-                        base-devel.cache-to=type=gha,scope=base-devel-${{ matrix.cpu }}
-                        build-php.cache-from=type=gha,scope=build-${{ matrix.cpu }}-php-${{ matrix.php_version }}
-                        build-php.cache-to=type=gha,scope=build-${{ matrix.cpu }}-php-${{ matrix.php_version }}
-                        fpm-internal-src.cache-from=type=gha,scope=fpm-internal-src
-                        fpm-internal-src.cache-to=type=gha,scope=fpm-internal-src
-                        console.cache-from=type=gha,scope=${{ matrix.cpu }}-console-${{ matrix.php_version }}
-                        console.cache-to=type=gha,scope=${{ matrix.cpu }}-console-${{ matrix.php_version }}
-                        php-fpm-dev.cache-from=type=gha,scope=${{ matrix.cpu }}-php-fpm-dev-${{ matrix.php_version }}
-                        php-fpm-dev.cache-to=type=gha,scope=${{ matrix.cpu }}-php-fpm-dev-${{ matrix.php_version }}
+                    USE_DEPOT: 1
+                    DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
 
             -   name: Test that layers can be exported
                 run: |
-                    make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }}
-                    make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }}-fpm
+                    make layer-php-${{ matrix.php_version }}
+                    make layer-php-${{ matrix.php_version }}-fpm
+                env:
+                    CPU: ${{ matrix.cpu }}
 
-            -   run: make -f cpu-${{ matrix.cpu }}.Makefile test-${{ matrix.php_version }}
+            -   name: Run tests
+                run: make test-${{ matrix.php_version }}
+                env:
+                    CPU: ${{ matrix.cpu }}
diff --git a/Makefile b/Makefile
@@ -2,46 +2,102 @@
 -include .env
 export # export all variables defined in .env
 
-# - Build all layers
-# - Publish all Docker images to Docker Hub
-# - Publish all layers to AWS Lambda
+# Define all the environment variables depending on the CPU
+# Set CPU= (empty) to build for x86
+# Set CPU=arm to build for ARM
+ifeq ($(CPU), arm) # if $CPU=="arm"
+  $(info "⚠️  Building for ARM") # Print a message
+  export CPU = arm
+  export CPU_PREFIX = arm-
+  export IMAGE_VERSION_SUFFIX = arm64
+  export DOCKER_PLATFORM = linux/arm64
+else
+  $(info "⚠️  Building for x86") # Print a message
+  export CPU = x86
+  export CPU_PREFIX =
+  export IMAGE_VERSION_SUFFIX = x86_64
+  export DOCKER_PLATFORM = linux/amd64
+endif
+
+# By default, Docker images are built using `docker buildx bake`
+# But we use https://depot.dev in CI (super fast) by setting USE_DEPOT=1
+ifeq ($(USE_DEPOT), 1) # if $USE_DEPOT=="1"
+  $(info "⚠️  Building using depot.dev") # Print a message
+  export BAKE_COMMAND = depot bake
+else
+  export BAKE_COMMAND = docker buildx bake
+endif
+
+
+# Build all Docker images and layers *locally*
+# Use this to test your changes
+default: docker-images layers
+
+
+# Build Docker images *locally*
+docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82
+docker-images-php-%:
+	PHP_VERSION=$* ${BAKE_COMMAND} --load
+
+
+# Build Lambda layers (zip files) *locally*
+layers: layer-php-80 layer-php-81 layer-php-82 layer-php-80-fpm layer-php-81-fpm layer-php-82-fpm
+	# Build the console layer only once (x86 and single PHP version)
+	@if [ ${CPU} = "x86" ]; then \
+		./utils/docker-zip-dir.sh bref/php-80-console-zip console; \
+	fi
+# This rule matches with a wildcard, for example `layer-php-80`.
+# The `$*` variable will contained the matched part, in this case `php-80`.
+layer-%:
+	./utils/docker-zip-dir.sh bref/${CPU_PREFIX}$* ${CPU_PREFIX}$*
+
+
+# Upload the layers to AWS Lambda
 # Uses the current AWS_PROFILE. Most users will not want to use this option
 # as this will publish all layers to all regions + publish all Docker images.
-everything:
-	$(MAKE) -f cpu-x86.Makefile everything
-	$(MAKE) -f cpu-arm.Makefile everything
+upload-layers: upload-layers-php-80 upload-layers-php-81 upload-layers-php-82
+	# Upload the console layer only once (x86 and single PHP version)
+	@if [ ${CPU} = "x86" ]; then \
+		LAYER_NAME=console $(MAKE) -C ./utils/lambda-publish publish-parallel; \
+	fi
+upload-layers-php-%:
+	# Upload the function layers to AWS
+	LAYER_NAME=${CPU_PREFIX}php-$* $(MAKE) -C ./utils/lambda-publish publish-parallel
+	# Upload the FPM layers to AWS
+	LAYER_NAME=${CPU_PREFIX}php-$*-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel
 
-# Build Docker images *locally*
-docker-images:
-	$(MAKE) -f cpu-x86.Makefile docker-images
-	$(MAKE) -f cpu-arm.Makefile docker-images
-
-# Build Lambda layers (zip files) *locally* (will also build Docker images)
-layers:
-	$(MAKE) -f cpu-x86.Makefile layers
-	$(MAKE) -f cpu-arm.Makefile layers
-
-# Upload the layers to AWS Lambda (will also build Docker images and layers)
-upload-layers:
-	$(MAKE) -f cpu-x86.Makefile upload-layers
-	$(MAKE) -f cpu-arm.Makefile upload-layers
-
-# Build and publish Docker images to Docker Hub.
-# Only publishes the `latest` version.
-# This process is executed when a merge to `main` happens.
-# When a release tag is created, GitHub Actions
-# will download the latest images, tag them with the version number
-# and re-upload them with the right tag.
-upload-to-docker-hub:
-	$(MAKE) -f cpu-x86.Makefile upload-to-docker-hub
-	$(MAKE) -f cpu-arm.Makefile upload-to-docker-hub
-
-test:
-	$(MAKE) -f cpu-x86.Makefile test
-	$(MAKE) -f cpu-arm.Makefile test
-
-clean:
-	$(MAKE) -f cpu-x86.Makefile clean
-	$(MAKE) -f cpu-arm.Makefile clean
-
-.PHONY: layers
+
+# Publish Docker images to Docker Hub.
+upload-to-docker-hub: upload-to-docker-hub-php-80 upload-to-docker-hub-php-81 upload-to-docker-hub-php-82
+upload-to-docker-hub-php-%:
+	# While in beta we tag and push the `:2` version, later we'll push `:latest` as well
+	for image in \
+	  "bref/${CPU_PREFIX}php-$*" "bref/${CPU_PREFIX}php-$*-fpm" "bref/${CPU_PREFIX}php-$*-console" \
+	  "bref/${CPU_PREFIX}build-php-$*" "bref/${CPU_PREFIX}php-$*-fpm-dev"; \
+	do \
+		docker tag $$image $$image:2 ; \
+		docker push $$image:2 ; \
+	done
+	# TODO: when v2 becomes "latest", we should also push "latest" tags
+	# We could actually use `docker push --all-tags` at the end probably?
+
+
+test: test-80 test-81 test-82
+test-%:
+	cd tests && $(MAKE) test-$*
+
+
+clean: clean-80 clean-81 clean-82
+	# Clear the build cache, else all images will be rebuilt using cached layers
+	docker builder prune
+	# Remove zip files
+	rm -f output/${CPU_PREFIX}*.zip
+clean-%:
+	# Clean Docker images to force rebuilding them
+	docker image rm --force bref/${CPU_PREFIX}build-php-$* \
+		bref/${CPU_PREFIX}php-$* \
+		bref/${CPU_PREFIX}php-$*-zip \
+		bref/${CPU_PREFIX}php-$*-fpm \
+		bref/${CPU_PREFIX}php-$*-fpm-zip \
+		bref/${CPU_PREFIX}php-$*-fpm-dev \
+		bref/${CPU_PREFIX}php-$*-console