Skip to content

Commit a30f3e0

Browse files
mnapolimnapoli
authored
Merge pull request #44 from brefphp/depot
2 parents 0501e0d + 8db39ac commit a30f3e0

File tree

17 files changed

+1234
-722
lines changed

17 files changed

+1234
-722
lines changed

‎.env.example

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,8 @@
77
# Limit the parallelization of layer publication.
88
# Default is 7, we recommend a lower number when publishing from a laptop.
99
MAX_PARALLEL_PUBLISH=3
10+
11+
# In the CI and on local machines in the core team we build using https://depot.dev
12+
# as it is much faster to build cross-platform images.
13+
# Do not uncomment this line, unless you have a depot.dev account.
14+
#USE_DEPOT=1

‎.github/workflows/release.yml

Lines changed: 40 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -15,57 +15,64 @@ permissions:
1515
contents: read # This is required for actions/checkout
1616

1717
jobs:
18-
release-x86:
19-
name: Publish x86 layers
20-
runs-on: ubuntu-latest
21-
steps:
22-
- uses: actions/checkout@v3
23-
- name: Configure AWS credentials
24-
uses: aws-actions/configure-aws-credentials@v1
25-
with:
26-
role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions
27-
role-session-name: bref-layer-publisher-github-actions
28-
aws-region: us-east-1
29-
- name: Configure Docker Hub credentials
30-
uses: docker/login-action@v2
31-
with:
32-
username: ${{ secrets.DOCKER_USERNAME }}
33-
password: ${{ secrets.DOCKER_PASSWORD }}
34-
- run: make -f cpu-x86.Makefile layers
35-
- run: make -f cpu-x86.Makefile test
36-
- run: make -f cpu-x86.Makefile upload-layers
37-
- run: make -f cpu-x86.Makefile upload-to-docker-hub
3818

39-
release-arm:
40-
name: Publish ARM layers
19+
release:
20+
name: Publish ${{ matrix.cpu }} layers
4121
runs-on: ubuntu-latest
22+
strategy:
23+
matrix:
24+
cpu:
25+
- x86
26+
- arm
4227
steps:
4328
- uses: actions/checkout@v3
29+
30+
# See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
31+
- name: Set up QEMU to run ARM images (that were built with Depot)
32+
uses: docker/setup-qemu-action@v2
33+
34+
- uses: depot/setup-action@v1
35+
4436
- name: Configure AWS credentials
4537
uses: aws-actions/configure-aws-credentials@v1
4638
with:
4739
role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions
4840
role-session-name: bref-layer-publisher-github-actions
4941
aws-region: us-east-1
42+
5043
- name: Configure Docker Hub credentials
5144
uses: docker/login-action@v2
5245
with:
5346
username: ${{ secrets.DOCKER_USERNAME }}
5447
password: ${{ secrets.DOCKER_PASSWORD }}
55-
# See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
56-
- name: Set up QEMU to build ARM images
57-
uses: docker/setup-qemu-action@v2
58-
- name: Set up Docker buildx to build ARM images
59-
uses: docker/setup-buildx-action@v2
60-
- run: make -f cpu-arm.Makefile layers
61-
- run: make -f cpu-arm.Makefile test
62-
- run: make -f cpu-arm.Makefile upload-layers
63-
- run: make -f cpu-arm.Makefile upload-to-docker-hub
48+
49+
- name: Build Docker images
50+
run: make docker-images
51+
env:
52+
CPU: ${{ matrix.cpu }}
53+
USE_DEPOT: 1
54+
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
55+
56+
- run: make layers
57+
env:
58+
CPU: ${{ matrix.cpu }}
59+
60+
- run: make test
61+
env:
62+
CPU: ${{ matrix.cpu }}
63+
64+
- run: make upload-layers
65+
env:
66+
CPU: ${{ matrix.cpu }}
67+
68+
- run: make upload-to-docker-hub
69+
env:
70+
CPU: ${{ matrix.cpu }}
6471

6572
update-layer-versions:
6673
name: Update layer versions in brefphp/bref
6774
runs-on: ubuntu-latest
68-
needs: [ release-x86, release-arm ]
75+
needs: [ release ]
6976
steps:
7077
- name: Trigger layer update in brefphp/bref
7178
uses: actions/github-script@v6
@@ -82,7 +89,7 @@ jobs:
8289
update-layer-js-versions:
8390
name: Update layer versions in brefphp/layers.js
8491
runs-on: ubuntu-latest
85-
needs: [ release-x86, release-arm ]
92+
needs: [ release ]
8693
steps:
8794
- name: Trigger release in brefphp/layers.js
8895
uses: actions/github-script@v6

‎.github/workflows/tests.yml

Lines changed: 14 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ on:
88

99
jobs:
1010
tests:
11-
name: Build and tests layers
11+
name: Build and tests PHP ${{ matrix.php_version }}, ${{ matrix.cpu }}
1212
runs-on: ubuntu-latest
1313
strategy:
1414
fail-fast: false
@@ -21,51 +21,29 @@ jobs:
2121
- 81
2222
- 82
2323
steps:
24-
2524
- uses: actions/checkout@v3
2625

2726
# See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
28-
- name: Set up QEMU to build ARM images
27+
- name: Set up QEMU to run ARM images (that were built with Depot)
2928
uses: docker/setup-qemu-action@v2
3029

31-
- name: Set up Docker buildx to use BuildKit features
32-
uses: docker/setup-buildx-action@v2
33-
with:
34-
# Sets up `docker build` command as an alias to `docker buildx`
35-
install: true
30+
- uses: depot/setup-action@v1
3631

3732
- name: Build Docker images
38-
uses: docker/bake-action@v2.3.0
33+
run: make docker-images-php-${{ matrix.php_version }}
3934
env:
40-
PHP_VERSION: ${{ matrix.php_version }}
4135
CPU: ${{ matrix.cpu }}
42-
CPU_PREFIX: ${{ (matrix.cpu == 'arm') && 'arm-' || '' }}
43-
IMAGE_VERSION_SUFFIX: ${{ (matrix.cpu == 'arm') && 'arm64' || 'x86_64' }}
44-
with:
45-
# This is needed to make the built images available in later steps
46-
# https://docs.docker.com/engine/reference/commandline/buildx_build/#load
47-
load: true
48-
# Cache Docker layers in GitHub Actions cache, scoped per image
49-
# https://github.com/docker/bake-action/issues/87#issuecomment-1184659151
50-
# We unfortunately don't use `mode=max` (which caches ALL layers instead of just tags)
51-
# nor do we cache all images because it creates a huge number of cache requests
52-
# and we get GitHub Actions cache timeouts:
53-
# https://github.com/moby/buildkit/issues/2804
54-
set: |
55-
base-devel.cache-from=type=gha,scope=base-devel-${{ matrix.cpu }}
56-
base-devel.cache-to=type=gha,scope=base-devel-${{ matrix.cpu }}
57-
build-php.cache-from=type=gha,scope=build-${{ matrix.cpu }}-php-${{ matrix.php_version }}
58-
build-php.cache-to=type=gha,scope=build-${{ matrix.cpu }}-php-${{ matrix.php_version }}
59-
fpm-internal-src.cache-from=type=gha,scope=fpm-internal-src
60-
fpm-internal-src.cache-to=type=gha,scope=fpm-internal-src
61-
console.cache-from=type=gha,scope=${{ matrix.cpu }}-console-${{ matrix.php_version }}
62-
console.cache-to=type=gha,scope=${{ matrix.cpu }}-console-${{ matrix.php_version }}
63-
php-fpm-dev.cache-from=type=gha,scope=${{ matrix.cpu }}-php-fpm-dev-${{ matrix.php_version }}
64-
php-fpm-dev.cache-to=type=gha,scope=${{ matrix.cpu }}-php-fpm-dev-${{ matrix.php_version }}
36+
USE_DEPOT: 1
37+
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
6538

6639
- name: Test that layers can be exported
6740
run: |
68-
make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }}
69-
make -f cpu-${{ matrix.cpu }}.Makefile layer-php-${{ matrix.php_version }}-fpm
41+
make layer-php-${{ matrix.php_version }}
42+
make layer-php-${{ matrix.php_version }}-fpm
43+
env:
44+
CPU: ${{ matrix.cpu }}
7045

71-
- run: make -f cpu-${{ matrix.cpu }}.Makefile test-${{ matrix.php_version }}
46+
- name: Run tests
47+
run: make test-${{ matrix.php_version }}
48+
env:
49+
CPU: ${{ matrix.cpu }}

‎Makefile

Lines changed: 96 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,46 +2,102 @@
22
-include .env
33
export # export all variables defined in .env
44

5-
# - Build all layers
6-
# - Publish all Docker images to Docker Hub
7-
# - Publish all layers to AWS Lambda
5+
# Define all the environment variables depending on the CPU
6+
# Set CPU= (empty) to build for x86
7+
# Set CPU=arm to build for ARM
8+
ifeq ($(CPU), arm) # if $CPU=="arm"
9+
$(info "⚠️ Building for ARM") # Print a message
10+
export CPU = arm
11+
export CPU_PREFIX = arm-
12+
export IMAGE_VERSION_SUFFIX = arm64
13+
export DOCKER_PLATFORM = linux/arm64
14+
else
15+
$(info "⚠️ Building for x86") # Print a message
16+
export CPU = x86
17+
export CPU_PREFIX =
18+
export IMAGE_VERSION_SUFFIX = x86_64
19+
export DOCKER_PLATFORM = linux/amd64
20+
endif
21+
22+
# By default, Docker images are built using `docker buildx bake`
23+
# But we use https://depot.dev in CI (super fast) by setting USE_DEPOT=1
24+
ifeq ($(USE_DEPOT), 1) # if $USE_DEPOT=="1"
25+
$(info "⚠️ Building using depot.dev") # Print a message
26+
export BAKE_COMMAND = depot bake
27+
else
28+
export BAKE_COMMAND = docker buildx bake
29+
endif
30+
31+
32+
# Build all Docker images and layers *locally*
33+
# Use this to test your changes
34+
default: docker-images layers
35+
36+
37+
# Build Docker images *locally*
38+
docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82
39+
docker-images-php-%:
40+
PHP_VERSION=$* ${BAKE_COMMAND} --load
41+
42+
43+
# Build Lambda layers (zip files) *locally*
44+
layers: layer-php-80 layer-php-81 layer-php-82 layer-php-80-fpm layer-php-81-fpm layer-php-82-fpm
45+
# Build the console layer only once (x86 and single PHP version)
46+
@if [ ${CPU} = "x86" ]; then \
47+
./utils/docker-zip-dir.sh bref/php-80-console-zip console; \
48+
fi
49+
# This rule matches with a wildcard, for example `layer-php-80`.
50+
# The `$*` variable will contained the matched part, in this case `php-80`.
51+
layer-%:
52+
./utils/docker-zip-dir.sh bref/${CPU_PREFIX}$* ${CPU_PREFIX}$*
53+
54+
55+
# Upload the layers to AWS Lambda
856
# Uses the current AWS_PROFILE. Most users will not want to use this option
957
# as this will publish all layers to all regions + publish all Docker images.
10-
everything:
11-
$(MAKE) -f cpu-x86.Makefile everything
12-
$(MAKE) -f cpu-arm.Makefile everything
58+
upload-layers: upload-layers-php-80 upload-layers-php-81 upload-layers-php-82
59+
# Upload the console layer only once (x86 and single PHP version)
60+
@if [ ${CPU} = "x86" ]; then \
61+
LAYER_NAME=console $(MAKE) -C ./utils/lambda-publish publish-parallel; \
62+
fi
63+
upload-layers-php-%:
64+
# Upload the function layers to AWS
65+
LAYER_NAME=${CPU_PREFIX}php-$* $(MAKE) -C ./utils/lambda-publish publish-parallel
66+
# Upload the FPM layers to AWS
67+
LAYER_NAME=${CPU_PREFIX}php-$*-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel
1368

14-
# Build Docker images *locally*
15-
docker-images:
16-
$(MAKE) -f cpu-x86.Makefile docker-images
17-
$(MAKE) -f cpu-arm.Makefile docker-images
18-
19-
# Build Lambda layers (zip files) *locally* (will also build Docker images)
20-
layers:
21-
$(MAKE) -f cpu-x86.Makefile layers
22-
$(MAKE) -f cpu-arm.Makefile layers
23-
24-
# Upload the layers to AWS Lambda (will also build Docker images and layers)
25-
upload-layers:
26-
$(MAKE) -f cpu-x86.Makefile upload-layers
27-
$(MAKE) -f cpu-arm.Makefile upload-layers
28-
29-
# Build and publish Docker images to Docker Hub.
30-
# Only publishes the `latest` version.
31-
# This process is executed when a merge to `main` happens.
32-
# When a release tag is created, GitHub Actions
33-
# will download the latest images, tag them with the version number
34-
# and re-upload them with the right tag.
35-
upload-to-docker-hub:
36-
$(MAKE) -f cpu-x86.Makefile upload-to-docker-hub
37-
$(MAKE) -f cpu-arm.Makefile upload-to-docker-hub
38-
39-
test:
40-
$(MAKE) -f cpu-x86.Makefile test
41-
$(MAKE) -f cpu-arm.Makefile test
42-
43-
clean:
44-
$(MAKE) -f cpu-x86.Makefile clean
45-
$(MAKE) -f cpu-arm.Makefile clean
46-
47-
.PHONY: layers
69+
70+
# Publish Docker images to Docker Hub.
71+
upload-to-docker-hub: upload-to-docker-hub-php-80 upload-to-docker-hub-php-81 upload-to-docker-hub-php-82
72+
upload-to-docker-hub-php-%:
73+
# While in beta we tag and push the `:2` version, later we'll push `:latest` as well
74+
for image in \
75+
"bref/${CPU_PREFIX}php-$*" "bref/${CPU_PREFIX}php-$*-fpm" "bref/${CPU_PREFIX}php-$*-console" \
76+
"bref/${CPU_PREFIX}build-php-$*" "bref/${CPU_PREFIX}php-$*-fpm-dev"; \
77+
do \
78+
docker tag $$image $$image:2 ; \
79+
docker push $$image:2 ; \
80+
done
81+
# TODO: when v2 becomes "latest", we should also push "latest" tags
82+
# We could actually use `docker push --all-tags` at the end probably?
83+
84+
85+
test: test-80 test-81 test-82
86+
test-%:
87+
cd tests && $(MAKE) test-$*
88+
89+
90+
clean: clean-80 clean-81 clean-82
91+
# Clear the build cache, else all images will be rebuilt using cached layers
92+
docker builder prune
93+
# Remove zip files
94+
rm -f output/${CPU_PREFIX}*.zip
95+
clean-%:
96+
# Clean Docker images to force rebuilding them
97+
docker image rm --force bref/${CPU_PREFIX}build-php-$* \
98+
bref/${CPU_PREFIX}php-$* \
99+
bref/${CPU_PREFIX}php-$*-zip \
100+
bref/${CPU_PREFIX}php-$*-fpm \
101+
bref/${CPU_PREFIX}php-$*-fpm-zip \
102+
bref/${CPU_PREFIX}php-$*-fpm-dev \
103+
bref/${CPU_PREFIX}php-$*-console

0 commit comments

Comments
 (0)