Refactor the build process into more images

The new `bref/build-php-7X` images will provide users a Docker images where they can compile PHP extensions.

Author: mnapoli

Makefile

@@ -1,16 +1,15 @@
 SHELL := /bin/bash
-TAG = latest
-.PHONY: layers
+.PHONY: publish layers docker-images
 
 # Publish the layers on AWS Lambda
 publish: layers
-	php publish.php
+	cd layers ; php publish.php
 
 # Build the layers
 layers: export/console.zip export/php-72.zip export/php-73.zip export/php-72-fpm.zip export/php-73-fpm.zip
 
 # The PHP runtimes
-export/php%.zip: build
+export/php%.zip: docker-images
 	PHP_VERSION=$$(echo $@ | cut -d'/' -f 2 | cut -d'.' -f 1);\
 	rm -f $@;\
 	mkdir export/tmp ; cd export/tmp ;\
@@ -22,14 +21,24 @@ export/console.zip: layers/console/bootstrap
 	rm -f export/console.zip
 	cd layers/console && zip ../../export/console.zip bootstrap
 
-# Compile PHP and its extensions
-build:
-	docker build -f ${PWD}/php-intermediary.Dockerfile -t bref/php-72-intermediary:latest $(shell helpers/docker_args.sh versions.ini php72) .
-	cd layers/fpm ; docker build -t bref/php-72-fpm:$(TAG) --build-arg LAYER_IMAGE=bref/php-72-intermediary:latest . ; cd ../..
-	cd layers/fpm-dev ; docker build -t bref/php-72-fpm-dev:$(TAG) --build-arg LAYER_IMAGE=bref/php-72-fpm:$(TAG) . ; cd ../..
-	cd layers/function ; docker build -t bref/php-72:$(TAG) --build-arg LAYER_IMAGE=bref/php-72-intermediary:latest . ; cd ../..
-	docker build -f ${PWD}/php-intermediary.Dockerfile -t bref/php-73-intermediary:latest $(shell helpers/docker_args.sh versions.ini php73) .
-	cd layers/fpm ; docker build -t bref/php-73-fpm:$(TAG) --build-arg LAYER_IMAGE=bref/php-73-intermediary:latest . ; cd ../..
-	cd layers/fpm-dev ; docker build -t bref/php-73-fpm-dev:$(TAG) --build-arg LAYER_IMAGE=bref/php-73-fpm:$(TAG) . ; cd ../..
-	cd layers/function ; docker build -t bref/php-73:$(TAG) --build-arg LAYER_IMAGE=bref/php-73-intermediary:latest . ; cd ../..
-	cd layers/web; docker build -t bref/fpm-dev-gateway:$(TAG) . ; cd ../..
+# Build Docker images
+docker-images:
+	# Build the base environment (without PHP)
+	cd base ; docker build --file base.Dockerfile -t bref/tmp/step-1/build-environment .
+	# Build the `bref/build-php-XX` images
+	# (build only the first `FROM` section of the Dockerfile)
+	cd base ; docker build --file php-72.Dockerfile -t bref/build-php-72 --target build-environment .
+	cd base ; docker build --file php-73.Dockerfile -t bref/build-php-73 --target build-environment .
+	# Build the whole Dockerfile to generate the cleaned images that will be used in the next step
+	cd base ; docker build --file php-72.Dockerfile -t bref/tmp/cleaned-build-php-72 .
+	cd base ; docker build --file php-73.Dockerfile -t bref/tmp/cleaned-build-php-73 .
+	# - function
+	cd layers/function ; docker build -t bref/php-72 --build-arg PHP_VERSION=72 .
+	cd layers/function ; docker build -t bref/php-73 --build-arg PHP_VERSION=73 .
+	# - fpm
+	cd layers/fpm ; docker build -t bref/php-72-fpm --build-arg PHP_VERSION=72 .
+	cd layers/fpm ; docker build -t bref/php-73-fpm --build-arg PHP_VERSION=73 .
+	# Other Docker images
+	cd layers/fpm-dev ; docker build -t bref/php-72-fpm-dev --build-arg PHP_VERSION=72 .
+	cd layers/fpm-dev ; docker build -t bref/php-73-fpm-dev --build-arg PHP_VERSION=73 .
+	cd layers/web; docker build -t bref/fpm-dev-gateway .

README.md

@@ -1,3 +1,36 @@
 This directory contains the scripts that create and publish the AWS Lambda runtimes for PHP.
 
 Read the [runtimes documentation](/docs/runtimes/README.md) to learn more.
+
+## How it works
+
+The scripts are written mainly in `Makefile`.
+
+Multiple Docker images are created:
+
+- `bref/tmp/...` images are created locally and not published online
+- `bref/...` images are published on Docker Hub
+
+Workflow:
+
+- 1: Create the `bref/tmp/step-1/build-environment` Docker image.
+    This image contains everything needed to compile PHP. This image is created standalone because
+    it is common for each PHP version. The next step involves a different Dockerfile per PHP version.
+
+- 2: Create the `bref/build-php-XX` images.
+    There is one image per PHP version. These images contain PHP compiled with all its extensions.
+    It is published so that anyone can use it to compile their own extensions.
+    It is not the final image because it contains too many things that need to be removed.
+
+- 3: Create the `bref/tmp/cleaned-build-php-XX` images.
+    There is one image per PHP version. These images contain PHP compiled with all its extensions,
+    but with all extra files removed. These images do not contain the bootstrap files and PHP config files.
+
+- 4: Create the `bref/php-XX`, `bref/php-XX-fpm`, `bref/php-XX-fpm-dev` images.
+    There is one image per PHP version. These images contain exactly what will be in the layers in `/opt`.
+
+- 5: Create the layers zip files in the `export/` directory.
+    We zip the `/opt` directory of each Docker image.
+
+- 6: Publish the layers on AWS.
+    We upload and publish the layers in every AWS region using the zip files.

base/README.md

@@ -0,0 +1 @@
+There is one Dockerfile per minor PHP version because each version can have slightly different configuration options.

php-intermediary.Dockerfile renamed to base/base.Dockerfile

@@ -1,6 +1,4 @@
-# The container we build here contains everything needed to compile PHP + PHP.
-#
-# It can be used as a base to compile extra extensions.
+# The container we build here contains everything needed to compile PHP.
 
 
 # Lambda instances use the amzn-ami-hvm-2018.03.0.20181129-x86_64-gp2 AMI, as
@@ -94,11 +92,11 @@ RUN mkdir -p ${BUILD_DIR}  \
 # https://github.com/madler/zlib/releases
 # Needed for:
 #   - openssl
+#   - curl
 #   - php
 # Used By:
 #   - xml2
-ARG zlib
-ENV VERSION_ZLIB=${zlib}
+ENV VERSION_ZLIB=1.2.11
 ENV ZLIB_BUILD_DIR=${BUILD_DIR}/xml2
 
 RUN set -xe; \
@@ -130,9 +128,9 @@ RUN set -xe; \
 # Needs:
 #   - zlib
 # Needed by:
+#   - curl
 #   - php
-ARG openssl
-ENV VERSION_OPENSSL=${openssl}
+ENV VERSION_OPENSSL=1.1.1a
 ENV OPENSSL_BUILD_DIR=${BUILD_DIR}/xml2
 ENV CA_BUNDLE_SOURCE="https://curl.haxx.se/ca/cacert.pem"
 ENV CA_BUNDLE="${INSTALL_DIR}/ssl/cert.pem"
@@ -173,8 +171,7 @@ RUN set -xe; \
 #   - OpenSSL
 # Needed by:
 #   - curl
-ARG libssh2
-ENV VERSION_LIBSSH2=${libssh2}
+ENV VERSION_LIBSSH2=1.8.0
 ENV LIBSSH2_BUILD_DIR=${BUILD_DIR}/libssh2
 
 RUN set -xe; \
@@ -207,11 +204,10 @@ RUN set -xe; \
 # # Needs:
 # #   - zlib
 # #   - OpenSSL
-# #   - curl
+# #   - libssh2
 # # Needed by:
 # #   - php
-ARG curl
-ENV VERSION_CURL=${curl}
+ENV VERSION_CURL=7.63.0
 ENV CURL_BUILD_DIR=${BUILD_DIR}/curl
 
 RUN set -xe; \
@@ -261,8 +257,7 @@ RUN set -xe; \
 #   - zlib
 # Needed by:
 #   - php
-ARG libxml2
-ENV VERSION_XML2=${libxml2}
+ENV VERSION_XML2=2.9.8
 ENV XML2_BUILD_DIR=${BUILD_DIR}/xml2
 
 RUN set -xe; \
@@ -300,8 +295,7 @@ RUN set -xe; \
 # https://github.com/nih-at/libzip/releases
 # Needed by:
 #   - php
-ARG libzip
-ENV VERSION_ZIP=${libzip}
+ENV VERSION_ZIP=1.5.1
 ENV ZIP_BUILD_DIR=${BUILD_DIR}/zip
 
 RUN set -xe; \
@@ -328,12 +322,11 @@ RUN set -xe; \
 ###############################################################################
 # LIBSODIUM Build
 # https://github.com/jedisct1/libsodium/releases
-# Uses:
+# Needs:
 #
 # Needed by:
 #   - php
-ARG libsodium
-ENV VERSION_LIBSODIUM=${libsodium}
+ENV VERSION_LIBSODIUM=1.0.16
 ENV LIBSODIUM_BUILD_DIR=${BUILD_DIR}/libsodium
 
 RUN set -xe; \
@@ -363,8 +356,7 @@ RUN set -xe; \
 #   - OpenSSL
 # Needed by:
 #   - php
-ARG postgres
-ENV VERSION_POSTGRES=${postgres}
+ENV VERSION_POSTGRES=9.6.11
 ENV POSTGRES_BUILD_DIR=${BUILD_DIR}/postgres
 
 RUN set -xe; \
@@ -386,164 +378,10 @@ RUN set -xe; cd ${POSTGRES_BUILD_DIR}/src/bin/pg_config && make -j $(nproc) && m
 RUN set -xe; cd ${POSTGRES_BUILD_DIR}/src/backend && make generated-headers
 RUN set -xe; cd ${POSTGRES_BUILD_DIR}/src/include && make install
 
-###############################################################################
-# PHP Build
-# https://github.com/php/php-src/releases
-# Needs:
-#   - zlib
-#   - libxml2
-#   - openssl
-#   - readline
-#   - sodium
-
-ARG php
-# Setup Build Variables
-ENV VERSION_PHP=${php}
-ENV PHP_BUILD_DIR=${BUILD_DIR}/php
-
-RUN set -xe; \
-    mkdir -p ${PHP_BUILD_DIR}; \
-# Download and upack the source code
-    curl -Ls https://github.com/php/php-src/archive/php-${VERSION_PHP}.tar.gz \
-  | tar xzC ${PHP_BUILD_DIR} --strip-components=1
-
-# Move into the unpackaged code directory
-WORKDIR  ${PHP_BUILD_DIR}/
-
 # Install some dev files for using old libraries already on the system
 # readline-devel : needed for the --with-libedit flag
 # gettext-devel : needed for the --with-gettext flag
 # libicu-devel : needed for
 # libpng-devel : needed for gd
 # libjpeg-devel : needed for gd
 RUN LD_LIBRARY_PATH= yum install -y readline-devel gettext-devel libicu-devel libpng-devel libjpeg-devel
-
-# Configure the build
-# -fstack-protector-strong : Be paranoid about stack overflows
-# -fpic : Make PHP's main executable position-independent (improves ASLR security mechanism, and has no performance impact on x86_64)
-# -fpie : Support Address Space Layout Randomization (see -fpic)
-# -O3 : Optimize for fastest binaries possible.
-# -I : Add the path to the list of directories to be searched for header files during preprocessing.
-# --enable-option-checking=fatal: make sure invalid --configure-flags are fatal errors instead of just warnings
-# --enable-ftp: because ftp_ssl_connect() needs ftp to be compiled statically (see https://github.com/docker-library/php/issues/236)
-# --enable-mbstring: because otherwise there's no way to get pecl to use it properly (see https://github.com/docker-library/php/issues/195)
-# --enable-maintainer-zts: build PHP as ZTS (Zend Thread Safe) to be able to use pthreads
-# --with-zlib and --with-zlib-dir: See https://stackoverflow.com/a/42978649/245552
-# --enable-opcache-file: allows to use the `opcache.file_cache` option
-#
-RUN set -xe \
- && ./buildconf --force \
- && CFLAGS="-fstack-protector-strong -fpic -fpie -O3 -I${INSTALL_DIR}/include -I/usr/include -ffunction-sections -fdata-sections" \
-    CPPFLAGS="-fstack-protector-strong -fpic -fpie -O3 -I${INSTALL_DIR}/include -I/usr/include -ffunction-sections -fdata-sections" \
-    LDFLAGS="-L${INSTALL_DIR}/lib64 -L${INSTALL_DIR}/lib -Wl,-O1 -Wl,--strip-all -Wl,--hash-style=both -pie" \
-    ./configure \
-        --build=x86_64-pc-linux-gnu \
-        --prefix=${INSTALL_DIR} \
-        --enable-option-checking=fatal \
-        --enable-maintainer-zts \
-        --enable-sockets \
-        --with-config-file-path=${INSTALL_DIR}/etc/php \
-        --with-config-file-scan-dir=${INSTALL_DIR}/etc/php/conf.d:/var/task/php/conf.d \
-        --enable-fpm \
-        --disable-cgi \
-        --enable-cli \
-        --disable-phpdbg \
-        --disable-phpdbg-webhelper \
-        --with-sodium \
-        --with-readline \
-        --with-openssl \
-        --with-zlib=${INSTALL_DIR} \
-        --with-zlib-dir=${INSTALL_DIR} \
-        --with-curl \
-        --enable-exif \
-        --enable-ftp \
-        --with-gettext \
-        --enable-mbstring \
-        --with-pdo-mysql=shared,mysqlnd \
-        --with-mysqli \
-        --enable-pcntl \
-        --enable-zip \
-        --enable-bcmath \
-        --with-pdo-pgsql=shared,${INSTALL_DIR} \
-        --enable-intl=shared \
-        --enable-opcache-file \
-        --enable-soap \
-        --with-gd \
-        --with-png-dir=${INSTALL_DIR} \
-        --with-jpeg-dir=${INSTALL_DIR}
-RUN make -j $(nproc)
-# Run `make install` and override PEAR's PHAR URL because pear.php.net is down
-RUN set -xe; \
- make install PEAR_INSTALLER_URL='https://github.com/pear/pearweb_phars/raw/master/install-pear-nozlib.phar'; \
- { find ${INSTALL_DIR}/bin ${INSTALL_DIR}/sbin -type f -perm +0111 -exec strip --strip-all '{}' + || true; }; \
- make clean; \
- cp php.ini-production ${INSTALL_DIR}/etc/php/php.ini
-
-RUN pecl install mongodb
-RUN pecl install redis
-RUN pecl install APCu
-
-ENV PTHREADS_BUILD_DIR=${BUILD_DIR}/pthreads
-
-# Build from master because there are no pthreads release compatible with PHP 7.3
-RUN set -xe; \
-    mkdir -p ${PTHREADS_BUILD_DIR}/bin; \
-    curl -Ls https://github.com/krakjoe/pthreads/archive/master.tar.gz \
-    | tar xzC ${PTHREADS_BUILD_DIR} --strip-components=1
-
-WORKDIR  ${PTHREADS_BUILD_DIR}/
-
-RUN set -xe; \
-    phpize \
- && ./configure \
- && make \
- && make install
-
-
-# Strip all the unneeded symbols from shared libraries to reduce size.
-RUN find ${INSTALL_DIR} -type f -name "*.so*" -o -name "*.a"  -exec strip --strip-unneeded {} \;
-RUN find ${INSTALL_DIR} -type f -executable -exec sh -c "file -i '{}' | grep -q 'x-executable; charset=binary'" \; -print|xargs strip --strip-all
-
-# Cleanup all the binaries we don't want.
-RUN find /opt/bref/bin -mindepth 1 -maxdepth 1 ! -name "php" ! -name "pecl" -exec rm {} \+
-
-# Cleanup all the files we don't want either
-# We do not support running pear functions in Lambda
-RUN rm -rf /opt/bref/lib/php/PEAR \
-  rm -rf /opt/bref/share/doc \
-  rm -rf /opt/bref/share/man \
-  rm -rf /opt/bref/share/gtk-doc \
-  rm -rf /opt/bref/include \
-  rm -rf /opt/bref/lib/php/test \
-  rm -rf /opt/bref/lib/php/doc \
-  rm -rf /opt/bref/lib/php/docs \
-  rm -rf /opt/bref/tests \
-  rm -rf /opt/bref/doc \
-  rm -rf /opt/bref/docs \
-  rm -rf /opt/bref/man \
-  rm -rf /opt/bref/www \
-  rm -rf /opt/bref/cfg \
-  rm -rf /opt/bref/libexec \
-  rm -rf /opt/bref/var \
-  rm -rf /opt/bref/data
-
-# Symlink all our binaries into /opt/bin so that Lambda sees them in the path.
-RUN mkdir -p /opt/bin
-RUN ln -s /opt/bref/bin/* /opt/bin
-RUN ln -s /opt/bref/sbin/* /opt/bin
-
-
-# Now we get rid of everything that is unnecessary. All the build tools, source code, and anything else
-# that might have created intermediate layers for docker. Back to base AmazonLinux we started with.
-FROM amazonlinux:2018.03
-ENV INSTALL_DIR="/opt/bref"
-ENV PATH="/opt/bin:${PATH}" \
-    LD_LIBRARY_PATH="${INSTALL_DIR}/lib64:${INSTALL_DIR}/lib"
-
-RUN mkdir -p /opt
-# Copy everything we built above into the same dir on the base AmazonLinux container.
-COPY --from=0 /opt /opt
-
-# Install zip: we will need it later to create the layers as zip files
-RUN LD_LIBRARY_PATH= yum -y install zip
-WORKDIR /var/task