diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d41f77b0..2b81e243 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,35 +9,6 @@ on: branches: [ master ] jobs: - test-default: - name: Test with default parameters - runs-on: ubuntu-latest - - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - # Run the integration test script - - name: Run integration tests - run: sh ./.integrationtest.sh - - test-bionic: - name: Test with a bionic rootfs - runs-on: ubuntu-latest - - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - # Switch to Ubuntu Bionic rootfs - - name: Switch to Ubuntu Bionic rootfs - run: | - echo -n "mendix/rootfs:bionic" > rootfs.version - echo -n "bionic" > docker-buildpack.version - - # Run the integration test script - - name: Run integration tests - run: sh ./.integrationtest.sh test-ubi8: name: Test with a ubi8 rootfs @@ -45,14 +16,8 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - # Switch to ubi8 rootfs - - name: Switch to ubi8 rootfs - run: | - echo -n "mendix/rootfs:ubi8" > rootfs.version - echo -n "ubi8" > docker-buildpack.version + - uses: actions/checkout@v3 # Run the integration test script - name: Run integration tests - run: sh ./.integrationtest.sh + run: sh tests/integrationtest.sh diff --git a/.integrationtest.sh b/.integrationtest.sh deleted file mode 100644 index 40b52425..00000000 --- a/.integrationtest.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eux -docker version -docker-compose version -make get-sample -make build-base-images -make build-image -tests/test-generic.sh tests/docker-compose-postgres.yml -#tests/test-generic.sh tests/docker-compose-sqlserver.yml -#tests/test-generic.sh tests/docker-compose-azuresql.yml diff --git a/Dockerfile b/Dockerfile index 9bd1ae9a..31c40eb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ # # Author: Mendix Digital Ecosystems, digitalecosystems@mendix.com # Version: 2.1.0 -ARG ROOTFS_IMAGE=mendix/rootfs:ubi8 -ARG BUILDER_ROOTFS_IMAGE=mendix/rootfs:bionic +ARG ROOTFS_IMAGE=mendix-rootfs:app +ARG BUILDER_ROOTFS_IMAGE=mendix-rootfs:builder # Build stage FROM ${BUILDER_ROOTFS_IMAGE} AS builder @@ -13,7 +13,7 @@ FROM ${BUILDER_ROOTFS_IMAGE} AS builder ARG BUILD_PATH=project ARG DD_API_KEY # CF buildpack version -ARG CF_BUILDPACK=v4.30.14 +ARG CF_BUILDPACK=v4.30.17 # CF buildpack download URL ARG CF_BUILDPACK_URL=https://github.com/mendix/cf-mendix-buildpack/releases/download/${CF_BUILDPACK}/cf-mendix-buildpack.zip @@ -36,6 +36,7 @@ ARG USER_UID=1001 # 6. Update permissions of /opt/mendix so that the app can run as a non-root user RUN mkdir -p /opt/mendix/buildpack /opt/mendix/build &&\ ln -s /root /home/vcap &&\ + mkdir -p /home/vcap/.local/bin && ln -s /etc/alternatives/pip3 /home/vcap/.local/bin/pip && pip3 install --upgrade pip &&\ echo "Downloading CF Buildpack from ${CF_BUILDPACK_URL}" &&\ curl -fsSL ${CF_BUILDPACK_URL} -o /tmp/cf-mendix-buildpack.zip && \ python3 -m zipfile -e /tmp/cf-mendix-buildpack.zip /opt/mendix/buildpack/ &&\ @@ -44,7 +45,7 @@ RUN mkdir -p /opt/mendix/buildpack /opt/mendix/build &&\ chmod -R g=u /opt/mendix # Copy python scripts which execute the buildpack (exporting the VCAP variables) -COPY scripts/compilation scripts/git /opt/mendix/buildpack/ +COPY scripts/compilation.py scripts/git /opt/mendix/buildpack/ # Copy project model/sources COPY $BUILD_PATH /opt/mendix/build @@ -68,10 +69,10 @@ ENV NGINX_CUSTOM_BIN_PATH=/usr/sbin/nginx # 7. Update ownership of /opt/mendix so that the app can run as a non-root user # 8. Update permissions of /opt/mendix so that the app can run as a non-root user RUN mkdir -p /tmp/buildcache /tmp/cf-deps /var/mendix/build /var/mendix/build/.local &&\ - chmod +rx /opt/mendix/buildpack/compilation /opt/mendix/buildpack/git /opt/mendix/buildpack/buildpack/stage.py &&\ + chmod +rx /opt/mendix/buildpack/compilation.py /opt/mendix/buildpack/git /opt/mendix/buildpack/buildpack/stage.py &&\ cd /opt/mendix/buildpack &&\ - ./compilation /opt/mendix/build /tmp/buildcache /tmp/cf-deps 0 &&\ - rm -fr /tmp/buildcache /tmp/javasdk /tmp/opt /tmp/downloads /opt/mendix/buildpack/compilation /opt/mendix/buildpack/git &&\ + ./compilation.py /opt/mendix/build /tmp/buildcache /tmp/cf-deps 0 &&\ + rm -fr /tmp/buildcache /tmp/javasdk /tmp/opt /tmp/downloads /opt/mendix/buildpack/compilation.py /opt/mendix/buildpack/git &&\ ln -s /opt/mendix/.java /opt/mendix/build &&\ chown -R ${USER_UID}:0 /opt/mendix /var/mendix &&\ chmod -R g=u /opt/mendix /var/mendix @@ -80,9 +81,6 @@ FROM ${ROOTFS_IMAGE} LABEL Author="Mendix Digital Ecosystems" LABEL maintainer="digitalecosystems@mendix.com" -# Uninstall build-time dependencies to remove potentially vulnerable libraries -ARG UNINSTALL_BUILD_DEPENDENCIES=true - # Set the user ID ARG USER_UID=1001 # Set the home path @@ -92,17 +90,11 @@ ENV HOME=/opt/mendix/build RUN chmod g=u /etc/passwd &&\ chown ${USER_UID}:0 /etc/passwd -# Uninstall Ubuntu packages which are only required during build time -RUN if [ "$UNINSTALL_BUILD_DEPENDENCIES" = "true" ] && grep -q ubuntu /etc/os-release ; then\ - DEBIAN_FRONTEND=noninteractive apt-mark manual libfontconfig1 && \ - DEBIAN_FRONTEND=noninteractive apt-get remove --purge --auto-remove -q -y wget curl libgdiplus ; \ - fi - # Add the buildpack modules ENV PYTHONPATH "/opt/mendix/buildpack/lib/:/opt/mendix/buildpack/:/opt/mendix/buildpack/lib/python3.6/site-packages/" # Copy start scripts -COPY scripts/startup scripts/vcap_application.json /opt/mendix/build/ +COPY scripts/startup.py scripts/vcap_application.json /opt/mendix/build/ # Create vcap home directory for Datadog configuration RUN mkdir -p /home/vcap /opt/datadog-agent/run &&\ @@ -114,7 +106,7 @@ RUN mkdir -p /home/vcap /opt/datadog-agent/run &&\ # 2. Update ownership of /opt/mendix so that the app can run as a non-root user # 3. Update permissions of /opt/mendix so that the app can run as a non-root user # 4. Ensure that running Java 8 as root will still be able to load offline licenses -RUN chmod +rx /opt/mendix/build/startup &&\ +RUN chmod +rx /opt/mendix/build/startup.py &&\ chown -R ${USER_UID}:0 /opt/mendix &&\ chmod -R g=u /opt/mendix &&\ ln -s /opt/mendix/.java /root @@ -139,4 +131,4 @@ WORKDIR /opt/mendix/build ENV PORT 8080 EXPOSE $PORT -ENTRYPOINT ["/opt/mendix/build/startup","/opt/mendix/buildpack/buildpack/start.py"] +ENTRYPOINT ["/opt/mendix/build/startup.py"] diff --git a/Dockerfile.rootfs.bionic b/Dockerfile.rootfs.bionic deleted file mode 100644 index d380c374..00000000 --- a/Dockerfile.rootfs.bionic +++ /dev/null @@ -1,27 +0,0 @@ -# Dockerfile to create a Mendix Docker image based on either the source code or -# Mendix Deployment Archive (aka mda file) -FROM ubuntu:bionic -#This version does a full build originating from the Ubuntu Docker images -LABEL Author="Mendix Digital Ecosystems" -LABEL maintainer="digitalecosystems@mendix.com" - -# When doing a full build: install dependencies & remove package lists -RUN apt-get -q -y update && \ - DEBIAN_FRONTEND=noninteractive apt-get upgrade -q -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget curl unzip libpq5 locales python3 python3-distutils libssl1.0.0 libgdiplus nginx-light libnginx-mod-stream binutils && \ - rm -rf /var/lib/apt/lists/* && \ - apt-get clean - -# Set the locale to UTF-8 (needed for proper python3 functioning) -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -# Set nginx permissions -RUN touch /run/nginx.pid && \ - chown -R 1001:0 /var/log/nginx /var/lib/nginx /run/nginx.pid &&\ - chmod -R g=u /var/log/nginx /var/lib/nginx /run/nginx.pid - -# Set python alias to python3 (required for Datadog) -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - diff --git a/Makefile b/Makefile deleted file mode 100644 index 5eed9471..00000000 --- a/Makefile +++ /dev/null @@ -1,32 +0,0 @@ -VERSION=$(shell cat docker-buildpack.version) -CF_BUILDPACK_VERSION=$(shell cat cf-buildpack.version) -ROOTFS_VERSION=$(shell cat rootfs.version) -ROOTFS_IMAGES=$(patsubst Dockerfile.rootfs.%, rootfs.%, $(wildcard Dockerfile.rootfs.*)) - -get-sample: - if [ -d build ]; then rm -rf build; fi - if [ -d downloads ]; then rm -rf downloads; fi - mkdir -p downloads build - wget https://s3-eu-west-1.amazonaws.com/mx-buildpack-ci/BuildpackTestApp-mx-7-16.mda -O downloads/application.mpk - unzip downloads/application.mpk -d build/ - -rootfs.%: Dockerfile.rootfs.% - docker build \ - -t mendix/rootfs:$* -f Dockerfile.rootfs.$* . - -build-base-images: $(ROOTFS_IMAGES) - -build-image: - docker build \ - --build-arg BUILD_PATH=build \ - --build-arg CF_BUILDPACK=$(CF_BUILDPACK_VERSION) \ - --build-arg ROOTFS_IMAGE=$(ROOTFS_VERSION) \ - -t mendix/mendix-buildpack:$(VERSION) . - -test-container: - tests/test-generic.sh tests/docker-compose-postgres.yml - tests/test-generic.sh tests/docker-compose-sqlserver.yml - tests/test-generic.sh tests/docker-compose-azuresql.yml - -run-container: - BUILDPACK_VERSION=$(VERSION) docker-compose -f tests/docker-compose-postgres.yml up diff --git a/README.md b/README.md index 55b4b683..b3b251bf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,11 @@ ![Test status](https://github.com/mendix/docker-mendix-buildpack/workflows/Test/badge.svg) -The Mendix Buildpack for Docker (aka docker-mendix-buildpack) is an example project you can use to build and run your Mendix Application in a [Docker](https://www.docker.com/) container. +The Mendix Buildpack for Docker (aka docker-mendix-buildpack) is an **example project** you can use to build and run your Mendix Application in a [Docker](https://www.docker.com/) container. + +**⚠️ Warning** If your pipeline is based on Docker Buildpack V4 or an earlier version, see the [upgrading from Docker Buildpack v4](upgrading-from-v4.md) document. To use Docker Buildpack v5, some changes will be required in your build process. + +For a Kubernetes native solution to run Mendix apps, see [Mendix for Private Cloud](https://www.mendix.com/evaluation-guide/app-lifecycle/mendix-for-private-cloud/). ## Try a sample mendix application @@ -13,13 +17,17 @@ Open a terminal and run the following code: ``` git clone --branch --config core.autocrlf=false https://github.com/mendix/docker-mendix-buildpack cd docker-mendix-buildpack -make get-sample -make build-image -make run-container +tests/integrationtest.sh ``` You can now open your browser [http://localhost:8080]([http://localhost:8080]) +### Scanning for vulnerabilities + +If you would like to run a vulnerability scanner (to get a baseline security score), please use the latest, LTS or MTS version of Mendix. + +Security patches are only available in the latest version of Mendix, and running a security scan on an outdated version might show CVEs that are already patched in the latest version. + ## Uses cases scenarios: This project is a goto reference for the following scenarios : @@ -31,35 +39,64 @@ This project is a goto reference for the following scenarios : ### Requirements -* Docker 17.05 (Installation [here](https://docs.docker.com/engine/installation/)) +* Docker 20.10 (Installation [here](https://docs.docker.com/engine/installation/)) * Earlier Docker versions are no longer compatible because they don't support multistage builds. - To use Docker versions below 17.05, download an earlier Mendix Docker Buildpack release, such as [v2.3.2](https://github.com/mendix/docker-mendix-buildpack/releases/tag/v2.3.2) -* For preparing, a local installation of wget (for macOS) + To use Docker versions below 20.10, download an earlier Mendix Docker Buildpack release, such as [v2.3.2](https://github.com/mendix/docker-mendix-buildpack/releases/tag/v2.3.2) +* For preparing, a local installation of `curl` * For local testing, make sure you can run the [docker-compose command](https://docs.docker.com/compose/install/) +* A Mendix app based on Mendix 7 or a later version ## Usage -### Compilation +### Preparation: rootfs + +To save build time, the build pack needs a prebuilt rootfs containing the base OS and additional packages. +This rootfs is based on [Red Hat Universal Base Image 8 minimal](https://developers.redhat.com/articles/ubi-faq) image. + +To build the rootfs, run the following commands + +```shell +docker build -t -f rootfs-builder.dockerfile . +docker build -t -f rootfs-app.dockerfile . +``` + +The `builder` image contains packages required to build an app; the `app` image contains a reduced package set, containing only packages required to run a Mendix app. + +For example: + +```shell +docker build -t mendix-rootfs:app -f rootfs-app.dockerfile . +docker build -t mendix-rootfs:builder -f rootfs-builder.dockerfile . +``` + +This command needs to be done at least once on the builder OS. + +Building images on a licensed RHEL host will enable access to additional packages and Red Hat support. + +To reuse the rootfs image, push it to a private repository: + +```shell +docker push +docker push +``` + +### Compile an app Before running the container, it is necessary to build the image with your application. This buildpack contains Dockerfile with a script that will compile your application using [cf-mendix-buildpack](https://github.com/mendix/cf-mendix-buildpack/). ``` -docker build +docker build \ --build-arg BUILD_PATH= \ - --build-arg ROOTFS_IMAGE= \ - --build-arg BUILDER_ROOTFS_IMAGE= \ - --build-arg CF_BUILDPACK= \ --tag mendix/mendix-buildpack:v1.2 . ``` For build you can provide next arguments: - **BUILD_PATH** indicates where the application model is located. It is a root directory of an unzipped .MDA or .MPK file. In the latter case, this is the directory where your .MPR file is located. Must be within [build context](https://docs.docker.com/engine/reference/commandline/build/#extended-description). Defaults to `./project`. -- **ROOTFS_IMAGE** is a type of rootfs image. Defaults to `mendix/rootfs:ubi8` (Red Hat Universal Base Image 8). To use Ubuntu 18.04, change this to `mendix/rootfs:bionic`. It's also possible to use a custom rootfs image as described in [Advanced feature: full-build](#advanced-feature-full-build). -- **BUILDER_ROOTFS_IMAGE** is a type of rootfs image used for downloading the Mendix app dependencies and compiling the Mendix app from source. Defaults to `mendix/rootfs:bionic`. It's also possible to use a custom rootfs image as described in [Advanced feature: full-build](#advanced-feature-full-build). -- **CF_BUILDPACK** is a version of CloudFoundry buildpack. Defaults to `v4.30.14`. For stable pipelines, it's recommended to use a fixed version from **v4.30.14** and later. CloudFoundry buildpack versions below **v4.30.14** are not supported. +- **ROOTFS_IMAGE** is a type of rootfs image. Defaults to `mendix-rootfs:app` (a locally prebuilt image). +- **BUILDER_ROOTFS_IMAGE** is a type of rootfs image used for downloading the Mendix app dependencies and compiling the Mendix app from source. Defaults to `mendix-rootfs:builder` (a locally prebuilt image). +- **CF_BUILDPACK** is a version of CloudFoundry buildpack. Defaults to `v4.30.17`. For stable pipelines, it's recommended to use a fixed version from **v4.30.17** and later. CloudFoundry buildpack versions below **v4.30.17** are not supported. - **EXCLUDE_LOGFILTER** will exclude the `mendix-logfilter` binary from the resulting Docker image if set to `true`. Defaults to `true`. Excluding `mendix-logfilter` will reduce the image size and remove a component that's not commonly used; the `LOG_RATELIMIT` environment variable option will be disabled. -- **UNINSTALL_BUILD_DEPENDENCIES** will uninstall packages which are not needed to launch an app, and are only used during the build phase. Defaults to `true`. This option will remove several libraries which are known to have unpatched CVE vulnerabilities. - **CF_BUILDPACK_URL** specifies the URL where the CF buildpack should be downloaded from (for example, a local mirror). Defaults to `https://github.com/mendix/cf-mendix-buildpack/releases/download/${CF_BUILDPACK}/cf-mendix-buildpack.zip`. Specifying **CF_BUILDPACK_URL** will override the version from **CF_BUILDPACK**. - **BLOBSTORE** can be used to specify an alternative buildpack resource server (instead of the default Mendix CDN). For more information, see the [CF Buildpack offline settings](https://github.com/mendix/cf-mendix-buildpack#offline-buildpack-settings). - **BUILDPACK_XTRACE** can be used to enable CF Buildpack [debug logging](https://github.com/mendix/cf-mendix-buildpack#logging-and-debugging). Set this variable to `true` to enable debug logging. @@ -190,7 +227,7 @@ environment: ### Configuring Custom Runtime Settings -To configure any of the advanced [Custom Runtime Settings](https://world.mendix.com/display/refguide6/Custom+Settings) you can use setting name prefixed with `MXRUNTIME_` as an environment variable. +To configure any of the advanced [Custom Runtime Settings](https://docs.mendix.com/refguide/custom-settings/) you can use setting name prefixed with `MXRUNTIME_` as an environment variable. For example, to configure the ConnectionPoolingMinIdle setting to value 10, you can set the following environment variable: @@ -290,25 +327,6 @@ In case your environment does not support multi-line environment variables, a Ba This string should be set into the CERTIFICATE_AUTHORITIES_BASE64 environment variable. -### Advanced feature: full-build - -To save build time, the build pack will normally use a pre-built rootfs from Docker Hub. This rootfs is prepared nightly by Mendix using [this](https://github.com/mendix/docker-mendix-buildpack/blob/master/Dockerfile.rootfs.bionic) Dockerfile. If you want to build the root-fs yourself you can use the following script: - -``` -docker build --build-arg BUILD_PATH= \ - -t -f Dockerfile.rootfs.bionic . -``` - -After that you can build the target image with the next command: - -``` -docker build - --build-arg BUILD_PATH= \ - --build-arg ROOTFS_IMAGE= \ - --build-arg BUILDER_ROOTFS_IMAGE= \ -``` - -t mendix/mendix-buildpack:v1.2 . - ### Industrial Edge Configuration File support @@ -349,11 +367,11 @@ Contributions are welcomed: This was built with the following: -* Docker version 18.06.3 +* Docker version 20.10 ### Versioning -We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/mendix/IBM-Watson-Connector-Kit/tags). +We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/mendix/docker-mendix-buildpack/tags). ## License diff --git a/cf-buildpack.version b/cf-buildpack.version deleted file mode 100644 index cfa278f8..00000000 --- a/cf-buildpack.version +++ /dev/null @@ -1 +0,0 @@ -v4.30.14 \ No newline at end of file diff --git a/docker-buildpack.version b/docker-buildpack.version deleted file mode 100644 index 512699eb..00000000 --- a/docker-buildpack.version +++ /dev/null @@ -1 +0,0 @@ -v3.5.3 \ No newline at end of file diff --git a/rootfs-app.dockerfile b/rootfs-app.dockerfile new file mode 100644 index 00000000..2a3ef9af --- /dev/null +++ b/rootfs-app.dockerfile @@ -0,0 +1,30 @@ +# Dockerfile to create a Mendix Docker image based on either the source code or +# Mendix Deployment Archive (aka mda file) +FROM --platform=linux/amd64 registry.access.redhat.com/ubi8/ubi-minimal:latest +#This version does a full build originating from the Ubuntu Docker images +LABEL Author="Mendix Digital Ecosystems" +LABEL maintainer="digitalecosystems@mendix.com" + +# Set the locale +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +# install dependencies & remove package lists +RUN microdnf update -y && \ + microdnf module enable nginx:1.20 -y && \ + microdnf install -y glibc-langpack-en python3 openssl nginx nginx-mod-stream fontconfig && \ + microdnf clean all && rm -rf /var/cache/yum + +# Set nginx permissions +RUN touch /run/nginx.pid && \ + chown -R 1001:0 /var/log/nginx /var/lib/nginx /run &&\ + chmod -R g=u /var/log/nginx /var/lib/nginx /run + +# Set python alias to python3 (required for Datadog) +RUN alternatives --set python /usr/bin/python3 + +# Set the user ID +ARG USER_UID=1001 + +# Create user (for non-OpenShift clusters) +RUN echo "mendix:x:${USER_UID}:${USER_UID}:mendix user:/opt/mendix/build:/sbin/nologin" >> /etc/passwd diff --git a/Dockerfile.rootfs.ubi8 b/rootfs-builder.dockerfile similarity index 60% rename from Dockerfile.rootfs.ubi8 rename to rootfs-builder.dockerfile index 6400434b..19d9bbb3 100644 --- a/Dockerfile.rootfs.ubi8 +++ b/rootfs-builder.dockerfile @@ -1,6 +1,6 @@ # Dockerfile to create a Mendix Docker image based on either the source code or # Mendix Deployment Archive (aka mda file) -FROM registry.access.redhat.com/ubi8/ubi-minimal:latest +FROM --platform=linux/amd64 registry.access.redhat.com/ubi8/ubi-minimal:latest #This version does a full build originating from the Ubuntu Docker images LABEL Author="Mendix Digital Ecosystems" LABEL maintainer="digitalecosystems@mendix.com" @@ -10,10 +10,10 @@ ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 # install dependencies & remove package lists -# doesn't include libpq5 python3-distutils libgdiplus -RUN microdnf update -y && \ +RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm &&\ + microdnf update -y && \ microdnf module enable nginx:1.20 -y && \ - microdnf install -y wget curl glibc-langpack-en python3 openssl tar gzip unzip nginx nginx-mod-stream binutils fontconfig && \ + microdnf install -y wget curl glibc-langpack-en python3 python3-setuptools openssl libgdiplus tar gzip unzip libpq nginx nginx-mod-stream binutils fontconfig && \ microdnf clean all && rm -rf /var/cache/yum # Set nginx permissions @@ -21,6 +21,9 @@ RUN touch /run/nginx.pid && \ chown -R 1001:0 /var/log/nginx /var/lib/nginx /run/nginx.pid &&\ chmod -R g=u /var/log/nginx /var/lib/nginx /run/nginx.pid +# Pretend to be Ubuntu to bypass CF Buildpack's check +RUN rm /etc/*-release && echo 'Ubuntu release 18.04 (Bionic)' > /etc/debian-release + # Set python alias to python3 (required for Datadog) RUN alternatives --set python /usr/bin/python3 diff --git a/rootfs.version b/rootfs.version deleted file mode 100644 index 8c78a7f6..00000000 --- a/rootfs.version +++ /dev/null @@ -1 +0,0 @@ -mendix/rootfs:ubi8 \ No newline at end of file diff --git a/scripts/compilation b/scripts/compilation.py similarity index 79% rename from scripts/compilation rename to scripts/compilation.py index 19b74276..4231bad3 100755 --- a/scripts/compilation +++ b/scripts/compilation.py @@ -2,18 +2,12 @@ import json import logging import os -import subprocess +import runpy import sys import shutil from buildpack import util from buildpack.core import java, runtime -from buildpack.stage import DOT_LOCAL_LOCATION - -BUILD_PATH = sys.argv[1] -CACHE_PATH = sys.argv[2] -DEPS_DIR = sys.argv[3] -DEPS_IDX = sys.argv[4] logging.basicConfig( level=logging.INFO, @@ -37,7 +31,7 @@ def export_vcap_services(): def call_buildpack_compilation(): logging.debug("Executing call_buildpack_compilation...") - return subprocess.check_call(["/opt/mendix/buildpack/buildpack/stage.py", BUILD_PATH, CACHE_PATH, DEPS_DIR, DEPS_IDX]) + return runpy.run_module("buildpack.stage", run_name="__main__") def fix_logfilter(): exclude_logfilter = os.getenv("EXCLUDE_LOGFILTER", "true").lower() == "true" @@ -47,14 +41,16 @@ def fix_logfilter(): else: os.chmod("/opt/mendix/build/.local/mendix-logfilter/mendix-logfilter", 0o0755) -def make_dependencies_reusable(): +def make_dependencies_reusable(compilation_globals): logging.info("Making dependencies reusable...") + dot_local_location = compilation_globals["DOT_LOCAL_LOCATION"] + build_path = compilation_globals["BUILD_DIR"] shutil.move("/opt/mendix/build/runtimes", "/var/mendix/build/") shutil.move("/opt/mendix/build/.local/usr", "/var/mendix/build/.local/") # separate cacerts from reusable jre components - jre = java._get_java_dependency(java.get_java_major_version(runtime.get_runtime_version(BUILD_PATH)), 'jre') + jre = java._get_java_dependency(java.get_java_major_version(runtime.get_runtime_version(build_path)), 'jre') jvm_location_reusable = os.path.join("/var/mendix/build/.local/", java._compose_jvm_target_dir(jre)) - jvm_location_customized = os.path.join(DOT_LOCAL_LOCATION, java._compose_jvm_target_dir(jre)) + jvm_location_customized = os.path.join(dot_local_location, java._compose_jvm_target_dir(jre)) cacerts_file_source = os.path.join(jvm_location_reusable, "lib", "security", "cacerts") cacerts_file_target = os.path.join(jvm_location_customized, "lib", "security", "cacerts") util.mkdir_p(os.path.dirname(cacerts_file_target)) @@ -64,8 +60,6 @@ def make_dependencies_reusable(): logging.info("Mendix project compilation phase...") export_vcap_services() - exit_code = call_buildpack_compilation() - if exit_code != 0: - sys.exit(exit_code) + compilation_globals = call_buildpack_compilation() fix_logfilter() - make_dependencies_reusable() + make_dependencies_reusable(compilation_globals) diff --git a/scripts/startup b/scripts/startup.py similarity index 81% rename from scripts/startup rename to scripts/startup.py index 0d35b1d1..b7f0b756 100755 --- a/scripts/startup +++ b/scripts/startup.py @@ -3,13 +3,10 @@ import logging import os import re -import subprocess -import signal +import runpy import sys import base64 -STARTUP_PATH = sys.argv[1] - logging.basicConfig( level=logging.INFO, stream=sys.stdout, @@ -85,35 +82,11 @@ def check_logfilter(): logging.warn("LOG_RATELIMIT is set, but the mendix-logfilter binary is missing. Rebuild Docker image with EXCLUDE_LOGFILTER=false to enable log filtering") del os.environ['LOG_RATELIMIT'] -def sigchld_handler(_signo, _stack_frame): - # reap zombies - logging.debug("Child process has exited, getting result") - (waitpid, result) = os.waitpid(-1, os.WNOHANG) - logging.debug("Child process %s has exited with result %s" % (waitpid, result)) - def call_buildpack_startup(): logging.debug("Executing call_buildpack_startup...") - signal.signal(signal.SIGCHLD, sigchld_handler) - - proc = subprocess.Popen(["python3", STARTUP_PATH], cwd='/opt/mendix/build') - - # Forward SIGTERM to allow a clean shutdown - def sig_forwarder(_signo, _stack_frame): - logging.debug("Forwarding SIGTERM to child process") - os.kill(proc.pid, _signo) - - signal.signal(signal.SIGTERM, sig_forwarder) - - try: - proc.wait() - except KeyboardInterrupt: - logging.debug("Interrupted by keyboard") - -def add_uid(): - logging.info("Adding uid to /etc/passwd") - with open('/etc/passwd','a') as passwd_file: - passwd_file.write('mendix:x:{uid}:{gid}:mendix user:/opt/mendix/build:/sbin/nologin\n'.format(uid=os.getuid(),gid=os.getgid())) + os.chdir('/opt/mendix/build') + runpy.run_module("buildpack.start", run_name="__main__") def get_welcome_header(): welcome_ascii_header = ''' @@ -135,6 +108,8 @@ def get_welcome_header(): digitalecosystems@mendix.com + For a Kubernetes native solution to run Mendix apps, + see Mendix for Private Cloud. ''' return welcome_ascii_header @@ -149,6 +124,4 @@ def get_welcome_header(): check_logfilter() export_encoded_cacertificates() - add_uid() call_buildpack_startup() - sys.exit(0) diff --git a/tests/docker-compose-postgres.yml b/tests/docker-compose-postgres.yml index 425418ee..044a9d4c 100644 --- a/tests/docker-compose-postgres.yml +++ b/tests/docker-compose-postgres.yml @@ -2,7 +2,7 @@ version: "2.3" services: mendixapp: - image: mendix/mendix-buildpack:$BUILDPACK_VERSION + image: mendix-testapp:$BUILDPACK_VERSION healthcheck: test: ["CMD", "curl", "-f", "http://localhost"] interval: 15s diff --git a/tests/integrationtest.sh b/tests/integrationtest.sh new file mode 100755 index 00000000..1062f360 --- /dev/null +++ b/tests/integrationtest.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eux +docker version +docker-compose version + +echo "Downloading test project" +mkdir -p downloads build +curl -L https://s3-eu-west-1.amazonaws.com/mx-buildpack-ci/BuildpackTestApp-mx-7-16.mda -o downloads/application.mpk +unzip downloads/application.mpk -d build/ + +echo "Building app rootfs" +docker build -t mendix-rootfs:app -f rootfs-app.dockerfile . + +echo "Building builder rootfs" +docker build -t mendix-rootfs:builder -f rootfs-builder.dockerfile . + +echo "Building test app" +export BUILDPACK_VERSION=`git rev-parse HEAD` +docker build \ + --build-arg BUILD_PATH=build \ + --build-arg BUILDER_ROOTFS_IMAGE=mendix-rootfs:builder \ + --build-arg ROOTFS_IMAGE=mendix-rootfs:app \ + -t mendix-testapp:$BUILDPACK_VERSION . + + +tests/test-generic.sh tests/docker-compose-postgres.yml +#tests/test-generic.sh tests/docker-compose-sqlserver.yml +#tests/test-generic.sh tests/docker-compose-azuresql.yml diff --git a/tests/stop.sh b/tests/stop.sh deleted file mode 100755 index e69de29b..00000000 diff --git a/tests/test-generic.sh b/tests/test-generic.sh index 18fa6a8c..e7f85ace 100755 --- a/tests/test-generic.sh +++ b/tests/test-generic.sh @@ -5,9 +5,6 @@ BASEDIR=$(dirname "$0") COMPOSEFILE=$1 TIMEOUT=180s -export BUILDPACK_VERSION="${BUILDPACK_VERSION:-$(cat $BASEDIR/../docker-buildpack.version)}" - -echo "Testing buildpack version ${BUILDPACK_VERSION}" echo "test.sh [TEST STARTED] starting docker-compose $COMPOSEFILE" docker-compose -f $COMPOSEFILE up & diff --git a/upgrading-from-v4.md b/upgrading-from-v4.md new file mode 100644 index 00000000..f620d062 --- /dev/null +++ b/upgrading-from-v4.md @@ -0,0 +1,66 @@ +# Upgrading from Docker Buildpack v4 + +Docker Buildpack v5 contains some breaking changes and will require some changes in your CI/CD pipeline: + +* rootfs images are no longer published to Docker Hub. +* rootfs images are now based on [Red Hat Universal Base Image 8 minimal](https://developers.redhat.com/articles/ubi-faq) image. + +## Building rootfs images + +In the past, Docker Buildpack offered two types of rootfs images: `bionic` and `ubi8`. + +* `bionic` images were based on Ubuntu 18.04 and were primarily used during the build phase - although it was also possible to build an app based on `bionic` by specifying a custom `--build-arg ROOTFS_IMAGE=mendix/rootfs:bionic` argument. +* `ubi8` images were based on Red Hat UBI8 minimal and contained only components that are required to run a Mendix app. + +With Docker Buildpack v5, all images are now based on ubi8-minimal: + +* `app` images contain only components that are required to run an app. Build-time components such as `tar`, compilers or Mono prerequisites are excluded - reducing the image size and excluding components that could increase the number of unpatched CVEs in the final image +* `builder` images contain additional components that are only required when compiling a Mendix app. + +You will need to update your pipelines to build these prerequisite images and (optionally) push them to your private registry. + +### Option 1: build rootfs images on every build + +If your build system uses a shared or local Docker instance with cache enabled, you can build the rootfs images on every build. +The local cache will skip rebuilding rootfs images if they've already been built previously. + +In this case, your build pipeline needs to be adjusted: + +1. Before building a Mendix app, build the rootfs images: + ```shell + docker build -t mendix-rootfs:app -f rootfs-app.dockerfile . + docker build -t mendix-rootfs:builder -f rootfs-builder.dockerfile . + ``` +2. When building the Mendix app itself, use the default values for `ROOTFS_IMAGE` and `BUILDER_ROOTFS_IMAGE` (remove any custom `--build-arg ROOTFS_IMAGE=...` and `--build-arg BUILDER_ROOTFS_IMAGE=...` arguments from the `docker build` command) + +### Option 2: build rootfs images centrally + +If your build system doesn't have a shared Docker instance, building rootfs images would be a better choice. + +In this case, you need to make the following changes in your CI/CD process: + +1. Create a CI/CD job that builds the rootfs images and pushes them to your private registry: + ```shell + docker build -t {your-private-registry}/mendix-rootfs:app -f rootfs-app.dockerfile . + docker push -t {your-private-registry}/mendix-rootfs:app + docker build -t {your-private-registry}/mendix-rootfs:builder -f rootfs-builder.dockerfile . + docker push -t {your-private-registry}/mendix-rootfs:builder + ``` + This CI/CD job can run periodically (for example, every 24 hours) or when a CVE scanner detects that a new patch is available. +2. In the CI/CD job, replace (or add) build args to use rootfs images from your private registry (in this example, `--build-arg ROOTFS_IMAGE={your-private-registry}/mendix-rootfs:app` and `--build-arg BUILDER_ROOTFS_IMAGE={your-private-registry}/mendix-rootfs:builder` specify to use the images built in step 1), for example: + ```shell + docker build \ + --build-arg BUILD_PATH= \ + --build-arg ROOTFS_IMAGE={your-private-registry}/mendix-rootfs:app \ + --build-arg BUILDER_ROOTFS_IMAGE={your-private-registry}/mendix-rootfs:builder \ + --tag {your-private-registry}/my-app:latest . + ``` + +## Migrating from Ubuntu to Red Hat UBI + +Ubuntu 18.04 Bionic is no longer supported and will no longer receive CVE patches or other updates. + +Docker Buildpack switched to Red Hat UBI8, since it's a de facto standard for enterprise applications. +UBI8 will receive security updates [until 2029](https://access.redhat.com/support/policy/updates/errata/#RHEL8_Planning_Guide). + +If your container requires additional binary packages, you will need to replace any Ubuntu packages with their RHEL alternatives.