Staged builds

.github/workflows/build.yml

@@ -5,19 +5,58 @@ on:
       - v1.x
       - v2.x
       - v3.x
+      - v4.x
 jobs:
-  build:
-    name: Publish latest
+  stage1:
+    name: Stage 1
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the Git repository
         uses: actions/checkout@v3
-      - name: Login to Docker Hub
+      - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and publish images
-        run: |
-          ./scripts/build -p .
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -p -g -s toolchain
+  stage2:
+    name: Stage 2
+    runs-on: ubuntu-latest
+    needs: stage1
+    steps:
+      - name: Checkout the Git repository
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -p -g -s base
+  stage3:
+    name: Stage 3
+    runs-on: ubuntu-latest
+    needs: stage2
+    strategy:
+      matrix:
+        target: [golang, python, qt, rust, dotnet6]
+    steps:
+      - name: Checkout the Git repository
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -p -g -s ${{ matrix.target }}

.github/workflows/pr.yml

@@ -5,15 +5,58 @@ on:
       - v1.x
       - v2.x
       - v3.x
+      - v4.x
 jobs:
-  build:
-    name: Publish latest
+  stage1:
+    name: Stage 1
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
     steps:
       - name: Checkout the Git repository
         uses: actions/checkout@v3
-      - name: Build images
-        run: |
-          ./scripts/build .
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -g -s toolchain
+  stage2:
+    name: Stage 2
+    runs-on: ubuntu-latest
+    needs: stage1
+    steps:
+      - name: Checkout the Git repository
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -g -s base
+  stage3:
+    name: Stage 3
+    runs-on: ubuntu-latest
+    needs: stage2
+    strategy:
+      matrix:
+        target: [golang, python, qt, rust, dotnet6]
+    steps:
+      - name: Checkout the Git repository
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -g -s ${{ matrix.target }}

.github/workflows/release.yml

@@ -4,19 +4,68 @@ on:
     tags:
       - '*'
 jobs:
-  release:
-    name: Publish a release
+  stage1:
+    name: Stage 1
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the Git repository
         uses: actions/checkout@v3
-      - name: Login to Docker Hub
+      - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and publish images
+      - name: Set up version
         run: |
-          version="$(echo "${{ github.ref }}" | cut -d / -f 3)"
-          ./scripts/build -p . "$version"
+          VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)"
+          echo "version=$VERSION" >> "$GITHUB_ENV"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -p -g -s toolchain -v ${{ env.version }}
+  stage2:
+    name: Stage 2
+    runs-on: ubuntu-latest
+    needs: stage1
+    steps:
+      - name: Checkout the Git repository
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up version
+        run: |
+          VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)"
+          echo "version=$VERSION" >> "$GITHUB_ENV"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -p -g -s base -v ${{ env.version }}
+  stage3:
+    name: Stage 3
+    runs-on: ubuntu-latest
+    needs: stage2
+    strategy:
+      matrix:
+        target: [golang, python, qt, rust, dotnet6]
+    steps:
+      - name: Checkout the Git repository
+        uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up version
+        run: |
+          VERSION="$(echo "${{ github.ref }}" | cut -d / -f 3)"
+          echo "version=$VERSION" >> "$GITHUB_ENV"
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push toolchain
+        run: ./scripts/build -p -g -s ${{ matrix.target }} -v ${{ env.version }}

base/Dockerfile

@@ -174,7 +174,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     && cd /root \
     && mkdir openssl \
     && cd openssl \
-    && curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -o openssl.tar.gz \
+    && curl https://www.openssl.org/source/openssl-1.1.1g.tar.gz -Lo openssl.tar.gz \
     && echo "ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46  openssl.tar.gz" > sha256sums \
     && sha256sum -c sha256sums \
     && tar --strip-components=1 -xf openssl.tar.gz \

scripts/build

@@ -1,95 +1,93 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
 
-set -e
-source "${BASH_SOURCE%/*}"/lib
+import argparse as ap
+import subprocess as sp
+import typing as tp
+import os
 
-usage="$0 [OPTION]... IMAGESDIR [VERSION]
-
-Build the Docker images under the IMAGESDIR directory.
+parser = ap.ArgumentParser(
+    description="""
+Build the Docker images under the IMAGESDIR directory (default: .).
 Label each built image with the given VERSION number (default: latest).
 The build takes advantage of any reusable layer from previously built images.
-
-Options:
-
-    -h              Show this help message.
-    -p              Publish each built image to the GitHub Container Registry
-                    (you need to be logged in to ghcr.io with the appropriate
-                    permissions for this to work)."
-
-helpflag=
-publishflag=
-
-while getopts hp name; do
-    case $name in
-        h) helpflag=1 ;;
-        p) publishflag=1 ;;
-        *) error "Invalid option. Use the -h flag for more information." ;;
-    esac
-done
-
-shift $((OPTIND - 1))
-
-if [[ -n $helpflag ]]; then
-    echo "$usage"
-    exit
-fi
-
-if [[ $# -eq 0 ]]; then
-    error "Missing IMAGESDIR arguments. Use the -h flag for more information."
-fi
-
-if [[ $# -gt 2 ]]; then
-    error "Extraneous arguments. Use the -h flag for more information."
-fi
-
-imagesdir="$1"
-version="$2"
-
-# Enable BuildKit for better cache behavior
-# See <https://docs.docker.com/engine/reference/builder/#buildkit>
-export DOCKER_BUILDKIT=1
-
-docker-build() {
-    from="$(image-name "$1" "$version")"
-    target="$(image-name "$2" "$version")"
-    status "Building image '$target'"
-    docker image build \
-        --build-arg BUILDKIT_INLINE_CACHE=1 \
-        --build-arg FROM="$from" \
-        --cache-from "$(image-name "$2")" \
-        --tag "$target" .
-
-    if [[ -n $publishflag ]]; then
-        docker image push "$target"
-    fi
+"""
+)
+
+builds = ["toolchain", "base", "golang", "python", "qt", "rust", "dotnet6"]
+requires = {
+    "toolchain": None,
+    "base": "toolchain",
+    "golang": "base",
+    "python": "base",
+    "qt": "base",
+    "rust": "base",
+    "dotnet6": "toolchain",
 }
 
-pushd "$imagesdir"/toolchain
-docker-build toolchain toolchain
-popd
-
-pushd "$imagesdir"/base
-docker-build toolchain base
-popd
-
-pushd "$imagesdir"/golang
-docker-build base golang
-popd
-
-pushd "$imagesdir"/python
-docker-build base python
-popd
-
-pushd "$imagesdir"/qt
-docker-build base qt
-popd
-
-pushd "$imagesdir"/rust
-docker-build base rust
-popd
-
-pushd "$imagesdir"/dotnet6
-docker-build toolchain dotnet6
-popd
-
-status "Done"
+parser.add_argument(
+    "-p", help="Publish built images to GitHub Container Registry", action="store_true"
+)
+parser.add_argument("-g", help="Use GitHub Actions for caching", action="store_true")
+parser.add_argument("-d", help="Set image directory", default=".", metavar="IMAGESDIR")
+parser.add_argument(
+    "-s", help="Just build single image given as argument", choices=builds
+)
+parser.add_argument(
+    "-v", help="Set version number", default="latest", metavar="VERSION"
+)
+
+
+def run_cmd(line: list[str]) -> None:
+    p = sp.Popen(
+        line,
+        stdout=sp.PIPE,
+        stderr=sp.STDOUT,
+        encoding="utf-8",
+        errors="replace",
+    )
+    while True:
+        realtime_output = p.stdout.readline()
+        if realtime_output == "" and p.poll() is not None:
+            break
+        if realtime_output:
+            print(realtime_output.strip(), flush=True)
+    if p.returncode != 0:
+        exit(p.returncode)
+
+
+def build(
+    image: str, version: str, publish: bool, github: bool, _from: tp.Optional[str] = None
+) -> None:
+    cmd: list[str] = [
+        "docker",
+        "buildx",
+        "build",
+        "--build-arg",
+        "BUILDKIT_INLINE_CACHE=1",
+    ]
+    if _from:
+        cmd.extend(["--build-arg", f"FROM=ghcr.io/toltec-dev/{_from}:{version}"])
+    if github:
+        cmd.extend(["--cache-from", f"type=gha,scope={image}"])
+        cmd.extend(["--cache-to", f"type=gha,mode=max,scope={image}"])
+    if publish:
+        cmd.extend(["--push"])
+    cmd.extend(["--tag", f"ghcr.io/toltec-dev/{image}:{version}", "."])
+    run_cmd(cmd)
+
+
+args = parser.parse_args()
+curdir = os.getcwd()
+os.chdir(args.d)
+workdir = os.getcwd()
+if args.s:
+    os.chdir(args.s)
+    build(args.s, args.v, args.p, args.g, requires[args.s])
+    os.chdir(workdir)
+else:
+    for i in builds:
+        print(f"Building {i}...")
+        os.chdir(i)
+        build(i, args.v, args.p, args.g, requires[i])
+        os.chdir(workdir)
+os.chdir(curdir)