Android SDK build scripts

.github/workflows/pull_request.yml

@@ -21,3 +21,151 @@ jobs:
           name: docker-logs
           path: |
             *.log
+
+   android-build:
+     name: Build Android ${{ matrix.build-type }} ${{ matrix.swift-version }} ${{ matrix.arch }} SDK
+     strategy:
+       fail-fast: false
+       matrix:
+         #build-type: ['docker']
+         build-type: ['docker', 'local']
+         # blank arch builds all (aarch64,x86_64,armv7)
+         #arch: ['']
+         # builds only x86_64 to speed up the validation
+         #arch: ['x86_64']
+         # build both the quick (x86_64) and complete (aarch64,x86_64,armv7) SDKs
+         arch: ['x86_64', '']
+         swift-version: ['release', 'swift-6.2-branch', 'development']
+     runs-on: ubuntu-24.04
+     steps:
+      - name: Free Disk Space
+        run: |
+          df -h
+          # brings available space from 25G to 32G
+          # otherwise we sometimes run out of space during the build
+          sudo rm -rf /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell /usr/share/dotnet /opt/ghc /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+          sudo docker builder prune -a
+          df -h
+      - name: Setup
+        id: config
+        run: |
+          # these variabes are used by build-docker and build-local
+          # to determine which Swift version to build for
+          echo "BUILD_SCHEME=${{ matrix.swift-version }}" >> $GITHUB_ENV
+          echo "TARGET_ARCHS=${{ matrix.arch }}" >> $GITHUB_ENV
+          echo "WORKDIR=${{ runner.temp }}/swift-android-sdk" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build Android SDK (Local)
+        if: ${{ matrix.build-type == 'local' }}
+        working-directory: swift-ci/sdks/android
+        run: |
+          sudo apt install -q ninja-build patchelf
+          ./build-local ${BUILD_SCHEME} ${WORKDIR}
+      - name: Build Android SDK (Docker)
+        if: ${{ matrix.build-type == 'docker' }}
+        working-directory: swift-ci/sdks/android
+        run: |
+          ./build-docker ${BUILD_SCHEME} ${WORKDIR}
+      - name: Install Host Toolchain
+        if: ${{ matrix.build-type == 'docker' }}
+        working-directory: swift-ci/sdks/android
+        run: |
+          # when building in a Docker container, we don't have a local host toolchain,
+          # but we need one in order to run the SDK validation tests, so we install it now
+          HOST_OS=ubuntu$(lsb_release -sr)
+          source ./scripts/toolchain-vars.sh
+          mkdir -p ${WORKDIR}/host-toolchain
+          ./scripts/install-swift.sh ${WORKDIR}/host-toolchain/$SWIFT_BASE/usr
+          ls ${WORKDIR}/host-toolchain
+          ${WORKDIR}/host-toolchain/*/usr/bin/swift --version
+      - name: Get artifact info
+        id: info
+        shell: bash
+        run: |
+          set -ex
+          SWIFT_ROOT=$(dirname ${WORKDIR}/host-toolchain/*/usr)
+          echo "swift-root=${SWIFT_ROOT}" >> $GITHUB_OUTPUT
+          echo "swift-path=${SWIFT_ROOT}/usr/bin/swift" >> $GITHUB_OUTPUT
+
+          ARTIFACT_PATH=$(realpath ${WORKDIR}/products/*.artifactbundle.tar.gz)
+          echo "artifact-path=${ARTIFACT_PATH}" >> $GITHUB_OUTPUT
+          echo "sdk-id=x86_64-unknown-linux-android28" >> $GITHUB_OUTPUT
+
+          ARTIFACT_EXT=".artifactbundle.tar.gz"
+          ARTIFACT_NAME="$(basename ${ARTIFACT_PATH} ${ARTIFACT_EXT})"
+          # depending on whether we are building locally or in a container, add a maker to the name
+          if [[ "${{ matrix.build-type }}" == 'local' ]]; then
+            ARTIFACT_NAME="${ARTIFACT_NAME}-local"
+          fi
+          # artifacts need a unique name so we suffix with the matrix arch(s)
+          if [[ ! -z "${{ matrix.arch }}" ]]; then
+            ARTIFACT_NAME="${ARTIFACT_NAME}-$(echo ${{ matrix.arch }} | tr ',' '-')"
+          fi
+          ARTIFACT_NAME="${ARTIFACT_NAME}${ARTIFACT_EXT}"
+
+          # There is no way to prevent even a single-file artifact from being zipped:
+          # https://github.com/actions/upload-artifact?tab=readme-ov-file#zip-archives
+          # so the actual artifact download will look like:
+          # swift-6.1-RELEASE_android-0.1-x86_64.artifactbundle.tar.gz.zip
+          echo "artifact-name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT
+      - name: Upload SDK artifactbundle
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: ${{ steps.info.outputs.artifact-name }}
+          path: ${{ steps.info.outputs.artifact-path }}
+      - name: Cleanup
+        run: |
+          # need to free up some space or else when installing we get: No space left on device
+          df -h
+          rm -rf ${WORKDIR}/{build,source}
+          sudo docker image prune --all --force
+          sudo docker builder prune -a
+          df -h
+      - name: Install artifactbundle
+        shell: bash
+        run: |
+          set -ex
+          ${{ steps.info.outputs.swift-path }} sdk install ${{ steps.info.outputs.artifact-path }}
+          ${{ steps.info.outputs.swift-path }} sdk configure --show-configuration $(${{ steps.info.outputs.swift-path }} sdk list | head -n 1) ${{ steps.info.outputs.sdk-id }}
+          # recent releases require that ANDROID_NDK_ROOT *not* be set
+          # see https://github.com/swiftlang/swift-driver/pull/1879
+          echo "ANDROID_NDK_ROOT=" >> $GITHUB_ENV
+
+      - name: Create Demo Project
+        run: |
+          cd ${{ runner.temp }}
+          mkdir DemoProject
+          cd DemoProject
+          ${{ steps.info.outputs.swift-path }} --version
+          ${{ steps.info.outputs.swift-path }} package init
+          echo 'import Foundation' >> Sources/DemoProject/DemoProject.swift
+          echo 'import FoundationEssentials' >> Sources/DemoProject/DemoProject.swift
+          echo 'import FoundationXML' >> Sources/DemoProject/DemoProject.swift
+          echo 'import FoundationNetworking' >> Sources/DemoProject/DemoProject.swift
+          echo 'import Dispatch' >> Sources/DemoProject/DemoProject.swift
+          echo 'import Android' >> Sources/DemoProject/DemoProject.swift
+      - name: Test Demo Project on Android
+        uses: skiptools/swift-android-action@main
+        with:
+          # only test for the complete arch SDK build to speed up CI
+          #run-tests: ${{ matrix.arch == '' }}
+          package-path: ${{ runner.temp }}/DemoProject
+          installed-sdk: ${{ steps.info.outputs.sdk-id }}
+          installed-swift: ${{ steps.info.outputs.swift-root }}
+
+      - name: Checkout swift-algorithms
+        uses: actions/checkout@v4
+        with:
+          repository: apple/swift-algorithms
+          path: swift-algorithms
+      - name: Test swift-algorithms
+        uses: skiptools/swift-android-action@main
+        with:
+          run-tests: ${{ matrix.arch == '' }}
+          package-path: swift-algorithms
+          installed-sdk: ${{ steps.info.outputs.sdk-id }}
+          installed-swift: ${{ steps.info.outputs.swift-root }}
+

.gitignore

@@ -0,0 +1 @@
+.*.swp

swift-ci/sdks/android/Dockerfile

@@ -0,0 +1,86 @@
+# ===----------------------------------------------------------------------===
+#
+#  Swift Android SDK: Docker-based build
+#
+# ===----------------------------------------------------------------------===
+
+FROM ubuntu:24.04
+
+# Architecture to build on (empty means x86-64)
+ARG OS_ARCH_SUFFIX=
+
+# the Swift toolchain URL to download
+ARG SWIFT_TOOLCHAIN_URL=
+
+# ............................................................................
+
+# Install development tools
+RUN apt-get -q update \
+    && DEBIAN_FRONTEND=noninteractive apt-get -q install -y \
+    build-essential \
+    cmake \
+    ninja-build \
+    python3 \
+    golang \
+    git \
+    gnupg2 \
+    libcurl4-openssl-dev  \
+    libedit-dev           \
+    libicu-dev            \
+    libncurses5-dev       \
+    libpython3-dev        \
+    libsqlite3-dev        \
+    libxml2-dev           \
+    rsync \
+    uuid-dev \
+    uuid-runtime \
+    tzdata \
+    curl \
+    unzip \
+    && rm -rf /var/lib/apt-lists/*
+
+# Install Swift
+ARG SWIFT_SIGNING_KEY=E813C892820A6FA13755B268F167DF1ACF9CE069
+ARG SWIFT_PLATFORM=ubuntu
+ARG OS_MAJOR_VER=24
+ARG OS_MINOR_VER=04
+
+ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
+    SWIFT_PLATFORM=$SWIFT_PLATFORM \
+    OS_MAJOR_VER=$OS_MAJOR_VER \
+    OS_MINOR_VER=$OS_MINOR_VER \
+    OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER
+
+COPY scripts/install-swift.sh /scripts/install-swift.sh
+RUN chmod ugo+x /scripts/install-swift.sh
+RUN /scripts/install-swift.sh /usr/local/swift
+ENV PATH="/usr/local/swift/bin:${PATH}"
+
+ARG ANDROID_NDK_VERSION=
+
+ENV ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION
+
+COPY scripts/install-ndk.sh /scripts/install-ndk.sh
+RUN chmod ugo+x /scripts/install-ndk.sh
+RUN /scripts/install-ndk.sh
+ENV ANDROID_NDK_HOME="/usr/local/ndk/${ANDROID_NDK_VERSION}"
+
+ENV SWIFT_VERSION=$SWIFT_VERSION \
+    LIBXML2_VERSION=$LIBXML2_VERSION \
+    CURL_VERSION=$CURL_VERSION \
+    BORINGSSL_VERSION=$BORINGSSL_VERSION \
+    ICU_VERSION=$ICU_VERSION \
+    ZLIB_VERSION=$ZLIB_VERSION
+
+ENV SWIFT_BUILD_DOCKER="1"
+
+COPY scripts /scripts
+RUN chmod ugo+x /scripts/*
+
+# Create a user
+RUN groupadd -g 998 build-user && \
+    useradd -m -r -u 998 -g build-user build-user
+
+USER build-user
+
+WORKDIR /home/build-user

swift-ci/sdks/android/README.md

@@ -0,0 +1,85 @@
+# Dockerfile-based build for Swift Android SDK
+
+This is a Dockerfile-based build set-up for the Swift Android SDK.
+
+The top-level `./build-docker` script will create a
+Docker container and install a host toolchain and the
+Android NDK, and then invoke `scripts/fetch-source.sh` which will
+fetch tagged sources for libxml2, curl, boringssl, and swift.
+
+It can be run with:
+
+```
+$ ./build-docker <version> <workdir>
+```
+
+for example:
+
+```
+$ ./build-docker release /tmp/android-sdk
+```
+
+This will create an Ubuntu 24.04 container with the necessary dependencies
+to build the Android SDK, including a Swift host toolchain and the
+Android NDK that will be used for cross-compilation.
+
+The `version` argument can be one of the following values:
+
+| version | Swift version example |
+| --- | --- |
+| `release` | swift-6.1-RELEASE |
+| `swift-6.2-branch` | swift-6.2-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd |
+| `development` | swift-DEVELOPMENT-SNAPSHOT-yyyy-mm-dd |
+
+> [!WARNING]
+> The workdir argument must not be located in a git repository (e.g., it cannot be the
+> current directory)
+
+## Running
+
+The top-level `./build-docker` script installs a host toolchain and the
+Android NDK, and then invokes `scripts/fetch-source.sh` which will
+fetch tagged sources for libxml2, curl, boringssl, and swift.
+
+It then applies some patches and invokes `scripts/build.sh`,
+which will build the sources for each of the specified
+architectures and then combines the SDKs into a single
+artifactbundle with targetTriples for each of the supported
+architectures (`aarch64`, `x86_64`, `aarmv7`)
+and Android API levels (28-35).
+
+## Specifying Architectures
+
+By default all the supported Android architectures
+will be built, but this can be reduced in order to speed
+up the build. This can be useful, e.g., as part of a CI that
+validates a pull request, as building a single architecture
+takes around 30 minutes on a standard ubuntu-24.04 GitHub runner,
+whereas building for all the architectures takes over an hour.
+
+To build an artifactbundle for just the `x86_64` architecture, run:
+
+```
+TARGET_ARCHS=x86_64 ./build-docker release /tmp/android-sdk
+```
+
+## Installing and validating the SDK
+
+The `.github/workflows/pull_request.yml` workflow
+will create and upload an installable SDK named something like:
+`swift-6.1-RELEASE_android-0.1.artifactbundle.tar.gz`
+
+The GitHub workflow will also install the SDK locally and use
+[swift-android-action](https://github.com/marketplace/actions/swift-android-action)
+to build and test various Swift packages in an Android emulator using the
+freshly-created SDK bundle.
+
+## Building locally
+
+Instead of building within a Docker container, the script can also
+perform the build locally on an Ubuntu 24.04 machine with all the
+build prerequisites already installed. This will generate
+the same artifacts in approximately half the time, and
+may be suitable to an already containerized envrionment (such as
+a GitHub runner).
+

swift-ci/sdks/android/build-docker

@@ -0,0 +1,67 @@
+#!/bin/bash -e
+#
+# ===----------------------------------------------------------------------===
+#
+#  Swift Android SDK: Docker Container Build Script
+#
+# ===----------------------------------------------------------------------===
+
+# default architectures to build for
+TARGET_ARCHS=${TARGET_ARCHS:-aarch64,x86_64,armv7}
+
+ANDROID_NDK_VERSION=android-ndk-r27c
+ANDROID_API=28
+
+export BUILD_SCHEME=${1}
+# note that WORKDIR must not be under the current checkout or the patches will fail to apply
+WORKDIR=${2}
+if [[ "${WORKDIR}" == '' ]]; then
+    echo "Usage: $(basename $0) <release/devel/trunk> <work directory>"
+    exit 1
+fi
+mkdir -p ${WORKDIR}
+WORKDIR=$(realpath ${WORKDIR})
+
+HOST_OS=ubuntu24.04
+source ./scripts/toolchain-vars.sh
+
+# Check-out and patch the sources
+./scripts/fetch-source.sh --source-dir ${WORKDIR}/source --swift-tag ${SWIFT_TAG}
+${WORKDIR}/source/swift-android-patches/apply-patches.sh ${WORKDIR}/source/swift-project
+
+mkdir -p ${WORKDIR}/products
+chmod ugo+rwx ${WORKDIR}/products
+
+if [[ "$DOCKER" == "" ]]; then
+    DOCKER=docker
+fi
+
+case $(arch) in
+    arm64|aarch64)
+        OS_ARCH_SUFFIX=-aarch64
+        ;;
+    amd64|x86_64)
+        OS_ARCH_SUFFIX=
+        ;;
+    *)
+        echo "Unknown architecture $(arch)"
+        exit 1
+        ;;
+esac
+
+CONTAINER_NAME="swift-android"
+
+# Build the Docker image
+$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX --build-arg SWIFT_TOOLCHAIN_URL=$SWIFT_TOOLCHAIN_URL --build-arg ANDROID_NDK_VERSION=$ANDROID_NDK_VERSION -t ${CONTAINER_NAME} .
+
+$DOCKER run -i --rm \
+          -v ${WORKDIR}/source:/source \
+          -v ${WORKDIR}/products:/products:rw \
+          ${CONTAINER_NAME} \
+          /scripts/build.sh \
+            --source-dir /source \
+            --products-dir /products \
+            --host-toolchain /usr/local/swift \
+            --android-api ${ANDROID_API} \
+            --ndk-home /usr/local/ndk/${ANDROID_NDK_VERSION} \
+            --archs ${TARGET_ARCHS}