From 37a05f97f3ee3cec8156e5a87076c0526477bdff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=AA=E3=81=A4=E3=81=8D?= Date: Wed, 23 Oct 2024 14:04:01 -0700 Subject: [PATCH] Enable AOT build for android-riscv64 (#2399) * Enable AOT build for android-riscv64 * Run arm64/arm linux builds on QEMU * Combine linux, linux-musl, and android release pipeline --- .github/workflows/build-android.yml | 80 ------------------- .github/workflows/build-linux-musl.yml | 71 ----------------- .github/workflows/build-linux.yml | 103 +++++++++++++++++-------- .github/workflows/release.yml | 10 +-- 4 files changed, 70 insertions(+), 194 deletions(-) delete mode 100644 .github/workflows/build-android.yml delete mode 100644 .github/workflows/build-linux-musl.yml diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml deleted file mode 100644 index 90405bbd0..000000000 --- a/.github/workflows/build-android.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Build for android - -on: - workflow_call: - workflow_dispatch: - -jobs: - build: - name: Build - - runs-on: ${{ matrix.runner }} - - strategy: - fail-fast: false - matrix: - include: - - arch: x64 - lib: lib64 - runner: ubuntu-latest - platform: linux/amd64 - - arch: ia32 - lib: lib - runner: ubuntu-latest - platform: linux/amd64 - - arch: arm64 - lib: lib64 - runner: linux-arm64 - platform: linux/arm64 - - arch: arm - lib: lib - runner: linux-arm64 - platform: linux/arm64 - # There is no docker image for riscv64 dart-sdk, build kernel snapshot instead. - - arch: riscv64 - lib: lib64 - runner: ubuntu-latest - platform: linux/amd64 # linux/riscv64 - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:master # need qemu >= 7.0.0 - - - name: Compile Protobuf - run: | - docker run --rm -i \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - docker.io/library/dart <<'EOF' - set -e - curl -fsSL -H "Authorization: Bearer ${{ github.token }}" "https://github.com/bufbuild/buf/releases/latest/download/buf-$(uname -s)-$(uname -m).tar.gz" | tar -xzC /usr/local --strip-components 1 - dart pub get - dart run grinder protobuf - EOF - - - name: Build - run: | - docker run --rm -i \ - --platform ${{ matrix.platform }} \ - --privileged \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - ghcr.io/dart-android/dart <<'EOF' - set -e - export DART_SDK=/system/${{ matrix.lib }}/dart - export PATH=$DART_SDK/bin:$PATH - dart pub get - dart run grinder pkg-standalone-android-${{ matrix.arch }} - EOF - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: build-android-${{ matrix.arch }} - path: build/*.tar.gz - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/build-linux-musl.yml b/.github/workflows/build-linux-musl.yml deleted file mode 100644 index 7b6a58f76..000000000 --- a/.github/workflows/build-linux-musl.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Build for linux-musl - -on: - workflow_call: - workflow_dispatch: - -jobs: - build: - name: Build - - runs-on: ${{ matrix.runner }} - - strategy: - fail-fast: false - matrix: - include: - - arch: x64 - runner: ubuntu-latest - platform: linux/amd64 - - arch: ia32 - runner: ubuntu-latest - platform: linux/amd64 - - arch: arm64 - runner: linux-arm64 - platform: linux/arm64 - - arch: arm - runner: linux-arm64 - platform: linux/arm/v7 - - arch: riscv64 - runner: ubuntu-latest - platform: linux/riscv64 - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:master - - - name: Compile Protobuf - run: | - docker run --rm -i \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - docker.io/library/dart <<'EOF' - set -e - curl -fsSL -H "Authorization: Bearer ${{ github.token }}" "https://github.com/bufbuild/buf/releases/latest/download/buf-$(uname -s)-$(uname -m).tar.gz" | tar -xzC /usr/local --strip-components 1 - dart pub get - dart run grinder protobuf - EOF - - - name: Build - run: | - docker run --rm -i \ - --platform ${{ matrix.platform }} \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - ghcr.io/dart-musl/dart <<'EOF' - set -e - dart pub get - dart run grinder pkg-standalone-linux-${{ matrix.arch }}-musl - EOF - - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: build-linux-musl-${{ matrix.arch }} - path: build/*.tar.gz - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 974636e9c..4aa12a3bc 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -8,76 +8,111 @@ jobs: build: name: Build - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - arch: x64 - runner: ubuntu-latest + - image: docker.io/library/dart platform: linux/amd64 - - arch: ia32 - runner: ubuntu-latest + target: linux-x64 + - image: docker.io/library/dart platform: linux/amd64 - - arch: arm64 - runner: linux-arm64 + target: linux-ia32 + - image: docker.io/library/dart platform: linux/arm64 - - arch: arm - runner: linux-arm64 + target: linux-arm64 + - image: docker.io/library/dart platform: linux/arm/v7 - - arch: riscv64 - runner: ubuntu-latest + target: linux-arm + - image: docker.io/library/debian:unstable-slim platform: linux/riscv64 + target: linux-riscv64 + - image: ghcr.io/dart-musl/dart + platform: linux/amd64 + target: linux-x64-musl + - image: ghcr.io/dart-musl/dart + platform: linux/amd64 + target: linux-ia32-musl + - image: ghcr.io/dart-musl/dart + platform: linux/arm64 + target: linux-arm64-musl + - image: ghcr.io/dart-musl/dart + platform: linux/arm/v7 + target: linux-arm-musl + - image: ghcr.io/dart-musl/dart + platform: linux/riscv64 + target: linux-riscv64-musl + - image: ghcr.io/dart-android/dart + platform: linux/amd64 + target: android-x64 + - image: ghcr.io/dart-android/dart + platform: linux/amd64 + target: android-ia32 + - image: ghcr.io/dart-android/dart + platform: linux/arm64 + target: android-arm64 + - image: ghcr.io/dart-android/dart + platform: linux/arm64 + target: android-arm + - image: ghcr.io/dart-android/dart + platform: linux/riscv64 + target: android-riscv64 steps: - uses: actions/checkout@v4 + - uses: ./.github/util/initialize + with: {github-token: "${{ github.token }}"} + - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - image: tonistiigi/binfmt:master + run: docker run --privileged --rm registry.fedoraproject.org/fedora-minimal /bin/sh -c "microdnf install --assumeyes --nodocs --setopt=install_weak_deps=False qemu-user-static systemd-udev && mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc && /usr/lib/systemd/systemd-binfmt --unregister && /usr/lib/systemd/systemd-binfmt" - - name: Compile Protobuf + - name: Build + if: matrix.image != 'ghcr.io/dart-android/dart' && matrix.image != 'docker.io/library/debian:unstable-slim' run: | docker run --rm -i \ + --platform ${{ matrix.platform }} \ --volume "$PWD:$PWD" \ --workdir "$PWD" \ - docker.io/library/dart <<'EOF' + ${{ matrix.image }} <<'EOF' set -e - curl -fsSL -H "Authorization: Bearer ${{ github.token }}" "https://github.com/bufbuild/buf/releases/latest/download/buf-$(uname -s)-$(uname -m).tar.gz" | tar -xzC /usr/local --strip-components 1 dart pub get - dart run grinder protobuf + dart run grinder pkg-standalone-${{ matrix.target }} EOF - name: Build - if: matrix.arch != 'riscv64' + if: matrix.image == 'ghcr.io/dart-android/dart' run: | docker run --rm -i \ - --platform ${{ matrix.platform }} \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - docker.io/library/dart:latest <<'EOF' + --privileged \ + --platform ${{ matrix.platform }} \ + --volume "$PWD:$PWD" \ + --workdir "$PWD" \ + ${{ matrix.image }} <<'EOF' set -e + export DART_SDK=/system/${{ endsWith(matrix.target, '64') && 'lib64' || 'lib' }}/dart + export PATH=$DART_SDK/bin:$PATH dart pub get - dart run grinder pkg-standalone-linux-${{ matrix.arch }} + dart run grinder pkg-standalone-${{ matrix.target }} EOF # https://github.com/dart-lang/dart-docker/issues/96#issuecomment-1669860829 # There is no official riscv64 dart container image yet, build on debian:unstable instead. # The setup is adopted from: https://github.com/dart-lang/dart-docker/blob/main/Dockerfile-debian.template - - name: Build riscv64 - if: matrix.arch == 'riscv64' + - name: Build + if: matrix.image == 'docker.io/library/debian:unstable-slim' run: | DART_CHANNEL=stable DART_VERSION=$(curl -fsSL https://storage.googleapis.com/dart-archive/channels/$DART_CHANNEL/release/latest/VERSION | yq .version) - curl -fsSLO https://storage.googleapis.com/dart-archive/channels/$DART_CHANNEL/release/$DART_VERSION/sdk/dartsdk-linux-${{ matrix.arch }}-release.zip + curl -fsSLO "https://storage.googleapis.com/dart-archive/channels/$DART_CHANNEL/release/$DART_VERSION/sdk/dartsdk-${{ matrix.target }}-release.zip" docker run --rm -i \ - --platform ${{ matrix.platform }} \ - --volume "$PWD:$PWD" \ - --workdir "$PWD" \ - docker.io/library/debian:unstable-slim <<'EOF' + --platform ${{ matrix.platform }} \ + --volume "$PWD:$PWD" \ + --workdir "$PWD" \ + ${{ matrix.image }} <<'EOF' set -e apt-get update apt-get install -y --no-install-recommends ca-certificates curl dnsutils git openssh-client unzip @@ -85,17 +120,17 @@ jobs: export DART_SDK=/usr/lib/dart export PATH=$DART_SDK/bin:/root/.pub-cache/bin:$PATH - SDK="dartsdk-linux-${{ matrix.arch }}-release.zip" + SDK="dartsdk-${{ matrix.target }}-release.zip" unzip "$SDK" && mv dart-sdk "$DART_SDK" && rm "$SDK" dart pub get - dart run grinder pkg-standalone-linux-${{ matrix.arch }} + dart run grinder pkg-standalone-${{ matrix.target }} EOF - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: build-linux-${{ matrix.arch }} + name: build-${{ matrix.target }} path: build/*.tar.gz if-no-files-found: error compression-level: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c151b6f4..28fc39218 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,18 +4,10 @@ on: workflow_call: jobs: - build_android: - uses: ./.github/workflows/build-android.yml - secrets: inherit - build_linux: uses: ./.github/workflows/build-linux.yml secrets: inherit - build_linux_musl: - uses: ./.github/workflows/build-linux-musl.yml - secrets: inherit - build_macos: uses: ./.github/workflows/build-macos.yml secrets: inherit @@ -27,7 +19,7 @@ jobs: release_github: name: Release Github runs-on: ubuntu-latest - needs: [build_android, build_linux, build_linux_musl, build_macos, build_windows] + needs: [build_linux, build_macos, build_windows] steps: - uses: actions/checkout@v4