Skip to content

Optimize CI runs, by avoiding unnecessary rebuilds #2833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 5, 2023
Merged

Optimize CI runs, by avoiding unnecessary rebuilds #2833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
208 changes: 44 additions & 164 deletions .github/workflows/push.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ jobs:
parallel: true
flag-name: run-${{ matrix.os }}-${{ matrix.python-version }}

ubuntu_build_apk:
name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
ubuntu_build:
name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
needs: [flake8]
runs-on: ${{ matrix.runs_on }}
continue-on-error: true
Expand All @@ -68,36 +68,41 @@ jobs:
target: testapps-with-scipy
- name: webview
target: testapps-webview
- name: service_library
target: testapps-service_library-aar
steps:
- name: Checkout python-for-android
uses: actions/checkout@v3
# helps with GitHub runner getting out of space
- name: Free disk space
run: |
df -h
sudo swapoff -a
sudo rm -f /swapfile
sudo apt -y clean
docker rmi $(docker image ls -aq)
df -h
- name: Pull docker image
run: |
make docker/pull
- name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
run: |
mkdir -p apks
make docker/run/make/with-artifact/apk/${{ matrix.bootstrap.target }}
- name: Rename apk artifact to include the build platform name
run: |
mv apks/${{ env.APK_ARTIFACT_FILENAME }} apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
- name: Upload apk artifact
uses: actions/upload-artifact@v1
- name: Build python-for-android docker image
run: |
docker build --tag=kivy/python-for-android .
- name: Build multi-arch ${{ matrix.bootstrap.target }} artifact with docker
run: |
docker run --name p4a-latest kivy/python-for-android make ${{ matrix.bootstrap.target }}
- name: Copy produced artifacts from docker container (*.apk, *.aab)
if: matrix.bootstrap.name != 'service_library'
run: |
mkdir -p dist
docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} dist/
docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} dist/
- name: Copy produced artifacts from docker container (*.aar)
if: matrix.bootstrap.name == 'service_library'
run: |
mkdir -p dist
docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAR_ARTIFACT_FILENAME }} dist/
- name: Rename artifacts to include the build platform name (*.apk, *.aab, *.aar)
run: |
if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi
if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi
if [ -f dist/${{ env.AAR_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAR_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAR_ARTIFACT_FILENAME }}; fi
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
path: apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts
path: dist

macos_build_apk:
name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
macos_build:
name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
needs: [flake8]
runs-on: ${{ matrix.runs_on }}
continue-on-error: true
Expand Down Expand Up @@ -137,146 +142,21 @@ jobs:
source ci/osx_ci.sh
arm64_set_path_and_python_version 3.9.7
make ${{ matrix.bootstrap.target }}
- name: Rename apk artifact to include the build platform name
run: |
mv testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
- name: Upload apk artifact
uses: actions/upload-artifact@v1
with:
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}

ubuntu_build_aab:
name: Unit test aab [ ${{ matrix.runs_on }} ]
needs: [flake8]
runs-on: ${{ matrix.runs_on }}
continue-on-error: true
strategy:
matrix:
runs_on: [ubuntu-latest]
bootstrap:
- name: sdl2
target: testapps-with-numpy-aab
- name: webview
target: testapps-webview-aab
steps:
- name: Checkout python-for-android
uses: actions/checkout@v3
# helps with GitHub runner getting out of space
- name: Free disk space
run: |
df -h
sudo swapoff -a
sudo rm -f /swapfile
sudo apt -y clean
docker rmi $(docker image ls -aq)
df -h
- name: Pull docker image
run: |
make docker/pull
- name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
run: |
mkdir -p aabs
make docker/run/make/with-artifact/aab/${{ matrix.bootstrap.target }}
- name: Rename artifact to include the build platform name
run: |
mv aabs/${{ env.AAB_ARTIFACT_FILENAME }} aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }}
- name: Upload apk artifact
uses: actions/upload-artifact@v1
with:
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }}
path: aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }}


ubuntu_build_aar:
name: Unit test aar [ ${{ matrix.runs_on }} ]
needs: [flake8]
runs-on: ${{ matrix.runs_on }}
continue-on-error: true
strategy:
matrix:
runs_on: [ubuntu-latest]
bootstrap:
- name: service_library
target: testapps-service_library-aar
steps:
- name: Checkout python-for-android
uses: actions/checkout@v3
# helps with GitHub runner getting out of space
- name: Free disk space
run: |
df -h
sudo swapoff -a
sudo rm -f /swapfile
sudo apt -y clean
docker rmi $(docker image ls -aq)
df -h
- name: Pull docker image
run: |
make docker/pull
- name: Build Android AAR Python 3 (arm64-v8a)
run: |
mkdir -p aars
make docker/run/make/with-artifact/aar/${{ matrix.bootstrap.target }}
- name: Rename artifact to include the build platform name
run: |
mv aars/${{ env.AAR_ARTIFACT_FILENAME }} aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }}
- name: Upload aar artifact
uses: actions/upload-artifact@v1
with:
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }}
path: aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }}


macos_build_aab:
name: Unit test aab [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
needs: [flake8]
runs-on: ${{ matrix.runs_on }}
continue-on-error: true
strategy:
matrix:
runs_on: [macos-latest, apple-silicon-m1]
bootstrap:
- name: sdl2
target: testapps-with-numpy-aab
- name: webview
target: testapps-webview-aab
env:
ANDROID_HOME: ${HOME}/.android
ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk
ANDROID_SDK_HOME: ${HOME}/.android/android-sdk
ANDROID_NDK_HOME: ${HOME}/.android/android-ndk
steps:
- name: Checkout python-for-android
uses: actions/checkout@v3
- name: Install python-for-android
run: |
source ci/osx_ci.sh
arm64_set_path_and_python_version 3.9.7
python3 -m pip install -e .
- name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental)
run: |
source ci/osx_ci.sh
arm64_set_path_and_python_version 3.9.7
python3 pythonforandroid/prerequisites.py
- name: Install dependencies (Legacy)
- name: Copy produced artifacts into dist/ (*.apk, *.aab)
run: |
source ci/osx_ci.sh
arm64_set_path_and_python_version 3.9.7
make --file ci/makefiles/osx.mk
- name: Build multi-arch sdl2 aab Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
run: |
source ci/osx_ci.sh
arm64_set_path_and_python_version 3.9.7
make ${{ matrix.bootstrap.target }}
- name: Rename sdl2 artifact to include the build platform name
mkdir -p dist
cp testapps/on_device_unit_tests/*.apk dist/
cp testapps/on_device_unit_tests/*.aab dist/
ls -l dist/
- name: Rename artifacts to include the build platform name (*.apk, *.aab)
run: |
mv testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}
- name: Upload sdl2 apk artifact
uses: actions/upload-artifact@v1
if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi
if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}
path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}
name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts
path: dist

ubuntu_rebuild_updated_recipes:
name: Test updated recipes for arch ${{ matrix.android_arch }} [ ubuntu-latest ]
Expand Down
67 changes: 28 additions & 39 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,47 +36,51 @@ rebuild_updated_recipes: virtualenv
ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \
$(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS)

testapps-with-numpy: virtualenv
testapps-with-numpy: testapps-with-numpy/debug/apk testapps-with-numpy/release/aab

# testapps-with-numpy/MODE/ARTIFACT
testapps-with-numpy/%: virtualenv
$(eval MODE := $(word 2, $(subst /, ,$@)))
$(eval ARTIFACT := $(word 3, $(subst /, ,$@)))
@echo Building testapps-with-numpy for $(MODE) mode and $(ARTIFACT) artifact
. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \
--arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 \
--permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)"

testapps-with-scipy: virtualenv
testapps-with-scipy: testapps-with-scipy/debug/apk testapps-with-scipy/release/aab

# testapps-with-scipy/MODE/ARTIFACT
testapps-with-scipy/%: virtualenv
$(eval MODE := $(word 2, $(subst /, ,$@)))
$(eval ARTIFACT := $(word 3, $(subst /, ,$@)))
@echo Building testapps-with-scipy for $(MODE) mode and $(ARTIFACT) artifact
. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
export LEGACY_NDK=$(ANDROID_NDK_HOME_LEGACY) && \
python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--requirements python3,scipy,kivy \
--arch=armeabi-v7a --arch=arm64-v8a

testapps-with-numpy-aab: virtualenv
. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \
--arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release \
--permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)"
testapps-webview: testapps-webview/debug/apk testapps-webview/release/aab

testapps-service_library-aar: virtualenv
# testapps-webview/MODE/ARTIFACT
testapps-webview/%: virtualenv
$(eval MODE := $(word 2, $(subst /, ,$@)))
$(eval ARTIFACT := $(word 3, $(subst /, ,$@)))
@echo Building testapps-webview for $(MODE) mode and $(ARTIFACT) artifact
. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--bootstrap service_library \
--requirements python3 \
--arch=arm64-v8a --arch=x86 --release

testapps-webview: virtualenv
. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--bootstrap webview \
--requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \
--arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86

testapps-webview-aab: virtualenv
testapps-service_library-aar: virtualenv
. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--bootstrap webview \
--requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \
--arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release
python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
--bootstrap service_library \
--requirements python3 \
--arch=arm64-v8a --arch=x86 --release

testapps/%: virtualenv
$(eval $@_APP_ARCH := $(shell basename $*))
Expand Down Expand Up @@ -106,21 +110,6 @@ docker/run/test: docker/build
docker/run/command: docker/build
docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)"

docker/run/make/with-artifact/apk/%: docker/build
docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1.apk ./apks
docker rm -fv p4a-latest

docker/run/make/with-artifact/aar/%: docker/build
docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aar ./aars
docker rm -fv p4a-latest

docker/run/make/with-artifact/aab/%: docker/build
docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aab ./aabs
docker rm -fv p4a-latest

docker/run/make/rebuild_updated_recipes: docker/build
docker run --name p4a-latest -e REBUILD_UPDATED_RECIPES_EXTRA_ARGS --env-file=.env $(DOCKER_IMAGE) make rebuild_updated_recipes

Expand Down