From ca8e4e8378bc489b0dda087dd9c0ed8f933ca3e2 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Wed, 7 Dec 2022 12:09:49 +0900 Subject: [PATCH] refactor wheel building and testing workflow (#3410) handle windows separately - it's cleaner that way --- .github/workflows/build-wheels.yml | 195 +++++++++++++++++++---------- 1 file changed, 126 insertions(+), 69 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index cd0548fe83..dab4957b2f 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -36,7 +36,8 @@ jobs: - name: Check Sphinx Gallery cache run: python docs/src/check_gallery.py - build: + + multibuild: timeout-minutes: 35 runs-on: ${{ matrix.os }} defaults: @@ -48,9 +49,6 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - os: [ubuntu-latest, macos-latest, windows-latest] - platform: [x64] include: # # We want the _oldest_ possible manylinux version to ensure our @@ -76,12 +74,12 @@ jobs: # - os: ubuntu-latest manylinux-version: 2010 - python-version: 3.8 + python-version: "3.8" build-depends: numpy==1.17.3 - os: ubuntu-latest manylinux-version: 2010 - python-version: 3.9 + python-version: "3.9" build-depends: numpy==1.19.3 - os: ubuntu-latest @@ -97,13 +95,13 @@ jobs: - os: macos-latest travis-os-name: osx manylinux-version: 1 - python-version: 3.8 + python-version: "3.8" build-depends: numpy==1.17.3 - os: macos-latest travis-os-name: osx manylinux-version: 1 - python-version: 3.9 + python-version: "3.9" build-depends: numpy==1.19.3 - os: macos-latest @@ -118,41 +116,23 @@ jobs: python-version: "3.11" build-depends: numpy==1.23.2 scipy==1.9.2 - - os: windows-latest - manylinux-version: 2010 - python-version: 3.8 - build-depends: numpy==1.17.3 - - - os: windows-latest - manylinux-version: 2010 - python-version: 3.9 - build-depends: numpy==1.19.3 - - - os: windows-latest - manylinux-version: 2010 - python-version: "3.10" - build-depends: numpy==1.22.2 scipy==1.8.0 - - - os: windows-latest - manylinux-version: 2010 - python-version: "3.11" - build-depends: numpy==1.23.2 scipy==1.9.2 - env: - PKG_NAME: gensim - REPO_DIR: gensim - BUILD_COMMIT: HEAD - PLAT: x86_64 - UNICODE_WIDTH: 32 - MB_PYTHON_VERSION: ${{ matrix.python-version }} # MB_PYTHON_VERSION is needed by Multibuild + SKIP_NETWORK_TESTS: 1 TEST_DEPENDS: pytest mock testfixtures + BUILD_DEPENDS: ${{ matrix.build-depends }} + + # + # For multibuild + # + BUILD_COMMIT: HEAD DOCKER_TEST_IMAGE: multibuild/xenial_x86_64 - TRAVIS_OS_NAME: ${{ matrix.travis-os-name }} - SKIP_NETWORK_TESTS: 1 MB_ML_VER: ${{ matrix.manylinux-version }} - WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }} - WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - BUILD_DEPENDS: ${{ matrix.build-depends }} + MB_PYTHON_VERSION: ${{ matrix.python-version }} # MB_PYTHON_VERSION is needed by Multibuild + PKG_NAME: gensim + PLAT: x86_64 + REPO_DIR: gensim + TRAVIS_OS_NAME: ${{ matrix.travis-os-name }} + UNICODE_WIDTH: 32 steps: - uses: actions/checkout@v3 @@ -175,8 +155,7 @@ jobs: run: | python -m pip install --upgrade pip pip install virtualenv - - name: Build Wheel (Multibuild) - if: matrix.os != 'windows-latest' + - name: Build Wheel run: | echo ::group::Set up Multibuild source multibuild/common_utils.sh @@ -191,31 +170,6 @@ jobs: build_wheel $REPO_DIR ${{ matrix.PLAT }} echo ::endgroup:: - # - # We can't use multibuild on Windows, so we have to roll our own build script. - # Adapted from - # https://github.com/RaRe-Technologies/gensim-wheels/commit/084b863390edee05bbe15d4ec05d1ab726e52202 - # - - name: Build Wheel (Windows) - if: matrix.os == 'windows-latest' - run: | - echo ::group::Set up dependencies - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - python -m pip install -U pip setuptools wheel wheelhouse_uploader ${{ env.BUILD_DEPENDS }} - echo ::endgroup:: - echo ::group::Build wheel - python setup.py bdist_wheel - echo ::endgroup - echo ::group::Install run - ls dist - python continuous_integration/install_wheel.py - echo ::endgroup:: - # - # For consistency with the multibuild step. - # - mv dist wheelhouse - - name: Prepare for testing run: | # @@ -232,8 +186,7 @@ jobs: # It also does not work under Windows. # So, we create our own simple test step here. # - - name: Install and Test Wheel (Linux, MacOS) - if: matrix.os != 'windows-latest' + - name: Install and Test Wheel run: | . test_environment/bin/activate python -m pip install --upgrade pip @@ -247,12 +200,113 @@ jobs: # pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim + - name: Upload wheels to s3://gensim-wheels + # + # Only do this if the credentials are set. + # This means that PRs will still build wheels, but not upload them. + # (PRs do not have access to secrets). + # + # The always() ensures this step runs even if a previous step fails. + # We want to upload wheels whenever possible (even if e.g. tests failed) + # because we don't want an innocuous test failure from blocking a release. + # + if: ${{ always() && env.WHEELHOUSE_UPLOADER_USERNAME && env.WHEELHOUSE_UPLOADER_SECRET }} + run: | + python -m pip install wheelhouse-uploader + ls wheelhouse/*.whl + python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn + env: + WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }} + WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + + # + # The build process for windows is different to that of Linux and MacOS. + # First, we cannot use multibuild (it does not support Windows). + # This means we have to write our own building and testing steps, but in a + # way it's simpler, because we don't need to care about configuring + # multibuild ourselves. + # Second, the syntax to enable virtual environments, etc. is different. + # + build_windows: + timeout-minutes: 35 + runs-on: windows-latest + defaults: + run: + shell: bash + + needs: [linters] + + strategy: + fail-fast: false + matrix: + include: + - python-version: "3.8" + build-depends: numpy==1.17.3 + + - python-version: "3.9" + build-depends: numpy==1.19.3 + + - python-version: "3.10" + build-depends: numpy==1.22.2 scipy==1.8.0 + + - python-version: "3.11" + build-depends: numpy==1.23.2 scipy==1.9.2 + + env: + SKIP_NETWORK_TESTS: 1 + TEST_DEPENDS: pytest mock testfixtures + BUILD_DEPENDS: ${{ matrix.build-depends }} + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install virtualenv + + - name: Build Wheel + run: | + echo ::group::Set up dependencies + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + python -m pip install -U pip setuptools wheel wheelhouse_uploader ${{ env.BUILD_DEPENDS }} + echo ::endgroup:: + echo ::group::Build wheel + python setup.py bdist_wheel + echo ::endgroup + echo ::group::Install run + ls dist + python continuous_integration/install_wheel.py + echo ::endgroup:: + # + # For consistency with the multibuild step. The wheel uploader expects + # the wheels to be under wheelhouse. + # + mv dist wheelhouse + + - name: Prepare for testing + run: | + # + # FIXME: Why are these eggs here? + # + # These eggs prevent the wheel from building and running on Py3.10 + # + find . -type f -name "*.egg" -exec rm -v {} \; + python -m venv test_environment + # # We need a separate testing step for windows because the command for # activating the virtual environment is slightly different # - name: Install and Test Wheel (Windows) - if: matrix.os == 'windows-latest' run: | test_environment/Scripts/activate.bat python -m pip install --upgrade pip @@ -277,3 +331,6 @@ jobs: python -m pip install wheelhouse-uploader ls wheelhouse/*.whl python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn + env: + WHEELHOUSE_UPLOADER_USERNAME: ${{ secrets.AWS_ACCESS_KEY_ID }} + WHEELHOUSE_UPLOADER_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}