From 6fae84ea6fed2c794cef55c3ff4754cca94f0e70 Mon Sep 17 00:00:00 2001 From: Stefan Behnel Date: Tue, 16 Nov 2021 10:52:37 +0100 Subject: [PATCH] Add Github Actions workflow scripts. --- .github/workflows/ci.yml | 317 ++++++++++++++++++++++++++ .github/workflows/wheel-manylinux.yml | 123 ++++++++++ Tools/ci-run.sh | 176 ++++++++++++++ 3 files changed, 616 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/wheel-manylinux.yml create mode 100644 Tools/ci-run.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..7a70f8517c2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,317 @@ +name: CI + +on: [push, pull_request] + +jobs: + ci: + strategy: + # Allows for matrix sub-jobs to fail without canceling the rest + fail-fast: false + + # MATRIX: + # ======= + # Required parameters: + # os the os to run on + # python-version the python version to use + # backend the backend to use + # env any additional env variables. Set to '{}' for none + # Optional parameters: + # allowed_failure whether the job is allowed to fail + # extra_hash extra hash str to differentiate from other caches with similar name (must always start with '-') + matrix: + # Tests [amd64] + # + # FIXME: 'cpp' tests seems to fail due to compilation errors (numpy_pythran_unit) + # in all python versions and test failures (builtin_float) in 3.5< + os: [ubuntu-18.04] + backend: [c, cpp] + python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev"] + env: [{}] + + include: + # Temporary - Allow failure on Python 3.11-dev jobs until they are considered stable + - python-version: 3.11-dev + allowed_failure: true + + # Ubuntu sub-jobs: + # ================ + # GCC 11 + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { GCC_VERSION: 11 } + extra_hash: "-gcc11" + - os: ubuntu-18.04 + python-version: 3.9 + backend: cpp + env: { GCC_VERSION: 11 } + extra_hash: "-gcc11" + # compile all modules + - os: ubuntu-18.04 + python-version: 2.7 + backend: c + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 2.7 + backend: cpp + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 3.9 + backend: cpp + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + # Linting + - os: ubuntu-18.04 + python-version: 3.7 + backend: "c,cpp" + env: { TEST_CODE_STYLE: 1, NO_CYTHON_COMPILE: 1 } + extra_hash: "-codestyle" + # Limited API + - os: ubuntu-18.04 + python-version: 3.6 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + - os: ubuntu-18.04 + python-version: 3.7 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + - os: ubuntu-18.04 + python-version: 3.8 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + # Type specs + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.8 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.7 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.6 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.5 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + allowed_failure: true + # Stackless + - os: ubuntu-18.04 + python-version: 2.7 + backend: c + env: { STACKLESS: true, PY: 2 } + extra_hash: "-stackless" + - os: ubuntu-18.04 + python-version: 3.6 + backend: c + env: { STACKLESS: true, PY: 3 } + extra_hash: "-stackless" + # Pypy + - os: ubuntu-18.04 + python-version: pypy-2.7 + backend: c + env: { NO_CYTHON_COMPILE: 1 } + - os: ubuntu-18.04 + python-version: pypy-3.7 + backend: c + env: { NO_CYTHON_COMPILE: 1 } + # Pypy [allowed-failures] - These specifically test known bugs + - os: ubuntu-18.04 + python-version: pypy-2.7 + backend: c + env: + { + NO_CYTHON_COMPILE: 1, + EXCLUDE: "--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs", + } + allowed_failure: true + extra_hash: "-allowed_failures" + - os: ubuntu-18.04 + python-version: pypy-3.7 + backend: c + env: { NO_CYTHON_COMPILE: 1, EXCLUDE: "--listfile=tests/pypy_bugs.txt bugs" } + allowed_failure: true + extra_hash: "-allowed_failures" + # Coverage - Disabled due to taking too long to run + # - os: ubuntu-18.04 + # python-version: 3.7 + # backend: "c,cpp" + # env: { COVERAGE: 1 } + # extra_hash: '-coverage' + + # MacOS sub-jobs + # ============== + # (C-only builds are used to create wheels) + - os: macos-10.15 + python-version: 2.7 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 2.7 + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.5 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.6 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.7 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.8 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.9 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.9 + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: "3.10" + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: "3.10" + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + + # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. + # From testing, the runs tend to take ~20/~30 minutes, so a limit of 40 minutes should be enough. This can always be + # changed in the future if needed. + timeout-minutes: 40 + runs-on: ${{ matrix.os }} + + env: + BACKEND: ${{ matrix.backend }} + OS_NAME: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + GCC_VERSION: 8 + USE_CCACHE: 1 + CCACHE_SLOPPINESS: "pch_defines,time_macros" + CCACHE_COMPRESS: 1 + CCACHE_MAXSIZE: "200M" + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache [ccache] + uses: pat-s/always-upload-cache@v2.1.3 + if: startsWith(runner.os, 'Linux') + with: + path: ~/.ccache + key: ${{ runner.os }}-ccache${{ matrix.extra_hash }}-${{ matrix.python-version }}-${{ matrix.backend == 'c' || matrix.backend == 'c,cpp' }}-${{ contains(matrix.backend, 'cpp') }}-${{ hashFiles('**/test-requirements*.txt', '**/ci.yml', '**/ci-run.sh') }} + + - name: Run CI + continue-on-error: ${{ matrix.allowed_failure || false }} + env: ${{ matrix.env }} + run: bash ./Tools/ci-run.sh + + - name: Upload HTML docs + uses: actions/upload-artifact@v2 + with: + name: htmldocs + path: docs/build/html + if-no-files-found: ignore + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-${{ runner.os }} + path: dist/*.whl + if-no-files-found: ignore + + + pycoverage: + runs-on: ubuntu-18.04 + + env: + BACKEND: c,cpp + OS_NAME: ubuntu-18.04 + PYTHON_VERSION: 3.9 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Run Coverage + env: { COVERAGE: 1, NO_CYTHON_COMPILE: 1 } + run: bash ./Tools/ci-run.sh + + - name: Upload Coverage Report + uses: actions/upload-artifact@v2 + with: + name: pycoverage_html + path: coverage-report-html + +# cycoverage: +# runs-on: ubuntu-18.04 +# +# env: +# BACKEND: c,cpp +# OS_NAME: ubuntu-18.04 +# PYTHON_VERSION: 3.9 +# +# steps: +# - name: Checkout repo +# uses: actions/checkout@v2 +# with: +# fetch-depth: 1 +# +# - name: Setup python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.9 +# +# - name: Run Coverage +# env: { COVERAGE: 1 } +# run: bash ./Tools/ci-run.sh +# +# - name: Upload Coverage Report +# uses: actions/upload-artifact@v2 +# with: +# name: cycoverage_html +# path: coverage-report-html diff --git a/.github/workflows/wheel-manylinux.yml b/.github/workflows/wheel-manylinux.yml new file mode 100644 index 00000000000..f95b15d5b6b --- /dev/null +++ b/.github/workflows/wheel-manylinux.yml @@ -0,0 +1,123 @@ +name: Linux wheel build + +on: + release: + types: [created] + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install build dependencies + run: pip install -U setuptools pip wheel + + - name: Make sdist and Python wheel + run: make sdist pywheel + + - name: Upload sdist + uses: actions/upload-artifact@v2 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: ignore + + - name: Upload Python wheel + uses: actions/upload-artifact@v2 + with: + name: wheel-Python + path: dist/*-none-any.whl + if-no-files-found: ignore + + manylinux1-i686: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux1_i686 + with: + python-versions: 'cp27-cp27m cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310' + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: dist/*-manylinux*.whl + if-no-files-found: ignore + + manylinux1-x86_64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux1_x86_64 + with: + python-versions: 'cp27-cp27m cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310' + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: wheels/*-manylinux*.whl + if-no-files-found: ignore + + manylinux2014-i686: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_i686 + with: + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310' + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: wheels/*-manylinux2014*.whl + if-no-files-found: ignore + + manylinux2014-x86_64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_x86_64 + with: + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310' + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: wheels/*-manylinux2014*.whl + if-no-files-found: ignore diff --git a/Tools/ci-run.sh b/Tools/ci-run.sh new file mode 100644 index 00000000000..2a8f2e22b73 --- /dev/null +++ b/Tools/ci-run.sh @@ -0,0 +1,176 @@ +#!/usr/bin/bash + +GCC_VERSION=${GCC_VERSION:=8} + +# Set up compilers +if [[ $TEST_CODE_STYLE == "1" ]]; then + echo "Skipping compiler setup" +elif [[ $OSTYPE == "linux-gnu"* ]]; then + echo "Setting up linux compiler" + echo "Installing requirements [apt]" + sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" + sudo apt update -y -q + sudo apt install -y -q ccache gdb python-dbg python3-dbg gcc-$GCC_VERSION || exit 1 + + ALTERNATIVE_ARGS="" + if [[ $BACKEND == *"cpp"* ]]; then + sudo apt install -y -q g++-$GCC_VERSION || exit 1 + ALTERNATIVE_ARGS="--slave /usr/bin/g++ g++ /usr/bin/g++-$GCC_VERSION" + fi + sudo /usr/sbin/update-ccache-symlinks + echo "/usr/lib/ccache" >> $GITHUB_PATH # export ccache to path + + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GCC_VERSION 60 $ALTERNATIVE_ARGS + + export CC="gcc" + if [[ $BACKEND == *"cpp"* ]]; then + sudo update-alternatives --set g++ /usr/bin/g++-$GCC_VERSION + export CXX="g++" + fi +elif [[ $OSTYPE == "darwin"* ]]; then + echo "Setting up macos compiler" + export CC="clang -Wno-deprecated-declarations" + export CXX="clang++ -stdlib=libc++ -Wno-deprecated-declarations" +else + echo "No setup specified for $OSTYPE" +fi + +# Set up miniconda +if [[ $STACKLESS == "true" ]]; then + echo "Installing stackless python" + #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt + conda config --add channels stackless + conda install --quiet --yes stackless || exit 1 +fi + +PYTHON_SYS_VERSION=$(python -c 'import sys; print(sys.version)') + +# Log versions in use +echo "====================" +echo "|VERSIONS INSTALLED|" +echo "====================" +echo "Python $PYTHON_SYS_VERSION" +if [[ $CC ]]; then + which ${CC%% *} + ${CC%% *} --version +fi +if [[ $CXX ]]; then + which ${CXX%% *} + ${CXX%% *} --version +fi +echo "====================" + +# Install python requirements +echo "Installing requirements [python]" +if [[ $PYTHON_VERSION == "2.7"* ]]; then + pip install wheel || exit 1 + pip install -r test-requirements-27.txt || exit 1 +elif [[ $PYTHON_VERSION == "3."[45]* ]]; then + python -m pip install wheel || exit 1 + python -m pip install -r test-requirements-34.txt || exit 1 +else + python -m pip install -U pip setuptools wheel || exit 1 + + if [[ $PYTHON_VERSION != *"-dev" || $COVERAGE == "1" ]]; then + python -m pip install -r test-requirements.txt || exit 1 + + if [[ $PYTHON_VERSION != "pypy"* && $PYTHON_VERSION != "3."[1]* ]]; then + python -m pip install -r test-requirements-cpython.txt || exit 1 + fi + fi +fi + +if [[ $TEST_CODE_STYLE == "1" ]]; then + STYLE_ARGS="--no-unit --no-doctest --no-file --no-pyregr --no-examples" + python -m pip install -r doc-requirements.txt || exit 1 +else + STYLE_ARGS="--no-code-style" + + # Install more requirements + if [[ $PYTHON_VERSION != *"-dev" ]]; then + if [[ $BACKEND == *"cpp"* ]]; then + echo "WARNING: Currently not installing pythran due to compatibility issues" + # python -m pip install pythran==0.9.5 || exit 1 + fi + + if [[ $BACKEND != "cpp" && $PYTHON_VERSION != "pypy"* && + $PYTHON_VERSION != "2"* && $PYTHON_VERSION != "3.4"* ]]; then + python -m pip install mypy || exit 1 + fi + fi +fi + +# Run tests +echo "==== Running tests ====" +ccache -s 2>/dev/null || true +export PATH="/usr/lib/ccache:$PATH" + +# Most modern compilers allow the last conflicting option +# to override the previous ones, so '-O0 -O3' == '-O3' +# This is true for the latest msvc, gcc and clang +CFLAGS="-O0 -ggdb -Wall -Wextra" + +if [[ $NO_CYTHON_COMPILE != "1" && $PYTHON_VERSION != "pypy"* ]]; then + + BUILD_CFLAGS="$CFLAGS -O2" + if [[ $PYTHON_SYS_VERSION == "2"* ]]; then + BUILD_CFLAGS="$BUILD_CFLAGS -fno-strict-aliasing" + fi + + SETUP_ARGS="" + if [[ $COVERAGE == "1" ]]; then + SETUP_ARGS="$SETUP_ARGS --cython-coverage" + fi + if [[ $CYTHON_COMPILE_ALL == "1" ]]; then + SETUP_ARGS="$SETUP_ARGS --cython-compile-all" + fi + SETUP_ARGS="$SETUP_ARGS + $(python -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")')" + + CFLAGS=$BUILD_CFLAGS \ + python setup.py build_ext -i $SETUP_ARGS || exit 1 + + # COVERAGE can be either "" (empty or not set) or "1" (when we set it) + # STACKLESS can be either "" (empty or not set) or "true" (when we set it) + # CYTHON_COMPILE_ALL can be either "" (empty or not set) or "1" (when we set it) + if [[ $COVERAGE != "1" && $STACKLESS != "true" && $BACKEND != *"cpp"* && + $CYTHON_COMPILE_ALL != "1" && $LIMITED_API == "" && $EXTRA_CFLAGS == "" ]]; then + python setup.py bdist_wheel || exit 1 + fi +fi + +if [[ $TEST_CODE_STYLE == "1" ]]; then + make -C docs html || exit 1 +elif [[ $PYTHON_VERSION != "pypy"* ]]; then + # Run the debugger tests in python-dbg if available + # (but don't fail, because they currently do fail) + PYTHON_DBG=$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])') + PYTHON_DBG="python$PYTHON_DBG-dbg" + if $PYTHON_DBG -V >&2; then + CFLAGS=$CFLAGS $PYTHON_DBG \ + runtests.py -vv --no-code-style Debugger --backends=$BACKEND + fi +fi + +RUNTESTS_ARGS="" +if [[ $COVERAGE == "1" ]]; then + RUNTESTS_ARGS="$RUNTESTS_ARGS --coverage --coverage-html --cython-only" +fi +if [[ $TEST_CODE_STYLE != "1" ]]; then + RUNTESTS_ARGS="$RUNTESTS_ARGS -j7" +fi + +export CFLAGS="$CFLAGS $EXTRA_CFLAGS" +python runtests.py \ + -vv $STYLE_ARGS \ + -x Debugger \ + --backends=$BACKEND \ + $LIMITED_API \ + $EXCLUDE \ + $RUNTESTS_ARGS + +EXIT_CODE=$? + +ccache -s 2>/dev/null || true + +exit $EXIT_CODE