From 90b7348c85384d80b2a1773c87da3f607b3eb288 Mon Sep 17 00:00:00 2001 From: Russell Martin Date: Sun, 3 Mar 2024 14:04:38 -0500 Subject: [PATCH] Revamp testing and coverage via tox --- .github/workflows/ci.yml | 259 +++++++++--------- changes/2440.misc.rst | 1 + core/pyproject.toml | 8 +- core/tests/test_paths.py | 32 ++- core/tests/testbed/customize/sitecustomize.py | 10 +- dummy/pyproject.toml | 8 +- testbed/tests/app/test_app.py | 2 +- tox.ini | 51 +++- 8 files changed, 218 insertions(+), 153 deletions(-) create mode 100644 changes/2440.misc.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9420e87ad..ac7f297c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: core: runs-on: ${{ matrix.platform }} - needs: [pre-commit, towncrier, package] + needs: [ pre-commit, towncrier, package ] continue-on-error: ${{ matrix.experimental }} strategy: fail-fast: false @@ -69,52 +69,58 @@ jobs: platform: [ "macos-12", "macos-14", "ubuntu-latest", "windows-latest" ] python-version: [ "3.8", "3.12", "3.13-dev" ] include: - - experimental: false - # Test Python 3.9-3.11 on Ubuntu only - - platform: "ubuntu-latest" - python-version: "3.9" - experimental: false - - platform: "ubuntu-latest" - python-version: "3.10" - experimental: false - - platform: "ubuntu-latest" - python-version: "3.11" - experimental: false - # Allow development Python to fail without failing entire job. - - python-version: "3.13-dev" - experimental: true + - experimental: false + # Test Python 3.9-3.11 on Ubuntu only + - platform: "ubuntu-latest" + python-version: "3.9" + experimental: false + - platform: "ubuntu-latest" + python-version: "3.10" + experimental: false + - platform: "ubuntu-latest" + python-version: "3.11" + experimental: false + # Allow development Python to fail without failing entire job. + - python-version: "3.13-dev" + experimental: true exclude: - # macos-14 (i.e. arm64) does not support Python 3.8 - - platform: "macos-14" - python-version: "3.8" - - # Pillow isn't available for Python 3.13 on Windows - - platform: "windows-latest" - python-version: "3.13-dev" + # macos-14 (i.e. arm64) does not support Python 3.8 + - platform: "macos-14" + python-version: "3.8" + # Pillow isn't available for Python 3.13 on Windows + - platform: "windows-latest" + python-version: "3.13-dev" steps: - - uses: actions/checkout@v4.1.1 + - name: Checkout + uses: actions/checkout@v4.1.1 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5.0.0 with: python-version: ${{ matrix.python-version }} - - name: Install dev dependencies + + - name: Install dev Dependencies run: | # We don't actually want to install toga-core; # we just want the dev extras so we have a known version of tox and coverage python -m pip install ./core[dev] - - name: Get packages + + - name: Get Packages uses: actions/download-artifact@v4.1.2 with: pattern: ${{ needs.package.outputs.artifact-name }}-* merge-multiple: true + - name: Test run: | # The $(ls ...) shell expansion is done in the Github environment; - # the value of TOGA_INSTALL_COMMAND will be a literal string, - # without any shell expansions to perform - TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl)" tox -e py + # the value of TOGA_INSTALL_COMMAND will be a literal string without any shell expansions to perform + TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl)" \ + tox -e py-cov + tox -e coverage-keep mv core/.coverage core/.coverage.${{ matrix.platform }}.${{ matrix.python-version }} - - name: Store coverage data + + - name: Store Coverage Data uses: actions/upload-artifact@v4.3.1 with: name: core-coverage-data-${{ matrix.platform }}-${{ matrix.python-version }} @@ -123,34 +129,32 @@ jobs: core-coverage: name: Combine & check core coverage. - runs-on: ubuntu-latest needs: core + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.1 + - name: Checkout + uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - - uses: actions/setup-python@v5.0.0 + + - name: Set up Python ${{ env.min_python_version }} + uses: actions/setup-python@v5.0.0 with: - # Use latest, so it understands all syntax. - python-version: ${{ env.max_python_version }} - - name: Install dev dependencies - run: | - # We don't actually want to install toga-core; - # we just want the dev extras so we have a known version of coverage - python -m pip install ./core[dev] - - name: Retrieve coverage data + # Use minimum version of python for coverage to avoid phantom branches + # https://github.com/nedbat/coveragepy/issues/1572#issuecomment-1522546425 + python-version: ${{ env.min_python_version }} + + - name: Retrieve Coverage Data uses: actions/download-artifact@v4.1.2 with: pattern: core-coverage-data-* path: core merge-multiple: true - - name: Generate coverage report - run: | - cd core - python -m coverage combine - python -m coverage html --skip-covered --skip-empty - python -m coverage report --rcfile ../pyproject.toml --fail-under=100 - - name: Upload HTML report if check failed. + + - name: Generate Coverage Report + run: tox -e coverage-html-fail + + - name: Upload HTML Report uses: actions/upload-artifact@v4.3.1 if: failure() with: @@ -158,84 +162,91 @@ jobs: path: core/htmlcov testbed: - runs-on: ${{ matrix.runs-on }} + name: Testbed needs: core + runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false matrix: backend: [ "macOS-x86_64", "macOS-arm64", "windows", "linux", "android", "iOS" ] include: - - pre-command: - briefcase-run-prefix: - briefcase-run-args: - setup-python: true - - - backend: "macOS-x86_64" - platform: "macOS" - runs-on: "macos-12" - app-user-data-path: $HOME/Library/Application Support/org.beeware.toga.testbed - - - backend: "macOS-arm64" - platform: "macOS" - runs-on: "macos-14" - app-user-data-path: $HOME/Library/Application Support/org.beeware.toga.testbed - - # We use a fixed Ubuntu version rather than `-latest` because at some point, - # `-latest` will be updated, but it will be a soft changeover, which would cause - # the system Python version to become inconsistent from run to run. - - backend: "linux" - platform: "linux" - runs-on: "ubuntu-22.04" - # The package list should be the same as in tutorial-0.rst, and the BeeWare - # tutorial, plus blackbox to provide a window manager. We need a window - # manager that is reasonably lightweight, honors full screen mode, and - # treats the window position as the top-left corner of the *window*, not the - # top-left corner of the window *content*. The default GNOME window managers of - # most distros meet these requirements, but they're heavyweight; flwm doesn't - # work either. Blackbox is the lightest WM we've found that works. - pre-command: | - sudo apt update -y - sudo apt install -y blackbox pkg-config python3-dev libgirepository1.0-dev libcairo2-dev gir1.2-webkit2-4.0 - - # Start Virtual X server - echo "Start X server..." - Xvfb :99 -screen 0 2048x1536x24 & - sleep 1 - - # Start Window manager - echo "Start window manager..." - DISPLAY=:99 blackbox & - sleep 1 - - briefcase-run-prefix: 'DISPLAY=:99' - setup-python: false # Use the system Python packages. - app-user-data-path: $HOME/.local/share/testbed - - - backend: "windows" - platform: "windows" - runs-on: "windows-latest" - app-user-data-path: $HOME\AppData\Local\Tiberius Yak\Toga Testbed\Data - - - backend: "iOS" - platform: "iOS" - runs-on: "macos-14" - briefcase-run-args: ' -d "iPhone SE (3rd generation)"' - app-user-data-path: $(xcrun simctl get_app_container booted org.beeware.toga.testbed data)/Documents - - - backend: "android" - platform: "android" - runs-on: "ubuntu-latest" - briefcase-run-args: " -d '{\"avd\":\"beePhone\",\"skin\":\"pixel_3a\"}' --Xemulator=-no-window --Xemulator=-no-snapshot --Xemulator=-no-audio --Xemulator=-no-boot-anim --shutdown-on-exit" - pre-command: | - # check if virtualization is supported... - sudo apt install -qq --no-install-recommends cpu-checker coreutils && echo "CPUs=$(nproc --all)" && kvm-ok - # allow access to KVM to run the emulator - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm + - pre-command: "" + briefcase-run-prefix: "" + briefcase-run-args: "" + setup-python: true + + - backend: "macOS-x86_64" + platform: "macOS" + runs-on: "macos-12" + app-user-data-path: $HOME/Library/Application Support/org.beeware.toga.testbed + + - backend: "macOS-arm64" + platform: "macOS" + runs-on: "macos-14" + app-user-data-path: $HOME/Library/Application Support/org.beeware.toga.testbed + + # We use a fixed Ubuntu version rather than `-latest` because at some point, + # `-latest` will be updated, but it will be a soft changeover, which would cause + # the system Python version to become inconsistent from run to run. + - backend: "linux" + platform: "linux" + runs-on: "ubuntu-22.04" + # The package list should be the same as in tutorial-0.rst, and the BeeWare + # tutorial, plus blackbox to provide a window manager. We need a window + # manager that is reasonably lightweight, honors full screen mode, and + # treats the window position as the top-left corner of the *window*, not the + # top-left corner of the window *content*. The default GNOME window managers of + # most distros meet these requirements, but they're heavyweight; flwm doesn't + # work either. Blackbox is the lightest WM we've found that works. + pre-command: | + sudo apt update -y + sudo apt install -y --no-install-recommends \ + blackbox pkg-config python3-dev libgirepository1.0-dev libcairo2-dev gir1.2-webkit2-4.0 + + # Start Virtual X server + echo "Start X server..." + Xvfb :99 -screen 0 2048x1536x24 & + sleep 1 + + # Start Window manager + echo "Start window manager..." + DISPLAY=:99 blackbox & + sleep 1 + + briefcase-run-prefix: 'DISPLAY=:99' + setup-python: false # Use the system Python packages. + app-user-data-path: $HOME/.local/share/testbed + + - backend: "windows" + platform: "windows" + runs-on: "windows-latest" + app-user-data-path: $HOME\AppData\Local\Tiberius Yak\Toga Testbed\Data + + - backend: "iOS" + platform: "iOS" + runs-on: "macos-14" + briefcase-run-args: ' -d "iPhone SE (3rd generation)"' + app-user-data-path: $(xcrun simctl get_app_container booted org.beeware.toga.testbed data)/Documents + + - backend: "android" + platform: "android" + runs-on: "ubuntu-latest" + briefcase-run-args: > + -d '{\"avd\":\"beePhone\",\"skin\":\"pixel_3a\"}' + --Xemulator=-no-window + --Xemulator=-no-snapshot + --Xemulator=-no-audio + --Xemulator=-no-boot-anim + --shutdown-on-exit + pre-command: | + # allow access to KVM to run the emulator + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm steps: - - uses: actions/checkout@v4.1.1 + - name: Checkout + uses: actions/checkout@v4.1.1 with: fetch-depth: 0 @@ -249,7 +260,7 @@ jobs: # * It doesn't have an Android build of Pillow yet. python-version: "3.10" - - name: Install dependencies + - name: Install Dependencies run: | ${{ matrix.pre-command }} # Use the development version of Briefcase @@ -259,22 +270,24 @@ jobs: - name: Test App working-directory: testbed timeout-minutes: 15 - run: ${{ matrix.briefcase-run-prefix }} briefcase run ${{ matrix.platform }} --test ${{ matrix.briefcase-run-args }} + run: | + ${{ matrix.briefcase-run-prefix }} \ + briefcase run ${{ matrix.platform }} --test ${{ matrix.briefcase-run-args }} - - name: Upload logs + - name: Upload Logs uses: actions/upload-artifact@v4.3.1 if: failure() with: name: testbed-failure-logs-${{ matrix.backend }} path: testbed/logs/* - - name: Copy app generated user data + - name: Copy App Generated User Data if: failure() && matrix.backend != 'android' run: | mkdir -p testbed/app_data cp -r "${{ matrix.app-user-data-path }}" testbed/app_data/testbed-app_data-${{ matrix.backend }} - - name: Upload app data + - name: Upload App Data uses: actions/upload-artifact@v4.3.1 if: failure() && matrix.backend != 'android' with: diff --git a/changes/2440.misc.rst b/changes/2440.misc.rst new file mode 100644 index 0000000000..1a4f164a13 --- /dev/null +++ b/changes/2440.misc.rst @@ -0,0 +1 @@ +The tox environments for tests and coverage were revamped for easier and more effective use. diff --git a/core/pyproject.toml b/core/pyproject.toml index ad432852d5..079446d04b 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -63,7 +63,7 @@ dependencies = [ ] [project.optional-dependencies] -# Extras used by developers *of* briefcase are pinned to specific versions to +# Extras used by developers *of* Toga are pinned to specific versions to # ensure environment consistency. dev = [ "coverage[toml] == 7.4.3", @@ -104,6 +104,9 @@ Documentation = "https://toga.readthedocs.io/en/latest/" Tracker = "https://github.com/beeware/toga/issues" Source = "https://github.com/beeware/toga" +[project.entry-points."toga.image_formats"] +pil = "toga.plugins.image_formats.PILConverter" + [tool.setuptools_scm] root = ".." @@ -127,6 +130,3 @@ asyncio_mode = "auto" filterwarnings = [ "error", ] - -[project.entry-points."toga.image_formats"] -pil = "toga.plugins.image_formats.PILConverter" diff --git a/core/tests/test_paths.py b/core/tests/test_paths.py index 9f1bf383d0..39973cda40 100644 --- a/core/tests/test_paths.py +++ b/core/tests/test_paths.py @@ -7,7 +7,7 @@ def run_app(args, cwd): - "Run a Toga app as a subprocess with coverage enabled and the Toga Dummy backend" + """Run a Toga app as a subprocess with coverage enabled and the Toga Dummy backend.""" # We need to do a full copy of the environment, then add our extra bits; # if we don't the Windows interpreter won't inherit SYSTEMROOT env = os.environ.copy() @@ -26,7 +26,7 @@ def run_app(args, cwd): env=env, text=True, ) - # When called as a subprocess, coverage drops it's coverage report in CWD. + # When called as a subprocess, coverage drops its coverage report in CWD. # Move it to the project root for combination with the main test report. for file in cwd.glob(".coverage*"): os.rename(file, Path(__file__).parent.parent / file.name) @@ -34,7 +34,7 @@ def run_app(args, cwd): def assert_paths(output, app_path, app_name): - "Assert the paths for the standalone app are consistent" + """Assert the paths for the standalone app are consistent.""" results = output.splitlines() assert f"app.paths.app={app_path.resolve()}" in results assert ( @@ -57,7 +57,7 @@ def assert_paths(output, app_path, app_name): def test_as_interactive(): - "At an interactive prompt, the app path is the current working directory" + """At an interactive prompt, the app path is the current working directory.""" # Spawn the interactive-mode mocking entry point cwd = Path(__file__).parent / "testbed" output = run_app(["interactive.py"], cwd=cwd) @@ -65,8 +65,8 @@ def test_as_interactive(): def test_simple_as_file_in_module(): - """When a simple app is started as `python app.py` inside a runnable module, the - app path is the folder holding app.py.""" + """When a simple app is started as `python app.py` inside a runnable module, the app + path is the folder holding app.py.""" # Spawn the simple testbed app using `app.py` cwd = Path(__file__).parent / "testbed/simple" output = run_app(["app.py"], cwd=cwd) @@ -74,8 +74,8 @@ def test_simple_as_file_in_module(): def test_simple_as_module(): - """When a simple apps is started as `python -m app` inside a runnable module, - the app path is the folder holding app.py.""" + """When a simple apps is started as `python -m app` inside a runnable module, the + app path is the folder holding app.py.""" # Spawn the simple testbed app using `-m app` cwd = Path(__file__).parent / "testbed/simple" output = run_app(["-m", "app"], cwd=cwd) @@ -83,7 +83,8 @@ def test_simple_as_module(): def test_simple_as_deep_file(): - "When a simple app is started as `python simple/app.py`, the app path is the folder holding app.py" + """When a simple app is started as `python simple/app.py`, the app path is the + folder holding app.py.""" # Spawn the simple testbed app using `simple/app.py` cwd = Path(__file__).parent / "testbed" output = run_app(["simple/app.py"], cwd=cwd) @@ -91,7 +92,8 @@ def test_simple_as_deep_file(): def test_simple_as_deep_module(): - "When a simple app is started as `python -m simple`, the app path is the folder holding app.py" + """When a simple app is started as `python -m simple`, the app path is the folder + holding app.py.""" # Spawn the simple testbed app using `-m simple` cwd = Path(__file__).parent / "testbed" output = run_app(["-m", "simple"], cwd=cwd) @@ -108,8 +110,8 @@ def test_subclassed_as_file_in_module(): def test_subclassed_as_module(): - """When a subclassed app is started as `python -m app` inside a runnable module, - the app path is the folder holding app.py.""" + """When a subclassed app is started as `python -m app` inside a runnable module, the + app path is the folder holding app.py.""" # Spawn the subclassed testbed app using `-m app` cwd = Path(__file__).parent / "testbed/subclassed" output = run_app(["-m", "app"], cwd=cwd) @@ -117,7 +119,8 @@ def test_subclassed_as_module(): def test_subclassed_as_deep_file(): - "When a subclassed app is started as `python simple/app.py`, the app path is the folder holding app.py" + """When a subclassed app is started as `python simple/app.py`, the app path is the + folder holding app.py.""" # Spawn the subclassed testbed app using `subclassed/app.py` cwd = Path(__file__).parent / "testbed" output = run_app(["subclassed/app.py"], cwd=cwd) @@ -125,7 +128,8 @@ def test_subclassed_as_deep_file(): def test_subclassed_as_deep_module(): - "When a subclassed app is started as `python -m simple`, the app path is the folder holding app.py" + """When a subclassed app is started as `python -m simple`, the app path is the + folder holding app.py.""" # Spawn the subclassed testbed app using `-m subclassed` cwd = Path(__file__).parent / "testbed" output = run_app(["-m", "subclassed"], cwd=cwd) diff --git a/core/tests/testbed/customize/sitecustomize.py b/core/tests/testbed/customize/sitecustomize.py index 6a1603a850..a57c0f0bd3 100644 --- a/core/tests/testbed/customize/sitecustomize.py +++ b/core/tests/testbed/customize/sitecustomize.py @@ -1,3 +1,9 @@ -import coverage +import os -cov = coverage.process_startup() +# Only set up the app processes for coverage *if* coverage is running. +# Otherwise, just importing coverage will force running coverage. +# The COVERAGE_RUN environment variable is set by coverage.py itself. +if os.environ.get("COVERAGE_RUN"): + import coverage + + coverage.process_startup() diff --git a/dummy/pyproject.toml b/dummy/pyproject.toml index 94106a9267..640747dc00 100644 --- a/dummy/pyproject.toml +++ b/dummy/pyproject.toml @@ -54,6 +54,10 @@ Source = "https://github.com/beeware/toga" [project.entry-points."toga.backends"] dummy = "toga_dummy" +[project.entry-points."toga.image_formats"] +dummy = "toga_dummy.plugins.image_formats.CustomImageConverter" +disabled = "toga_dummy.plugins.image_formats.DisabledImageConverter" + [tool.setuptools_scm] root = ".." @@ -61,7 +65,3 @@ root = ".." dependencies = [ "toga-core == {version}", ] - -[project.entry-points."toga.image_formats"] -dummy = "toga_dummy.plugins.image_formats.CustomImageConverter" -disabled = "toga_dummy.plugins.image_formats.DisabledImageConverter" diff --git a/testbed/tests/app/test_app.py b/testbed/tests/app/test_app.py index 11043a98c3..067c6466cf 100644 --- a/testbed/tests/app/test_app.py +++ b/testbed/tests/app/test_app.py @@ -17,7 +17,7 @@ def mock_app_exit(monkeypatch, app): return app_exit -# Mobile platforms have different windowing characterics, so they have different tests. +# Mobile platforms have different windowing characteristics, so they have different tests. if toga.platform.current_platform in {"iOS", "android"}: #################################################################################### # Mobile platform tests diff --git a/tox.ini b/tox.ini index 1e989cc585..0cd146c6c7 100644 --- a/tox.ini +++ b/tox.ini @@ -6,20 +6,61 @@ extend-ignore = # See https://github.com/PyCQA/pycodestyle/issues/373 E203, +[tox] +# Add Python 3.13 once a wheel for Pillow is available +envlist = towncrier-check,docs-lint,pre-commit,py{38,39,310,311,312}-cov,coverage +labels = + test = py{38,39,310,311,312}-cov,coverage + test38 = py38-cov,coverage38 + test39 = py39-cov,coverage39 + test310 = py310-cov,coverage310 + test311 = py311-cov,coverage311 + test312 = py312-cov,coverage312 + test313 = py313-cov,coverage313 +skip_missing_interpreters = True + +[testenv:pre-commit] +skip_install = True +deps = {tox_root}{/}core[dev] +commands = pre-commit run --all-files --show-diff-on-failure --color=always + # The leading comma generates the "py" environment. -[testenv:py{,38,39,310,311,312,313}] +[testenv:py{,38,39,310,311,312,313}{,-cov}] +depends = pre-commit +changedir = core skip_install = True setenv = TOGA_BACKEND = toga_dummy -changedir = core allowlist_externals = bash commands = # TOGA_INSTALL_COMMAND is set to a bash command by the CI workflow. {env:TOGA_INSTALL_COMMAND:python -m pip install .[dev] {tox_root}{/}dummy} - {env:test_command_prefix:} coverage run -m pytest -vv {posargs} - coverage combine - coverage report --rcfile {tox_root}{/}pyproject.toml + cov : python -m coverage run -m pytest {posargs:-vv --color yes} + !cov: python -m pytest {posargs:-vv --color yes} + +[testenv:coverage{,38,39,310,311,312,313}{,-html}{,-keep}{,-fail}] +depends = pre-commit,py{,38,39,310,311,312,313}{,-cov} +changedir = core +skip_install = True +# by default, coverage should run on oldest supported Python for testing platform coverage. +# however, coverage for a particular Python version should match the version used for pytest. +base_python = + coverage: py38,py39,py310,py311,py312,py313 + coverage38: py38 + coverage39: py39 + coverage310: py310 + coverage311: py311 + coverage312: py312 + coverage313: py313 +deps = {tox_root}{/}core[dev] +setenv = + keep: COMBINE_FLAGS = --keep + fail: REPORT_FAIL_COND = --fail-under=100 +commands = + -python -m coverage combine {env:COMBINE_FLAGS} + html: python -m coverage html --skip-covered --skip-empty --rcfile {tox_root}{/}pyproject.toml + python -m coverage report --rcfile {tox_root}{/}pyproject.toml {env:REPORT_FAIL_COND} [testenv:towncrier{,-check}] skip_install = True