Test suite clean up (#3385)

* Split apart test suite into those run on different numbers of ranks. Testing is now using MPI 'on the outside' which should allow us to stop relying on MPICH.
* Various fixes for parallel tests now they are not run in isolation.
* Added `firedrake-run-split-tests` script which other packages may wish to use.

Author: JDBetteridge

.github/workflows/build-mac.yml

@@ -46,14 +46,16 @@ jobs:
             --venv-name firedrake_venv \
             --disable-ssh \
             || (cat firedrake-install.log && /bin/false)
+      - name: Install test dependencies
+        run: |
+          . ../firedrake_venv/bin/activate
+          python -m pip install pytest-timeout
+
       - name: Run smoke tests
         run: |
           . ../firedrake_venv/bin/activate
-          python -m pytest -v tests/firedrake/regression/ -k "poisson_strong or stokes_mini or dg_advection"
-          # also test for 'problem libraries' (spatialindex and libsupermesh)
-          python -m pytest -v tests/firedrake/regression/test_locate_cell.py
-          python -m pytest -v tests/firedrake/supermesh/test_assemble_mixed_mass_matrix.py
-        timeout-minutes: 30
+          make check CHECK_PYTEST_ARGS="--timeout 60"
+        timeout-minutes: 10
       - name: Post-run cleanup
         if: ${{ always() }}
         run: |

.github/workflows/build.yml

@@ -21,14 +21,14 @@ env:
   RELEASE_TAG: latest
 
 jobs:
-  build:
-    name: "Build Firedrake"
-    # Run on our self-hosted machines
+  test:
+    name: "Run Firedrake tests (Linux)"
     runs-on: [self-hosted, Linux]
     container:
       image: firedrakeproject/firedrake-env:latest
     strategy:
-      # Don't immediately kill real if complex fails and vice versa.
+      # We want to know all of the tests which fail, so don't kill real if
+      # complex fails and vice-versa
       fail-fast: false
       matrix:
         include:
@@ -47,13 +47,16 @@ jobs:
       OPENBLAS_NUM_THREADS: 1
       COMPLEX: ${{ matrix.complex }}
       RDMAV_FORK_SAFE: 1
+      EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=1800 --timeout-method=thread -o faulthandler_timeout=1860 tests/firedrake
     steps:
       - uses: actions/checkout@v4
+
       - name: Cleanup
         if: ${{ always() }}
         run: |
           cd ..
           rm -rf firedrake_venv
+
       - name: Build Firedrake
         run: |
           cd ..
@@ -85,58 +88,105 @@ jobs:
             --install gadopt \
             --install asQ \
             || (cat firedrake-install.log && /bin/false)
+
       - name: Install test dependencies
+        id: build
         run: |
+          sudo apt update
+          sudo apt -y install parallel
           . ../firedrake_venv/bin/activate
           python "$(which firedrake-clean)"
-          python -m pip install \
-            pytest-xdist pytest-timeout ipympl
+          python -m pip install pytest-timeout ipympl pytest-split
           python -m pip list
-      - name: Test Firedrake
+
+      - name: Run tests (nprocs = 1)
         run: |
           . ../firedrake_venv/bin/activate
-          echo OMP_NUM_THREADS is "$OMP_NUM_THREADS"
-          echo OPENBLAS_NUM_THREADS is "$OPENBLAS_NUM_THREADS"
-          python -m pytest -v tests/firedrake/test_0init.py
-          python -m pytest \
-            --durations=200 \
-            --timeout=1800 \
-            --timeout-method=thread \
-            -o faulthandler_timeout=1860 \
-            -n 12 --dist worksteal \
-            --junit-xml=firedrake.xml \
-            -sv tests/firedrake
-        timeout-minutes: 120
+          firedrake-run-split-tests 1 12 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake1_{#}.xml"
+
+      - name: Run tests (nprocs = 2)
+        # Run even if earlier tests failed
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 2 6 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake2_{#}.xml"
+
+      - name: Run tests (nprocs = 3)
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 3 4 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake3_{#}.xml"
+
+      - name: Run tests (nprocs = 4)
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 4 3 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake4_{#}.xml"
+
+      - name: Run tests (nprocs = 5)
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 5 2 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake5_{#}.xml"
+
+      - name: Run tests (nprocs = 6)
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 6 2 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake6_{#}.xml"
+
+      - name: Run tests (nprocs = 7)
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 7 1 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake7_{#}.xml"
+
+      - name: Run tests (nprocs = 8)
+        if: ${{ success() || steps.build.conclusion == 'success' }}
+        run: |
+          . ../firedrake_venv/bin/activate
+          firedrake-run-split-tests 8 1 "$EXTRA_PYTEST_ARGS --junit-xml=firedrake8_{#}.xml"
+
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v5.0.0-a02
         # To avoid permissions issues do not run with forked repos
         # (see https://github.com/mikepenz/action-junit-report/issues/23)
         if: ${{ always() && (github.ref != 'refs/heads/master') && (github.event.pull_request.head.repo.full_name == github.repository) }}
         with:
-          report_paths: '/__w/firedrake/firedrake/firedrake.xml'
+          report_paths: 'firedrake*.xml'
           comment: true
           check_name: "Firedrake ${{ matrix.scalar-type }}"
           updateComment: true
           flaky_summary: true
+
+      - name: Upload log files
+        uses: actions/upload-artifact@v4
+        with:
+          name: firedrake-logs-${{ matrix.scalar-type }}
+          path: pytest_*.log
+
       - name: Test pyadjoint
         if: ${{ matrix.scalar-type == 'real' }}
         run: |
           . ../firedrake_venv/bin/activate
           cd ../firedrake_venv/src/pyadjoint
           python -m pytest \
+            --strict-markers \
             --durations=200 \
             --timeout=600 \
             --timeout-method=thread \
             -o faulthandler_timeout=660 \
             -n 12 --dist worksteal \
             -sv tests/firedrake_adjoint
         timeout-minutes: 30
+
       - name: Cleanup
         # Belt and braces: clean up before and after the run.
         if: ${{ always() }}
         run: |
           cd ..
           rm -rf firedrake_venv
+
   docker_tag:
     name: "Set the Docker release tag"
     runs-on: [self-hosted, Linux]
@@ -153,14 +203,15 @@ jobs:
           echo The release tag is "$RELEASE_TAG"
     outputs:
       tag: ${{ env.RELEASE_TAG }}
+
   docker:
     name: "Build Docker containers"
     # Only run on master, but always generate firedrake-env image,
     # even if build fails (see docker.yml)
     if: ${{ (github.ref == 'refs/heads/master') && always() }}
-    needs: [build, docker_tag]
+    needs: [test, docker_tag]
     uses: ./.github/workflows/docker.yml
     with:
       tag: ${{ needs.docker_tag.outputs.tag }}
-      status: ${{ needs.build.result }}
+      status: ${{ needs.test.result }}
     secrets: inherit

.github/workflows/pip-mac.yml

@@ -23,6 +23,7 @@ jobs:
     # Only run this action if we are pushing to master or the PR is labelled "macOS"
     if: ${{ (github.ref == 'refs/heads/master') || contains(github.event.pull_request.labels.*.name, 'macOS') }}
     env:
+      FIREDRAKE_CI_TESTS: 1
       OMP_NUM_THREADS: 1
       OPENBLAS_NUM_THREADS: 1
     steps:
@@ -98,12 +99,8 @@ jobs:
         run: |
           source pip_venv/bin/activate
           cd pip_venv/src/firedrake
-          python -m pytest --timeout=1800 -v tests/firedrake/regression \
-            -k "poisson_strong or stokes_mini or dg_advection"
-          # also test for 'problem libraries' (spatialindex and libsupermesh)
-          python -m pytest -v tests/firedrake/regression/test_locate_cell.py
-          python -m pytest -v tests/firedrake/supermesh/test_assemble_mixed_mass_matrix.py
-        timeout-minutes: 30
+          make check CHECK_PYTEST_ARGS="--timeout 60"
+        timeout-minutes: 10
 
       - name: Cleanup (post)
         if: ${{ always() }}

.github/workflows/pip.yml

@@ -83,16 +83,8 @@ jobs:
         run: |
           source pip_venv/bin/activate
           cd pip_venv/src/firedrake
-          pytest -v tests/firedrake/test_0init.py
-          pytest \
-            --durations=200 \
-            --timeout=1800 \
-            --timeout-method=thread \
-            -o faulthandler_timeout=1860 \
-            -n 12 --dist worksteal \
-            --junit-xml=firedrake.xml \
-            -sv tests/firedrake/regression -k "poisson_strong or stokes_mini or dg_advection"
-        timeout-minutes: 120
+          make check CHECK_PYTEST_ARGS="--timeout 60"
+        timeout-minutes: 10
 
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v5.0.0-a02