pytest-gpu.yml

# Runner information:
# OpenACC on NVidia runs on `sarlaac`
# OpenMP on NVidia runs on `kimogila`
# OpenMP on AMD runs on `wampa`

name: CI-gpu

env:
  OUTPUT_PATH: ${{ github.workspace }}
  RESOURCE_GROUP: CI-gpu

on:
  # Trigger the workflow on push or pull request,
  # but only for the master branch
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  # Push-button activation
  workflow_dispatch:
    inputs:
      tags:
        description: 'Run GPU tests'

jobs:

  build:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.tags }}

    env:
      DEVITO_ARCH: ${{ matrix.arch }}
      DEVITO_PLATFORM: ${{ matrix.platform }}
      DEVITO_LANGUAGE: ${{ matrix.language }}
      OMPI_CC: ${{ matrix.arch }}

    strategy:
      # Prevent all builds from terminating if one fails
      fail-fast: false

      matrix:
        name: [
          pytest-gpu-omp-nvidia,
          pytest-gpu-acc-nvidia,
          pytest-gpu-omp-amd
        ]
        include:
        - name: pytest-gpu-omp-nvidia
          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py"
          base: "devitocodes/bases:nvidia-clang"
          tags: ["self-hosted", "gpu", "docker", "openmp", "kimogila"]
          test_drive_cmd: "nvidia-smi"
          flags: '--gpus all --rm --name testrun-clang-nvidia'

        - name: pytest-gpu-acc-nvidia
          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py"
          base: "devitocodes/bases:nvidia-nvc"
          tags: ["self-hosted", "gpu",  "docker", "v1004"]
          test_drive_cmd: "nvidia-smi"
          flags: '--gpus all --rm --name testrun-nvc'

        - name: pytest-gpu-omp-amd
          # We only test the OpenMP generation due to issue with the offloading discussed in
          # https://github.com/ROCm-Developer-Tools/aomp/issues/397
          test_files: "tests/test_gpu_openmp.py"
          tags: ["self-hosted", "gpu", "docker", "openmp", "aomp"]
          base: "devitocodes/bases:amd"
          test_drive_cmd: "rocm-smi"
          # Attach the AMD GPU devices `/dev` and add user to video and render (109 on wampa) group
          # Options from https://rocmdocs.amd.com/en/latest/ROCm_Virtualization_Containers/ROCm-Virtualization-&-Containers.html
          flags: "--network=host --device=/dev/kfd --device=/dev/dri --ipc=host --group-add video --group-add 109 --security-opt seccomp=unconfined --rm --name testrun-amd"

    steps:
    - name: Checkout devito
      uses: actions/checkout@v3

    - name: Build docker image
      run: |
          docker build . --rm --pull --file docker/Dockerfile.devito --tag devito_img --build-arg base=${{ matrix.base }}

    - name: Remove dangling layers
      run: docker system prune -f

    - name: Test with pytest
      run: |
        # Remove all existing containers for safety
        docker ps -aq | xargs -r docker rm -f
        # Run tests
        docker run ${{ matrix.flags }} devito_img ${{ matrix.test_drive_cmd }}
        docker run ${{ matrix.flags }} -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} devito_img pytest --cov --cov-config=.coveragerc --cov-report=xml ${{ matrix.test_files }}

    - name: Test examples
      run: |
        docker run ${{ matrix.flags }} devito_img pytest examples/seismic/acoustic/acoustic_example.py
        docker run ${{ matrix.flags }} devito_img pytest examples/seismic/elastic/elastic_example.py
        docker run ${{ matrix.flags }} devito_img pytest examples/seismic/tti/tti_example.py
        docker run ${{ matrix.flags }} devito_img pytest examples/seismic/viscoacoustic/viscoacoustic_example.py
        docker run ${{ matrix.flags }} devito_img pytest examples/seismic/viscoelastic/viscoelastic_example.py

    - name: Test examples with MPI
      if: matrix.name != 'pytest-gpu-omp-amd'
      run: |
        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 devito_img mpirun -n 2 pytest examples/seismic/acoustic/acoustic_example.py
        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 devito_img mpirun -n 2 pytest examples/seismic/elastic/elastic_example.py
        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 devito_img mpirun -n 2 pytest examples/seismic/tti/tti_example.py
        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 devito_img mpirun -n 2 pytest examples/seismic/viscoacoustic/viscoacoustic_example.py
        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 devito_img mpirun -n 2 pytest examples/seismic/viscoelastic/viscoelastic_example.py