test.yml

# This GitHub workflow will setup and run various kinds of tests with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: test

on:
  schedule:
    # The schedule event always (and only) runs on the master branch.
    - # cron (in UTC): minute hour day_of_month month day_of_week
      cron: '00 22 * * SAT'
  pull_request:  # When creating a PR targeting these branches
    branches:
    - master
    - stable_*
  push:  # When merging a PR targeting these branches (direct push is disabled)
    branches:
    - master
    - stable_*

jobs:

  set_matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.select_matrix.outputs.matrix }}
    steps:
    - name: "Select matrix"
      id: select_matrix
      # Select full matrix when scheduled or when releasing, and normal matrix
      # otherwise. The matrix is defined as a JSON string.
      # This technique documented in:
      #    https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional
      # TODO: Find a way to define this with less escapes.
      run: |
        if [[ "${{ github.event_name }}" == "schedule" || "${{ github.head_ref }}" =~ ^release_ ]]; then \
          echo "matrix={ \
            \"os\": [ \"ubuntu-latest\", \"macos-latest\" ], \
            \"python-version\": [ \"3.5\", \"3.6\", \"3.7\", \"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\" ], \
            \"package_level\": [ \"minimum\", \"latest\", \"ansible\" ], \
            \"exclude\": [ \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"ansible\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"ansible\" \
              } \
            ], \
            \"include\": [ \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"2.7\", \
                \"package_level\": \"minimum\", \
                \"container\": {\"image\": \"python:2.7.18-buster\"} \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"2.7\", \
                \"package_level\": \"latest\", \
                \"container\": {\"image\": \"python:2.7.18-buster\"} \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"2.7\", \
                \"package_level\": \"ansible\", \
                \"container\": {\"image\": \"python:2.7.18-buster\"} \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"ansible\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"ansible\" \
              } \
            ] \
          }" >> $GITHUB_OUTPUT; \
        else \
          echo "matrix={ \
            \"os\": [ \"ubuntu-latest\" ], \
            \"python-version\": [ \"3.11\" ], \
            \"package_level\": [ \"minimum\", \"latest\" ], \
            \"include\": [ \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"2.7\", \
                \"package_level\": \"minimum\", \
                \"container\": {\"image\": \"python:2.7.18-buster\"} \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"2.7\", \
                \"package_level\": \"latest\", \
                \"container\": {\"image\": \"python:2.7.18-buster\"} \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.5\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"ubuntu-20.04\", \
                \"python-version\": \"3.6\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.7\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.8\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.9\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.9\", \
                \"package_level\": \"ansible\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.10\", \
                \"package_level\": \"minimum\" \
              }, \
              { \
                \"os\": \"ubuntu-latest\", \
                \"python-version\": \"3.12\", \
                \"package_level\": \"latest\" \
              }, \
              { \
                \"os\": \"macos-latest\", \
                \"python-version\": \"3.11\", \
                \"package_level\": \"latest\" \
              } \
            ] \
          }" >> $GITHUB_OUTPUT; \
        fi
    - name: Show matrix in JSON
      run: echo '${{ steps.select_matrix.outputs.matrix }}'

  test:
    needs: set_matrix
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix: ${{ fromJson(needs.set_matrix.outputs.matrix) }}
    runs-on: ${{ matrix.os }}
    container: ${{ matrix.container }}
    env:
      PIP_DISABLE_PIP_VERSION_CHECK: 1
      PIP_NO_PYTHON_VERSION_WARNING: 1
    steps:
    - name: Set run type (normal, scheduled, release)
      id: set-run-type
      uses: actions/github-script@v6
      with:
        result-encoding: string
        script: |
          var result
          if ("${{ github.event_name }}" == "schedule") {
            result = "scheduled"
          } else if ("${{ github.head_ref }}".match(/^release_/)) {
            result = "release"
          } else {
            result = "normal"
          }
          console.log(result)
          return result
    - name: Checkout repo
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      if: ${{ ! ( matrix.python-version == '2.7' ) }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Display initial Python packages
      run: |
        echo "Installed Python packages:"
        pip list
    - name: Install the package and its dependents
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make install
    - name: Show installed package versions
      run: |
        echo "Installed Python packages:"
        pip list
    - name: Development setup
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make develop
    - name: Show installed package versions
      run: |
        echo "Installed Python packages:"
        pip list
    - name: Show package dependency tree
      run: |
        echo "Package dependency tree of installed Python packages:"
        python -m pipdeptree --all
    - name: Run docs
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make docs
    - name: Check that module .rst files are up to date
      # Because the Ansible rst generator changed its output over time, we only
      # check this on the latest Python release with latest package levels.
      if: ${{ matrix.python-version == '3.11' && matrix.package_level == 'latest' }}
      run: |
        bash -c "if ! git diff --exit-code HEAD docs/source/modules; then
          echo 'Error: Module .rst files are not up to date in this PR. Run make docslocal to update.';
          exit 1;
        fi"
    - name: Run check
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make check
    - name: Run sanity
      # Notes on running 'make sanity':
      # * Excluded on Python 2.7 because that would require pylint 1.x
      if: ${{ ! ( matrix.python-version == '2.7' ) }}
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make sanity
    - name: Run ansible_lint
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make ansible_lint
    - name: Run unit/function test
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
        # TESTCASES: test_partition.py
      run: |
        make test
    - name: Run mocked end2end test
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
        # TESTCASES: test_partition.py
      run: |
        make end2end_mocked
    - name: Report coverage result locally
      run: |
        coverage report
    - name: Send coverage result to coveralls.io
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        COVERALLS_PARALLEL: true
        COVERALLS_FLAG_NAME: "${{ matrix.os }},${{ matrix.python-version }},${{ matrix.package_level }}"
        COVERALLS_SERVICE_NAME: github
        COVERALLS_SERVICE_JOB_ID: "${{ github.run_id }}"
        COVERALLS_SERVICE_NUMBER: "${{ github.workflow }}-${{ github.run_number }}"
      run: |
        coveralls
    - name: Run check_reqs
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make check_reqs
    - name: Run safety
      env:
        PACKAGE_LEVEL: ${{ matrix.package_level }}
        RUN_TYPE: ${{ steps.set-run-type.outputs.result }}
      run: |
        make safety

  test_finish:
    needs: test
    runs-on: ubuntu-latest
    container: python:3-slim
    steps:
    - name: Install coveralls
      run: |
        pip3 install --upgrade coveralls
    - name: Send coverage finish to coveralls.io
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        COVERALLS_SERVICE_NUMBER: "${{ github.workflow }}-${{ github.run_number }}"
      run: |
        coveralls --finish