tests.yml

name: Tests

on:
  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#push
  push:
    branches:
      - main
      - feature/**
      - '[0-9].*.x'  # e.g., 3.24.x
      - '[0-9][0-9].*.x'  # e.g., 23.3.x

  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request
  pull_request:

  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch
  workflow_dispatch:

  # no payload
  schedule:
      # https://crontab.guru/#37_18_*_*_*
    - cron: 37 18 * * *

concurrency:
  # Concurrency group that uses the workflow name and PR number if available
  # or commit SHA as a fallback. If a new build is triggered under that
  # concurrency group while a previous build is running it will be canceled.
  # Repeated pushes to a PR will cancel all previous builds, while multiple
  # merges to main will not cancel.
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  # https://conda.github.io/conda-libmamba-solver/user-guide/configuration/#advanced-options
  CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED: true

jobs:
  # detect whether any code changes are included in this PR
  changes:
    runs-on: ubuntu-latest
    permissions:
      # necessary to detect changes
      # https://github.com/dorny/paths-filter#supported-workflows
      pull-requests: read
    outputs:
      code: ${{ steps.filter.outputs.code }}
    steps:
      - name: Checkout Source
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
        # dorny/paths-filter needs git clone for non-PR events
        # https://github.com/dorny/paths-filter#supported-workflows
        if: github.event_name != 'pull_request'

      - name: Filter Changes
        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
        id: filter
        with:
          filters: |
            code:
              - 'conda_build/**'
              - 'tests/**'
              - '*.py'
              - 'recipe/**'
              - '.github/workflows/tests.yml'

  # linux test suite
  linux:
    # only run test suite if there are code changes
    needs: changes
    if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true'

    runs-on: ubuntu-latest
    defaults:
      run:
        # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell
        shell: bash -el {0}
    strategy:
      fail-fast: false
      matrix:
        # test all lower versions (w/ stable conda) and upper version (w/ canary conda)
        python-version: ['3.9', '3.10', '3.11']
        conda-version: [release]
        test-type: [serial, parallel]
        include:
          # minimum Python/conda combo
          - python-version: '3.8'
            conda-version: 23.7.0
            test-type: serial
          - python-version: '3.8'
            conda-version: 23.7.0
            test-type: parallel
          # maximum Python/conda combo
          - python-version: '3.12'
            conda-version: canary
            test-type: serial
          - python-version: '3.12'
            conda-version: canary
            test-type: parallel
    env:
      CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev::' || '' }}
      CONDA_VERSION: ${{ contains('canary|release', matrix.conda-version) && 'conda' || format('conda={0}', matrix.conda-version) }}
      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}

    steps:
      - name: Checkout Source
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
        with:
          fetch-depth: 0

      - name: Hash + Timestamp
        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV

      - name: Cache Conda
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
        with:
          path: ~/conda_pkgs_dir
          key: cache-${{ env.HASH }}

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca
        with:
          condarc-file: .github/condarc
          run-post: false  # skip post cleanup

      - name: Conda Install
        run: >
          conda install
          --yes
          --file tests/requirements.txt
          --file tests/requirements-${{ runner.os }}.txt
          --file tests/requirements-ci.txt
          python=${{ matrix.python-version }}
          ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }}

      # TODO: how can we remove this step?
      - name: Install Self
        run: pip install -e .

      - name: Conda Info
        # view test env info (not base)
        run: python -m conda info --verbose

      - name: Conda List
        run: conda list --show-channel-urls

      - name: Run Tests
        run: >
          pytest
          --cov=conda_build
          -n auto
          -m "${{ env.PYTEST_MARKER }}"

      - name: Upload Coverage
        uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed
        with:
          flags: ${{ runner.os }},${{ runner.arch }},${{ matrix.python-version }},${{ matrix.test-type }}

      - name: Upload Test Results
        if: '!cancelled()'
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: test-results-${{ env.HASH }}
          path: |
            .coverage
            test-report.xml
          retention-days: 1  # temporary, combined in aggregate below

  # linux benchmarks
  linux-benchmarks:
    # only run test suite if there are code changes
    needs: changes
    if: needs.changes.outputs.code == 'true'

    runs-on: ubuntu-latest
    defaults:
      run:
        # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell
        shell: bash -el {0}  # bash exit immediately on error + login shell
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.12']

    steps:
      - name: Checkout Source
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
        with:
          fetch-depth: 0

      - name: Hash + Timestamp
        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-benchmark-$(date -u "+%Y%m")" >> $GITHUB_ENV

      - name: Cache Conda
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
        with:
          path: ~/conda_pkgs_dir
          key: cache-${{ env.HASH }}

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca
        with:
          condarc-file: .github/condarc
          run-post: false  # skip post cleanup

      - name: Conda Install
        run: >
          conda install
          --yes
          --file tests/requirements.txt
          --file tests/requirements-${{ runner.os }}.txt
          --file tests/requirements-ci.txt
          python=${{ matrix.python-version }}
          ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }}

      - name: Install CodSpeed
        run: pip install git+https://github.com/kenodegard/pytest-codspeed.git@fix-outerr-redirects#egg=pytest-codspeed

      # TODO: how can we remove this step?
      - name: Install Self
        run: pip install -e .

      - name: Conda Info
        # view test env info (not base)
        run: python -m conda info --verbose

      - name: Conda Config
        run: conda config --show-sources

      - name: Conda List
        run: conda list --show-channel-urls

      - name: Run Benchmarks
        uses: CodSpeedHQ/action@0b631f8998f2389eb5144632b6f9f8fabd33a86e
        with:
          token: ${{ secrets.CODSPEED_TOKEN }}
          run: $CONDA/envs/test/bin/pytest --codspeed

  # windows test suite
  windows:
    # only run test suite if there are code changes
    needs: changes
    if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true'

    runs-on: windows-2019
    strategy:
      fail-fast: false
      matrix:
        # test lower version (w/ stable conda) and upper version (w/ canary conda)
        python-version: ['3.8']
        conda-version: [release]
        test-type: [serial, parallel]
        include:
          - python-version: '3.12'
            conda-version: canary
            test-type: serial
          - python-version: '3.12'
            conda-version: canary
            test-type: parallel
    env:
      ErrorActionPreference: Stop  # powershell exit on first error
      CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial and not slow' }}

    steps:
      - name: Checkout Source
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
        with:
          fetch-depth: 0

      - name: Hash + Timestamp
        shell: bash  # use bash to run date command
        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV

      - name: Cache Conda
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
        with:
          path: ~/conda_pkgs_dir
          key: cache-${{ env.HASH }}

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca
        with:
          condarc-file: .github\condarc
          run-post: false  # skip post cleanup

      - name: Choco Install
        run: choco install visualstudio2017-workload-vctools

      - name: Conda Install
        run: >
          conda install
          --yes
          --file tests\requirements.txt
          --file tests\requirements-${{ runner.os }}.txt
          --file tests\requirements-ci.txt
          python=${{ matrix.python-version }}
          ${{ env.CONDA_CHANNEL_LABEL }}::conda

      # TODO: how can we remove this step?
      - name: Install Self
        run: pip install -e .

      - name: Conda Info
        # view test env info (not base)
        run: python -m conda info --verbose

      - name: Conda List
        run: conda list --show-channel-urls

      - name: Run Tests
        # Windows is sensitive to long paths, using `--basetemp=${{ runner.temp }} to
        # keep the test directories shorter
        run: >
          pytest
          --cov=conda_build
          --basetemp=${{ runner.temp }}
          -n auto
          -m "${{ env.PYTEST_MARKER }}"

      - name: Upload Coverage
        uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed
        with:
          flags: ${{ runner.os }},${{ runner.arch }},${{ matrix.python-version }},${{ matrix.test-type }}

      - name: Upload Test Results
        if: '!cancelled()'
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: test-results-${{ env.HASH }}
          path: |
            .coverage
            test-report.xml
          retention-days: 1  # temporary, combined in aggregate below

  # macos test suite
  macos:
    # only run test suite if there are code changes
    needs: changes
    if: github.event_name == 'schedule' || needs.changes.outputs.code == 'true'

    # Old macOS needed for old SDK (see xcode step below)
    # This is needed for some MACOSX_DEPLOYMENT_TARGET tests
    # We could also install SDKs from a external provider in the future
    # if we want to update this runner to a non-deprecated version
    runs-on: macos-11
    defaults:
      run:
        # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell
        shell: bash -el {0}
    strategy:
      fail-fast: false
      matrix:
        # test lower version (w/ stable conda) and upper version (w/ canary conda)
        python-version: ['3.8']
        conda-version: [release]
        test-type: [serial, parallel]
        include:
          - python-version: '3.12'
            conda-version: canary
            test-type: serial
          - python-version: '3.12'
            conda-version: canary
            test-type: parallel
    env:
      CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}

    steps:
      - name: Checkout Source
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
        with:
          fetch-depth: 0

      - name: Hash + Timestamp
        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV

      - name: Cache Conda
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
        with:
          path: ~/conda_pkgs_dir
          key: cache-${{ env.HASH }}

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca
        with:
          condarc-file: .github/condarc
          run-post: false  # skip post cleanup

      - name: Xcode Install
        run: sudo xcode-select --switch /Applications/Xcode_11.7.app

      - name: Conda Install
        run: >
          conda install
          --yes
          --file tests/requirements.txt
          --file tests/requirements-${{ runner.os }}.txt
          --file tests/requirements-ci.txt
          python=${{ matrix.python-version }}
          ${{ env.CONDA_CHANNEL_LABEL }}::conda

      # TODO: how can we remove this step?
      - name: Install Self
        run: pip install -e .

      - name: Conda Info
        # view test env info (not base)
        run: python -m conda info --verbose

      - name: Conda List
        run: conda list --show-channel-urls

      - name: Run Tests
        run: >
          pytest
          --cov=conda_build
          -n auto
          -m "${{ env.PYTEST_MARKER }}"

      - name: Upload Coverage
        uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed
        with:
          flags: ${{ runner.os }},${{ runner.arch }},${{ matrix.python-version }},${{ matrix.test-type }}

      - name: Upload Test Results
        if: '!cancelled()'
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: test-results-${{ env.HASH }}
          path: |
            .coverage
            test-report.xml
          retention-days: 1  # temporary, combined in aggregate below

  # aggregate and upload
  aggregate:
    # only aggregate test suite if there are code changes
    needs: [changes, linux, linux-benchmarks, windows, macos]
    if: >-
      !cancelled()
      && (
        github.event_name == 'schedule'
        || needs.changes.outputs.code == 'true'
      )

    runs-on: ubuntu-latest
    steps:
      - name: Download Artifacts
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e

      - name: Upload Combined Test Results
        # provides one downloadable archive of all matrix run test results for further analysis
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
        with:
          name: test-results-${{ github.sha }}-all
          path: test-results-*

      - name: Test Summary
        uses: test-summary/action@032c8a9cec6aaa3c20228112cae6ca10a3b29336
        with:
          paths: test-results-*/test-report.xml

  # required check
  analyze:
    needs: [linux, linux-benchmarks, windows, macos, aggregate]
    if: '!cancelled()'

    runs-on: ubuntu-latest
    steps:
      - name: Determine Success
        uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
        with:
          # permit jobs to be skipped if there are no code changes (see changes job)
          allowed-skips: ${{ toJSON(needs) }}
          jobs: ${{ toJSON(needs) }}

  # canary builds
  build:
    needs: [analyze]
    # only build canary build if
    # - prior steps succeeded,
    # - this is the main repo, and
    # - we are on the main, feature, or release branch
    if: >-
      !cancelled()
      && !github.event.repository.fork
      && (
        github.ref_name == 'main'
        || startsWith(github.ref_name, 'feature/')
        || endsWith(github.ref_name, '.x')
      )
    strategy:
      matrix:
        include:
          - runner: ubuntu-latest
            subdir: linux-64
          - runner: macos-13
            subdir: osx-64
          - runner: macos-14  # FUTURE: Use -latest
            subdir: osx-arm64
          - runner: windows-latest
            subdir: win-64
    runs-on: ${{ matrix.runner }}
    steps:
      # Clean checkout of specific git ref needed for package metadata version
      # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR:
      - name: Checkout Source
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
        with:
          ref: ${{ github.ref }}
          clean: true
          fetch-depth: 0

      # Explicitly use Python 3.12 since each of the OSes has a different default Python
      - name: Setup Python
        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
        with:
          python-version: '3.12'

      - name: Detect Label
        shell: python
        run: |
          from pathlib import Path
          from re import match
          from os import environ

          if "${{ github.ref_name }}" == "main":
              # main branch commits are uploaded to the dev label
              label = "dev"
          elif "${{ github.ref_name }}".startswith("feature/"):
              # feature branch commits are uploaded to a custom label
              label = "${{ github.ref_name }}"
          else:
              # release branch commits are added to the rc label
              # see https://github.com/conda/infrastructure/issues/760
              _, name = "${{ github.repository }}".split("/")
              label = f"rc-{name}-${{ github.ref_name }}"

          Path(environ["GITHUB_ENV"]).write_text(f"ANACONDA_ORG_LABEL={label}")

      - name: Create & Upload
        uses: conda/actions/canary-release@1e442e090ad28c9b0f85697105703a303320ffd1
        with:
          package-name: ${{ github.event.repository.name }}
          subdir: ${{ matrix.subdir }}
          anaconda-org-channel: conda-canary
          anaconda-org-label: ${{ env.ANACONDA_ORG_LABEL }}
          anaconda-org-token: ${{ secrets.ANACONDA_ORG_CONDA_CANARY_TOKEN }}