pr_tests.yaml

name: Run all tests

on:
  schedule:
    - cron: "0 0 * * *"
  pull_request:
    types:
      - opened
      - synchronize
      - ready_for_review
  # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions
  merge_group:
    types:
      - checks_requested

env:
  ALL_PYTEST_MARKERS: "not nats and not kafka and not confluent and not rabbit and not redis"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  pre-commit-check:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    env:
      SKIP: "docs"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: |
            3.8
            3.9
            3.10
      - name: Set $PY environment variable
        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ hashFiles('.pre-commit-config.yaml') }}
      - uses: pre-commit/action@v3.0.1

  test-basic:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        pydantic-version: ["pydantic-v1", "pydantic-v2"]
      fail-fast: false

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-uv
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - name: Install Pydantic v1
        if: matrix.pydantic-version == 'pydantic-v1'
        run: uv pip install --system "pydantic>=1.10.0,<2.0.0"
      - name: Install Pydantic v2
        if: matrix.pydantic-version == 'pydantic-v2'
        run: uv pip install --system --pre "pydantic>=2.0.0b2,<3.0.0"
      - run: mkdir coverage
      - name: Test
        run: >
          bash scripts/test.sh -vv
          -m "(slow and (${{env.ALL_PYTEST_MARKERS}})) or (${{env.ALL_PYTEST_MARKERS}})"
        env:
          COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.pydantic-version }}
          CONTEXT: ${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.pydantic-version }}
      - name: Store coverage files
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.pydantic-version }}
          path: coverage
          if-no-files-found: error
          include-hidden-files: true

  test-macos-latest:
    if: github.event.pull_request.draft == false
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "(slow and (${{env.ALL_PYTEST_MARKERS}})) or (${{env.ALL_PYTEST_MARKERS}})"

  test-windows-latest:
    if: github.event.pull_request.draft == false
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "(slow and (${{env.ALL_PYTEST_MARKERS}})) or (${{env.ALL_PYTEST_MARKERS}})"

  test-kafka-smoke:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[kafka,test-core,cli]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "not kafka"
          tests/brokers/kafka/test_test_client.py

  test-kafka-real:
    if: github.event.pull_request.draft == false
    needs:
      - test-basic
      - test-kafka-smoke
    runs-on: ubuntu-latest
    services:
      kafka:
        image: bitnami/kafka:3.5.0
        ports:
          - 9092:9092
        env:
          KAFKA_ENABLE_KRAFT: "true"
          KAFKA_CFG_NODE_ID: "1"
          KAFKA_CFG_PROCESS_ROLES: "broker,controller"
          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
          KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
          KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092"
          KAFKA_BROKER_ID: "1"
          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
          ALLOW_PLAINTEXT_LISTENER: "true"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - run: mkdir coverage
      - name: Test
        run: >
          bash scripts/test.sh
          -m "(slow and kafka) or kafka"
        env:
          COVERAGE_FILE: coverage/.coverage.kafka-py
          CONTEXT: kafka-py
      - name: Store coverage files
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.kafka-py
          path: coverage
          if-no-files-found: error
          include-hidden-files: true

  test-confluent-smoke:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[confluent,test-core,cli]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "not confluent"
          tests/brokers/confluent/test_test_client.py

  test-confluent-real:
    if: github.event.pull_request.draft == false
    needs:
      - test-basic
      - test-confluent-smoke
    runs-on: ubuntu-latest
    services:
      kafka:
        image: bitnami/kafka:3.5.0
        ports:
          - 9092:9092
        env:
          KAFKA_ENABLE_KRAFT: "true"
          KAFKA_CFG_NODE_ID: "1"
          KAFKA_CFG_PROCESS_ROLES: "broker,controller"
          KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
          KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
          KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
          KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092"
          KAFKA_BROKER_ID: "1"
          KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
          ALLOW_PLAINTEXT_LISTENER: "true"
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - run: mkdir coverage
      - name: Test
        run: >
          bash scripts/test.sh -vv
          -m "(slow and confluent) or confluent"
        env:
          COVERAGE_FILE: coverage/.coverage.confluent-py
          CONTEXT: confluent-py
      - name: Store coverage files
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.confluent-py
          path: coverage
          if-no-files-found: error
          include-hidden-files: true

  test-rabbit-smoke:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[rabbit,test-core,cli]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "not rabbit"
          tests/brokers/rabbit/test_test_client.py

  test-rabbit-real:
    if: github.event.pull_request.draft == false
    needs:
      - test-basic
      - test-rabbit-smoke
    runs-on: ubuntu-latest
    services:
      rabbitmq:
        image: rabbitmq:alpine
        ports:
          - 5672:5672
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - run: mkdir coverage
      - name: Test
        run: >
          bash scripts/test.sh
          -m "(slow and rabbit) or rabbit"
        env:
          COVERAGE_FILE: coverage/.coverage.rabbit-py
          CONTEXT: rabbit-py
      - name: Store coverage files
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.rabbit-py
          path: coverage
          if-no-files-found: error
          include-hidden-files: true

  test-nats-smoke:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[nats,test-core,cli]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "not nats"
          tests/brokers/nats/test_test_client.py

  test-nats-real:
    if: github.event.pull_request.draft == false
    needs:
      - test-basic
      - test-nats-smoke
    runs-on: ubuntu-latest
    services:
      nats:
        image: diementros/nats:js
        ports:
          - 4222:4222
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - run: mkdir coverage
      - name: Test
        run: >
          bash scripts/test.sh
          -m "(slow and nats) or nats"
        env:
          COVERAGE_FILE: coverage/.coverage.nats-py
          CONTEXT: nats-py
      - name: Store coverage files
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.nats-py
          path: coverage
          if-no-files-found: error
          include-hidden-files: true

  test-redis-smoke:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[redis,test-core,cli]
      - name: Test
        run: >
          bash scripts/test.sh
          -m "not redis"
          tests/brokers/redis/test_test_client.py

  test-redis-real:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-latest
    needs:
      - test-basic
      - test-redis-smoke
    services:
      nats:
        image: redis:alpine
        ports:
          - 6379:6379
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install Dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip install uv
          uv pip install --system .[optionals,testing]
      - run: mkdir coverage
      - name: Test
        run: >
          bash scripts/test.sh
          -m "(slow and redis) or redis"
        env:
          COVERAGE_FILE: coverage/.coverage.redis-py
          CONTEXT: redis-py
      - name: Store coverage files
        uses: actions/upload-artifact@v4
        with:
          name: .coverage.redis-py
          path: coverage
          if-no-files-found: error
          include-hidden-files: true

  coverage-combine:
    if: github.event.pull_request.draft == false
    needs:
      - test-basic
      - test-kafka-real
      - test-confluent-real
      - test-rabbit-real
      - test-nats-real
      - test-redis-real
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"

      - name: Get coverage files
        uses: actions/download-artifact@v4
        with:
          pattern: .coverage*
          path: coverage
          merge-multiple: true

      - run: |
          python -m pip install uv
          uv pip install --system coverage[toml]

      - run: ls -la coverage
      - run: coverage combine coverage
      - run: coverage report
      - run: coverage html --show-contexts --title "FastStream coverage for ${{ github.sha }}"

      - name: Store coverage html
        uses: actions/upload-artifact@v4
        with:
          name: coverage-html
          path: htmlcov

  # https://github.com/marketplace/actions/alls-green#why
  check: # This job does nothing and is only used for the branch protection
    if: github.event.pull_request.draft == false

    needs:
      - pre-commit-check
      - coverage-combine
      - test-macos-latest
      - test-windows-latest

    runs-on: ubuntu-latest

    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1 # nosemgrep
        with:
          jobs: ${{ toJSON(needs) }}