main.yml

# Copyright (c) ONNX Project Contributors
#
# SPDX-License-Identifier: Apache-2.0

name: CI

on:
  schedule:
    - cron: '0 0 * * *'  # every day at midnight for reporting code coverage to codecov
  push:
    branches:
      - main
  pull_request:
  merge_group:
  workflow_dispatch:

permissions:  # set top-level default permissions as security best practice
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true

jobs:
  test:
    name: Test ${{ matrix.os }}, ${{ matrix.python_version }}, ${{ matrix.protobuf_type }}, debug=${{ matrix.debug_build }}, onnx_ml=${{ matrix.onnx_ml }}, doc=${{ matrix.documentation }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python_version: ['3.12', '3.11', '3.10', '3.9', '3.8']
        include:
          - python_version: '3.12'
            onnx_ml: 1
            debug_build: 1
            documentation: 1
            protobuf_type: 'Internal'
          - python_version: '3.11'
            onnx_ml: 1
            debug_build: 0
            documentation: 0
            protobuf_type: 'External'
          - python_version: '3.10'
            onnx_ml: 0
            debug_build: 0
            documentation: 0
            protobuf_type: 'Internal'
          - python_version: '3.9'
            onnx_ml: 1
            debug_build: 0
            documentation: 0
            protobuf_type: 'External'
          - python_version: '3.8'
            onnx_ml: 0
            debug_build: 0
            documentation: 0
            protobuf_type: 'Internal'
    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938

      - name: Set up Python
        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
        with:
          python-version: ${{ matrix.python_version }}

      - name: Show versions
        run: |
          python --version
          cmake --version

      - name: Install external protobuf - Linux
        if: matrix.protobuf_type == 'External' && matrix.os == 'ubuntu-latest'
        run: |
          sudo apt-get install libprotobuf-dev protobuf-compiler

      - name: Install external protobuf - MacOS
        if: matrix.protobuf_type == 'External' && matrix.os == 'macos-latest'
        run: |
          source workflow_scripts/protobuf/build_protobuf_unix.sh 3 $(pwd)/protobuf/protobuf_install

      - name: Set up MSBuild
        if: matrix.os == 'windows-latest'
        uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # v2.0.0
        with:
          msbuild-architecture: x64

      - name: Install external protobuf - Windows
        if: matrix.protobuf_type == 'External' && matrix.os == 'windows-latest'
        run: |
          workflow_scripts/protobuf/build_protobuf_win.ps1

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r requirements-release.txt
          python -m pip install protobuf==3.20.2
          git submodule update --init --recursive

      - name: Build and install ONNX - Linux
        if: matrix.os == 'ubuntu-latest'
        run: |
          if [ "${{ matrix.protobuf_type }}" == "External" ]; then
            export CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
          fi
          pip install -e ".[reference]" -v
        env:
          DEBUG: ${{ matrix.debug_build }}
          ONNX_ML: ${{ matrix.onnx_ml }}
          ONNX_BUILD_TESTS: 1
          CMAKE_ARGS: "-DONNX_WERROR=ON -DCMAKE_CXX_FLAGS='-fsanitize=undefined -fno-sanitize-recover=all '"

      - name: Build and install ONNX - MacOS
        if: matrix.os == 'macos-latest'
        run: |
          pip install -e ".[reference]" -v
        env:
          DEBUG: ${{ matrix.debug_build }}
          ONNX_ML: ${{ matrix.onnx_ml }}
          ONNX_BUILD_TESTS: 1
          CMAKE_ARGS: "-DONNX_WERROR=ON"

      - name: Build and install ONNX - Windows
        if: matrix.os == 'windows-latest' && matrix.python_version != '3.9' && matrix.python_version != '3.8' && matrix.python_version != '3.12'
        run: |
          python setup.py build_ext --inplace
          python setup.py install
        env:
          DEBUG: ${{ matrix.debug_build }}
          ONNX_ML: ${{ matrix.onnx_ml }}
          ONNX_BUILD_TESTS: 1
          CMAKE_ARGS: "-DONNX_WERROR=OFF -DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DONNX_USE_LITE_PROTO=ON"

      - name: Build and install ONNX - Windows
        if: matrix.os == 'windows-latest' && (matrix.python_version == '3.9' || matrix.python_version == '3.8' || matrix.python_version == '3.12')
        run: |
          pip install -e . -v
        env:
          DEBUG: ${{ matrix.debug_build }}
          ONNX_ML: ${{ matrix.onnx_ml }}
          ONNX_BUILD_TESTS: 1
          CMAKE_ARGS: "-DONNX_WERROR=OFF -DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DONNX_USE_LITE_PROTO=ON"

      - name: pip freeze
        run: |
          pip freeze

      - name: Run Python tests
        run: |
          pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junitxml junit.xml -n auto --dist loadscope

      - name: Run C++ tests
        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
        run: |
          export LD_LIBRARY_PATH="./.setuptools-cmake-build/:$LD_LIBRARY_PATH"
          ./.setuptools-cmake-build/onnx_gtests

      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      # Note that the test data should be generated with numpy>=2.0.
      # numpy 1.x and numpy 2.0 produce slightly different numerical values.
      - name: Test backend test data
        if: matrix.documentation == 1 && matrix.os == 'ubuntu-latest'
        run: |
          python onnx/backend/test/cmd_tools.py generate-data --clean
          git status
          git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
          if [ $? -ne 0 ]; then
            echo "git diff for test generation returned failures. Please check updated node test files"
            exit 1
          fi
          git diff --exit-code --diff-filter=ADR -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3'
          if [ $? -ne 0 ]; then
            echo "Test generation returned failures. Please check the number of node test files (input_*.pb or output_*.pb)"
            exit 1
          fi
          pip uninstall -y pillow
          python onnx/backend/test/cmd_tools.py generate-data --clean
          git status
          git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
          if [ $? -ne 0 ]; then
            echo "git diff for test generation without pillow returned failures. Please check updated node test files"
            exit 1
          fi

      - name: Test documentation
        if: matrix.documentation == 1
        run: |
          pip install -r docs/docsgen/source/requirements.txt
          cd docs/docsgen && make text
        continue-on-error: true

      - name: Run Python tests with numpy<2.0
        if: matrix.python_version == '3.11' || matrix.python_version == '3.12'
        run: |
          pip install "numpy<2.0" "pillow<10"
          pytest -s