From d4da8e25832056b6e29d773e27c0b8b7b35b918b Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Wed, 18 Sep 2024 06:28:03 +0100 Subject: [PATCH] [CI] [GHA] Drop Python 3.8 testing in GHA (#26586) ### Tickets: - *151796* --- .github/dockerfiles/docker_tag | 2 +- .../dockerfiles/ov_build/fedora_29/Dockerfile | 14 +- .../ov_build/ubuntu_20_04_x64/Dockerfile | 13 +- .../{fedora_32 => fedora_33}/Dockerfile | 2 +- .../ov_test/ubuntu_20_04_x64/Dockerfile | 11 + .github/workflows/fedora_29.yml | 6 +- .github/workflows/job_build_linux.yml | 2 +- .github/workflows/job_build_windows.yml | 2 +- .github/workflows/linux_arm64.yml | 255 ++---------------- .github/workflows/py_checks.yml | 2 +- .../send_workflows_to_opentelemetry.yml | 4 +- .github/workflows/ubuntu_20.yml | 2 +- 12 files changed, 57 insertions(+), 258 deletions(-) rename .github/dockerfiles/ov_test/{fedora_32 => fedora_33}/Dockerfile (91%) diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 61dd89f6e199f0..413b6019a3b0be 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-26588 \ No newline at end of file +pr-26586 \ No newline at end of file diff --git a/.github/dockerfiles/ov_build/fedora_29/Dockerfile b/.github/dockerfiles/ov_build/fedora_29/Dockerfile index 6d03a31937ea43..17797df62f08db 100644 --- a/.github/dockerfiles/ov_build/fedora_29/Dockerfile +++ b/.github/dockerfiles/ov_build/fedora_29/Dockerfile @@ -28,9 +28,9 @@ RUN chmod +x /install_build_dependencies.sh && \ # Setup Python RUN cd /usr/src && \ - wget https://www.python.org/ftp/python/3.8.9/Python-3.8.9.tar.xz && \ - tar xvf Python-3.8.9.tar.xz -RUN cd /usr/src/Python-3.8.9 && \ + wget https://www.python.org/ftp/python/3.9.9/Python-3.9.9.tar.xz && \ + tar xvf Python-3.9.9.tar.xz +RUN cd /usr/src/Python-3.9.9 && \ ./configure --enable-optimizations --enable-loadable-sqlite-extensions --prefix=/usr && \ make altinstall @@ -46,15 +46,15 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ ENV PATH="$SCCACHE_HOME:$PATH" -# Use Python 3.8 as default -RUN python3.8 -m venv venv +# Use Python 3.9 as default +RUN python3.9 -m venv venv ENV PATH="/venv/bin:$PATH" -RUN alternatives --install /usr/bin/python python /usr/bin/python3.8 10 +RUN alternatives --install /usr/bin/python python /usr/bin/python3.9 10 # Setup pip ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.9 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile index 3b860ac892a735..886ed15a634974 100644 --- a/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64/Dockerfile @@ -22,6 +22,10 @@ RUN apt-get update && \ git \ gpg-agent \ tzdata \ + # Pythons + python3.9-dev \ + python3.9-venv \ + python3.9-distutils \ # For Java API default-jdk \ # Compiler \ @@ -51,10 +55,17 @@ RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} ENV PATH="$SCCACHE_HOME:$PATH" + # Setup pip ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.9 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py +# Use Python 3.9 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.9 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_test/fedora_32/Dockerfile b/.github/dockerfiles/ov_test/fedora_33/Dockerfile similarity index 91% rename from .github/dockerfiles/ov_test/fedora_32/Dockerfile rename to .github/dockerfiles/ov_test/fedora_33/Dockerfile index ef0a21b8743ff4..c059c82c7d3cf2 100644 --- a/.github/dockerfiles/ov_test/fedora_32/Dockerfile +++ b/.github/dockerfiles/ov_test/fedora_33/Dockerfile @@ -1,5 +1,5 @@ ARG REGISTRY="docker.io" -FROM ${REGISTRY}/library/fedora:32 +FROM ${REGISTRY}/library/fedora:33 USER root diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile index 279e2e137b63dd..38c377724f3ffe 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -22,6 +22,10 @@ RUN apt-get update && \ git \ gpg-agent \ tzdata \ + # Python + python3.9-dev \ + python3.9-venv \ + python3.9-distutils \ libhdf5-dev \ && \ rm -rf /var/lib/apt/lists/* @@ -36,6 +40,13 @@ RUN chmod +x /install_openvino_dependencies.sh && \ ENV PIP_VERSION="24.0" RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.9 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ rm -f get-pip.py +# Use Python 3.9 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.9 -m venv venv +ENV PATH="/venv/bin:$PATH" + ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} +ENV PIP_INSTALL_PATH=/venv/lib/python3.9/site-packages diff --git a/.github/workflows/fedora_29.yml b/.github/workflows/fedora_29.yml index d76ccb8ec58b42..a79b0f86af28f3 100644 --- a/.github/workflows/fedora_29.yml +++ b/.github/workflows/fedora_29.yml @@ -1,4 +1,4 @@ -name: Fedora 29 (RHEL 8.4), Python 3.8 +name: Fedora 29 (RHEL 8.4), Python 3.9 on: workflow_dispatch: pull_request: @@ -64,7 +64,7 @@ jobs: with: images: | ov_build/fedora_29 - ov_test/fedora_32 + ov_test/fedora_33 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -104,7 +104,7 @@ jobs: shell: bash runs-on: aks-linux-4-cores-16gb container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.fedora_32 }} + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.fedora_33 }} env: RPM_PACKAGES_DIR: /__w/openvino/packages/ diff --git a/.github/workflows/job_build_linux.yml b/.github/workflows/job_build_linux.yml index 0b76af69ffb0ec..244bb0ca2ca113 100644 --- a/.github/workflows/job_build_linux.yml +++ b/.github/workflows/job_build_linux.yml @@ -121,7 +121,7 @@ jobs: ${{ env.OPENVINO_REPO }} ${{ env.OPENVINO_CONTRIB_REPO }} product_type: ${{ env.PRODUCT_TYPE }} - target_arch: 'intel64' + target_arch: ${{ inputs.arch }} build_type: 'release' save_to: ${{ env.MANIFEST_PATH }} diff --git a/.github/workflows/job_build_windows.yml b/.github/workflows/job_build_windows.yml index 0f5debf3c6c26e..81283ecfafb729 100644 --- a/.github/workflows/job_build_windows.yml +++ b/.github/workflows/job_build_windows.yml @@ -91,7 +91,7 @@ jobs: ${{ env.OPENVINO_REPO }} ${{ env.OPENVINO_CONTRIB_REPO }} product_type: ${{ env.PRODUCT_TYPE }} - target_arch: 'intel64' + target_arch: 'x86_64' build_type: ${{ inputs.build-type }} save_to: ${{ env.MANIFEST_PATH }} diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index ab9fab0313fb7e..7c0282b5519705 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -74,104 +74,22 @@ jobs: changed_components: ${{ needs.Smart_CI.outputs.changed_components }} Build: - needs: [ Smart_CI, Docker ] - timeout-minutes: 150 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }} - volumes: - - /mount:/mount - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING - env: - CMAKE_BUILD_TYPE: 'Release' - CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - SCCACHE_IGNORE_SERVER_IO_ERROR: 1 - SCCACHE_SERVER_PORT: 35555 - SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt - SCCACHE_LOG: warn - GITHUB_WORKSPACE: '/__w/openvino/openvino' - OPENVINO_REPO: /__w/openvino/openvino/openvino - OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib - INSTALL_DIR: /__w/openvino/openvino/openvino_install - INSTALL_DIR_JS: /__w/openvino/openvino/openvino_install/js - INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install - DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install - BUILD_DIR: /__w/openvino/openvino/openvino_build - SCCACHE_AZURE_KEY_PREFIX: 'ubuntu20_aarch64_Release' - ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime - ARTIFACTS_SHARE: "/mount/build-artifacts" - MANIFEST_PATH: '/__w/openvino/openvino/manifest.yml' - PRODUCT_TYPE: 'public_linux_ubuntu_20_04_arm64_release' + needs: [ Docker, Smart_CI ] if: "!needs.smart_ci.outputs.skip_workflow" - - steps: - - name: Clone OpenVINO - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - path: ${{ env.OPENVINO_REPO }} - submodules: 'true' - - - name: Clone OpenVINO Contrib - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - repository: 'openvinotoolkit/openvino_contrib' - path: ${{ env.OPENVINO_CONTRIB_REPO }} - submodules: 'true' - ref: ${{ env.TARGET_BRANCH }} - - - name: Generate product manifest and set CI_BUILD_NUMBER & CI_BUILD_DEV_TAG - id: create_manifest - uses: ./openvino/.github/actions/create_manifest - with: - repos: | - ${{ env.OPENVINO_REPO }} - ${{ env.OPENVINO_CONTRIB_REPO }} - product_type: ${{ env.PRODUCT_TYPE }} - target_arch: 'aarch64' - build_type: 'release' - save_to: ${{ env.MANIFEST_PATH }} - - # - # Print system info - # - - - name: System info - uses: ./openvino/.github/actions/system_info - - # - # Dependencies - # - - - name: Install python dependencies - run: | - # For Python API: build and wheel packaging - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - - # For running ONNX frontend unit tests - python3 -m pip install --force-reinstall -r ${OPENVINO_REPO}/src/frontends/onnx/tests/requirements.txt - - # For running TensorFlow frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow/tests/requirements.txt - - # For running TensorFlow Lite frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/tensorflow_lite/tests/requirements.txt - - # For running Paddle frontend unit tests - python3 -m pip install -r ${OPENVINO_REPO}/src/frontends/paddle/tests/requirements.txt - - # - # Build - # - - - name: CMake configure - OpenVINO - run: | - cmake \ - -G "${{ env.CMAKE_GENERATOR }}" \ + uses: ./.github/workflows/job_build_linux.yml + with: + runner: 'aks-linux-16-cores-arm' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + event-name: ${{ github.event_name }} + os: 'ubuntu_20_04' + arch: 'arm64' + build-js: true + build-contrib: true + build-debian-packages: false + build-rpm-packages: false + cmake-options: |- + -G "Ninja Multi-Config" \ -DENABLE_CPPLINT=OFF \ -DENABLE_NCC_STYLE=OFF \ -DENABLE_TESTS=ON \ @@ -179,148 +97,7 @@ jobs: -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCPACK_GENERATOR=TGZ \ - -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ - -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \ - -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ - -DOV_CPU_AARCH64_USE_MULTI_ISA=ON \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - - - name: Clean sccache stats - run: ${SCCACHE_PATH} --zero-stats - - - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats - - - name: Cmake install - OpenVINO - run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${DEVELOPER_PACKAGE_DIR} -DCOMPONENT=developer_package -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - - - name: Pack Artifacts - run: | - - # Add the ONNX Runtime version and skip tests list to the archive to use in the ONNX Runtime Job - # w/o the need to checkout repository - - cp -R ${ONNX_RUNTIME_UTILS} ${INSTALL_DIR} - - pushd ${INSTALL_DIR} - tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * - popd - - pushd ${DEVELOPER_PACKAGE_DIR} - tar -czvf ${BUILD_DIR}/openvino_developer_package.tar.gz * - popd - - pushd ${INSTALL_TEST_DIR} - tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz * - popd - - - name: Build Debian packages - if: ${{ 'false' }} - run: | - /usr/bin/python3.8 -m pip install -U pip - /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - cmake -UPYTHON* \ - -DENABLE_PYTHON_PACKAGING=ON \ - -DENABLE_TESTS=OFF \ - -DPython3_EXECUTABLE=/usr/bin/python3.8 \ - -DCPACK_GENERATOR=DEB \ - ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package - - - name: Cmake & Build - OpenVINO Contrib - run: | - cmake \ - -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \ - -DOPENVINO_EXTRA_MODULES="${OPENVINO_CONTRIB_REPO}/modules/java_api;${OPENVINO_CONTRIB_REPO}/modules/custom_operations" \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: CMake configure, build and install - OpenVINO JS API - if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - run: | - cmake \ - -DCPACK_GENERATOR=NPM \ - -DENABLE_SYSTEM_TBB=OFF -UTBB* \ - -DENABLE_INTEL_GPU=OFF \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - - cmake --build ${BUILD_DIR} --parallel - - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake - - # - # Upload build artifacts and logs - # - - name: Upload build logs - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - if: always() - with: - name: build_logs - path: ${{ env.SCCACHE_ERROR_LOG }} - if-no-files-found: 'ignore' - - - name: Upload openvino package - if: ${{ always() }} - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: openvino_package - path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz - if-no-files-found: 'error' - - - name: Upload openvino developer package - if: ${{ always() }} - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: openvino_developer_package - path: ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz - if-no-files-found: 'error' - - - name: Upload openvino js package - if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: openvino_js_package - path: ${{ env.INSTALL_DIR_JS }} - if-no-files-found: 'error' - - - name: Upload openvino debian packages - if: ${{ 'false' }} - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: openvino_debian_packages - path: ${{ env.BUILD_DIR }}/*.deb - if-no-files-found: 'error' - - - name: Upload openvino tests package - if: ${{ always() }} - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: openvino_tests - path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz - if-no-files-found: 'error' - - - name: Store artifacts to a shared drive - id: store_artifacts - if: ${{ always() }} - uses: ./openvino/.github/actions/store_artifacts - with: - artifacts: | - ${{ env.BUILD_DIR }}/openvino_package.tar.gz - ${{ env.BUILD_DIR }}/openvino_developer_package.tar.gz - ${{ env.BUILD_DIR }}/openvino_tests.tar.gz - ${{ env.MANIFEST_PATH }} - storage_dir: ${{ env.PRODUCT_TYPE }} - storage_root: ${{ env.ARTIFACTS_SHARE }} + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON Debian_Packages: name: Debian Packages diff --git a/.github/workflows/py_checks.yml b/.github/workflows/py_checks.yml index e709a233531260..db0918d0eb61c0 100644 --- a/.github/workflows/py_checks.yml +++ b/.github/workflows/py_checks.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Python uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: - python-version: '3.8' + python-version: '3.9' - name: Install dependencies run: python -m pip install -r src/bindings/python/requirements_test.txt diff --git a/.github/workflows/send_workflows_to_opentelemetry.yml b/.github/workflows/send_workflows_to_opentelemetry.yml index 3bda0df32c79a9..8f70389e645906 100644 --- a/.github/workflows/send_workflows_to_opentelemetry.yml +++ b/.github/workflows/send_workflows_to_opentelemetry.yml @@ -12,8 +12,8 @@ on: - Code coverage - Coverity (Ubuntu 20.04, Python 3.11) - Debian 10 ARM - - Fedora 29 (RHEL 8.4), Python 3.8 - - Linux (Ubuntu 20.04, Python 3.8) + - Fedora 29 (RHEL 8.4), Python 3.9 + - Linux (Ubuntu 20.04, Python 3.9) - Linux (Ubuntu 22.04, Python 3.11) - Linux (Ubuntu 24.04, Python 3.12) - Linux ARM64 (Ubuntu 20.04, Python 3.11) diff --git a/.github/workflows/ubuntu_20.yml b/.github/workflows/ubuntu_20.yml index 96a42ab197207f..5d74284b8c16fc 100644 --- a/.github/workflows/ubuntu_20.yml +++ b/.github/workflows/ubuntu_20.yml @@ -1,4 +1,4 @@ -name: Linux (Ubuntu 20.04, Python 3.8) +name: Linux (Ubuntu 20.04, Python 3.9) on: schedule: # at 00:00 on Wednesday and Saturday